diff --git a/reimplement/set9.c b/reimplement/set9.c new file mode 100644 index 0000000..3f8da85 --- /dev/null +++ b/reimplement/set9.c @@ -0,0 +1,935 @@ +#include +#include +#include +#include +#include + +#include "rpmlib.h" +#ifdef SELF_TEST +#undef NDEBUG +#include +#endif +#include "set.h" +#include "system.h" + +#define CACHE_SIZE 512 +#define PIVOT_SIZE 486 + +struct set { + size_t cnt; + size_t symbols_cap; + size_t strings_len; + size_t strings_cap; + char* strings; + struct symbols { + size_t offset; + unsigned hash; + }* symbols_v; +}; + +struct set* set_new() { + struct set* set = xmalloc(sizeof *set); + set->cnt = 0; + set->symbols_cap = 0; + set->strings_len = 0; + set->strings_cap = 0; + set->strings = NULL; + set->symbols_v = NULL; + + return set; +} + +void set_add(struct set* set, const char* sym) { + if (set->cnt == set->symbols_cap) { + set->symbols_cap += 1024; + set->symbols_v = xrealloc(set->symbols_v, sizeof(*set->symbols_v) * set->symbols_cap); + } + + size_t length = strlen(sym) + 1; + size_t required = set->strings_len + length; + if (required > set->strings_cap) { + size_t capacity = set->strings_cap ? set->strings_cap : 4096; + while (capacity < required) capacity *= 2; + + set->strings = xrealloc(set->strings, capacity); + set->strings_cap = capacity; + } + + set->symbols_v[set->cnt].offset = set->strings_len; + set->symbols_v[set->cnt].hash = 0; + memcpy(set->strings + set->strings_len, sym, length); + set->strings_len = required; + set->cnt++; + + return; +} + +struct set* set_free(struct set* set) { + if (set) { + _free(set->strings); + _free(set->symbols_v); + set = _free(set); + } + + return NULL; +} + +// --- + +static unsigned hash(const char* str) { + unsigned hash = 0x9e3779b9; + const unsigned char* p = (const unsigned char*)str; + + while (*p) { + hash += *p++; + hash += (hash << 10); + hash ^= (hash >> 6); + } + + hash += (hash << 3); + hash ^= (hash >> 11); + hash += (hash << 15); + + return hash; +} + +int cmp(const void* arg1, const void* arg2) { + const struct symbols* s1 = arg1; + const struct symbols* s2 = arg2; + + if (s1->hash > s2->hash) return 1; + if (s2->hash > s1->hash) return -1; + + return 0; +} + +static void sort_symbols(struct symbols* values, size_t count, int bpp) { + if (count < 128) { + qsort(values, count, sizeof(*values), cmp); + return; + } + + struct symbols temporary[count]; + struct symbols* source = values; + struct symbols* destination = temporary; + unsigned passes = ((unsigned)bpp + 7) / 8; + + for (unsigned pass = 0; pass < passes; ++pass) { + size_t offsets[256] = {0}; + unsigned shift = pass * 8; + for (size_t i = 0; i < count; ++i) ++offsets[(source[i].hash >> shift) & 0xffu]; + + size_t position = 0; + for (size_t i = 0; i < 256; ++i) { + size_t bucket_count = offsets[i]; + offsets[i] = position; + position += bucket_count; + } + + for (size_t i = 0; i < count; ++i) { + unsigned bucket = (source[i].hash >> shift) & 0xffu; + destination[offsets[bucket]++] = source[i]; + } + + struct symbols* swap = source; + source = destination; + destination = swap; + } + + if (source != values) memcpy(values, source, count * sizeof(*values)); + + return; +} + +// --- + +static int log2i(int n) { + int m = 0; + while (n /= 2) m++; + + return m; +} + +// Calculate Mshift paramter for encoding. +static int encode_golomb_Mshift(int cnt, int bpp) { + // XXX Slightly better Mshift estimations are probably possible. + // Recheck "Compression and coding algorithms" by Moffat & Turpin. + int Mshift = bpp - log2i(cnt) - 1; + + // Adjust out-of-range values. + Mshift = (Mshift < 7) ? 7 : Mshift; + Mshift = (Mshift > 31) ? 31 : Mshift; + assert(Mshift < bpp); + + return Mshift; +} + +// Estimate how many bits can be filled up. +static inline int encode_golomb_size(int cnt, int Mshift) { + // XXX No precise estimation. However, we do not expect unary-encoded bits + // to take more than binary-encoded Mshift bits. + return Mshift * 2 * cnt + 16; +} + +// Estimate base62 buffer size required to encode a given number of bits. +static inline int encode_base62_size(int bit_cnt) { + // In the worst case, which is ZxZxZx..., five bits can make a character; + // the remaining bits can make a character, too. And the string must be + // null-terminated. + return bit_cnt / 5 + 2; +} + +static int encode_set_size(int cnt, int bpp) { + int Mshift = encode_golomb_Mshift(cnt, bpp); + int bit_cnt = encode_golomb_size(cnt, Mshift); + // two leading characters are special + return 2 + encode_base62_size(bit_cnt); +} + +// --- + +// Main base62 encoding routine: pack bit_arr into base62 string. +/* + * Base62 routines - encode bits with alnum characters. + * + * This is a base64-based base62 implementation. Values 0..61 are encoded + * with '0'..'9', 'a'..'z', and 'A'..'Z'. However, 'Z' is special: it will + * also encode 62 and 63. To achieve this, 'Z' will occupy two high bits in + * the next character. Thus 'Z' can be interpreted as an escape character + * (which indicates that the next character must be handled specially). + * Note that setting high bits to "00", "01" or "10" cannot contribute + * to another 'Z' (which would require high bits set to "11"). This is + * how multiple escapes are avoided. + */ + +static char* bits_to_char(int c, char* base62) { + assert(c >= 0 && c <= 61); + + if (c < 10) { + *base62++ = c + '0'; + } else if (c < 36) { + *base62++ = c - 10 + 'a'; + } else if (c < 62) { + *base62++ = c - 36 + 'A'; + } + + return base62; +} + +// --- + +static inline char encode_bpp(int bpp) { return bpp - 7 + 'a'; } + +struct encode_writer { + uint64_t bits; + unsigned filled; + unsigned escaped; + unsigned pending_high; + char* output; +}; + +static inline void encode_writer_digit(struct encode_writer* writer, unsigned value) { + assert(value < 62); + + if (value < 10) { + *writer->output++ = (char)('0' + value); + } else if (value < 36) { + *writer->output++ = (char)('a' + value - 10); + } else { + *writer->output++ = (char)('A' + value - 36); + } +} + +static inline void encode_writer_flush(struct encode_writer* writer) { + for (;;) { + unsigned width = writer->escaped ? 4u : 6u; + if (writer->filled < width) return; + + unsigned value = (unsigned)writer->bits & ((1u << width) - 1); + writer->bits >>= width; + writer->filled -= width; + + if (writer->escaped) { + encode_writer_digit(writer, writer->pending_high | value); + writer->escaped = 0; + } else if (value >= 61) { + encode_writer_digit(writer, 61); + writer->pending_high = (value - 61) << 4; + writer->escaped = 1; + } else { + encode_writer_digit(writer, value); + } + } +} + +static inline void encode_writer_zeros(struct encode_writer* writer, unsigned count) { + while (count) { + unsigned take = count > 56 ? 56 : count; + writer->filled += take; + count -= take; + + encode_writer_flush(writer); + } + + return; +} + +static inline void encode_writer_put(struct encode_writer* writer, uint64_t value, unsigned width) { + writer->bits |= value << writer->filled; + writer->filled += width; + encode_writer_flush(writer); + + return; +} + +static int encode_set(int cnt, const unsigned* hash_arr, int bpp, char* base62_str) { + const unsigned Mshift = (unsigned)encode_golomb_Mshift(cnt, bpp); + const unsigned mask = (1u << Mshift) - 1; + char* const start = base62_str; + unsigned previous = 0; + + *base62_str++ = encode_bpp(bpp); + *base62_str++ = encode_bpp((int)Mshift); + struct encode_writer writer = {.output = base62_str}; + + for (int i = 0; i < cnt; ++i) { + unsigned current = hash_arr[i]; + unsigned delta = current - previous; + previous = current; + + encode_writer_zeros(&writer, delta >> Mshift); + encode_writer_put(&writer, 1, 1); + encode_writer_put(&writer, delta & mask, Mshift); + } + + encode_writer_flush(&writer); + if (writer.filled || writer.escaped) { + unsigned value = (unsigned)writer.bits; + if (writer.escaped) value |= writer.pending_high; + encode_writer_digit(&writer, value); + } + + *writer.output = '\0'; + + return (int)(writer.output - start); +} + +const char* set_fini(struct set* set, int bpp) { + // Implementation for finalizing the set + + assert(set != NULL); + assert(set->cnt > 0); + assert(bpp >= 10 && bpp <= 32); + + unsigned mask = (bpp < 32) ? (1u << bpp) - 1 : ~0u; + + for (size_t i = 0; i < set->cnt; ++i) { + set->symbols_v[i].hash = hash(set->strings + set->symbols_v[i].offset) & mask; + } + + sort_symbols(set->symbols_v, set->cnt, bpp); + + // warn on hash collizions + for (size_t i = 0; i < set->cnt - 1; ++i) { + if (set->symbols_v[i].hash != set->symbols_v[i + 1].hash) continue; + const char* left = set->strings + set->symbols_v[i].offset; + const char* right = set->strings + set->symbols_v[i + 1].offset; + if (!strcmp(left, right)) continue; + + fprintf(stderr, "warning: hash collision: %s %s\n", left, right); + } + + unsigned unique_hash[set->cnt]; + size_t unique_cnt = 0; + + // delete duplicates + for (size_t i = 0; i < set->cnt; ++i) { + while (i + 1 < set->cnt && set->symbols_v[i].hash == set->symbols_v[i + 1].hash) { + ++i; + } + unique_hash[unique_cnt++] = set->symbols_v[i].hash; + } + + char base62_str[encode_set_size(unique_cnt, bpp)]; + encode_set(unique_cnt, unique_hash, bpp, base62_str); + + return xstrdup(base62_str); +} + +// --- + +struct set_meta { + const char* str; + const char* payload; + size_t len; + size_t payload_len; + int bpp; + int Mshift; + int bit_capacity; + int value_capacity; +}; + +static int set_meta_init(const char* str, struct set_meta* meta) { + // len >= 3 + if (!str[0] || !str[1] || !str[2]) return -4; + + int bpp = str[0] + 7 - 'a'; + if (bpp < 10 || bpp > 32) return -1; + + int Mshift = str[1] + 7 - 'a'; + if (Mshift < 7 || Mshift > 31) return -2; + if (Mshift >= bpp) return -3; + + *meta = (struct set_meta){ + .str = str, + .payload = str + 2, + .bpp = bpp, + .Mshift = Mshift, + }; + + return 0; +} + +static int set_meta_fini(struct set_meta* meta) { + size_t len = strlen(meta->str); + + size_t payload_len = len - 2; + + int bit_capacity = (int)payload_len * 6; + int value_capacity = bit_capacity / (meta->Mshift + 1); + + if (value_capacity < 1) return -4; + + meta->len = len; + meta->payload_len = payload_len; + meta->bit_capacity = bit_capacity; + meta->value_capacity = value_capacity; + + return 0; +} +// UCHAR_MAX == 255 +static const unsigned char char_to_num[255 + 1] = {[0] = 0xff, /* конец строки */ + + [1 ...('0' - 1)] = 0xee, + + ['0'] = 0, + ['1'] = 1, + ['2'] = 2, + ['3'] = 3, + ['4'] = 4, + ['5'] = 5, + ['6'] = 6, + ['7'] = 7, + ['8'] = 8, + ['9'] = 9, + + [('9' + 1)...('A' - 1)] = 0xee, + + ['A'] = 36, + ['B'] = 37, + ['C'] = 38, + ['D'] = 39, + ['E'] = 40, + ['F'] = 41, + ['G'] = 42, + ['H'] = 43, + ['I'] = 44, + ['J'] = 45, + ['K'] = 46, + ['L'] = 47, + ['M'] = 48, + ['N'] = 49, + ['O'] = 50, + ['P'] = 51, + ['Q'] = 52, + ['R'] = 53, + ['S'] = 54, + ['T'] = 55, + ['U'] = 56, + ['V'] = 57, + ['W'] = 58, + ['X'] = 59, + ['Y'] = 60, + ['Z'] = 61, + + [('Z' + 1)...('a' - 1)] = 0xee, + + ['a'] = 10, + ['b'] = 11, + ['c'] = 12, + ['d'] = 13, + ['e'] = 14, + ['f'] = 15, + ['g'] = 16, + ['h'] = 17, + ['i'] = 18, + ['j'] = 19, + ['k'] = 20, + ['l'] = 21, + ['m'] = 22, + ['n'] = 23, + ['o'] = 24, + ['p'] = 25, + ['q'] = 26, + ['r'] = 27, + ['s'] = 28, + ['t'] = 29, + ['u'] = 30, + ['v'] = 31, + ['w'] = 32, + ['x'] = 33, + ['y'] = 34, + ['z'] = 35, + + [('z' + 1)... 255] = 0xee}; + +static char* put6bits(int c, char* bit_pt) { + *bit_pt++ = (c >> 0) & 1; + *bit_pt++ = (c >> 1) & 1; + *bit_pt++ = (c >> 2) & 1; + *bit_pt++ = (c >> 3) & 1; + *bit_pt++ = (c >> 4) & 1; + *bit_pt++ = (c >> 5) & 1; + + return bit_pt; +} + +static char* put4bits(int c, char* bit_pt) { + *bit_pt++ = (c >> 0) & 1; + *bit_pt++ = (c >> 1) & 1; + *bit_pt++ = (c >> 2) & 1; + *bit_pt++ = (c >> 3) & 1; + + return bit_pt; +} + +// Decode base62 and Golomb-Rice in one pass. Base62 is LSB-first; a Z escape contributes 10 stream +// bits. +static inline int decode_chunk(const unsigned char** input, uint64_t* chunk, unsigned* width) { + unsigned value = char_to_num[*(*input)++]; + + if (value < 61) { + *chunk = value; + *width = 6; + return 1; + } + if (value == 0xff) return 0; + if (value == 0xee) return -1; + + unsigned escaped = char_to_num[*(*input)++]; + if (escaped == 0xff) return -2; + if (escaped == 0xee) return -3; + + unsigned high = escaped & 0x30u; + if (high == 0x30u) return -4; + + *chunk = (61u + (high >> 4)) | ((uint64_t)(escaped & 0x0fu) << 6); + *width = 10; + + return 1; +} + +static int decode_set(const struct set_meta* meta, unsigned* hash_arr) { + const unsigned char* input = (const unsigned char*)meta->payload; + const unsigned Mshift = (unsigned)meta->Mshift; + const uint64_t mask = (UINT64_C(1) << Mshift) - 1; + uint64_t bits = 0; + unsigned filled = 0; + unsigned q = 0; + unsigned previous = 0; + int count = 0; + + for (;;) { + // Unary quotient: zero bits terminated by one. + for (;;) { + if (filled == 0) { + uint64_t chunk; + unsigned width; + int rc = decode_chunk(&input, &chunk, &width); + + if (rc < 0) return rc; + if (rc == 0) return q <= 5 ? count : -10; + + bits = chunk; + filled = width; + } + + if (bits == 0) { + q += filled; + filled = 0; + continue; + } + + unsigned zeros = (unsigned)__builtin_ctzll(bits); + if (zeros >= filled) { + q += filled; + bits = 0; + filled = 0; + continue; + } + + q += zeros; + bits >>= zeros + 1; + filled -= zeros + 1; + break; + } + + // Fixed-width remainder. At most 31+10 bits are held at once. + while (filled < Mshift) { + uint64_t chunk; + unsigned width; + int rc = decode_chunk(&input, &chunk, &width); + if (rc < 0) return rc; + if (rc == 0) return -11; + bits |= chunk << filled; + filled += width; + } + + unsigned delta = (q << Mshift) | (unsigned)(bits & mask); + bits >>= Mshift; + filled -= Mshift; + q = 0; + + previous += delta; + hash_arr[count++] = previous; + } +} + +// Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates. +static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp); + +static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigned** hash_pt, + unsigned cache_id) { + enum { CACHE_BUCKETS = 1024 }; + struct cache_ent { + struct cache_ent* bucket_next; + struct cache_ent* newer; + struct cache_ent* older; + char* str; + unsigned* hash_arr; + uint32_t fingerprint; + int len; + int cnt; + int target_bpp; + }; + + static unsigned cache_count[2]; + static struct cache_ent* buckets[2][CACHE_BUCKETS]; + static struct cache_ent* newest[2]; + static struct cache_ent* oldest[2]; + + assert(cache_id < 2); + + const unsigned char* str = (const unsigned char*)meta->str; + uint32_t fp = (uint32_t)str[0] | ((uint32_t)str[2] << 8) | ((uint32_t)str[3] << 16); + uint32_t mixed = fp ^ ((uint32_t)target_bpp * UINT32_C(0x85ebca6b)); + mixed ^= mixed >> 11; + mixed *= UINT32_C(0x9e3779b1); + mixed ^= mixed >> 16; + unsigned bucket = mixed & (CACHE_BUCKETS - 1); + + for (struct cache_ent* ent = buckets[cache_id][bucket]; ent; ent = ent->bucket_next) { + if (ent->fingerprint != fp || ent->target_bpp != target_bpp || strcmp(meta->str, ent->str) != 0) + continue; + + if (ent != newest[cache_id]) { + if (ent->newer) ent->newer->older = ent->older; + if (ent->older) ent->older->newer = ent->newer; + if (ent == oldest[cache_id]) oldest[cache_id] = ent->newer; + + ent->newer = NULL; + ent->older = newest[cache_id]; + newest[cache_id]->newer = ent; + newest[cache_id] = ent; + } + + *hash_pt = ent->hash_arr; + + return ent->cnt; + } + + if (set_meta_fini(meta) < 0) return -4; + + int len = (int)meta->len; + int capacity = meta->value_capacity; + struct cache_ent* ent = xmalloc(sizeof(*ent) + (size_t)(capacity) * sizeof(unsigned) + len + 1); + ent->hash_arr = (unsigned*)(ent + 1); + ent->str = (char*)(ent->hash_arr + capacity); + + int cnt = decode_set(meta, ent->hash_arr); + if (cnt <= 0) { + _free(ent); + return cnt; + } + + if (target_bpp < meta->bpp) { + unsigned temporary[capacity]; + unsigned* current = ent->hash_arr; + unsigned* destination = temporary; + + for (int bpp = meta->bpp - 1; bpp >= target_bpp; --bpp) { + cnt = downsample_set(cnt, current, destination, bpp); + unsigned* swap = current; + current = destination; + destination = swap; + } + + if (current != ent->hash_arr) { + memcpy(ent->hash_arr, current, (size_t)cnt * sizeof(*current)); + } + } + + memcpy(ent->str, meta->str, (size_t)len + 1); + ent->fingerprint = fp; + ent->len = len; + ent->cnt = cnt; + ent->target_bpp = target_bpp; + + if (cache_count[cache_id] == CACHE_SIZE) { + struct cache_ent* victim = oldest[cache_id]; + oldest[cache_id] = victim->newer; + if (oldest[cache_id]) oldest[cache_id]->older = NULL; + if (victim == newest[cache_id]) newest[cache_id] = NULL; + + uint32_t victim_mixed = + victim->fingerprint ^ ((uint32_t)victim->target_bpp * UINT32_C(0x85ebca6b)); + victim_mixed ^= victim_mixed >> 11; + victim_mixed *= UINT32_C(0x9e3779b1); + victim_mixed ^= victim_mixed >> 16; + unsigned victim_bucket = victim_mixed & (CACHE_BUCKETS - 1); + struct cache_ent** link = &buckets[cache_id][victim_bucket]; + while (*link != victim) link = &(*link)->bucket_next; + *link = victim->bucket_next; + _free(victim); + } else { + ++cache_count[cache_id]; + } + + ent->bucket_next = buckets[cache_id][bucket]; + buckets[cache_id][bucket] = ent; + ent->newer = NULL; + ent->older = newest[cache_id]; + if (newest[cache_id]) { + newest[cache_id]->newer = ent; + } else { + oldest[cache_id] = ent; + } + newest[cache_id] = ent; + + *hash_pt = ent->hash_arr; + + return cnt; +} + +// Reduce a set of (bpp + 1) values to a set of bpp values. +static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp) { + unsigned mask = (1u << bpp) - 1; + + // find the first element with high bit set + int l = 0; + int u = cnt; + while (l < u) { + int i = (l + u) / 2; + + if (hash_pt[i] <= mask) { + l = i + 1; + } else { + u = i; + } + } + + // initialize parts + const unsigned* ds_start = ds_pt; + const unsigned *v1 = hash_pt + 0, *v1_end = hash_pt + u; + const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + cnt; + + // merge v1 and v2 into w + if (v1 < v1_end && v2 < v2_end) { + unsigned v1_val = *v1; + unsigned v2_val = *v2 & mask; + + while (1) { + if (v1_val < v2_val) { + *ds_pt++ = v1_val; + v1++; + + if (v1 == v1_end) break; + + v1_val = *v1; + } else if (v2_val < v1_val) { + *ds_pt++ = v2_val; + v2++; + + if (v2 == v2_end) break; + + v2_val = *v2 & mask; + } else { + *ds_pt++ = v1_val; + v1++; + v2++; + + if (v1 == v1_end) break; + if (v2 == v2_end) break; + + v1_val = *v1; + v2_val = *v2 & mask; + } + } + } + + // append what's left + while (v1 < v1_end) *ds_pt++ = *v1++; + while (v2 < v2_end) *ds_pt++ = *v2++ & mask; + + return ds_pt - ds_start; +} + +static const unsigned* step_lower_bound(const unsigned* first, const unsigned* last, unsigned value, + size_t jump) { + const size_t count = (size_t)(last - first); + + if (count == 0 || first[0] >= value) { + return first; + } + + if (jump == 0) { + jump = 1; + } + + size_t position = 0; + size_t step = jump; + + while (step != 0) { + if (step > count - position - 1) { + step /= 2; + continue; + } + + const size_t next = position + step; + + if (first[next] < value) { + position = next; + } else { + step /= 2; + } + } + + return first + position + 1; +} + +static int sorted_subset(const unsigned* small, size_t small_count, const unsigned* large, + size_t large_count) { + const unsigned* const small_end = small + small_count; + const unsigned* const large_end = large + large_count; + size_t jump = large_count / small_count; + + // Dense sets favor a conventional merge; sparse sets skip by approximately + // the mean distance between required values and then refine the last block. + if (jump < 4) { + while (small < small_end) { + unsigned value = *small++; + while (large < large_end && *large < value) ++large; + if (large == large_end || *large != value) return 0; + ++large; + } + + return 1; + } + + while (small < small_end) { + unsigned value = *small++; + large = step_lower_bound(large, large_end, value, jump); + if (large == large_end || *large != value) return 0; + ++large; + } + + return 1; +} + +// main API routine +int rpmsetcmp(const char* str1, const char* str2) { + if (strncmp(str1, "set:", 4) == 0) str1 += 4; + if (strncmp(str2, "set:", 4) == 0) str2 += 4; + + struct set_meta meta1; + struct set_meta meta2; + + if (set_meta_init(str1, &meta1) < 0) return -3; + if (set_meta_init(str2, &meta2) < 0) return -4; + + int target_bpp = meta1.bpp < meta2.bpp ? meta1.bpp : meta2.bpp; + + // Decode and cache the first operand at the comparison precision. + const unsigned* hash_arr1 = NULL; + int cnt1 = cache_decode_set(&meta1, target_bpp, &hash_arr1, 0); + if (cnt1 < 0) return -3; + + // Metadata for both operands has already been validated, and set1 has been + // decoded, so this preserves set8's malformed-input error precedence. + if (str1 == str2 || strcmp(str1, str2) == 0) return 0; + + // Requirement sets are frequently reused by dependency solvers too. + const unsigned* hash_arr2 = NULL; + int cnt2 = cache_decode_set(&meta2, target_bpp, &hash_arr2, 1); + if (cnt2 < 0) return -4; + + // Cardinality determines which strict-inclusion result is even possible. + // For equal cardinalities, sorted unique sets are equal iff their bytes match. + if (cnt1 == cnt2) { + return memcmp(hash_arr1, hash_arr2, (size_t)cnt1 * sizeof(*hash_arr1)) == 0 ? 0 : -2; + } + if (cnt1 > cnt2) { + return sorted_subset(hash_arr2, (size_t)cnt2, hash_arr1, (size_t)cnt1) ? 1 : -2; + } + return sorted_subset(hash_arr1, (size_t)cnt1, hash_arr2, (size_t)cnt2) ? -1 : -2; +} + +// --- + +#ifdef SELF_TEST +int main(void) { + struct set* set1 = set_new(); + set_add(set1, "mama"); + set_add(set1, "myla"); + set_add(set1, "ramu"); + const char* str10 = set_fini(set1, 16); + fprintf(stderr, "set10=%s\n", str10); + + int cmp; + struct set* set2 = set_new(); + set_add(set2, "myla"); + set_add(set2, "mama"); + const char* str20 = set_fini(set2, 16); + fprintf(stderr, "set20=%s\n", str20); + cmp = rpmsetcmp(str10, str20); + assert(cmp == 1); + + set_add(set2, "ramu"); + const char* str21 = set_fini(set2, 16); + fprintf(stderr, "set21=%s\n", str21); + cmp = rpmsetcmp(str10, str21); + assert(cmp == 0); + + set_add(set2, "baba"); + const char* str22 = set_fini(set2, 16); + cmp = rpmsetcmp(str10, str22); + assert(cmp == -1); + + set_add(set1, "deda"); + const char* str11 = set_fini(set1, 16); + cmp = rpmsetcmp(str11, str22); + assert(cmp == -2); + + set1 = set_free(set1); + set2 = set_free(set2); + str10 = _free(str10); + str11 = _free(str11); + str20 = _free(str20); + str21 = _free(str21); + str22 = _free(str22); + + fprintf(stderr, "%s: api test OK\n", __FILE__); + + return 0; +} +#endif diff --git a/scripts/old/check_alt_set_impl.py b/scripts/old/check_alt_set_impl.py deleted file mode 100755 index 8f2b91a..0000000 --- a/scripts/old/check_alt_set_impl.py +++ /dev/null @@ -1,429 +0,0 @@ -#!/usr/bin/env python3 -"""Check package compatibility using this repo's Python set implementation. - -This script intentionally does *not* compare with ALT's existing set.c-produced -set strings. It uses ALT only as a source of real binary RPMs: - -* Provided labels: `nm --dynamic -j -U ` -* Required labels: `nm --dynamic -j -u ` - -Both sides are encoded with `reimplement/set.py`, then compared with that same -implementation's `rpmsetcmp()`. In other words, it checks whether the current -Python implementation is internally useful for real ALT package symbol labels. -""" - -from __future__ import annotations - -import argparse -import contextlib -import json -import re -import shutil -import subprocess -import sys -import tempfile -import urllib.parse -import urllib.request -from dataclasses import dataclass -from pathlib import Path -from typing import Iterable - -REPO_ROOT = Path(__file__).resolve().parents[1] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - -from reimplement import set as rpmset # noqa: E402 - -API_BASE = "https://rdb.altlinux.org/api" -DEFAULT_PROVIDERS = ["glibc-core", "zlib", "libssl3", "libcrypto3"] -DEFAULT_REQUIRERS = ["coreutils", "curl", "openssl"] - - -@dataclass(frozen=True) -class PackageRPM: - name: str - pkghash: str - rpm_path: Path - extract_dir: Path - members: list[str] - - -@dataclass(frozen=True) -class LabelSet: - role: str - package: str - member: str - labels: tuple[str, ...] - set_string: str - - @property - def label_count(self) -> int: - return len(self.labels) - - @property - def set_len(self) -> int: - return len(self.set_string) - - -@dataclass(frozen=True) -class CompatibilityResult: - provider_package: str - provider_member: str - requirer_package: str - requirer_member: str - provider_labels: int - required_labels: int - cmp_result: int - status: str - - -def api_json(path: str, params: dict[str, object] | None = None) -> dict: - url = API_BASE + path - if params: - url += "?" + urllib.parse.urlencode(params, doseq=True) - req = urllib.request.Request(url, headers={"User-Agent": "arsv-alt-set-compat/1.0"}) - with urllib.request.urlopen(req, timeout=60) as response: - return json.load(response) - - -def get_pkghash(package: str, branch: str, arch: str) -> str: - data = api_json( - "/site/pkghash_by_binary_name", - {"branch": branch, "name": package, "arch": arch}, - ) - return str(data["pkghash"]) - - -def package_download_url(pkghash: str, branch: str, arch: str) -> str: - data = api_json( - f"/site/package_downloads_bin/{pkghash}", - {"branch": branch, "arch": arch}, - ) - downloads = data.get("downloads") or [] - if not downloads or not downloads[0].get("packages"): - raise RuntimeError(f"no download URL for pkghash={pkghash}") - return downloads[0]["packages"][0]["url"] - - -def run_text(command: list[str], cwd: Path | None = None, check: bool = True) -> str: - proc = subprocess.run(command, cwd=cwd, text=True, capture_output=True) - if check and proc.returncode != 0: - raise RuntimeError(f"{' '.join(command)} failed with {proc.returncode}: {proc.stderr.strip()}") - return proc.stdout - - -def download_file(url: str, destination: Path) -> None: - req = urllib.request.Request(url, headers={"User-Agent": "arsv-alt-set-compat/1.0"}) - with urllib.request.urlopen(req, timeout=120) as response, destination.open("wb") as out: - shutil.copyfileobj(response, out) - - -def rpm_members(rpm_path: Path) -> list[str]: - return [line for line in run_text(["bsdtar", "-tf", str(rpm_path)]).splitlines() if line] - - -def is_shared_library_member(member: str) -> bool: - name = Path(member).name - return not member.endswith("/") and re.search(r"(?:^|/)lib[^/]*\.so(?:\.|$)", member) is not None and ".debug" not in name - - -def select_provider_members(members: Iterable[str]) -> list[str]: - """Files whose defined dynamic symbols are Provided labels.""" - return [member for member in members if is_shared_library_member(member)] - - -def select_requirer_members(members: Iterable[str]) -> list[str]: - """Files whose undefined dynamic symbols are Required labels.""" - executable_prefixes = ("./bin/", "./usr/bin/", "./sbin/", "./usr/sbin/", "./usr/lib/systemd/") - selected = [] - for member in members: - if member.endswith("/"): - continue - if is_shared_library_member(member) or member.startswith(executable_prefixes): - selected.append(member) - return selected - - -def extract_members(rpm_path: Path, extract_dir: Path, members: Iterable[str]) -> None: - unique_members = sorted(set(members)) - if unique_members: - run_text(["bsdtar", "-xf", str(rpm_path), "-C", str(extract_dir), *unique_members]) - - -def nm_symbols(path: Path, mode: str) -> list[str]: - command = ["nm", "--dynamic", "-j", "-U", str(path)] if mode == "provided" else ["nm", "--dynamic", "-u", str(path)] - proc = subprocess.run(command, text=True, capture_output=True) - if proc.returncode != 0: - return [] - if mode == "required": - return parse_required_nm_output(proc.stdout) - return [line.strip() for line in proc.stdout.splitlines() if line.strip()] - - -def parse_required_nm_output(output: str) -> list[str]: - """Parse `nm --dynamic -u` output, ignoring weak undefined references. - - Plain `nm -j -u` discards the symbol type, but real ALT RPMs contain weak - undefined hooks like `__gmon_start__` and `_ITM_*`. Those are optional ELF - references, not hard Required labels, so keep only strong `U` entries. - """ - symbols = [] - for line in output.splitlines(): - parts = line.split() - if len(parts) < 2: - continue - symbol_type, symbol = parts[-2], parts[-1] - if symbol_type == "U": - symbols.append(symbol) - return symbols - - -def normalize_required_symbol(symbol: str) -> str: - """Normalize `foo@VER` from undefined nm output to provider-like `foo@@VER`. - - `nm -u` prints required versioned symbols with a single `@`, while defined - default-version symbols commonly use `@@`. This normalization is for this - script's compatibility model only; it is not a set.c compatibility shim. - """ - if "@@" in symbol or "@" not in symbol: - return symbol - name, version = symbol.split("@", 1) - if not name or not version: - return symbol - return f"{name}@@{version}" - - -def labels_to_set_string(labels: Iterable[str], bpp: int) -> str | None: - item_set = rpmset.set_new() - for label in labels: - rpmset.set_add(item_set, label) - # reimplement/set.py prints collision warnings to stdout; keep this script's - # stdout machine-readable and route those warnings to stderr instead. - with contextlib.redirect_stdout(sys.stderr): - return rpmset.set_fini(item_set, bpp) - - -def generate_label_set(role: str, member: str, labels: Iterable[str], bpp: int, package: str = "") -> LabelSet: - unique_labels = tuple(sorted(set(label for label in labels if label))) - set_string = labels_to_set_string(unique_labels, bpp) - if set_string is None: - raise ValueError(f"no labels for {role} {package}:{member}") - return LabelSet(role=role, package=package, member=member, labels=unique_labels, set_string=set_string) - - -def compare_status(cmp_result: int) -> str: - # provider is first argument; compatible means provider is equal or superset. - if cmp_result in (0, 1): - return "compatible" - return "incompatible" - - -def compare_label_sets(provider: LabelSet, requirer: LabelSet) -> CompatibilityResult: - cmp_result = rpmset.rpmsetcmp(provider.set_string, requirer.set_string) - return CompatibilityResult( - provider_package=provider.package, - provider_member=provider.member, - requirer_package=requirer.package, - requirer_member=requirer.member, - provider_labels=provider.label_count, - required_labels=requirer.label_count, - cmp_result=cmp_result, - status=compare_status(cmp_result), - ) - - -def build_dependency_results(provider_sets: list[LabelSet], requirer_sets: list[LabelSet], bpp: int) -> list[CompatibilityResult]: - """Compare each library only with the symbols actually required from it. - - A package executable/library has one undefined-symbol list containing symbols - required from all of its DT_NEEDED libraries. Comparing that whole list with - one provider library gives false ``-2`` results: e.g. ``/usr/bin/curl`` needs - symbols from libc, libssl, libcrypto, zlib, etc., and no single library is - supposed to provide all of them. - - For this local set.py check, keep the symbol-level ground truth around and - split every requirer's labels by provider library: provider labels ∩ required - labels. Each non-empty subset is then encoded as the requirement for exactly - that provider and compared with ``rpmsetcmp(provider, required_subset)``. - """ - results: list[CompatibilityResult] = [] - for requirer in requirer_sets: - required_labels = set(requirer.labels) - for provider in provider_sets: - required_from_provider = sorted(required_labels.intersection(provider.labels)) - if not required_from_provider: - continue - split_requirer = generate_label_set( - "required", - requirer.member, - required_from_provider, - bpp, - package=requirer.package, - ) - results.append(compare_label_sets(provider, split_requirer)) - return results - - -def fetch_package_rpm(package: str, branch: str, arch: str, workdir: Path) -> PackageRPM: - pkghash = get_pkghash(package, branch, arch) - rpm_url = package_download_url(pkghash, branch, arch) - rpm_path = workdir / Path(urllib.parse.urlparse(rpm_url).path).name - if not rpm_path.exists(): - download_file(rpm_url, rpm_path) - members = rpm_members(rpm_path) - extract_dir = workdir / f"extract-{package.replace('/', '_').replace('+', '_')}" - extract_dir.mkdir(parents=True, exist_ok=True) - return PackageRPM(package, pkghash, rpm_path, extract_dir, members) - - -def build_provider_sets(package_rpm: PackageRPM, bpp: int, max_files: int) -> list[LabelSet]: - members = select_provider_members(package_rpm.members)[:max_files] - extract_members(package_rpm.rpm_path, package_rpm.extract_dir, members) - sets = [] - for member in members: - labels = nm_symbols(package_rpm.extract_dir / member, "provided") - if labels: - sets.append(generate_label_set("provided", member, labels, bpp, package_rpm.name)) - return sets - - -def build_requirer_sets( - package_rpm: PackageRPM, - bpp: int, - max_files: int, - normalize_versions: bool, -) -> list[LabelSet]: - members = select_requirer_members(package_rpm.members)[:max_files] - extract_members(package_rpm.rpm_path, package_rpm.extract_dir, members) - sets = [] - for member in members: - labels = nm_symbols(package_rpm.extract_dir / member, "required") - if normalize_versions: - labels = [normalize_required_symbol(label) for label in labels] - if labels: - sets.append(generate_label_set("required", member, labels, bpp, package_rpm.name)) - return sets - - -def aggregate_label_sets(role: str, package_names: list[str], label_sets: list[LabelSet], bpp: int) -> LabelSet: - labels = [label for label_set in label_sets for label in label_set.labels] - return generate_label_set(role, "+".join(package_names), labels, bpp, package="aggregate") - - -def print_label_sets(title: str, label_sets: list[LabelSet]) -> None: - print(f"\n# {title}") - print("role\tpackage\tmember\tlabels\tset_len\tset_prefix") - for label_set in label_sets: - print( - f"{label_set.role}\t{label_set.package}\t{label_set.member}\t" - f"{label_set.label_count}\t{label_set.set_len}\t{label_set.set_string[:48]}" - ) - - -def print_results(results: list[CompatibilityResult]) -> None: - print("\n# compatibility") - print("status\tcmp\tprovider_pkg\tprovider_member\tprovider_labels\trequirer_pkg\trequirer_member\trequired_labels") - for result in results: - print( - f"{result.status}\t{result.cmp_result}\t{result.provider_package}\t{result.provider_member}\t" - f"{result.provider_labels}\t{result.requirer_package}\t{result.requirer_member}\t{result.required_labels}" - ) - summary = {status: sum(1 for result in results if result.status == status) for status in sorted({r.status for r in results})} - print(f"summary: {summary}") - - -def parse_package_list(values: list[str] | None) -> list[str]: - packages = [] - for value in values or []: - packages.extend(part for part in value.split(",") if part) - return packages - - -def parse_args(argv: list[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Generate Provided/Required set strings from ALT RPM labels with reimplement/set.py and compare them." - ) - parser.add_argument("packages", nargs="*", help="packages used as both providers and requirers if explicit lists are omitted") - parser.add_argument("--provider", action="append", help="provider package; can be repeated or comma-separated") - parser.add_argument("--requirer", action="append", help="requirer package; can be repeated or comma-separated") - parser.add_argument("--branch", default="sisyphus", help="ALT repository branch/packageset") - parser.add_argument("--arch", default="x86_64", help="binary package architecture") - parser.add_argument("--bpp", type=int, default=32, help="bits per hash used by local set.py") - parser.add_argument("--max-provider-files", type=int, default=64, help="max provider ELF files per package") - parser.add_argument("--max-requirer-files", type=int, default=64, help="max requirer ELF files per package") - parser.add_argument("--all-pairs", action="store_true", help="compare every provider file set with every requirer file set") - parser.add_argument( - "--no-normalize-required-version", - action="store_true", - help="keep nm -u single-@ required symbols unchanged instead of converting foo@VER to foo@@VER", - ) - parser.add_argument("--keep-workdir", action="store_true", help="keep downloaded RPMs/extracted files") - parser.add_argument("--workdir", type=Path, help="directory for downloads/extraction") - return parser.parse_args(argv) - - -def main(argv: list[str]) -> int: - args = parse_args(argv) - missing = [cmd for cmd in ("bsdtar", "nm") if shutil.which(cmd) is None] - if missing: - print(f"missing required command(s): {', '.join(missing)}", file=sys.stderr) - return 2 - - positional = args.packages or [] - providers = parse_package_list(args.provider) or positional or DEFAULT_PROVIDERS - requirers = parse_package_list(args.requirer) or positional or DEFAULT_REQUIRERS - - cleanup = False - if args.workdir: - workdir = args.workdir - workdir.mkdir(parents=True, exist_ok=True) - else: - workdir = Path(tempfile.mkdtemp(prefix="arsv-alt-set-compat-")) - cleanup = not args.keep_workdir - - try: - provider_sets: list[LabelSet] = [] - requirer_sets: list[LabelSet] = [] - for package in providers: - provider_sets.extend( - build_provider_sets(fetch_package_rpm(package, args.branch, args.arch, workdir), args.bpp, args.max_provider_files) - ) - for package in requirers: - requirer_sets.extend( - build_requirer_sets( - fetch_package_rpm(package, args.branch, args.arch, workdir), - args.bpp, - args.max_requirer_files, - normalize_versions=not args.no_normalize_required_version, - ) - ) - - print(f"branch: {args.branch}") - print(f"arch: {args.arch}") - print(f"bpp: {args.bpp}") - print(f"providers: {', '.join(providers)}") - print(f"requirers: {', '.join(requirers)}") - print(f"required symbol version normalization: {not args.no_normalize_required_version}") - print_label_sets("generated Provided sets", provider_sets) - print_label_sets("generated Required sets", requirer_sets) - - if not provider_sets or not requirer_sets: - print("\nNo comparable sets generated.", file=sys.stderr) - return 1 - - if args.all_pairs: - results = [compare_label_sets(provider, requirer) for provider in provider_sets for requirer in requirer_sets] - else: - results = build_dependency_results(provider_sets, requirer_sets, args.bpp) - print_results(results) - finally: - if cleanup: - shutil.rmtree(workdir, ignore_errors=True) - else: - print(f"workdir: {workdir}") - - return 0 if all(result.status == "compatible" for result in results) else 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/old/check_arch_set_impl.py b/scripts/old/check_arch_set_impl.py deleted file mode 100755 index 86c55b6..0000000 --- a/scripts/old/check_arch_set_impl.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python3 -"""Check reimplement/set.py against ELF libraries installed on this Arch Linux system. - -This is the local-system analogue of check_alt_set_impl.py. It treats shared -libraries as providers (defined dynamic symbols) and executables/shared objects -as requirers (undefined dynamic symbols). Required labels are split by provider -library before comparing, so each library is checked only against symbols that -it actually exports. -""" - -from __future__ import annotations - -import argparse -import os -import shutil -import sys -from pathlib import Path - -SCRIPT_DIR = Path(__file__).resolve().parent -if str(SCRIPT_DIR) not in sys.path: - sys.path.insert(0, str(SCRIPT_DIR)) - -import check_alt_set_impl as compat # noqa: E402 - -DEFAULT_PROVIDER_DIRS = [Path("/usr/lib")] -DEFAULT_REQUIRER_DIRS = [Path("/usr/bin"), Path("/usr/lib")] - - -def parse_path_list(values: list[str] | None, defaults: list[Path]) -> list[Path]: - paths: list[Path] = [] - for value in values or []: - paths.extend(Path(part) for part in value.split(",") if part) - return paths or defaults - - -def is_shared_library_path(path: Path) -> bool: - name = path.name - return ".debug" not in name and name.startswith("lib") and ".so" in name - - -def iter_files(paths: list[Path], recursive: bool) -> list[Path]: - files: list[Path] = [] - seen: set[Path] = set() - for root in paths: - if root.is_file(): - candidates = [root] - elif recursive: - candidates = (path for path in root.rglob("*") if path.is_file()) - else: - candidates = (path for path in root.iterdir() if path.is_file()) if root.is_dir() else [] - for path in candidates: - try: - resolved = path.resolve() - except OSError: - continue - if resolved in seen: - continue - seen.add(resolved) - files.append(path) - return sorted(files) - - -def executable_or_library(path: Path) -> bool: - return is_shared_library_path(path) or os.access(path, os.X_OK) - - -def build_local_provider_sets(paths: list[Path], bpp: int, recursive: bool, max_files: int) -> list[compat.LabelSet]: - provider_files = [path for path in iter_files(paths, recursive) if is_shared_library_path(path)][:max_files] - sets: list[compat.LabelSet] = [] - for path in provider_files: - labels = compat.nm_symbols(path, "provided") - if labels: - sets.append(compat.generate_label_set("provided", str(path), labels, bpp, package="arch-local")) - return sets - - -def build_local_requirer_sets( - paths: list[Path], - bpp: int, - recursive: bool, - max_files: int, - normalize_versions: bool, -) -> list[compat.LabelSet]: - requirer_files = [path for path in iter_files(paths, recursive) if executable_or_library(path)][:max_files] - sets: list[compat.LabelSet] = [] - for path in requirer_files: - labels = compat.nm_symbols(path, "required") - if normalize_versions: - labels = [compat.normalize_required_symbol(label) for label in labels] - if labels: - sets.append(compat.generate_label_set("required", str(path), labels, bpp, package="arch-local")) - return sets - - -def print_unmatched_required_labels(provider_sets: list[compat.LabelSet], requirer_sets: list[compat.LabelSet], limit: int) -> None: - provided = {label for provider in provider_sets for label in provider.labels} - rows: list[tuple[str, int, list[str]]] = [] - for requirer in requirer_sets: - missing = sorted(set(requirer.labels).difference(provided)) - if missing: - rows.append((requirer.member, len(missing), missing[:limit])) - - print("\n# required labels not exported by scanned provider libraries") - print("requirer\tmissing_labels\texamples") - for member, count, examples in rows[:limit]: - print(f"{member}\t{count}\t{', '.join(examples)}") - if len(rows) > limit: - print(f"... {len(rows) - limit} more requirer files with unmatched labels") - - -def parse_args(argv: list[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Generate Provided/Required set strings from local Arch Linux ELF files and compare them with reimplement/set.py." - ) - parser.add_argument("--provider-dir", action="append", help="directory/file containing provider libraries; repeat or comma-separate") - parser.add_argument("--requirer-dir", action="append", help="directory/file containing requirer ELF files; repeat or comma-separate") - parser.add_argument("--bpp", type=int, default=32, help="bits per hash used by local set.py") - parser.add_argument("--max-provider-files", type=int, default=256, help="max provider libraries to inspect") - parser.add_argument("--max-requirer-files", type=int, default=256, help="max requirer files to inspect") - parser.add_argument("--recursive", action="store_true", help="scan directories recursively") - parser.add_argument("--all-pairs", action="store_true", help="compare every provider set with every full requirer set") - parser.add_argument( - "--no-normalize-required-version", - action="store_true", - help="keep nm -u single-@ required symbols unchanged instead of converting foo@VER to foo@@VER", - ) - parser.add_argument("--unmatched-limit", type=int, default=20, help="max unmatched-label rows/examples to print") - return parser.parse_args(argv) - - -def main(argv: list[str]) -> int: - args = parse_args(argv) - if shutil.which("nm") is None: - print("missing required command: nm", file=sys.stderr) - return 2 - - provider_paths = parse_path_list(args.provider_dir, DEFAULT_PROVIDER_DIRS) - requirer_paths = parse_path_list(args.requirer_dir, DEFAULT_REQUIRER_DIRS) - - provider_sets = build_local_provider_sets(provider_paths, args.bpp, args.recursive, args.max_provider_files) - requirer_sets = build_local_requirer_sets( - requirer_paths, - args.bpp, - args.recursive, - args.max_requirer_files, - normalize_versions=not args.no_normalize_required_version, - ) - - print("system: arch-local") - print(f"bpp: {args.bpp}") - print(f"provider paths: {', '.join(str(path) for path in provider_paths)}") - print(f"requirer paths: {', '.join(str(path) for path in requirer_paths)}") - print(f"required symbol version normalization: {not args.no_normalize_required_version}") - compat.print_label_sets("generated Provided sets", provider_sets) - compat.print_label_sets("generated Required sets", requirer_sets) - - if not provider_sets or not requirer_sets: - print("\nNo comparable sets generated.", file=sys.stderr) - return 1 - - if args.all_pairs: - results = [compat.compare_label_sets(provider, requirer) for provider in provider_sets for requirer in requirer_sets] - else: - results = compat.build_dependency_results(provider_sets, requirer_sets, args.bpp) - compat.print_results(results) - print_unmatched_required_labels(provider_sets, requirer_sets, args.unmatched_limit) - - return 0 if results and all(result.status == "compatible" for result in results) else 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/old/compare_binary_with_lib_sets.py b/scripts/old/compare_binary_with_lib_sets.py deleted file mode 100755 index 181aecf..0000000 --- a/scripts/old/compare_binary_with_lib_sets.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python3 -"""Compare one binary's required set string with a few provider libraries. - -Usage: - scripts/compare_binary_with_lib_sets.py [--bpp N] BINARY LIB.so [LIB.so ...] - -For every library, this script compares: - set(defined dynamic symbols from LIB) vs - set(required dynamic symbols from BINARY that LIB provides) -""" - -from __future__ import annotations - -import argparse -import contextlib -import shutil -import subprocess -import sys -from pathlib import Path -from typing import Iterable - -REPO_ROOT = Path(__file__).resolve().parents[1] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - -from reimplement import set as rpmset # noqa: E402 - - -def run_nm(command: list[str]) -> str: - proc = subprocess.run(command, text=True, capture_output=True) - if proc.returncode != 0: - raise RuntimeError(f"{' '.join(command)} failed with {proc.returncode}: {proc.stderr.strip()}") - return proc.stdout - - -def normalize_required_symbol(symbol: str) -> str: - """Normalize nm's required `foo@VER` form to provider-like `foo@@VER`.""" - if "@@" in symbol or "@" not in symbol: - return symbol - name, version = symbol.split("@", 1) - if not name or not version: - return symbol - return f"{name}@@{version}" - - -def required_symbols(path: Path, normalize_versions: bool) -> set[str]: - """Return strong undefined dynamic symbols required by one ELF file.""" - output = run_nm(["nm", "--dynamic", "-u", str(path)]) - symbols: set[str] = set() - for line in output.splitlines(): - parts = line.split() - if len(parts) < 2 or parts[-2] != "U": - continue - symbol = parts[-1] - symbols.add(normalize_required_symbol(symbol) if normalize_versions else symbol) - return symbols - - -def provided_symbols(path: Path) -> set[str]: - """Return defined dynamic symbols provided by one ELF shared library.""" - output = run_nm(["nm", "--dynamic", "-j", "-U", str(path)]) - return {line.strip() for line in output.splitlines() if line.strip()} - - -def labels_to_set_string(labels: Iterable[str], bpp: int) -> str | None: - item_set = rpmset.set_new() - for label in sorted(set(labels)): - rpmset.set_add(item_set, label) - # set.py may print hash-collision warnings; keep stdout tabular. - with contextlib.redirect_stdout(sys.stderr): - return rpmset.set_fini(item_set, bpp) - - -def compare_library(binary_required: set[str], library: Path, bpp: int) -> tuple[str, str, int, int, str, str]: - provided = provided_symbols(library) - required_from_library = binary_required.intersection(provided) - provider_set = labels_to_set_string(provided, bpp) - required_set = labels_to_set_string(required_from_library, bpp) - - if not provided: - return "no-provided-symbols", "", 0, 0, "-", "-" - if not required_from_library: - return "not-required", "", len(provided), 0, provider_set or "-", "-" - - assert provider_set is not None - assert required_set is not None - cmp_result = rpmset.rpmsetcmp(provider_set, required_set) - status = "compatible" if cmp_result in (0, 1) else "incompatible" - return status, str(cmp_result), len(provided), len(required_from_library), provider_set, required_set - - -def parse_args(argv: list[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Compare one binary against a few libraries using local set.py set strings.") - parser.add_argument("binary", type=Path, help="ELF executable/shared object with required dynamic symbols") - parser.add_argument("libraries", nargs="+", type=Path, help="provider shared libraries to compare against") - parser.add_argument("--bpp", type=int, default=32, help="bits per hash used by local set.py") - parser.add_argument( - "--no-normalize-required-version", - action="store_true", - help="keep nm -u single-@ required symbols unchanged instead of converting foo@VER to foo@@VER", - ) - return parser.parse_args(argv) - - -def main(argv: list[str]) -> int: - args = parse_args(argv) - if shutil.which("nm") is None: - print("missing required command: nm", file=sys.stderr) - return 2 - - required = required_symbols(args.binary, normalize_versions=not args.no_normalize_required_version) - if not required: - print(f"no strong dynamic required symbols found in {args.binary}", file=sys.stderr) - return 1 - - print(f"binary\t{args.binary}") - print(f"bpp\t{args.bpp}") - print(f"required_symbols\t{len(required)}") - print("status\tcmp\tlib\tprovided\trequired_from_lib\tprovider_set\trequired_set") - - failed = False - for library in args.libraries: - try: - status, cmp_result, provided_count, required_count, provider_set, required_set = compare_library( - required, library, args.bpp - ) - except RuntimeError as exc: - print(f"error\t\t{library}\t0\t0\t-\t-", flush=True) - print(exc, file=sys.stderr) - failed = True - continue - - print( - f"{status}\t{cmp_result}\t{library}\t{provided_count}\t{required_count}\t{provider_set}\t{required_set}" - ) - failed = failed or status in {"incompatible", "no-provided-symbols"} - - return 1 if failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/old/compare_cmp_realization.py b/scripts/old/compare_cmp_realization.py deleted file mode 100755 index 28029a0..0000000 --- a/scripts/old/compare_cmp_realization.py +++ /dev/null @@ -1,401 +0,0 @@ -#!/usr/bin/env python3 -"""Benchmark rpmsetcmp from a selected set.c on hardcoded ELF pairs.""" - -from __future__ import annotations - -import argparse -import csv -import os -import shlex -import shutil -import statistics -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Sequence - -import compare_realization as common - - -REPO_ROOT = Path(__file__).resolve().parents[1] -RUN_CMP_SOURCE = REPO_ROOT / "scripts" / "run_cmp.cpp" -DEFAULT_RPM_ROOT = REPO_ROOT / "rpm-build" -DEFAULT_RESULT_ROOT = REPO_ROOT / "res_cmp" - -LIBC_CANDIDATES = ( - "/usr/lib/libc.so.6", - "/lib64/libc.so.6", - "/usr/lib64/libc.so.6", - "/lib/libc.so.6", - "/lib/x86_64-linux-gnu/libc.so.6", -) - - -@dataclass(frozen=True) -class ComparisonCase: - name: str - binary_candidates: tuple[str, ...] - library_candidates: tuple[str, ...] - - -# The benchmark corpus is intentionally fixed here rather than supplied via CLI. -HARDCODED_CASES = ( - ComparisonCase("true-libc", ("/usr/bin/true", "/bin/true"), LIBC_CANDIDATES), - ComparisonCase("ls-libc", ("/usr/bin/ls", "/bin/ls"), LIBC_CANDIDATES), - ComparisonCase("bash-libc", ("/usr/bin/bash", "/bin/bash"), LIBC_CANDIDATES), - ComparisonCase( - "python-libc", - ("/usr/bin/python3", "/usr/local/bin/python3"), - LIBC_CANDIDATES, - ), -) - -SUMMARY_FIELDS = ( - "case", - "binary", - "library", - "runs", - "cmp_result", - "outputs_consistent", - "median_rpmsetcmp_ns", -) - - -@dataclass(frozen=True) -class ResolvedCase: - name: str - binary: Path - library: Path - - -@dataclass(frozen=True) -class PreparedCase: - case: ResolvedCase - provider_set: str - required_set: str - - -@dataclass(frozen=True) -class ComparisonSummary: - prepared: PreparedCase - runs: int - cmp_result: int - outputs_consistent: bool - median_ns: int | float - - def as_row(self) -> dict[str, str]: - return { - "case": self.prepared.case.name, - "binary": str(self.prepared.case.binary), - "library": str(self.prepared.case.library), - "runs": str(self.runs), - "cmp_result": str(self.cmp_result), - "outputs_consistent": "yes" if self.outputs_consistent else "no", - "median_rpmsetcmp_ns": common.format_number(self.median_ns), - } - - -def parse_args(argv: Sequence[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( - "Compile a selected set.c and benchmark only rpmsetcmp on the " - "hardcoded binary/library corpus." - ) - ) - parser.add_argument("set_c", type=Path, help="path to the set.c implementation") - parser.add_argument( - "-n", - "--runs", - type=common.positive_int, - default=10, - help="rpmsetcmp process runs per hardcoded case (default: 10)", - ) - parser.add_argument( - "--rpm-root", - type=Path, - default=DEFAULT_RPM_ROOT, - help="ALT rpm source root", - ) - parser.add_argument( - "--res-dir", - type=Path, - default=DEFAULT_RESULT_ROOT, - help="result root (default: repository res_cmp/)", - ) - parser.add_argument("--name", help="implementation directory name under res_cmp/") - parser.add_argument( - "--bpp", - type=common.bpp_value, - help="force one bpp value while preparing provider/required sets", - ) - parser.add_argument("--cc", default="cc", help="C compiler (default: cc)") - parser.add_argument("--cxx", default="g++", help="C++ compiler (default: g++)") - return parser.parse_args(argv) - - -def first_existing(candidates: Sequence[str]) -> Path | None: - for candidate in candidates: - path = Path(candidate) - if path.is_file(): - return path.resolve() - return None - - -def resolve_cases(cases: Sequence[ComparisonCase]) -> list[ResolvedCase]: - resolved: list[ResolvedCase] = [] - for case in cases: - binary = first_existing(case.binary_candidates) - library = first_existing(case.library_candidates) - if binary is None or library is None: - missing = "binary" if binary is None else "library" - print(f"warning: skipping {case.name}: no hardcoded {missing} path exists", file=sys.stderr) - continue - resolved.append(ResolvedCase(common.safe_name(case.name), binary, library)) - if not resolved: - raise RuntimeError("none of the hardcoded binary/library cases is available") - return resolved - - -def compile_cmp_runner(build_dir: Path, rpm_root: Path, cxx: str) -> Path: - runner = build_dir / "run_cmp" - command = [ - cxx, - "-O2", - "-std=c++17", - "-Wall", - "-Wextra", - "-Werror", - str(RUN_CMP_SOURCE), - str(build_dir / "set.o"), - str(build_dir / "rpmmalloc.o"), - "-o", - str(runner), - ] - log: list[str] = ["\n[rpmsetcmp runner]", "$ " + shlex.join(command)] - completed = subprocess.run(command, cwd=rpm_root, text=True, capture_output=True) - if completed.stdout: - log.append(completed.stdout.rstrip()) - if completed.stderr: - log.append(completed.stderr.rstrip()) - log.append(f"[exit {completed.returncode}]") - build_log = build_dir / "build.log" - with build_log.open("a", encoding="utf-8") as stream: - stream.write("\n".join(log) + "\n") - if completed.returncode != 0: - raise RuntimeError(f"run_cmp compilation failed; see {build_log}") - return runner - - -def run_set_once( - runner: Path, - target: Path, - sets_dir: Path, - slug: str, - bpp: int | None, -) -> common.ParsedRun: - command = [str(runner)] - if bpp is not None: - command.extend(("--bpp", str(bpp))) - command.append(str(target)) - completed = subprocess.run(command, text=True, capture_output=True) - stdout_path = sets_dir / f"{slug}.tsv" - stderr_path = sets_dir / f"{slug}.stderr" - stdout_path.write_text(completed.stdout, encoding="utf-8") - stderr_path.write_text(completed.stderr, encoding="utf-8") - if completed.returncode != 0: - raise RuntimeError(f"run_set failed for {target}; see {stderr_path}") - return common.parse_run_set_output(completed.stdout) - - -def same_file(left: str, right: Path) -> bool: - try: - return os.path.samefile(left, right) - except OSError: - return Path(left).resolve() == right.resolve() - - -def prepare_cases( - cases: Sequence[ResolvedCase], - run_set: Path, - sets_dir: Path, - bpp: int | None, -) -> list[PreparedCase]: - sets_dir.mkdir(parents=True) - cache: dict[Path, common.ParsedRun] = {} - used_slugs: set[str] = set() - - def inspect(target: Path) -> common.ParsedRun: - parsed = cache.get(target) - if parsed is not None: - return parsed - slug = common.target_slug(target, used_slugs) - parsed = run_set_once(run_set, target, sets_dir, slug, bpp) - cache[target] = parsed - return parsed - - prepared: list[PreparedCase] = [] - for case in cases: - library_run = inspect(case.library) - provider_rows = [row for row in library_run.rows if row.get("role") == "provided"] - if len(provider_rows) != 1: - raise RuntimeError( - f"expected one provider set for {case.library}, got {len(provider_rows)}" - ) - - binary_run = inspect(case.binary) - required_rows = [ - row - for row in binary_run.rows - if row.get("role") == "required" - and row.get("object") - and same_file(row["object"], case.library) - ] - if len(required_rows) != 1: - raise RuntimeError( - f"expected one {case.library.name} requirement from {case.binary}, " - f"got {len(required_rows)}" - ) - - provider_row = provider_rows[0] - required_row = required_rows[0] - if provider_row["bpp"] != required_row["bpp"]: - raise RuntimeError( - f"bpp mismatch for {case.name}: provider={provider_row['bpp']} " - f"required={required_row['bpp']}" - ) - prepared.append( - PreparedCase( - case=case, - provider_set=provider_row["set"], - required_set=required_row["set"], - ) - ) - return prepared - - -def parse_cmp_output(output: str) -> tuple[int, int]: - values: dict[str, str] = {} - for line in output.splitlines(): - if "\t" not in line: - continue - key, value = line.split("\t", 1) - values[key] = value - try: - result = int(values["cmp_result"]) - elapsed = int(values["rpmsetcmp_ns"]) - except (KeyError, ValueError) as exception: - raise RuntimeError("invalid run_cmp output") from exception - if elapsed < 0: - raise RuntimeError("run_cmp returned a negative duration") - return result, elapsed - - -def benchmark_case( - runner: Path, - prepared: PreparedCase, - runs: int, - output_dir: Path, -) -> ComparisonSummary: - output_dir.mkdir(parents=True) - results: list[int] = [] - timings: list[int] = [] - for run_number in range(1, runs + 1): - completed = subprocess.run( - [str(runner), prepared.provider_set, prepared.required_set], - text=True, - capture_output=True, - ) - stem = f"run-{run_number:03d}" - stdout_path = output_dir / f"{stem}.tsv" - stderr_path = output_dir / f"{stem}.stderr" - stdout_path.write_text(completed.stdout, encoding="utf-8") - stderr_path.write_text(completed.stderr, encoding="utf-8") - if completed.returncode != 0: - raise RuntimeError( - f"run_cmp failed for {prepared.case.name} on run {run_number}; " - f"see {stderr_path}" - ) - result, elapsed = parse_cmp_output(completed.stdout) - results.append(result) - timings.append(elapsed) - - return ComparisonSummary( - prepared=prepared, - runs=runs, - cmp_result=results[0], - outputs_consistent=len(set(results)) == 1, - median_ns=statistics.median(timings), - ) - - -def write_summary(path: Path, summaries: Sequence[ComparisonSummary]) -> None: - with path.open("w", newline="", encoding="utf-8") as stream: - writer = csv.DictWriter(stream, fieldnames=list(SUMMARY_FIELDS), delimiter="\t") - writer.writeheader() - for summary in summaries: - writer.writerow(summary.as_row()) - - -def print_summary(summaries: Sequence[ComparisonSummary]) -> None: - print("\t".join(SUMMARY_FIELDS)) - for summary in summaries: - row = summary.as_row() - print("\t".join(row[field] for field in SUMMARY_FIELDS)) - - -def write_metadata( - path: Path, - set_source: Path, - rpm_root: Path, - runs: int, - cases: Sequence[ResolvedCase], -) -> None: - lines = [ - f"set_c\t{set_source}", - f"rpm_root\t{rpm_root}", - f"runs\t{runs}", - ] - for case in cases: - lines.append(f"case\t{case.name}\t{case.binary}\t{case.library}") - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def main(argv: Sequence[str]) -> int: - args = parse_args(argv) - try: - common.require_commands((args.cc, args.cxx, "eu-readelf", "eu-elfclassify", "nm")) - set_source = common.resolve_file(args.set_c, "set.c") - rpm_root = common.resolve_directory(args.rpm_root, "rpm root") - common.resolve_file(rpm_root / "system.h", "system.h") - cases = resolve_cases(HARDCODED_CASES) - name = common.safe_name(args.name) if args.name else common.default_name(set_source) - result_dir = args.res_dir.expanduser().resolve() / name - if result_dir.exists(): - shutil.rmtree(result_dir) - build_dir = result_dir / "build" - sets_dir = result_dir / "sets" - runs_dir = result_dir / "runs" - build_dir.mkdir(parents=True) - runs_dir.mkdir() - - run_set = common.compile_runner(set_source, rpm_root, build_dir, args.cc, args.cxx) - run_cmp = compile_cmp_runner(build_dir, rpm_root, args.cxx) - prepared = prepare_cases(cases, run_set, sets_dir, args.bpp) - summaries = [ - benchmark_case(run_cmp, item, args.runs, runs_dir / item.case.name) - for item in prepared - ] - - write_summary(result_dir / "summary.tsv", summaries) - write_metadata(result_dir / "metadata.tsv", set_source, rpm_root, args.runs, cases) - print_summary(summaries) - print(f"\nresults\t{result_dir}") - return 0 - except (OSError, RuntimeError, ValueError) as exception: - print(f"compare_cmp_realization: {exception}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/old/compare_realization.py b/scripts/old/compare_realization.py deleted file mode 100755 index 9d0962b..0000000 --- a/scripts/old/compare_realization.py +++ /dev/null @@ -1,518 +0,0 @@ -#!/usr/bin/env python3 -"""Build one set.c implementation and benchmark it through run_set.cpp. - -The benchmark stores every raw run_set.cpp stdout/stderr stream and a TSV -summary with median set.c API timings under res//. -""" - -from __future__ import annotations - -import argparse -import csv -import hashlib -import io -import shlex -import shutil -import statistics -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Iterable, Sequence - -REPO_ROOT = Path(__file__).resolve().parents[1] -RUN_SET_SOURCE = REPO_ROOT / "scripts" / "run_set.cpp" -DEFAULT_RPM_ROOT = REPO_ROOT / "rpm-build" - -TIMING_FIELDS = ( - "set_new_ns", - "set_add_total_ns", - "set_fini_ns", - "set_free_ns", - "set_api_total_ns", -) -SUMMARY_FIELDS = ( - "target", - "kind", - "runs", - "sets_per_run", - "outputs_consistent", - *(f"median_{field}" for field in TIMING_FIELDS), -) - -# Each tuple describes one logical target. The first existing path is used. -DEFAULT_TARGET_GROUPS = ( - ("/usr/bin/true", "/bin/true"), - ("/usr/bin/ls", "/bin/ls"), - ("/usr/bin/bash", "/bin/bash"), - ("/usr/bin/python3", "/usr/local/bin/python3"), - ("/usr/lib/libc.so.6", "/lib64/libc.so.6", "/lib/x86_64-linux-gnu/libc.so.6"), - ("/usr/lib/libm.so.6", "/lib64/libm.so.6", "/lib/x86_64-linux-gnu/libm.so.6"), - ("/usr/lib/libz.so.1", "/usr/lib64/libz.so.1", "/lib/x86_64-linux-gnu/libz.so.1"), - ( - "/usr/lib/libstdc++.so.6", - "/usr/lib64/libstdc++.so.6", - "/lib/x86_64-linux-gnu/libstdc++.so.6", - ), -) - - -@dataclass(frozen=True) -class ParsedRun: - kind: str - rows: tuple[dict[str, str], ...] - totals: dict[str, int] - output_signature: tuple[tuple[str, ...], ...] - - -@dataclass(frozen=True) -class TargetSummary: - target: Path - kind: str - runs: int - sets_per_run: int - outputs_consistent: bool - medians: dict[str, int | float] - - def as_row(self) -> dict[str, str]: - row = { - "target": str(self.target), - "kind": self.kind, - "runs": str(self.runs), - "sets_per_run": str(self.sets_per_run), - "outputs_consistent": "yes" if self.outputs_consistent else "no", - } - row.update( - { - f"median_{field}": format_number(value) - for field, value in self.medians.items() - } - ) - return row - - -def positive_int(value: str) -> int: - number = int(value) - if number < 1: - raise argparse.ArgumentTypeError("must be greater than zero") - return number - - -def bpp_value(value: str) -> int: - number = int(value) - if not 10 <= number <= 32: - raise argparse.ArgumentTypeError("must be in [10, 32]") - return number - - -def safe_name(value: str) -> str: - name = "".join( - character if character.isalnum() or character in "._-" else "-" - for character in value - ) - name = name.strip(".-") - if not name or name in {".", ".."}: - raise ValueError(f"invalid result name: {value!r}") - return name - - -def default_name(set_source: Path) -> str: - return safe_name(f"{set_source.parent.name}-{set_source.stem}") - - -def parse_args(argv: Sequence[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( - "Compile a selected set.c, run scripts/run_set.cpp repeatedly on ELF targets, " - "and store raw outputs plus median set.c timings." - ) - ) - parser.add_argument("set_c", type=Path, help="path to the set.c implementation") - parser.add_argument( - "-n", - "--runs", - type=positive_int, - default=10, - help="runs per ELF target (default: 10)", - ) - parser.add_argument( - "--rpm-root", type=Path, default=DEFAULT_RPM_ROOT, help="ALT rpm source root" - ) - parser.add_argument( - "--res-dir", - type=Path, - default=REPO_ROOT / "res", - help="result root (default: repository res/)", - ) - parser.add_argument("--name", help="implementation directory name under res/") - parser.add_argument( - "--target", - action="append", - type=Path, - default=[], - help="ELF target; repeat to override the built-in binary/library set", - ) - parser.add_argument( - "--bpp", type=bpp_value, help="force one bpp value for every run_set invocation" - ) - parser.add_argument("--cc", default="cc", help="C compiler (default: cc)") - parser.add_argument("--cxx", default="g++", help="C++ compiler (default: g++)") - return parser.parse_args(argv) - - -def resolve_file(path: Path, description: str) -> Path: - resolved = path.expanduser().resolve() - if not resolved.is_file(): - raise RuntimeError(f"{description} is not a file: {path}") - return resolved - - -def resolve_directory(path: Path, description: str) -> Path: - resolved = path.expanduser().resolve() - if not resolved.is_dir(): - raise RuntimeError(f"{description} is not a directory: {path}") - return resolved - - -def select_default_targets() -> list[Path]: - selected: list[Path] = [] - seen: set[Path] = set() - for candidates in DEFAULT_TARGET_GROUPS: - for candidate in candidates: - path = Path(candidate) - if not path.is_file(): - continue - resolved = path.resolve() - if resolved not in seen: - selected.append(resolved) - seen.add(resolved) - break - return selected - - -def resolve_targets(explicit: Iterable[Path]) -> list[Path]: - supplied = list(explicit) - targets = ( - [resolve_file(path, "target") for path in supplied] - if supplied - else select_default_targets() - ) - unique: list[Path] = [] - seen: set[Path] = set() - for target in targets: - if target not in seen: - unique.append(target) - seen.add(target) - if not unique: - raise RuntimeError("no ELF targets were found; pass at least one --target") - return unique - - -def require_commands(commands: Iterable[str]) -> None: - missing = [command for command in commands if shutil.which(command) is None] - if missing: - raise RuntimeError("missing required commands: " + ", ".join(missing)) - - -def create_header_shim(rpm_root: Path, include_dir: Path) -> None: - include_dir.mkdir(parents=True, exist_ok=True) - by_name: dict[str, Path] = {} - for header in sorted(rpm_root.glob("*/*.h")): - existing = by_name.get(header.name) - if existing is not None and existing.resolve() != header.resolve(): - raise RuntimeError( - f"duplicate rpm header basename {header.name}: {existing} and {header}" - ) - by_name[header.name] = header - if not by_name: - raise RuntimeError(f"no */*.h headers found below {rpm_root}") - for name, source in by_name.items(): - (include_dir / name).symlink_to(source.resolve()) - - -def run_build_command( - command: Sequence[str], cwd: Path, log: list[str] -) -> subprocess.CompletedProcess[str]: - log.append("$ " + shlex.join(str(part) for part in command)) - completed = subprocess.run(command, cwd=cwd, text=True, capture_output=True) - if completed.stdout: - log.append(completed.stdout.rstrip()) - if completed.stderr: - log.append(completed.stderr.rstrip()) - log.append(f"[exit {completed.returncode}]") - return completed - - -def compile_runner( - set_source: Path, - rpm_root: Path, - build_dir: Path, - cc: str, - cxx: str, -) -> Path: - include_dir = build_dir / "include" / "rpm" - create_header_shim(rpm_root, include_dir) - set_object = build_dir / "set.o" - malloc_object = build_dir / "rpmmalloc.o" - runner = build_dir / "run_set" - rpmmalloc_source = resolve_file(rpm_root / "rpmio" / "rpmmalloc.c", "rpmmalloc.c") - log: list[str] = [f"set_c={set_source}", f"rpm_root={rpm_root}"] - - common_flags = [ - "-O3", - "-std=gnu11", - "-include", - "stddef.h", - f"-I{rpm_root}", - f"-I{include_dir}", - ] - compatibility_flags = [ - "-D_GNU_SOURCE=1", - "-DSTDC_HEADERS=1", - "-DHAVE_STRING_H=1", - "-DHAVE_SETENV=1", - "-DHAVE_STPCPY=1", - "-DHAVE_STPNCPY=1", - "-DHAVE_S_IFSOCK=1", - "-DHAVE_S_ISLNK=1", - "-DHAVE_S_ISSOCK=1", - ] - - def attempt(extra_flags: Sequence[str], label: str) -> bool: - log.append(f"\n[{label}]") - set_command = [ - cc, - *common_flags, - *extra_flags, - "-c", - str(set_source), - "-o", - str(set_object), - ] - malloc_command = [ - cc, - *common_flags, - *extra_flags, - "-include", - "stdarg.h", - "-c", - str(rpmmalloc_source), - "-o", - str(malloc_object), - ] - link_command = [ - cxx, - "-O3", - "-std=c++17", - "-Wall", - "-Wextra", - "-Werror", - str(RUN_SET_SOURCE), - str(set_object), - str(malloc_object), - "-o", - str(runner), - ] - for command in (set_command, malloc_command, link_command): - if run_build_command(command, rpm_root, log).returncode != 0: - return False - return True - - succeeded = attempt((), "minimal ALT-style build") - if not succeeded: - set_object.unlink(missing_ok=True) - malloc_object.unlink(missing_ok=True) - runner.unlink(missing_ok=True) - succeeded = attempt(compatibility_flags, "modern libc compatibility retry") - - build_log = build_dir / "build.log" - build_log.write_text("\n".join(log) + "\n", encoding="utf-8") - if not succeeded: - raise RuntimeError(f"compilation failed; see {build_log}") - return runner - - -def parse_run_set_output(output: str) -> ParsedRun: - lines = output.splitlines() - try: - table_start = next( - index for index, line in enumerate(lines) if line.startswith("role\t") - ) - except StopIteration as exception: - raise RuntimeError("run_set output has no TSV result table") from exception - - metadata: dict[str, str] = {} - for line in lines[:table_start]: - if "\t" not in line: - continue - key, value = line.split("\t", 1) - metadata[key] = value - rows = tuple( - csv.DictReader(io.StringIO("\n".join(lines[table_start:])), delimiter="\t") - ) - if not rows: - raise RuntimeError("run_set output contains no result rows") - - totals = {field: 0 for field in TIMING_FIELDS} - for row in rows: - for field in TIMING_FIELDS: - try: - totals[field] += int(row[field]) - except (KeyError, ValueError) as exception: - raise RuntimeError(f"invalid {field} in run_set output") from exception - signature_fields = ("role", "object", "labels", "bpp", "set") - signature = tuple(tuple(row[field] for field in signature_fields) for row in rows) - return ParsedRun( - kind=metadata.get("kind", "unknown"), - rows=rows, - totals=totals, - output_signature=signature, - ) - - -def target_slug(target: Path, used: set[str]) -> str: - base = safe_name(target.name) - candidate = base - if candidate in used: - digest = hashlib.sha256(str(target).encode()).hexdigest()[:8] - candidate = f"{base}-{digest}" - used.add(candidate) - return candidate - - -def benchmark_target( - runner: Path, - target: Path, - runs: int, - output_dir: Path, - bpp: int | None, -) -> TargetSummary: - output_dir.mkdir(parents=True, exist_ok=True) - parsed_runs: list[ParsedRun] = [] - for run_number in range(1, runs + 1): - command = [str(runner)] - if bpp is not None: - command.extend(("--bpp", str(bpp))) - command.append(str(target)) - completed = subprocess.run(command, text=True, capture_output=True) - stem = f"run-{run_number:03d}" - (output_dir / f"{stem}.tsv").write_text(completed.stdout, encoding="utf-8") - (output_dir / f"{stem}.stderr").write_text(completed.stderr, encoding="utf-8") - if completed.returncode != 0: - raise RuntimeError( - f"run_set failed for {target} on run {run_number}; " - f"see {output_dir / f'{stem}.stderr'}" - ) - parsed_runs.append(parse_run_set_output(completed.stdout)) - - signatures = {parsed.output_signature for parsed in parsed_runs} - kinds = {parsed.kind for parsed in parsed_runs} - row_counts = {len(parsed.rows) for parsed in parsed_runs} - medians = { - field: statistics.median(parsed.totals[field] for parsed in parsed_runs) - for field in TIMING_FIELDS - } - return TargetSummary( - target=target, - kind=parsed_runs[0].kind if len(kinds) == 1 else "inconsistent", - runs=runs, - sets_per_run=len(parsed_runs[0].rows) if len(row_counts) == 1 else -1, - outputs_consistent=len(signatures) == 1 - and len(kinds) == 1 - and len(row_counts) == 1, - medians=medians, - ) - - -def format_number(value: int | float) -> str: - number = float(value) - return str(int(number)) if number.is_integer() else f"{number:.1f}" - - -def write_summary(path: Path, summaries: Iterable[TargetSummary]) -> None: - with path.open("w", newline="", encoding="utf-8") as stream: - writer = csv.DictWriter(stream, fieldnames=SUMMARY_FIELDS, delimiter="\t") - writer.writeheader() - for summary in summaries: - writer.writerow(summary.as_row()) - - -def print_summary(summaries: Iterable[TargetSummary]) -> None: - fields = ( - "target", - "kind", - "runs", - "sets_per_run", - "outputs_consistent", - "median_set_api_total_ns", - ) - print("\t".join(fields)) - for summary in summaries: - row = summary.as_row() - print("\t".join(row[field] for field in fields)) - - -def write_metadata( - path: Path, - set_source: Path, - rpm_root: Path, - runner: Path, - runs: int, - targets: Iterable[Path], -) -> None: - lines = [ - f"set_c\t{set_source}", - f"rpm_root\t{rpm_root}", - f"runner\t{runner}", - f"runs\t{runs}", - ] - lines.extend(f"target\t{target}" for target in targets) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def main(argv: Sequence[str]) -> int: - args = parse_args(argv) - try: - require_commands((args.cc, args.cxx, "eu-readelf", "eu-elfclassify", "nm")) - set_source = resolve_file(args.set_c, "set.c") - rpm_root = resolve_directory(args.rpm_root, "rpm root") - resolve_file(rpm_root / "system.h", "system.h") - targets = resolve_targets(args.target) - name = safe_name(args.name) if args.name else default_name(set_source) - result_root = args.res_dir.expanduser().resolve() - result_dir = result_root / name - if result_dir.exists(): - shutil.rmtree(result_dir) - build_dir = result_dir / "build" - runs_dir = result_dir / "runs" - build_dir.mkdir(parents=True) - runs_dir.mkdir() - - runner = compile_runner(set_source, rpm_root, build_dir, args.cc, args.cxx) - used_slugs: set[str] = set() - summaries: list[TargetSummary] = [] - for target in targets: - slug = target_slug(target, used_slugs) - summaries.append( - benchmark_target(runner, target, args.runs, runs_dir / slug, args.bpp) - ) - - write_summary(result_dir / "summary.tsv", summaries) - write_metadata( - result_dir / "metadata.tsv", - set_source, - rpm_root, - runner, - args.runs, - targets, - ) - print_summary(summaries) - print(f"\nresults\t{result_dir}") - return 0 - except (OSError, RuntimeError, ValueError) as exception: - print(f"compare_realization: {exception}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/old/compare_sisyphus_set_versions.sh b/scripts/old/compare_sisyphus_set_versions.sh deleted file mode 100755 index d89bf2e..0000000 --- a/scripts/old/compare_sisyphus_set_versions.sh +++ /dev/null @@ -1,611 +0,0 @@ -#!/bin/bash -set -euo pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd) -NEWSET_C="$REPO_ROOT/reimplement/newset.c" -NEWSET_COMPAT="$SCRIPT_DIR/newset_compat.h" -NEWSET_WRAPPER="$SCRIPT_DIR/newset_mkset.c" -PACKAGE_PARSER="$SCRIPT_DIR/parse_sisyphus_packages.awk" -RESUME_PARSER="$SCRIPT_DIR/completed_set_versions.awk" -PAYLOAD_HELPERS="$SCRIPT_DIR/rpm_payload_safety.sh" - -# shellcheck source=scripts/rpm_payload_safety.sh -source "$PAYLOAD_HELPERS" - -MIRROR=https://ftp.altlinux.org/pub/distributions/ALTLinux -REPORT=sisyphus-set-compare.tsv -LIMIT= -ALL=0 -RESUME=0 -KEEP_WORK=0 -BUILD_MKSET= -PACKAGES=() - -usage() -{ - cat <<'EOF' -Usage: - compare_sisyphus_set_versions.sh --all [OPTIONS] - compare_sisyphus_set_versions.sh --limit N [OPTIONS] - compare_sisyphus_set_versions.sh --package NAME [--package NAME ...] [OPTIONS] - compare_sisyphus_set_versions.sh --build-mkset PATH - -Scopes: - --all Process every x86_64/noarch package record. - --limit N Process only the first N records (testing). - --package NAME Process one named package; may be repeated (testing). - -Options: - --report FILE Streaming TSV report (default: sisyphus-set-compare.tsv). - --resume Skip completed package/architecture/version records. - --keep-work Preserve the temporary working directory. - --mirror URL ALT repository root. - --build-mkset PATH Build only the mkset-compatible newset.c wrapper and exit. - -h, --help Show this help. - -The report gets a START row before processing and DEPENDENCY/SUMMARY rows are -appended after every package, so it can be monitored while the script runs. -No repository-wide run is possible without the explicit --all option. -EOF -} - -die() -{ - printf 'error: %s\n' "$*" >&2 - exit 2 -} - -while (($#)); do - case "$1" in - --all) ALL=1; shift ;; - --limit) (($# >= 2)) || die '--limit requires N'; LIMIT=$2; shift 2 ;; - --package) (($# >= 2)) || die '--package requires NAME'; PACKAGES+=("$2"); shift 2 ;; - --report) (($# >= 2)) || die '--report requires FILE'; REPORT=$2; shift 2 ;; - --resume) RESUME=1; shift ;; - --keep-work) KEEP_WORK=1; shift ;; - --mirror) (($# >= 2)) || die '--mirror requires URL'; MIRROR=${2%/}; shift 2 ;; - --build-mkset) (($# >= 2)) || die '--build-mkset requires PATH'; BUILD_MKSET=$2; shift 2 ;; - -h|--help) usage; exit 0 ;; - *) die "unknown option: $1" ;; - esac -done - -build_mkset() -( -{ - local output=$1 build - build=$(mktemp -d "${TMPDIR:-/tmp}/arsv-newset-build.XXXXXX") - trap 'rm -rf "$build"' EXIT - : >"$build/rpmlib.h" - : >"$build/system.h" - mkdir -p "$(dirname -- "$output")" - - "${CC:-cc}" -O2 -std=gnu11 -D_GNU_SOURCE \ - -I"$build" -include "$NEWSET_COMPAT" \ - -c "$NEWSET_C" -o "$build/newset.o" - "${CC:-cc}" -O2 -std=gnu11 -D_GNU_SOURCE \ - "$NEWSET_WRAPPER" "$build/newset.o" -o "$output" - chmod 755 "$output" -} -) - -if [[ -n $BUILD_MKSET ]]; then - build_mkset "$BUILD_MKSET" - exit 0 -fi - -[[ $MIRROR == http://* || $MIRROR == https://* ]] || die '--mirror must use http:// or https://' -[[ $MIRROR != *[$'\t\r\n ']* ]] || die '--mirror must not contain whitespace' -for package in "${PACKAGES[@]}"; do - [[ $package =~ ^[A-Za-z0-9][A-Za-z0-9+_.-]*$ ]] || die "invalid package name: $package" -done - -scope_count=$ALL -[[ -n $LIMIT ]] && scope_count=$((scope_count + 1)) -((${#PACKAGES[@]} > 0)) && scope_count=$((scope_count + 1)) -((scope_count == 1)) || die 'use --all, --limit, or --package (exactly one scope)' -if [[ -n $LIMIT && ! $LIMIT =~ ^[1-9][0-9]*$ ]]; then - die '--limit must be a positive integer' -fi - -for command in apt-get apt-cache rpm rpmquery rpm2cpio cpio curl md5sum awk sed sort find cp cc realpath; do - command -v "$command" >/dev/null || die "required command not found: $command" -done - -rpmlibdir=$(rpm --eval '%_rpmlibdir') -[[ -x $rpmlibdir/find-provides ]] || die "$rpmlibdir/find-provides is missing (install rpm-build)" -[[ -x $rpmlibdir/find-requires ]] || die "$rpmlibdir/find-requires is missing (install rpm-build)" - -case $REPORT in - /*) ;; - *) REPORT="$PWD/$REPORT" ;; -esac -mkdir -p "$(dirname -- "$REPORT")" -if [[ ! -s $REPORT ]]; then - printf 'timestamp\trecord\tpackage\tarchitecture\tstatus\tside\tcapability\toperator\texpected\tgenerated\tdetail\n' >"$REPORT" -fi - -clean_field() -{ - local value=${1-} - value=${value//$'\t'/ } - value=${value//$'\r'/ } - value=${value//$'\n'/ } - printf '%s' "$value" -} - -report_row() -{ - local fields=() field row - (($# == 11)) || { - printf 'internal error: report row has %s fields, expected 11\n' "$#" >&2 - return 1 - } - for field in "$@"; do - fields+=("$(clean_field "$field")") - done - if [[ ${fields[1]} == SUMMARY ]]; then - case ${fields[10]} in - '') fields[10]='complete=1' ;; - *'; ') fields[10]="${fields[10]}complete=1" ;; - *) fields[10]="${fields[10]}; complete=1" ;; - esac - fi - row=${fields[0]} - for field in "${fields[@]:1}"; do - printf -v row '%s\t%s' "$row" "$field" - done - printf '%s\n' "$row" >>"$REPORT" -} - -timestamp() -{ - date -u '+%Y-%m-%dT%H:%M:%SZ' -} - -WORK=$(mktemp -d "${TMPDIR:-/tmp}/arsv-sisyphus-sets.XXXXXX") -cleanup() -{ - if ((KEEP_WORK)); then - printf 'work directory preserved: %s\n' "$WORK" >&2 - else - rm -rf "$WORK" - fi -} -trap cleanup EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -mkdir -p "$WORK/apt/lists/partial" "$WORK/apt/archives/partial" "$WORK/tools" -: >"$WORK/apt/apt.conf" -: >"$WORK/apt/status" -printf '%s\n' \ - "rpm [alt] $MIRROR Sisyphus/x86_64 classic" \ - "rpm [alt] $MIRROR Sisyphus/noarch classic" >"$WORK/apt/sources.list" - -APT_OPTIONS=( - -o 'Dir::Etc::main=-' - -o 'Dir::Etc::parts=-' - -o "Dir::Etc::sourcelist=$WORK/apt/sources.list" - -o 'Dir::Etc::sourceparts=-' - -o 'Dir::Etc::preferences=-' - -o 'Dir::Etc::preferencesparts=-' - -o "Dir::State::lists=$WORK/apt/lists" - -o "Dir::State::status=$WORK/apt/status" - -o "Dir::Cache::archives=$WORK/apt/archives" - -o "Dir::Cache::pkgcache=$WORK/apt/pkgcache.bin" - -o "Dir::Cache::srcpkgcache=$WORK/apt/srcpkgcache.bin" -) - -apt_get() -{ - APT_CONFIG="$WORK/apt/apt.conf" apt-get "${APT_OPTIONS[@]}" "$@" -} - -apt_cache() -{ - APT_CONFIG="$WORK/apt/apt.conf" apt-cache "${APT_OPTIONS[@]}" "$@" -} - -NEW_MKSET="$WORK/new-mkset" -build_mkset "$NEW_MKSET" -cp -as "$rpmlibdir"/. "$WORK/tools/" -rm -f "$WORK/tools/mkset" -ln -s "$NEW_MKSET" "$WORK/tools/mkset" - -report_row "$(timestamp)" RUN - - start - - - - - "updating isolated Sisyphus indexes" -if ! apt_get update >"$WORK/apt-update.log" 2>&1; then - detail=$(sed -n '/./{p;q;}' "$WORK/apt-update.log") - report_row "$(timestamp)" RUN - - apt_update_error - - - - - "$detail" - exit 1 -fi - -QUEUE="$WORK/packages.tsv" -if ! apt_cache dumpavail | - awk -f "$PACKAGE_PARSER" | - LC_ALL=C sort -t$'\034' -k1,1 -k2,2 >"$QUEUE"; then - report_row "$(timestamp)" RUN - - metadata_error - - - - - 'unable to parse apt-cache dumpavail' - exit 1 -fi - -SELECTED="$WORK/selected.tsv" -if ((${#PACKAGES[@]})); then - : >"$SELECTED" - for package in "${PACKAGES[@]}"; do - if ! awk -F '\034' -v package="$package" '$1 == package { print; found=1 } END { exit !found }' \ - "$QUEUE" >>"$SELECTED"; then - die "package not found in Sisyphus x86_64/noarch: $package" - fi - done -elif [[ -n $LIMIT ]]; then - sed -n "1,${LIMIT}p" "$QUEUE" >"$SELECTED" -else - cp "$QUEUE" "$SELECTED" -fi - -selected_count=$(wc -l <"$SELECTED") -report_row "$(timestamp)" RUN - - ready - - - - - "selected packages: $selected_count" -printf 'selected packages: %s; report: %s\n' "$selected_count" "$REPORT" - -declare -A COMPLETED=() -if ((RESUME)); then - while IFS=$'\t' read -r package architecture version; do - COMPLETED["$package"$'\t'"$architecture"$'\t'"$version"]=1 - done < <(awk -f "$RESUME_PARSER" "$REPORT") -fi - -first_log_line() -{ - local file=$1 - [[ -s $file ]] || return 0 - sed -n '/./{p;q;}' "$file" | cut -c1-500 -} - -last_log_line() -{ - local file=$1 - [[ -s $file ]] || return 0 - awk 'NF { line=$0 } END { print line }' "$file" | cut -c1-500 -} - -extract_rpm() -{ - local package_file=$1 root=$2 log=$3 mode=$4 - local metadata="$root/../.arsv-file-metadata" - local patterns="$root/../.arsv-nonlink-patterns" - local symlinks="$root/../.arsv-symlinks" - local field_separator=$'\034' record_separator=$'\035' record - local filename link_target relative archive_name - local query_format="[%{FILENAMES}${field_separator}%{FILELINKTOS}${record_separator}]" - - if ! rpmquery -p --qf "$query_format" \ - "$package_file" >"$metadata" 2>>"$log"; then - return 1 - fi - case $mode in - symlinks) : >"$symlinks" ;; - files) : >"$patterns" ;; - *) printf 'internal error: invalid RPM extraction mode: %s\n' "$mode" >>"$log"; return 1 ;; - esac - - while IFS= read -r -d "$record_separator" record; do - if ! rpm_split_file_record "$record"; then - printf 'unsupported RPM filename or symlink record\n' >>"$log" - return 1 - fi - filename=$RPM_FILENAME - link_target=$RPM_LINK_TARGET - [[ -n $filename && $filename == /* ]] || { - printf 'unsafe RPM filename: %q\n' "$filename" >>"$log" - return 1 - } - relative=${filename#/} - if [[ $relative == '..' || $relative == ../* || $relative == */../* || $relative == */.. ]]; then - printf 'unsafe RPM filename: %q\n' "$filename" >>"$log" - return 1 - fi - - if [[ -n $link_target && $link_target != '(none)' ]]; then - if [[ $mode == symlinks ]]; then - printf '%s\t%s\n' "$relative" "$link_target" >>"$symlinks" - fi - elif [[ $mode == files ]]; then - archive_name=./$relative - if ! rpm_cpio_literal_pattern "$archive_name" >>"$patterns"; then - printf 'unsupported RPM filename: %q\n' "$filename" >>"$log" - return 1 - fi - fi - done <"$metadata" - - if [[ $mode == files ]]; then - if ! rpm2cpio "$package_file" | - cpio -it --quiet --no-absolute-filenames 2>>"$log" | - while IFS= read -r member; do - case $member in - /*|../*|*/../*|*/..) - printf 'unsafe cpio member: %q\n' "$member" >>"$log" - exit 1 - ;; - esac - done; then - return 1 - fi - - if [[ -s $patterns ]]; then - if ! rpm2cpio "$package_file" | - (cd "$root" && cpio -idm --quiet --no-absolute-filenames -E "$patterns") \ - 2>>"$log"; then - return 1 - fi - fi - return 0 - fi - - cat "$symlinks" -} - -normalize_expected() -{ - local side=$1 package_file=$2 output=$3 - if [[ $side == provides ]]; then - rpmquery -p --qf \ - '[%{PROVIDENAME}\t%{PROVIDEFLAGS:depflags}\t%{PROVIDEVERSION}\n]' \ - "$package_file" - else - rpmquery -p --qf \ - '[%{REQUIRENAME}\t%{REQUIREFLAGS:depflags}\t%{REQUIREVERSION}\n]' \ - "$package_file" - fi | awk -F '\t' '$3 ~ /^set:/ { print $1 "\t" $2 "\t" $3 }' | - LC_ALL=C sort -u >"$output" -} - -normalize_generated() -{ - local side=$1 input=$2 output=$3 - if [[ $side == provides ]]; then - awk '$2 == "=" && $3 ~ /^set:/ { print $1 "\t" $2 "\t" $3 }' "$input" - else - awk '$2 == ">=" && $3 ~ /^set:/ { print $1 "\t" $2 "\t" $3 }' "$input" - fi | LC_ALL=C sort -u >"$output" -} - -compare_side() -{ - local side=$1 package=$2 architecture=$3 expected=$4 generated=$5 output=$6 - if ! awk -F '\t' -v OFS='\034' ' - FILENAME == ARGV[1] { key=$1 "\034" $2; expected[key]=$3; keys[key]=1; next } - { key=$1 "\034" $2; generated[key]=$3; keys[key]=1 } - END { - for (key in keys) { - split(key, parts, "\034") - e=expected[key]; g=generated[key] - if (e == "") status="extra_generated" - else if (g == "") status="missing_generated" - else if (e == g) status="match" - else status="mismatch" - print status, parts[1], parts[2], e, g - } - } - ' "$expected" "$generated" | - LC_ALL=C sort -t$'\034' -k2,2 -k3,3 >"$output"; then - return 1 - fi - - # A non-whitespace separator preserves empty expected/generated fields. - while IFS=$'\034' read -r status capability operator expected_set generated_set; do - [[ -n $status ]] || continue - report_row "$(timestamp)" DEPENDENCY "$package" "$architecture" "$status" \ - "$side" "$capability" "$operator" "$expected_set" "$generated_set" - - done <"$output" -} - -process_package() -{ - local package=$1 architecture=$2 version=$3 filename=$4 md5=$5 - local has_provides=$6 has_requires=$7 - local key=$package$'\t'$architecture$'\t'$version - if [[ -n ${COMPLETED[$key]-} ]]; then - printf 'skip completed: %s.%s\n' "$package" "$architecture" - return 0 - fi - - report_row "$(timestamp)" START "$package" "$architecture" processing - - - - - "$version" - printf 'process: %s.%s\n' "$package" "$architecture" - - if ((has_provides == 0 && has_requires == 0)); then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" no_set_metadata \ - - - - - - 'APT metadata contains no set: Provides/Requires' - return 0 - fi - - local package_work="$WORK/package" - rm -rf "$package_work" - mkdir -p "$package_work/root" "$package_work/archives/partial" - local target="$package_work/target.rpm" - local url="$MIRROR/Sisyphus/files/$architecture/RPMS/$filename" - local log="$package_work/package.log" - - if ! curl --silent --show-error -fL --retry 3 --retry-delay 2 \ - -o "$target" "$url" >"$log" 2>&1; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" download_error \ - - - - - - "$(first_log_line "$log")" - return 0 - fi - if [[ -n $md5 ]]; then - local actual_md5 - if ! actual_md5=$(md5sum "$target" | awk '{print $1}'); then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" checksum_error \ - - - - "$md5" - "$filename" - return 0 - fi - if [[ $actual_md5 != "$md5" ]]; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" checksum_error \ - - - - "$md5" "$actual_md5" "$filename" - return 0 - fi - fi - - local -a payload_rpms=("$target") - if ((has_requires)); then - mkdir -p "$package_work/root/var/lib/rpm" - if ! rpm --root "$package_work/root" --initdb >>"$log" 2>&1; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" rpmdb_error \ - - - - - - "$(last_log_line "$log")" - return 0 - fi - if ! apt_get -y -d \ - -o "Dir::Cache::archives=$package_work/archives" \ - -o "RPM::RootDir=$package_work/root" \ - install "$target" >>"$log" 2>&1; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" dependency_download_error \ - - - - - - "$(last_log_line "$log")" - return 0 - fi - local dependency_rpm - for dependency_rpm in "$package_work"/archives/*.rpm; do - [[ -f $dependency_rpm ]] || continue - payload_rpms+=("$dependency_rpm") - done - fi - - local payload_rpm extraction_status - local symlink_manifest="$package_work/symlinks.tsv" - : >"$symlink_manifest" - for payload_rpm in "${payload_rpms[@]}"; do - if ! extract_rpm "$payload_rpm" "$package_work/root" "$log" symlinks \ - >>"$symlink_manifest"; then - if [[ $payload_rpm == "$target" ]]; then - extraction_status=extract_error - else - extraction_status=dependency_extract_error - fi - report_row "$(timestamp)" SUMMARY "$package" "$architecture" "$extraction_status" \ - - - - - - "$(last_log_line "$log")" - return 0 - fi - done - if ! rpm_install_symlink_manifest "$package_work/root" "$symlink_manifest" "$log"; then - if ((has_requires)); then - extraction_status=dependency_extract_error - else - extraction_status=extract_error - fi - report_row "$(timestamp)" SUMMARY "$package" "$architecture" "$extraction_status" \ - - - - - - "$(last_log_line "$log")" - return 0 - fi - - for payload_rpm in "${payload_rpms[@]}"; do - if ! extract_rpm "$payload_rpm" "$package_work/root" "$log" files; then - if [[ $payload_rpm == "$target" ]]; then - extraction_status=extract_error - else - extraction_status=dependency_extract_error - fi - report_row "$(timestamp)" SUMMARY "$package" "$architecture" "$extraction_status" \ - - - - - - "$(last_log_line "$log")" - return 0 - fi - done - - if ! rpmquery -p --qf '[%{FILENAMES}\n]' "$target" | - awk -v root="$package_work/root" \ - '{ if (substr($0,1,1)=="/") print root $0; else print root "/" $0 }' | - while IFS= read -r path; do - if [[ -e $path || -L $path ]]; then - printf '%s\n' "$path" - fi - done >"$package_work/files"; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" file_list_error \ - - - - - - 'rpmquery failed while reading package filenames' - return 0 - fi - - local scanner_env=( - "RPM_BUILD_ROOT=$package_work/root" - "RPM_SUBPACKAGE_NAME=$package" - "RPM_PACKAGE_NAME=$package" - "RPMB_TOOLS_DIR=$WORK/tools" - "RPMB_LIB_DIR=$rpmlibdir" - "RPMB_AUTODEPS_DIR=$rpmlibdir" - 'RPM_FINDPROV_METHOD=none,lib' - 'RPM_FINDREQ_METHOD=none,lib' - 'RPM_SCRIPTS_DEBUG=0' - ) - - local side generator expected generated comparison - local total_mismatches=0 detail='' - for side in provides requires; do - [[ $side == provides && $has_provides == 1 ]] || - [[ $side == requires && $has_requires == 1 ]] || continue - - generator="$package_work/generated.$side.raw" - expected="$package_work/expected.$side.tsv" - generated="$package_work/generated.$side.tsv" - comparison="$package_work/comparison.$side.tsv" - if ! normalize_expected "$side" "$target" "$expected"; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" metadata_error \ - "$side" - - - - 'rpmquery failed while reading set dependencies' - return 0 - fi - - if [[ $side == provides ]]; then - if ! env "${scanner_env[@]}" "$rpmlibdir/find-provides" \ - <"$package_work/files" >"$generator" 2>"$package_work/$side.log"; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" generator_error \ - "$side" - - - - "$(first_log_line "$package_work/$side.log")" - return 0 - fi - else - if ! env "${scanner_env[@]}" "$rpmlibdir/find-requires" \ - <"$package_work/files" >"$generator" 2>"$package_work/$side.log"; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" generator_error \ - "$side" - - - - "$(first_log_line "$package_work/$side.log")" - return 0 - fi - fi - - if ! normalize_generated "$side" "$generator" "$generated"; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" generator_output_error \ - "$side" - - - - 'unable to normalize generator output' - return 0 - fi - if ! compare_side "$side" "$package" "$architecture" "$expected" "$generated" "$comparison"; then - report_row "$(timestamp)" SUMMARY "$package" "$architecture" comparison_error \ - "$side" - - - - 'unable to compare generated dependencies' - return 0 - fi - - local expected_count generated_count mismatch_count extra_count - expected_count=$(wc -l <"$expected") - generated_count=$(wc -l <"$generated") - mismatch_count=$(awk -F '\034' \ - '$1 == "mismatch" || $1 == "missing_generated" { count++ } END { print count+0 }' \ - "$comparison") - extra_count=$(awk -F '\034' '$1 == "extra_generated" { count++ } END { print count+0 }' \ - "$comparison") - total_mismatches=$((total_mismatches + mismatch_count + extra_count)) - detail+="$side expected=$expected_count generated=$generated_count differences=$mismatch_count extras=$extra_count; " - done - - local status=match - ((total_mismatches == 0)) || status=mismatch - report_row "$(timestamp)" SUMMARY "$package" "$architecture" "$status" - - - - - "$detail" - sync -f "$REPORT" 2>/dev/null || true -} - -while IFS=$'\034' read -r package architecture version filename md5 has_provides has_requires extra; do - if [[ -n ${extra-} || -z $package || -z $architecture || -z $version || -z $filename || - ! $has_provides =~ ^[01]$ || ! $has_requires =~ ^[01]$ ]]; then - report_row "$(timestamp)" RUN "$package" "$architecture" queue_error - - - - - \ - 'invalid package queue record' - continue - fi - process_package "$package" "$architecture" "$version" "$filename" "$md5" \ - "$has_provides" "$has_requires" -done <"$SELECTED" - -report_row "$(timestamp)" RUN - - complete - - - - - "processed selection: $selected_count" -printf 'complete; report: %s\n' "$REPORT" diff --git a/scripts/old/completed_set_versions.awk b/scripts/old/completed_set_versions.awk deleted file mode 100644 index 95f2369..0000000 --- a/scripts/old/completed_set_versions.awk +++ /dev/null @@ -1,10 +0,0 @@ -BEGIN { FS = "\t" } - -NF == 11 && $2 == "START" { - version[$3 "\034" $4] = $11 -} - -NF == 11 && $2 == "SUMMARY" && $11 ~ /(^|; )complete=1$/ && \ - version[$3 "\034" $4] != "" { - print $3 "\t" $4 "\t" version[$3 "\034" $4] -} diff --git a/scripts/old/newset_mkset.c b/scripts/old/newset_mkset.c deleted file mode 100644 index 3215b73..0000000 --- a/scripts/old/newset_mkset.c +++ /dev/null @@ -1,63 +0,0 @@ -/* mkset-compatible wrapper around reimplement/newset.c. - * The stdin/argv contract matches rpm-build/tools/mkset.c. */ -#include -#include -#include - -struct set; -struct set *set_new(void); -void set_add(struct set *set, const char *symbol); -const char *set_fini(struct set *set, int bpp); -struct set *set_free(struct set *set); - -int main(int argc, char **argv) -{ - if (argc != 2) { - fprintf(stderr, "usage: %s BPP\n", argv[0]); - return 2; - } - - char *end = NULL; - errno = 0; - long parsed_bpp = strtol(argv[1], &end, 10); - if (errno || !end || *end || parsed_bpp < 10 || parsed_bpp > 32) { - fprintf(stderr, "invalid BPP: %s (expected 10..32)\n", argv[1]); - return 2; - } - int bpp = (int)parsed_bpp; - - struct set *set = set_new(); - char *line = NULL; - size_t allocated = 0; - ssize_t length; - int added = 0; - - while ((length = getline(&line, &allocated, stdin)) >= 0) { - if (length > 0 && line[length - 1] == '\n') - line[--length] = '\0'; - if (length == 0) - continue; - set_add(set, line); - ++added; - } - - if (!added) { - fputs("mkset: no symbols on standard input\n", stderr); - free(line); - set_free(set); - return 2; - } - const char *encoded = set_fini(set, bpp); - if (!encoded) { - fputs("mkset: unable to encode set\n", stderr); - free(line); - set_free(set); - return 1; - } - printf("set:%s\n", encoded); - - free((void *)encoded); - free(line); - set_free(set); - return 0; -} diff --git a/scripts/old/parse_sisyphus_packages.awk b/scripts/old/parse_sisyphus_packages.awk deleted file mode 100644 index feb628e..0000000 --- a/scripts/old/parse_sisyphus_packages.awk +++ /dev/null @@ -1,39 +0,0 @@ -BEGIN { - RS = "" - FS = "\n" - OFS = "\034" -} - -{ - package = architecture = version = filename = md5 = "" - has_provides = has_requires = 0 - dependency_field = "" - - for (i = 1; i <= NF; ++i) { - if ($i ~ /^Package: /) - package = substr($i, 10) - else if ($i ~ /^Architecture: /) - architecture = substr($i, 15) - else if ($i ~ /^Version: /) - version = substr($i, 10) - else if ($i ~ /^Filename: /) - filename = substr($i, 11) - else if ($i ~ /^MD5Sum: /) - md5 = substr($i, 9) - - if ($i ~ /^Provides: /) - dependency_field = "provides" - else if ($i ~ /^(Pre-Depends|Depends): /) - dependency_field = "requires" - else if ($i !~ /^[[:space:]]/) - dependency_field = "" - - if (dependency_field == "provides" && index($i, "set:")) - has_provides = 1 - if (dependency_field == "requires" && index($i, "set:")) - has_requires = 1 - } - - if (package != "" && architecture != "" && version != "" && filename != "") - print package, architecture, version, filename, md5, has_provides, has_requires -} diff --git a/scripts/old/rpm_payload_safety.sh b/scripts/old/rpm_payload_safety.sh deleted file mode 100644 index 370f5c2..0000000 --- a/scripts/old/rpm_payload_safety.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -rpm_safe_symlink_target() -{ - local root=$1 relative_path=$2 link_target=$3 - local canonical_root link_directory resolved_target - - canonical_root=$(realpath -m -- "$root") || return 1 - link_directory=$(realpath -m -- "$canonical_root/$(dirname -- "$relative_path")") || return 1 - - if [[ $link_target == /* ]]; then - resolved_target=$(realpath -m -- "$canonical_root/${link_target#/}") || return 1 - else - resolved_target=$(realpath -m -- "$link_directory/$link_target") || return 1 - fi - - [[ $resolved_target == "$canonical_root" || $resolved_target == "$canonical_root"/* ]] || return 1 - - if [[ $link_target == /* ]]; then - realpath -m --relative-to="$link_directory" -- "$resolved_target" - else - printf '%s\n' "$link_target" - fi -} - -rpm_cpio_literal_pattern() -{ - local input=$1 output= character index - - [[ $input != *$'\n'* && $input != *$'\r'* ]] || return 1 - for ((index = 0; index < ${#input}; ++index)); do - character=${input:index:1} - case $character in - '[') output+='[[]' ;; - ']') output+='[]]' ;; - '*') output+='[*]' ;; - '?') output+='[?]' ;; - '\\') output+='[\\]' ;; - *) output+=$character ;; - esac - done - printf '%s\n' "$output" -} - -rpm_split_file_record() -{ - local record=$1 separator=$'\034' - - RPM_FILENAME= - RPM_LINK_TARGET= - [[ $record == *"$separator"* ]] || return 1 - RPM_FILENAME=${record%%"$separator"*} - RPM_LINK_TARGET=${record#*"$separator"} - [[ $RPM_LINK_TARGET != *"$separator"* ]] || return 1 - [[ $RPM_FILENAME != *$'\t'* && $RPM_FILENAME != *$'\n'* && $RPM_FILENAME != *$'\r'* ]] || return 1 - [[ $RPM_LINK_TARGET != *$'\t'* && $RPM_LINK_TARGET != *$'\n'* && $RPM_LINK_TARGET != *$'\r'* ]] || return 1 -} - -rpm_install_symlink_manifest() -{ - local root=$1 manifest=$2 log=$3 separator=$'\034' - local canonical_root sorted row relative link_target safe_target destination parent existing - - canonical_root=$(realpath -m -- "$root") || return 1 - - sorted=$(mktemp "${manifest}.sorted.XXXXXX") || return 1 - if ! awk -F '\t' -v separator="$separator" ' - NF >= 2 { - path=$1 - depth=gsub(/\//, "/", path) - printf "%08d%s%s\n", depth, separator, $0 - } - ' "$manifest" | LC_ALL=C sort -t "$separator" -k1,1n -k2,2 >"$sorted"; then - rm -f -- "$sorted" - return 1 - fi - - while IFS=$separator read -r _depth row; do - IFS=$'\t' read -r relative link_target <<<"$row" - [[ -n $relative && -n $link_target ]] || continue - destination=$canonical_root/$relative - parent=$(realpath -m -- "$(dirname -- "$destination")") || { - rm -f -- "$sorted" - return 1 - } - [[ $parent == "$canonical_root" || $parent == "$canonical_root"/* ]] || { - printf 'escaping RPM symlink parent rejected: %q\n' "/$relative" >>"$log" - rm -f -- "$sorted" - return 1 - } - if ! safe_target=$(rpm_safe_symlink_target "$canonical_root" "$relative" "$link_target"); then - printf 'escaping RPM symlink rejected: %q -> %q\n' "/$relative" "$link_target" >>"$log" - rm -f -- "$sorted" - return 1 - fi - mkdir -p -- "$parent" || { - rm -f -- "$sorted" - return 1 - } - - if [[ -L $destination ]]; then - existing=$(readlink -- "$destination") || { - rm -f -- "$sorted" - return 1 - } - if [[ $existing == "$safe_target" ]]; then - continue - fi - printf 'conflicting RPM symlinks: %q -> %q and %q\n' \ - "/$relative" "$existing" "$safe_target" >>"$log" - rm -f -- "$sorted" - return 1 - elif [[ -e $destination ]]; then - printf 'RPM symlink conflicts with existing path: %q\n' "/$relative" >>"$log" - rm -f -- "$sorted" - return 1 - fi - if ! ln -s -- "$safe_target" "$destination"; then - printf 'unable to create RPM symlink: %q -> %q\n' "/$relative" "$safe_target" >>"$log" - rm -f -- "$sorted" - return 1 - fi - done <"$sorted" - - rm -f -- "$sorted" -} diff --git a/scripts/old/run_cmp.cpp b/scripts/old/run_cmp.cpp deleted file mode 100644 index 467ab3b..0000000 --- a/scripts/old/run_cmp.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#include -#include - -extern "C" { -#include "../rpm-build/lib/set.h" -} - -namespace { - -using Clock = std::chrono::steady_clock; -using Nanoseconds = std::chrono::nanoseconds; - -void usage(const char* program) -{ - std::cerr << "Usage: " << program << " PROVIDER_SET REQUIRED_SET\n"; -} - -} // namespace - -int main(int argc, char** argv) -{ - if (argc != 3) { - usage(argv[0]); - return 2; - } - - const std::string provider_set = argv[1]; - const std::string required_set = argv[2]; - - const auto start = Clock::now(); - const int result = rpmsetcmp(provider_set.c_str(), required_set.c_str()); - const auto finish = Clock::now(); - const std::int64_t elapsed = - std::chrono::duration_cast(finish - start).count(); - - std::cout << "provider_set\t" << provider_set << '\n'; - std::cout << "required_set\t" << required_set << '\n'; - std::cout << "cmp_result\t" << result << '\n'; - std::cout << "rpmsetcmp_ns\t" << elapsed << '\n'; - - if (result == -3 || result == -4) { - std::cerr << "run_cmp: rpmsetcmp rejected an input set (result=" << result << ")\n"; - return 1; - } - return 0; -} diff --git a/scripts/old/run_set.cpp b/scripts/old/run_set.cpp deleted file mode 100644 index 4c03c9f..0000000 --- a/scripts/old/run_set.cpp +++ /dev/null @@ -1,585 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -extern "C" { -#include "../rpm-build/lib/set.h" -} - -namespace { - -using Clock = std::chrono::steady_clock; -using Nanoseconds = std::chrono::nanoseconds; -using LabelSet = std::set; - -struct CommandResult { - int exit_code; - std::string output; -}; - -struct Timings { - std::int64_t set_new_ns = 0; - std::int64_t set_add_total_ns = 0; - std::int64_t set_fini_ns = 0; - std::int64_t set_free_ns = 0; - - std::int64_t total_ns() const - { - return set_new_ns + set_add_total_ns + set_fini_ns + set_free_ns; - } -}; - -struct SetResult { - std::string value; - Timings timings; -}; - -struct Options { - std::filesystem::path input; - std::optional bpp; -}; - -std::int64_t elapsed_ns(Clock::time_point start, Clock::time_point finish) -{ - return std::chrono::duration_cast(finish - start).count(); -} - -std::string shortened(const std::string& value, std::size_t limit = 1200) -{ - if (value.size() <= limit) { - return value; - } - return value.substr(0, limit) + "\n... output truncated ..."; -} - -CommandResult run_command(const std::vector& command, - const std::map& environment = {}) -{ - if (command.empty()) { - throw std::runtime_error("empty command"); - } - - int output_pipe[2]; - if (pipe(output_pipe) != 0) { - throw std::runtime_error("pipe failed: " + std::string(std::strerror(errno))); - } - - const pid_t child = fork(); - if (child < 0) { - const int saved_errno = errno; - close(output_pipe[0]); - close(output_pipe[1]); - throw std::runtime_error("fork failed: " + std::string(std::strerror(saved_errno))); - } - - if (child == 0) { - close(output_pipe[0]); - if (dup2(output_pipe[1], STDOUT_FILENO) < 0 || dup2(output_pipe[1], STDERR_FILENO) < 0) { - _exit(126); - } - close(output_pipe[1]); - - setenv("LC_ALL", "C", 1); - for (const auto& [name, value] : environment) { - setenv(name.c_str(), value.c_str(), 1); - } - - std::vector argv; - argv.reserve(command.size() + 1); - for (const std::string& argument : command) { - argv.push_back(const_cast(argument.c_str())); - } - argv.push_back(nullptr); - execvp(argv[0], argv.data()); - _exit(127); - } - - close(output_pipe[1]); - std::string output; - char buffer[16384]; - while (true) { - const ssize_t count = read(output_pipe[0], buffer, sizeof(buffer)); - if (count > 0) { - output.append(buffer, static_cast(count)); - continue; - } - if (count < 0 && errno == EINTR) { - continue; - } - if (count < 0) { - const int saved_errno = errno; - close(output_pipe[0]); - waitpid(child, nullptr, 0); - throw std::runtime_error("read from child failed: " + std::string(std::strerror(saved_errno))); - } - break; - } - close(output_pipe[0]); - - int status = 0; - while (waitpid(child, &status, 0) < 0) { - if (errno != EINTR) { - throw std::runtime_error("waitpid failed: " + std::string(std::strerror(errno))); - } - } - - int exit_code = 128; - if (WIFEXITED(status)) { - exit_code = WEXITSTATUS(status); - } else if (WIFSIGNALED(status)) { - exit_code += WTERMSIG(status); - } - return {exit_code, std::move(output)}; -} - -CommandResult checked_command(const std::vector& command, - const std::map& environment = {}) -{ - CommandResult result = run_command(command, environment); - if (result.exit_code != 0) { - std::ostringstream message; - message << command.front() << " exited with " << result.exit_code; - if (!result.output.empty()) { - message << ":\n" << shortened(result.output); - } - throw std::runtime_error(message.str()); - } - return result; -} - -std::string strip_symbol_version(std::string symbol) -{ - const std::size_t at = symbol.find('@'); - if (at != std::string::npos) { - symbol.resize(at); - } - return symbol; -} - -bool allowed_symbol_type(const std::string& type) -{ - static const std::unordered_set allowed = { - "NOTYPE", "OBJECT", "FUNC", "COMMON", "TLS", "IFUNC", "GNU_IFUNC", - }; - return allowed.count(type) != 0; -} - -bool allowed_symbol_binding(const std::string& binding) -{ - static const std::unordered_set allowed = { - "GLOBAL", "WEAK", "UNIQUE", "GNU_UNIQUE", - }; - return allowed.count(binding) != 0; -} - -bool allowed_symbol_visibility(const std::string& visibility) -{ - return visibility == "DEFAULT" || visibility == "PROTECTED"; -} - -bool special_symbol(const std::string& symbol) -{ - static const std::unordered_set ignored = { - "__bss_start", "_edata", "_end", "_fini", "_init", - }; - return ignored.count(symbol) != 0; -} - -LabelSet provided_labels(const std::filesystem::path& library) -{ - const CommandResult result = checked_command( - {"eu-readelf", "--wide", "--dyn-syms", library.string()}); - LabelSet labels; - std::istringstream output(result.output); - std::string line; - while (std::getline(output, line)) { - std::istringstream fields(line); - std::string number; - std::string value; - std::string size; - std::string type; - std::string binding; - std::string visibility; - std::string section; - std::string symbol; - if (!(fields >> number >> value >> size >> type >> binding >> visibility >> section >> symbol)) { - continue; - } - if (number.empty() || number.back() != ':') { - continue; - } - - std::uint64_t symbol_value = 0; - try { - symbol_value = std::stoull(value, nullptr, 16); - } catch (const std::exception&) { - continue; - } - if (symbol_value == 0 && type != "TLS") { - continue; - } - if (!allowed_symbol_type(type) || !allowed_symbol_binding(binding) || - !allowed_symbol_visibility(visibility)) { - continue; - } - - symbol = strip_symbol_version(std::move(symbol)); - if (symbol.empty() || special_symbol(symbol) || - symbol.find_first_of("()@") != std::string::npos) { - continue; - } - labels.insert(std::move(symbol)); - } - return labels; -} - -LabelSet weak_undefined_labels(const std::filesystem::path& executable) -{ - const CommandResult result = checked_command({"nm", "--dynamic", executable.string()}); - LabelSet weak; - std::istringstream output(result.output); - std::string line; - while (std::getline(output, line)) { - std::istringstream fields(line); - std::vector parts; - std::string part; - while (fields >> part) { - parts.push_back(part); - } - if (parts.size() != 2 || parts[0].size() != 1 || - std::string("wWvV").find(parts[0][0]) == std::string::npos) { - continue; - } - weak.insert(strip_symbol_version(parts[1])); - } - return weak; -} - -std::string program_interpreter(const std::filesystem::path& executable) -{ - const CommandResult result = checked_command( - {"eu-readelf", "--program-headers", executable.string()}); - const std::string marker = "[Requesting program interpreter: "; - const std::size_t start = result.output.find(marker); - if (start == std::string::npos) { - throw std::runtime_error("ELF executable has no PT_INTERP entry: " + executable.string()); - } - const std::size_t value_start = start + marker.size(); - const std::size_t end = result.output.find(']', value_start); - if (end == std::string::npos || end == value_start) { - throw std::runtime_error("cannot parse PT_INTERP for " + executable.string()); - } - return result.output.substr(value_start, end - value_start); -} - -bool same_path(const std::string& lhs, const std::filesystem::path& rhs) -{ - std::error_code error; - if (std::filesystem::equivalent(lhs, rhs, error)) { - return true; - } - return std::filesystem::path(lhs).lexically_normal() == rhs.lexically_normal(); -} - -std::map required_labels_by_provider( - const std::filesystem::path& executable) -{ - const std::string interpreter = program_interpreter(executable); - const std::map environment = { - {"LD_BIND_NOW", "1"}, - {"LD_DEBUG", "bindings"}, - {"LD_TRACE_LOADED_OBJECTS", "1"}, - {"LD_WARN", "1"}, - }; - const CommandResult result = checked_command( - {interpreter, executable.string()}, environment); - const LabelSet weak = weak_undefined_labels(executable); - std::map labels_by_provider; - - std::istringstream output(result.output); - std::string line; - while (std::getline(output, line)) { - const std::string binding_marker = "binding file "; - const std::size_t binding = line.find(binding_marker); - if (binding == std::string::npos) { - continue; - } - const std::size_t source_start = binding + binding_marker.size(); - const std::size_t source_end = line.find(" [", source_start); - const std::size_t to = source_end == std::string::npos - ? std::string::npos - : line.find(" to ", source_end); - const std::size_t provider_start = to == std::string::npos - ? std::string::npos - : to + std::string(" to ").size(); - const std::size_t provider_end = provider_start == std::string::npos - ? std::string::npos - : line.find(" [", provider_start); - const std::size_t symbol_marker = provider_end == std::string::npos - ? std::string::npos - : line.find(" symbol ", provider_end); - if (source_end == std::string::npos || to == std::string::npos || - provider_start == std::string::npos || provider_end == std::string::npos || - symbol_marker == std::string::npos) { - continue; - } - - const std::string source = line.substr(source_start, source_end - source_start); - const std::string provider = line.substr(provider_start, provider_end - provider_start); - if (!same_path(source, executable) || source == provider) { - continue; - } - - std::size_t symbol_start = symbol_marker + std::string(" symbol ").size(); - if (symbol_start < line.size() && (line[symbol_start] == '`' || line[symbol_start] == '\'')) { - ++symbol_start; - } - std::size_t symbol_end = line.find('\'', symbol_start); - if (symbol_end == std::string::npos) { - symbol_end = line.find_first_of(" \t[", symbol_start); - } - if (symbol_end == std::string::npos) { - symbol_end = line.size(); - } - std::string symbol = strip_symbol_version( - line.substr(symbol_start, symbol_end - symbol_start)); - if (symbol.empty() || weak.count(symbol) != 0) { - continue; - } - labels_by_provider[provider].insert(std::move(symbol)); - } - - for (auto it = labels_by_provider.begin(); it != labels_by_provider.end();) { - if (it->second.empty()) { - it = labels_by_provider.erase(it); - } else { - ++it; - } - } - return labels_by_provider; -} - -int suggested_bpp(std::size_t provider_label_count) -{ - if (provider_label_count < 1) { - provider_label_count = 1; - } - std::size_t value = provider_label_count - 1; - int bits = 0; - while (value != 0) { - ++bits; - value >>= 1; - } - return std::min(32, bits + 10); -} - -SetResult build_set(const LabelSet& labels, int bpp) -{ - if (labels.empty()) { - throw std::runtime_error("cannot build a set from zero labels"); - } - - Timings timings; - auto start = Clock::now(); - struct set* value = set_new(); - auto finish = Clock::now(); - timings.set_new_ns = elapsed_ns(start, finish); - if (value == nullptr) { - throw std::runtime_error("set_new returned NULL"); - } - - start = Clock::now(); - for (const std::string& label : labels) { - set_add(value, label.c_str()); - } - finish = Clock::now(); - timings.set_add_total_ns = elapsed_ns(start, finish); - - start = Clock::now(); - const char* payload = set_fini(value, bpp); - finish = Clock::now(); - timings.set_fini_ns = elapsed_ns(start, finish); - if (payload == nullptr) { - set_free(value); - throw std::runtime_error("set_fini returned NULL"); - } - std::string encoded = "set:" + std::string(payload); - std::free(const_cast(payload)); - - start = Clock::now(); - value = set_free(value); - finish = Clock::now(); - timings.set_free_ns = elapsed_ns(start, finish); - (void)value; - return {std::move(encoded), timings}; -} - -enum class ElfKind { - executable, - shared_library, -}; - -ElfKind classify_elf(const std::filesystem::path& input) -{ - const CommandResult executable = run_command( - {"eu-elfclassify", "--executable", input.string()}); - if (executable.exit_code == 0) { - return ElfKind::executable; - } - if (executable.exit_code == 127) { - throw std::runtime_error("eu-elfclassify is required but was not found"); - } - - const CommandResult shared = run_command( - {"eu-elfclassify", "--shared", input.string()}); - if (shared.exit_code == 0) { - return ElfKind::shared_library; - } - throw std::runtime_error("input is not a supported dynamic ELF executable or shared library: " + - input.string()); -} - -int parse_bpp(const std::string& value) -{ - std::size_t parsed = 0; - int bpp = 0; - try { - bpp = std::stoi(value, &parsed); - } catch (const std::exception&) { - throw std::runtime_error("invalid bpp: " + value); - } - if (parsed != value.size() || bpp < 10 || bpp > 32) { - throw std::runtime_error("bpp must be an integer in [10, 32]"); - } - return bpp; -} - -void usage(const char* program) -{ - std::cerr << "Usage: " << program << " [--bpp 10..32] ELF_PATH\n"; -} - -Options parse_options(int argc, char** argv) -{ - Options options; - for (int index = 1; index < argc; ++index) { - const std::string argument = argv[index]; - if (argument == "--help" || argument == "-h") { - usage(argv[0]); - std::exit(0); - } - if (argument == "--bpp") { - if (++index >= argc) { - throw std::runtime_error("--bpp requires a value"); - } - options.bpp = parse_bpp(argv[index]); - continue; - } - if (!argument.empty() && argument.front() == '-') { - throw std::runtime_error("unknown option: " + argument); - } - if (!options.input.empty()) { - throw std::runtime_error("exactly one ELF path is required"); - } - options.input = argument; - } - if (options.input.empty()) { - throw std::runtime_error("ELF path is required"); - } - return options; -} - -void print_header(const std::filesystem::path& input, ElfKind kind, - const std::optional& bpp) -{ - std::cout << "input\t" << input.string() << '\n'; - std::cout << "kind\t" - << (kind == ElfKind::executable ? "executable" : "shared-library") << '\n'; - std::cout << "bpp_mode\t" << (bpp.has_value() ? "override" : "auto") << '\n'; - std::cout << "role\tobject\tlabels\tbpp\tset\tset_new_ns\tset_add_total_ns\t" - "set_fini_ns\tset_free_ns\tset_api_total_ns\n"; -} - -void print_row(const std::string& role, const std::string& object, - std::size_t label_count, int bpp, const SetResult& result) -{ - const Timings& timings = result.timings; - std::cout << role << '\t' << object << '\t' << label_count << '\t' << bpp << '\t' - << result.value << '\t' << timings.set_new_ns << '\t' - << timings.set_add_total_ns << '\t' << timings.set_fini_ns << '\t' - << timings.set_free_ns << '\t' << timings.total_ns() << '\n'; -} - -int run(const Options& options) -{ - std::error_code error; - const std::filesystem::path input = std::filesystem::canonical(options.input, error); - if (error || !std::filesystem::is_regular_file(input)) { - throw std::runtime_error("input is not a readable regular file: " + options.input.string()); - } - if (input.string().find_first_of("\t\r\n") != std::string::npos) { - throw std::runtime_error("input path contains characters unsupported by TSV output"); - } - - const ElfKind kind = classify_elf(input); - print_header(input, kind, options.bpp); - - if (kind == ElfKind::shared_library) { - const LabelSet labels = provided_labels(input); - if (labels.empty()) { - throw std::runtime_error("no provided labels found in " + input.string()); - } - const int bpp = options.bpp.value_or(suggested_bpp(labels.size())); - print_row("provided", input.string(), labels.size(), bpp, build_set(labels, bpp)); - return 0; - } - - const std::map requirements = required_labels_by_provider(input); - if (requirements.empty()) { - throw std::runtime_error("no external dynamic symbol bindings found in " + input.string()); - } - for (const auto& [provider, labels] : requirements) { - std::size_t provider_count = 0; - try { - provider_count = provided_labels(provider).size(); - } catch (const std::exception& exception) { - std::cerr << "warning: cannot inspect provider " << provider << ": " - << exception.what() << '\n'; - } - const int bpp = options.bpp.value_or( - suggested_bpp(provider_count == 0 ? labels.size() : provider_count)); - print_row("required", provider, labels.size(), bpp, build_set(labels, bpp)); - } - return 0; -} - -} // namespace - -int main(int argc, char** argv) -{ - try { - return run(parse_options(argc, argv)); - } catch (const std::exception& exception) { - std::cerr << "run_set: " << exception.what() << '\n'; - usage(argv[0]); - return 2; - } -} diff --git a/scripts/run-setc-bench.sh b/scripts/run-setc-bench.sh index 759294c..6935513 100755 --- a/scripts/run-setc-bench.sh +++ b/scripts/run-setc-bench.sh @@ -11,7 +11,7 @@ CPU=0 RESET_WORK=1 # 0 — продолжить готовые сборки, 1 — начать всё заново. COLLECT_PERF=1 PERF_RECORD=1 # Отдельный профильный прогон; не входит в среднее время. -PERF_FREQUENCY=99 +PERF_FREQUENCY=499 PERF_EVENTS='task-clock,context-switches,cpu-migrations,page-faults,minor-faults,major-faults,cycles,instructions,branches,branch-misses,cache-references,cache-misses,L1-dcache-loads,L1-dcache-load-misses,LLC-loads,LLC-load-misses' PACKAGER='krosh ' APT_SOURCE=/etc/apt/sources.list.d/alt.list @@ -147,7 +147,7 @@ append_perf_stat() record_profile() { - local operation=$1 variant=$2 perf_dir=$3 status data report + local operation=$1 variant=$2 perf_dir=$3 dso_name=$4 status data report local libdir="$variant/lib/usr/lib64" local root="$COMMON/root" local -a command @@ -187,17 +187,39 @@ record_profile() --input "$data" \ >"$report" \ 2>"$perf_dir/$operation.report.stderr" || true + + perf --buildid-dir "$perf_dir/buildid-cache" report \ + --stdio \ + --no-children \ + --inline \ + --percent-limit 0 \ + --sort dso,symbol,srcline \ + --input "$data" \ + >"$perf_dir/$operation.lines.txt" \ + 2>"$perf_dir/$operation.lines.stderr" || true + + # Без TUI: perf 6.18 может упасть при выборе символа без self-samples. + perf --buildid-dir "$perf_dir/buildid-cache" annotate \ + --stdio \ + --dsos "$dso_name" \ + --input "$data" \ + >"$perf_dir/$operation.annotate.txt" \ + 2>"$perf_dir/$operation.annotate.stderr" || true fi } prepare_perf_symbols() { - local perf_dir=$1 debug_file=$2 debuginfo_rpm=$3 + local perf_dir=$1 runtime_file=$2 debug_file=$3 debuginfo_rpm=$4 + local unstripped="$perf_dir/librpm.unstripped" mkdir -p "$perf_dir/buildid-cache" cp -a "$debuginfo_rpm" "$perf_dir/" + # Split-debug ELF содержит DWARF, но не байты .text для annotate. + # eu-unstrip объединяет runtime-код и matching debuginfo с тем же Build ID. + eu-unstrip -o "$unstripped" "$runtime_file" "$debug_file" perf --buildid-dir "$perf_dir/buildid-cache" buildid-cache \ - --add "$debug_file" \ + --add "$unstripped" \ >"$perf_dir/buildid-cache.stdout" \ 2>"$perf_dir/buildid-cache.stderr" } @@ -205,6 +227,7 @@ prepare_perf_symbols() benchmark_variant() { local variant=$1 result=$2 debug_file=$3 debuginfo_rpm=$4 + local runtime_file=$5 dso_name=$6 local operation run average label status_text perf_result perf_dir local -a times statuses local -a operations=( @@ -228,7 +251,8 @@ benchmark_variant() if ((PERF_RECORD)); then rm -rf "$perf_dir" mkdir -p "$perf_dir" - prepare_perf_symbols "$perf_dir" "$debug_file" "$debuginfo_rpm" + prepare_perf_symbols "$perf_dir" "$runtime_file" \ + "$debug_file" "$debuginfo_rpm" fi fi @@ -267,13 +291,13 @@ benchmark_variant() if ((COLLECT_PERF && PERF_RECORD)); then printf '%s: perf record\n' "$operation" - record_profile "$operation" "$variant" "$perf_dir" + record_profile "$operation" "$variant" "$perf_dir" "$dso_name" fi done } for command in git gear-hsh hsh rpm rpmquery rpm2cpio cpio apt-get apt-cache \ - taskset awk sed date sha256sum ldd readelf; do + taskset awk sed date sha256sum ldd readelf readlink eu-unstrip; do command -v "$command" >/dev/null || fail "required command not found: $command" done if ((COLLECT_PERF)); then @@ -424,6 +448,8 @@ for setc in "${setc_files[@]}"; do libdir="$variant/lib/usr/lib64" [[ -e $libdir/librpm.so.7 && -e $libdir/librpmio.so.7 ]] || \ fail "librpm libraries were not extracted for $filename" + runtime_file=$(readlink -f "$libdir/librpm.so.7") + dso_name=$(basename "$runtime_file") LD_LIBRARY_PATH="$libdir" ldd "$APT_GET" | \ grep -F "$libdir/librpm.so.7" >/dev/null || \ fail "apt-get does not load the built librpm for $filename" @@ -446,7 +472,7 @@ for setc in "${setc_files[@]}"; do fi benchmark_variant "$variant" "$RESULT_DIR/$result_name" \ - "$debug_file" "$debuginfo_rpm" + "$debug_file" "$debuginfo_rpm" "$runtime_file" "$dso_name" done printf '\nResults:\n'