add new implement, remove old tests
This commit is contained in:
@@ -0,0 +1,935 @@
|
||||
#include <assert.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
#include "rpmlib.h"
|
||||
#ifdef SELF_TEST
|
||||
#undef NDEBUG
|
||||
#include <stdio.h>
|
||||
#endif
|
||||
#include "set.h"
|
||||
#include "system.h"
|
||||
|
||||
#define CACHE_SIZE 512
|
||||
#define PIVOT_SIZE 486
|
||||
|
||||
struct set {
|
||||
size_t cnt;
|
||||
size_t symbols_cap;
|
||||
size_t strings_len;
|
||||
size_t strings_cap;
|
||||
char* strings;
|
||||
struct symbols {
|
||||
size_t offset;
|
||||
unsigned hash;
|
||||
}* symbols_v;
|
||||
};
|
||||
|
||||
struct set* set_new() {
|
||||
struct set* set = xmalloc(sizeof *set);
|
||||
set->cnt = 0;
|
||||
set->symbols_cap = 0;
|
||||
set->strings_len = 0;
|
||||
set->strings_cap = 0;
|
||||
set->strings = NULL;
|
||||
set->symbols_v = NULL;
|
||||
|
||||
return set;
|
||||
}
|
||||
|
||||
void set_add(struct set* set, const char* sym) {
|
||||
if (set->cnt == set->symbols_cap) {
|
||||
set->symbols_cap += 1024;
|
||||
set->symbols_v = xrealloc(set->symbols_v, sizeof(*set->symbols_v) * set->symbols_cap);
|
||||
}
|
||||
|
||||
size_t length = strlen(sym) + 1;
|
||||
size_t required = set->strings_len + length;
|
||||
if (required > set->strings_cap) {
|
||||
size_t capacity = set->strings_cap ? set->strings_cap : 4096;
|
||||
while (capacity < required) capacity *= 2;
|
||||
|
||||
set->strings = xrealloc(set->strings, capacity);
|
||||
set->strings_cap = capacity;
|
||||
}
|
||||
|
||||
set->symbols_v[set->cnt].offset = set->strings_len;
|
||||
set->symbols_v[set->cnt].hash = 0;
|
||||
memcpy(set->strings + set->strings_len, sym, length);
|
||||
set->strings_len = required;
|
||||
set->cnt++;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
struct set* set_free(struct set* set) {
|
||||
if (set) {
|
||||
_free(set->strings);
|
||||
_free(set->symbols_v);
|
||||
set = _free(set);
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
static unsigned hash(const char* str) {
|
||||
unsigned hash = 0x9e3779b9;
|
||||
const unsigned char* p = (const unsigned char*)str;
|
||||
|
||||
while (*p) {
|
||||
hash += *p++;
|
||||
hash += (hash << 10);
|
||||
hash ^= (hash >> 6);
|
||||
}
|
||||
|
||||
hash += (hash << 3);
|
||||
hash ^= (hash >> 11);
|
||||
hash += (hash << 15);
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
int cmp(const void* arg1, const void* arg2) {
|
||||
const struct symbols* s1 = arg1;
|
||||
const struct symbols* s2 = arg2;
|
||||
|
||||
if (s1->hash > s2->hash) return 1;
|
||||
if (s2->hash > s1->hash) return -1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void sort_symbols(struct symbols* values, size_t count, int bpp) {
|
||||
if (count < 128) {
|
||||
qsort(values, count, sizeof(*values), cmp);
|
||||
return;
|
||||
}
|
||||
|
||||
struct symbols temporary[count];
|
||||
struct symbols* source = values;
|
||||
struct symbols* destination = temporary;
|
||||
unsigned passes = ((unsigned)bpp + 7) / 8;
|
||||
|
||||
for (unsigned pass = 0; pass < passes; ++pass) {
|
||||
size_t offsets[256] = {0};
|
||||
unsigned shift = pass * 8;
|
||||
for (size_t i = 0; i < count; ++i) ++offsets[(source[i].hash >> shift) & 0xffu];
|
||||
|
||||
size_t position = 0;
|
||||
for (size_t i = 0; i < 256; ++i) {
|
||||
size_t bucket_count = offsets[i];
|
||||
offsets[i] = position;
|
||||
position += bucket_count;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
unsigned bucket = (source[i].hash >> shift) & 0xffu;
|
||||
destination[offsets[bucket]++] = source[i];
|
||||
}
|
||||
|
||||
struct symbols* swap = source;
|
||||
source = destination;
|
||||
destination = swap;
|
||||
}
|
||||
|
||||
if (source != values) memcpy(values, source, count * sizeof(*values));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
static int log2i(int n) {
|
||||
int m = 0;
|
||||
while (n /= 2) m++;
|
||||
|
||||
return m;
|
||||
}
|
||||
|
||||
// Calculate Mshift paramter for encoding.
|
||||
static int encode_golomb_Mshift(int cnt, int bpp) {
|
||||
// XXX Slightly better Mshift estimations are probably possible.
|
||||
// Recheck "Compression and coding algorithms" by Moffat & Turpin.
|
||||
int Mshift = bpp - log2i(cnt) - 1;
|
||||
|
||||
// Adjust out-of-range values.
|
||||
Mshift = (Mshift < 7) ? 7 : Mshift;
|
||||
Mshift = (Mshift > 31) ? 31 : Mshift;
|
||||
assert(Mshift < bpp);
|
||||
|
||||
return Mshift;
|
||||
}
|
||||
|
||||
// Estimate how many bits can be filled up.
|
||||
static inline int encode_golomb_size(int cnt, int Mshift) {
|
||||
// XXX No precise estimation. However, we do not expect unary-encoded bits
|
||||
// to take more than binary-encoded Mshift bits.
|
||||
return Mshift * 2 * cnt + 16;
|
||||
}
|
||||
|
||||
// Estimate base62 buffer size required to encode a given number of bits.
|
||||
static inline int encode_base62_size(int bit_cnt) {
|
||||
// In the worst case, which is ZxZxZx..., five bits can make a character;
|
||||
// the remaining bits can make a character, too. And the string must be
|
||||
// null-terminated.
|
||||
return bit_cnt / 5 + 2;
|
||||
}
|
||||
|
||||
static int encode_set_size(int cnt, int bpp) {
|
||||
int Mshift = encode_golomb_Mshift(cnt, bpp);
|
||||
int bit_cnt = encode_golomb_size(cnt, Mshift);
|
||||
// two leading characters are special
|
||||
return 2 + encode_base62_size(bit_cnt);
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
// Main base62 encoding routine: pack bit_arr into base62 string.
|
||||
/*
|
||||
* Base62 routines - encode bits with alnum characters.
|
||||
*
|
||||
* This is a base64-based base62 implementation. Values 0..61 are encoded
|
||||
* with '0'..'9', 'a'..'z', and 'A'..'Z'. However, 'Z' is special: it will
|
||||
* also encode 62 and 63. To achieve this, 'Z' will occupy two high bits in
|
||||
* the next character. Thus 'Z' can be interpreted as an escape character
|
||||
* (which indicates that the next character must be handled specially).
|
||||
* Note that setting high bits to "00", "01" or "10" cannot contribute
|
||||
* to another 'Z' (which would require high bits set to "11"). This is
|
||||
* how multiple escapes are avoided.
|
||||
*/
|
||||
|
||||
static char* bits_to_char(int c, char* base62) {
|
||||
assert(c >= 0 && c <= 61);
|
||||
|
||||
if (c < 10) {
|
||||
*base62++ = c + '0';
|
||||
} else if (c < 36) {
|
||||
*base62++ = c - 10 + 'a';
|
||||
} else if (c < 62) {
|
||||
*base62++ = c - 36 + 'A';
|
||||
}
|
||||
|
||||
return base62;
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
static inline char encode_bpp(int bpp) { return bpp - 7 + 'a'; }
|
||||
|
||||
struct encode_writer {
|
||||
uint64_t bits;
|
||||
unsigned filled;
|
||||
unsigned escaped;
|
||||
unsigned pending_high;
|
||||
char* output;
|
||||
};
|
||||
|
||||
static inline void encode_writer_digit(struct encode_writer* writer, unsigned value) {
|
||||
assert(value < 62);
|
||||
|
||||
if (value < 10) {
|
||||
*writer->output++ = (char)('0' + value);
|
||||
} else if (value < 36) {
|
||||
*writer->output++ = (char)('a' + value - 10);
|
||||
} else {
|
||||
*writer->output++ = (char)('A' + value - 36);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void encode_writer_flush(struct encode_writer* writer) {
|
||||
for (;;) {
|
||||
unsigned width = writer->escaped ? 4u : 6u;
|
||||
if (writer->filled < width) return;
|
||||
|
||||
unsigned value = (unsigned)writer->bits & ((1u << width) - 1);
|
||||
writer->bits >>= width;
|
||||
writer->filled -= width;
|
||||
|
||||
if (writer->escaped) {
|
||||
encode_writer_digit(writer, writer->pending_high | value);
|
||||
writer->escaped = 0;
|
||||
} else if (value >= 61) {
|
||||
encode_writer_digit(writer, 61);
|
||||
writer->pending_high = (value - 61) << 4;
|
||||
writer->escaped = 1;
|
||||
} else {
|
||||
encode_writer_digit(writer, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void encode_writer_zeros(struct encode_writer* writer, unsigned count) {
|
||||
while (count) {
|
||||
unsigned take = count > 56 ? 56 : count;
|
||||
writer->filled += take;
|
||||
count -= take;
|
||||
|
||||
encode_writer_flush(writer);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
static inline void encode_writer_put(struct encode_writer* writer, uint64_t value, unsigned width) {
|
||||
writer->bits |= value << writer->filled;
|
||||
writer->filled += width;
|
||||
encode_writer_flush(writer);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
static int encode_set(int cnt, const unsigned* hash_arr, int bpp, char* base62_str) {
|
||||
const unsigned Mshift = (unsigned)encode_golomb_Mshift(cnt, bpp);
|
||||
const unsigned mask = (1u << Mshift) - 1;
|
||||
char* const start = base62_str;
|
||||
unsigned previous = 0;
|
||||
|
||||
*base62_str++ = encode_bpp(bpp);
|
||||
*base62_str++ = encode_bpp((int)Mshift);
|
||||
struct encode_writer writer = {.output = base62_str};
|
||||
|
||||
for (int i = 0; i < cnt; ++i) {
|
||||
unsigned current = hash_arr[i];
|
||||
unsigned delta = current - previous;
|
||||
previous = current;
|
||||
|
||||
encode_writer_zeros(&writer, delta >> Mshift);
|
||||
encode_writer_put(&writer, 1, 1);
|
||||
encode_writer_put(&writer, delta & mask, Mshift);
|
||||
}
|
||||
|
||||
encode_writer_flush(&writer);
|
||||
if (writer.filled || writer.escaped) {
|
||||
unsigned value = (unsigned)writer.bits;
|
||||
if (writer.escaped) value |= writer.pending_high;
|
||||
encode_writer_digit(&writer, value);
|
||||
}
|
||||
|
||||
*writer.output = '\0';
|
||||
|
||||
return (int)(writer.output - start);
|
||||
}
|
||||
|
||||
const char* set_fini(struct set* set, int bpp) {
|
||||
// Implementation for finalizing the set
|
||||
|
||||
assert(set != NULL);
|
||||
assert(set->cnt > 0);
|
||||
assert(bpp >= 10 && bpp <= 32);
|
||||
|
||||
unsigned mask = (bpp < 32) ? (1u << bpp) - 1 : ~0u;
|
||||
|
||||
for (size_t i = 0; i < set->cnt; ++i) {
|
||||
set->symbols_v[i].hash = hash(set->strings + set->symbols_v[i].offset) & mask;
|
||||
}
|
||||
|
||||
sort_symbols(set->symbols_v, set->cnt, bpp);
|
||||
|
||||
// warn on hash collizions
|
||||
for (size_t i = 0; i < set->cnt - 1; ++i) {
|
||||
if (set->symbols_v[i].hash != set->symbols_v[i + 1].hash) continue;
|
||||
const char* left = set->strings + set->symbols_v[i].offset;
|
||||
const char* right = set->strings + set->symbols_v[i + 1].offset;
|
||||
if (!strcmp(left, right)) continue;
|
||||
|
||||
fprintf(stderr, "warning: hash collision: %s %s\n", left, right);
|
||||
}
|
||||
|
||||
unsigned unique_hash[set->cnt];
|
||||
size_t unique_cnt = 0;
|
||||
|
||||
// delete duplicates
|
||||
for (size_t i = 0; i < set->cnt; ++i) {
|
||||
while (i + 1 < set->cnt && set->symbols_v[i].hash == set->symbols_v[i + 1].hash) {
|
||||
++i;
|
||||
}
|
||||
unique_hash[unique_cnt++] = set->symbols_v[i].hash;
|
||||
}
|
||||
|
||||
char base62_str[encode_set_size(unique_cnt, bpp)];
|
||||
encode_set(unique_cnt, unique_hash, bpp, base62_str);
|
||||
|
||||
return xstrdup(base62_str);
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
struct set_meta {
|
||||
const char* str;
|
||||
const char* payload;
|
||||
size_t len;
|
||||
size_t payload_len;
|
||||
int bpp;
|
||||
int Mshift;
|
||||
int bit_capacity;
|
||||
int value_capacity;
|
||||
};
|
||||
|
||||
static int set_meta_init(const char* str, struct set_meta* meta) {
|
||||
// len >= 3
|
||||
if (!str[0] || !str[1] || !str[2]) return -4;
|
||||
|
||||
int bpp = str[0] + 7 - 'a';
|
||||
if (bpp < 10 || bpp > 32) return -1;
|
||||
|
||||
int Mshift = str[1] + 7 - 'a';
|
||||
if (Mshift < 7 || Mshift > 31) return -2;
|
||||
if (Mshift >= bpp) return -3;
|
||||
|
||||
*meta = (struct set_meta){
|
||||
.str = str,
|
||||
.payload = str + 2,
|
||||
.bpp = bpp,
|
||||
.Mshift = Mshift,
|
||||
};
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int set_meta_fini(struct set_meta* meta) {
|
||||
size_t len = strlen(meta->str);
|
||||
|
||||
size_t payload_len = len - 2;
|
||||
|
||||
int bit_capacity = (int)payload_len * 6;
|
||||
int value_capacity = bit_capacity / (meta->Mshift + 1);
|
||||
|
||||
if (value_capacity < 1) return -4;
|
||||
|
||||
meta->len = len;
|
||||
meta->payload_len = payload_len;
|
||||
meta->bit_capacity = bit_capacity;
|
||||
meta->value_capacity = value_capacity;
|
||||
|
||||
return 0;
|
||||
}
|
||||
// UCHAR_MAX == 255
|
||||
static const unsigned char char_to_num[255 + 1] = {[0] = 0xff, /* конец строки */
|
||||
|
||||
[1 ...('0' - 1)] = 0xee,
|
||||
|
||||
['0'] = 0,
|
||||
['1'] = 1,
|
||||
['2'] = 2,
|
||||
['3'] = 3,
|
||||
['4'] = 4,
|
||||
['5'] = 5,
|
||||
['6'] = 6,
|
||||
['7'] = 7,
|
||||
['8'] = 8,
|
||||
['9'] = 9,
|
||||
|
||||
[('9' + 1)...('A' - 1)] = 0xee,
|
||||
|
||||
['A'] = 36,
|
||||
['B'] = 37,
|
||||
['C'] = 38,
|
||||
['D'] = 39,
|
||||
['E'] = 40,
|
||||
['F'] = 41,
|
||||
['G'] = 42,
|
||||
['H'] = 43,
|
||||
['I'] = 44,
|
||||
['J'] = 45,
|
||||
['K'] = 46,
|
||||
['L'] = 47,
|
||||
['M'] = 48,
|
||||
['N'] = 49,
|
||||
['O'] = 50,
|
||||
['P'] = 51,
|
||||
['Q'] = 52,
|
||||
['R'] = 53,
|
||||
['S'] = 54,
|
||||
['T'] = 55,
|
||||
['U'] = 56,
|
||||
['V'] = 57,
|
||||
['W'] = 58,
|
||||
['X'] = 59,
|
||||
['Y'] = 60,
|
||||
['Z'] = 61,
|
||||
|
||||
[('Z' + 1)...('a' - 1)] = 0xee,
|
||||
|
||||
['a'] = 10,
|
||||
['b'] = 11,
|
||||
['c'] = 12,
|
||||
['d'] = 13,
|
||||
['e'] = 14,
|
||||
['f'] = 15,
|
||||
['g'] = 16,
|
||||
['h'] = 17,
|
||||
['i'] = 18,
|
||||
['j'] = 19,
|
||||
['k'] = 20,
|
||||
['l'] = 21,
|
||||
['m'] = 22,
|
||||
['n'] = 23,
|
||||
['o'] = 24,
|
||||
['p'] = 25,
|
||||
['q'] = 26,
|
||||
['r'] = 27,
|
||||
['s'] = 28,
|
||||
['t'] = 29,
|
||||
['u'] = 30,
|
||||
['v'] = 31,
|
||||
['w'] = 32,
|
||||
['x'] = 33,
|
||||
['y'] = 34,
|
||||
['z'] = 35,
|
||||
|
||||
[('z' + 1)... 255] = 0xee};
|
||||
|
||||
static char* put6bits(int c, char* bit_pt) {
|
||||
*bit_pt++ = (c >> 0) & 1;
|
||||
*bit_pt++ = (c >> 1) & 1;
|
||||
*bit_pt++ = (c >> 2) & 1;
|
||||
*bit_pt++ = (c >> 3) & 1;
|
||||
*bit_pt++ = (c >> 4) & 1;
|
||||
*bit_pt++ = (c >> 5) & 1;
|
||||
|
||||
return bit_pt;
|
||||
}
|
||||
|
||||
static char* put4bits(int c, char* bit_pt) {
|
||||
*bit_pt++ = (c >> 0) & 1;
|
||||
*bit_pt++ = (c >> 1) & 1;
|
||||
*bit_pt++ = (c >> 2) & 1;
|
||||
*bit_pt++ = (c >> 3) & 1;
|
||||
|
||||
return bit_pt;
|
||||
}
|
||||
|
||||
// Decode base62 and Golomb-Rice in one pass. Base62 is LSB-first; a Z escape contributes 10 stream
|
||||
// bits.
|
||||
static inline int decode_chunk(const unsigned char** input, uint64_t* chunk, unsigned* width) {
|
||||
unsigned value = char_to_num[*(*input)++];
|
||||
|
||||
if (value < 61) {
|
||||
*chunk = value;
|
||||
*width = 6;
|
||||
return 1;
|
||||
}
|
||||
if (value == 0xff) return 0;
|
||||
if (value == 0xee) return -1;
|
||||
|
||||
unsigned escaped = char_to_num[*(*input)++];
|
||||
if (escaped == 0xff) return -2;
|
||||
if (escaped == 0xee) return -3;
|
||||
|
||||
unsigned high = escaped & 0x30u;
|
||||
if (high == 0x30u) return -4;
|
||||
|
||||
*chunk = (61u + (high >> 4)) | ((uint64_t)(escaped & 0x0fu) << 6);
|
||||
*width = 10;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
|
||||
const unsigned char* input = (const unsigned char*)meta->payload;
|
||||
const unsigned Mshift = (unsigned)meta->Mshift;
|
||||
const uint64_t mask = (UINT64_C(1) << Mshift) - 1;
|
||||
uint64_t bits = 0;
|
||||
unsigned filled = 0;
|
||||
unsigned q = 0;
|
||||
unsigned previous = 0;
|
||||
int count = 0;
|
||||
|
||||
for (;;) {
|
||||
// Unary quotient: zero bits terminated by one.
|
||||
for (;;) {
|
||||
if (filled == 0) {
|
||||
uint64_t chunk;
|
||||
unsigned width;
|
||||
int rc = decode_chunk(&input, &chunk, &width);
|
||||
|
||||
if (rc < 0) return rc;
|
||||
if (rc == 0) return q <= 5 ? count : -10;
|
||||
|
||||
bits = chunk;
|
||||
filled = width;
|
||||
}
|
||||
|
||||
if (bits == 0) {
|
||||
q += filled;
|
||||
filled = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
unsigned zeros = (unsigned)__builtin_ctzll(bits);
|
||||
if (zeros >= filled) {
|
||||
q += filled;
|
||||
bits = 0;
|
||||
filled = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
q += zeros;
|
||||
bits >>= zeros + 1;
|
||||
filled -= zeros + 1;
|
||||
break;
|
||||
}
|
||||
|
||||
// Fixed-width remainder. At most 31+10 bits are held at once.
|
||||
while (filled < Mshift) {
|
||||
uint64_t chunk;
|
||||
unsigned width;
|
||||
int rc = decode_chunk(&input, &chunk, &width);
|
||||
if (rc < 0) return rc;
|
||||
if (rc == 0) return -11;
|
||||
bits |= chunk << filled;
|
||||
filled += width;
|
||||
}
|
||||
|
||||
unsigned delta = (q << Mshift) | (unsigned)(bits & mask);
|
||||
bits >>= Mshift;
|
||||
filled -= Mshift;
|
||||
q = 0;
|
||||
|
||||
previous += delta;
|
||||
hash_arr[count++] = previous;
|
||||
}
|
||||
}
|
||||
|
||||
// Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates.
|
||||
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp);
|
||||
|
||||
static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigned** hash_pt,
|
||||
unsigned cache_id) {
|
||||
enum { CACHE_BUCKETS = 1024 };
|
||||
struct cache_ent {
|
||||
struct cache_ent* bucket_next;
|
||||
struct cache_ent* newer;
|
||||
struct cache_ent* older;
|
||||
char* str;
|
||||
unsigned* hash_arr;
|
||||
uint32_t fingerprint;
|
||||
int len;
|
||||
int cnt;
|
||||
int target_bpp;
|
||||
};
|
||||
|
||||
static unsigned cache_count[2];
|
||||
static struct cache_ent* buckets[2][CACHE_BUCKETS];
|
||||
static struct cache_ent* newest[2];
|
||||
static struct cache_ent* oldest[2];
|
||||
|
||||
assert(cache_id < 2);
|
||||
|
||||
const unsigned char* str = (const unsigned char*)meta->str;
|
||||
uint32_t fp = (uint32_t)str[0] | ((uint32_t)str[2] << 8) | ((uint32_t)str[3] << 16);
|
||||
uint32_t mixed = fp ^ ((uint32_t)target_bpp * UINT32_C(0x85ebca6b));
|
||||
mixed ^= mixed >> 11;
|
||||
mixed *= UINT32_C(0x9e3779b1);
|
||||
mixed ^= mixed >> 16;
|
||||
unsigned bucket = mixed & (CACHE_BUCKETS - 1);
|
||||
|
||||
for (struct cache_ent* ent = buckets[cache_id][bucket]; ent; ent = ent->bucket_next) {
|
||||
if (ent->fingerprint != fp || ent->target_bpp != target_bpp || strcmp(meta->str, ent->str) != 0)
|
||||
continue;
|
||||
|
||||
if (ent != newest[cache_id]) {
|
||||
if (ent->newer) ent->newer->older = ent->older;
|
||||
if (ent->older) ent->older->newer = ent->newer;
|
||||
if (ent == oldest[cache_id]) oldest[cache_id] = ent->newer;
|
||||
|
||||
ent->newer = NULL;
|
||||
ent->older = newest[cache_id];
|
||||
newest[cache_id]->newer = ent;
|
||||
newest[cache_id] = ent;
|
||||
}
|
||||
|
||||
*hash_pt = ent->hash_arr;
|
||||
|
||||
return ent->cnt;
|
||||
}
|
||||
|
||||
if (set_meta_fini(meta) < 0) return -4;
|
||||
|
||||
int len = (int)meta->len;
|
||||
int capacity = meta->value_capacity;
|
||||
struct cache_ent* ent = xmalloc(sizeof(*ent) + (size_t)(capacity) * sizeof(unsigned) + len + 1);
|
||||
ent->hash_arr = (unsigned*)(ent + 1);
|
||||
ent->str = (char*)(ent->hash_arr + capacity);
|
||||
|
||||
int cnt = decode_set(meta, ent->hash_arr);
|
||||
if (cnt <= 0) {
|
||||
_free(ent);
|
||||
return cnt;
|
||||
}
|
||||
|
||||
if (target_bpp < meta->bpp) {
|
||||
unsigned temporary[capacity];
|
||||
unsigned* current = ent->hash_arr;
|
||||
unsigned* destination = temporary;
|
||||
|
||||
for (int bpp = meta->bpp - 1; bpp >= target_bpp; --bpp) {
|
||||
cnt = downsample_set(cnt, current, destination, bpp);
|
||||
unsigned* swap = current;
|
||||
current = destination;
|
||||
destination = swap;
|
||||
}
|
||||
|
||||
if (current != ent->hash_arr) {
|
||||
memcpy(ent->hash_arr, current, (size_t)cnt * sizeof(*current));
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(ent->str, meta->str, (size_t)len + 1);
|
||||
ent->fingerprint = fp;
|
||||
ent->len = len;
|
||||
ent->cnt = cnt;
|
||||
ent->target_bpp = target_bpp;
|
||||
|
||||
if (cache_count[cache_id] == CACHE_SIZE) {
|
||||
struct cache_ent* victim = oldest[cache_id];
|
||||
oldest[cache_id] = victim->newer;
|
||||
if (oldest[cache_id]) oldest[cache_id]->older = NULL;
|
||||
if (victim == newest[cache_id]) newest[cache_id] = NULL;
|
||||
|
||||
uint32_t victim_mixed =
|
||||
victim->fingerprint ^ ((uint32_t)victim->target_bpp * UINT32_C(0x85ebca6b));
|
||||
victim_mixed ^= victim_mixed >> 11;
|
||||
victim_mixed *= UINT32_C(0x9e3779b1);
|
||||
victim_mixed ^= victim_mixed >> 16;
|
||||
unsigned victim_bucket = victim_mixed & (CACHE_BUCKETS - 1);
|
||||
struct cache_ent** link = &buckets[cache_id][victim_bucket];
|
||||
while (*link != victim) link = &(*link)->bucket_next;
|
||||
*link = victim->bucket_next;
|
||||
_free(victim);
|
||||
} else {
|
||||
++cache_count[cache_id];
|
||||
}
|
||||
|
||||
ent->bucket_next = buckets[cache_id][bucket];
|
||||
buckets[cache_id][bucket] = ent;
|
||||
ent->newer = NULL;
|
||||
ent->older = newest[cache_id];
|
||||
if (newest[cache_id]) {
|
||||
newest[cache_id]->newer = ent;
|
||||
} else {
|
||||
oldest[cache_id] = ent;
|
||||
}
|
||||
newest[cache_id] = ent;
|
||||
|
||||
*hash_pt = ent->hash_arr;
|
||||
|
||||
return cnt;
|
||||
}
|
||||
|
||||
// Reduce a set of (bpp + 1) values to a set of bpp values.
|
||||
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp) {
|
||||
unsigned mask = (1u << bpp) - 1;
|
||||
|
||||
// find the first element with high bit set
|
||||
int l = 0;
|
||||
int u = cnt;
|
||||
while (l < u) {
|
||||
int i = (l + u) / 2;
|
||||
|
||||
if (hash_pt[i] <= mask) {
|
||||
l = i + 1;
|
||||
} else {
|
||||
u = i;
|
||||
}
|
||||
}
|
||||
|
||||
// initialize parts
|
||||
const unsigned* ds_start = ds_pt;
|
||||
const unsigned *v1 = hash_pt + 0, *v1_end = hash_pt + u;
|
||||
const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + cnt;
|
||||
|
||||
// merge v1 and v2 into w
|
||||
if (v1 < v1_end && v2 < v2_end) {
|
||||
unsigned v1_val = *v1;
|
||||
unsigned v2_val = *v2 & mask;
|
||||
|
||||
while (1) {
|
||||
if (v1_val < v2_val) {
|
||||
*ds_pt++ = v1_val;
|
||||
v1++;
|
||||
|
||||
if (v1 == v1_end) break;
|
||||
|
||||
v1_val = *v1;
|
||||
} else if (v2_val < v1_val) {
|
||||
*ds_pt++ = v2_val;
|
||||
v2++;
|
||||
|
||||
if (v2 == v2_end) break;
|
||||
|
||||
v2_val = *v2 & mask;
|
||||
} else {
|
||||
*ds_pt++ = v1_val;
|
||||
v1++;
|
||||
v2++;
|
||||
|
||||
if (v1 == v1_end) break;
|
||||
if (v2 == v2_end) break;
|
||||
|
||||
v1_val = *v1;
|
||||
v2_val = *v2 & mask;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// append what's left
|
||||
while (v1 < v1_end) *ds_pt++ = *v1++;
|
||||
while (v2 < v2_end) *ds_pt++ = *v2++ & mask;
|
||||
|
||||
return ds_pt - ds_start;
|
||||
}
|
||||
|
||||
static const unsigned* step_lower_bound(const unsigned* first, const unsigned* last, unsigned value,
|
||||
size_t jump) {
|
||||
const size_t count = (size_t)(last - first);
|
||||
|
||||
if (count == 0 || first[0] >= value) {
|
||||
return first;
|
||||
}
|
||||
|
||||
if (jump == 0) {
|
||||
jump = 1;
|
||||
}
|
||||
|
||||
size_t position = 0;
|
||||
size_t step = jump;
|
||||
|
||||
while (step != 0) {
|
||||
if (step > count - position - 1) {
|
||||
step /= 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
const size_t next = position + step;
|
||||
|
||||
if (first[next] < value) {
|
||||
position = next;
|
||||
} else {
|
||||
step /= 2;
|
||||
}
|
||||
}
|
||||
|
||||
return first + position + 1;
|
||||
}
|
||||
|
||||
static int sorted_subset(const unsigned* small, size_t small_count, const unsigned* large,
|
||||
size_t large_count) {
|
||||
const unsigned* const small_end = small + small_count;
|
||||
const unsigned* const large_end = large + large_count;
|
||||
size_t jump = large_count / small_count;
|
||||
|
||||
// Dense sets favor a conventional merge; sparse sets skip by approximately
|
||||
// the mean distance between required values and then refine the last block.
|
||||
if (jump < 4) {
|
||||
while (small < small_end) {
|
||||
unsigned value = *small++;
|
||||
while (large < large_end && *large < value) ++large;
|
||||
if (large == large_end || *large != value) return 0;
|
||||
++large;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
while (small < small_end) {
|
||||
unsigned value = *small++;
|
||||
large = step_lower_bound(large, large_end, value, jump);
|
||||
if (large == large_end || *large != value) return 0;
|
||||
++large;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// main API routine
|
||||
int rpmsetcmp(const char* str1, const char* str2) {
|
||||
if (strncmp(str1, "set:", 4) == 0) str1 += 4;
|
||||
if (strncmp(str2, "set:", 4) == 0) str2 += 4;
|
||||
|
||||
struct set_meta meta1;
|
||||
struct set_meta meta2;
|
||||
|
||||
if (set_meta_init(str1, &meta1) < 0) return -3;
|
||||
if (set_meta_init(str2, &meta2) < 0) return -4;
|
||||
|
||||
int target_bpp = meta1.bpp < meta2.bpp ? meta1.bpp : meta2.bpp;
|
||||
|
||||
// Decode and cache the first operand at the comparison precision.
|
||||
const unsigned* hash_arr1 = NULL;
|
||||
int cnt1 = cache_decode_set(&meta1, target_bpp, &hash_arr1, 0);
|
||||
if (cnt1 < 0) return -3;
|
||||
|
||||
// Metadata for both operands has already been validated, and set1 has been
|
||||
// decoded, so this preserves set8's malformed-input error precedence.
|
||||
if (str1 == str2 || strcmp(str1, str2) == 0) return 0;
|
||||
|
||||
// Requirement sets are frequently reused by dependency solvers too.
|
||||
const unsigned* hash_arr2 = NULL;
|
||||
int cnt2 = cache_decode_set(&meta2, target_bpp, &hash_arr2, 1);
|
||||
if (cnt2 < 0) return -4;
|
||||
|
||||
// Cardinality determines which strict-inclusion result is even possible.
|
||||
// For equal cardinalities, sorted unique sets are equal iff their bytes match.
|
||||
if (cnt1 == cnt2) {
|
||||
return memcmp(hash_arr1, hash_arr2, (size_t)cnt1 * sizeof(*hash_arr1)) == 0 ? 0 : -2;
|
||||
}
|
||||
if (cnt1 > cnt2) {
|
||||
return sorted_subset(hash_arr2, (size_t)cnt2, hash_arr1, (size_t)cnt1) ? 1 : -2;
|
||||
}
|
||||
return sorted_subset(hash_arr1, (size_t)cnt1, hash_arr2, (size_t)cnt2) ? -1 : -2;
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
#ifdef SELF_TEST
|
||||
int main(void) {
|
||||
struct set* set1 = set_new();
|
||||
set_add(set1, "mama");
|
||||
set_add(set1, "myla");
|
||||
set_add(set1, "ramu");
|
||||
const char* str10 = set_fini(set1, 16);
|
||||
fprintf(stderr, "set10=%s\n", str10);
|
||||
|
||||
int cmp;
|
||||
struct set* set2 = set_new();
|
||||
set_add(set2, "myla");
|
||||
set_add(set2, "mama");
|
||||
const char* str20 = set_fini(set2, 16);
|
||||
fprintf(stderr, "set20=%s\n", str20);
|
||||
cmp = rpmsetcmp(str10, str20);
|
||||
assert(cmp == 1);
|
||||
|
||||
set_add(set2, "ramu");
|
||||
const char* str21 = set_fini(set2, 16);
|
||||
fprintf(stderr, "set21=%s\n", str21);
|
||||
cmp = rpmsetcmp(str10, str21);
|
||||
assert(cmp == 0);
|
||||
|
||||
set_add(set2, "baba");
|
||||
const char* str22 = set_fini(set2, 16);
|
||||
cmp = rpmsetcmp(str10, str22);
|
||||
assert(cmp == -1);
|
||||
|
||||
set_add(set1, "deda");
|
||||
const char* str11 = set_fini(set1, 16);
|
||||
cmp = rpmsetcmp(str11, str22);
|
||||
assert(cmp == -2);
|
||||
|
||||
set1 = set_free(set1);
|
||||
set2 = set_free(set2);
|
||||
str10 = _free(str10);
|
||||
str11 = _free(str11);
|
||||
str20 = _free(str20);
|
||||
str21 = _free(str21);
|
||||
str22 = _free(str22);
|
||||
|
||||
fprintf(stderr, "%s: api test OK\n", __FILE__);
|
||||
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
@@ -1,429 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check package compatibility using this repo's Python set implementation.
|
||||
|
||||
This script intentionally does *not* compare with ALT's existing set.c-produced
|
||||
set strings. It uses ALT only as a source of real binary RPMs:
|
||||
|
||||
* Provided labels: `nm --dynamic -j -U <shared-library>`
|
||||
* Required labels: `nm --dynamic -j -u <elf-file>`
|
||||
|
||||
Both sides are encoded with `reimplement/set.py`, then compared with that same
|
||||
implementation's `rpmsetcmp()`. In other words, it checks whether the current
|
||||
Python implementation is internally useful for real ALT package symbol labels.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from reimplement import set as rpmset # noqa: E402
|
||||
|
||||
API_BASE = "https://rdb.altlinux.org/api"
|
||||
DEFAULT_PROVIDERS = ["glibc-core", "zlib", "libssl3", "libcrypto3"]
|
||||
DEFAULT_REQUIRERS = ["coreutils", "curl", "openssl"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackageRPM:
|
||||
name: str
|
||||
pkghash: str
|
||||
rpm_path: Path
|
||||
extract_dir: Path
|
||||
members: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LabelSet:
|
||||
role: str
|
||||
package: str
|
||||
member: str
|
||||
labels: tuple[str, ...]
|
||||
set_string: str
|
||||
|
||||
@property
|
||||
def label_count(self) -> int:
|
||||
return len(self.labels)
|
||||
|
||||
@property
|
||||
def set_len(self) -> int:
|
||||
return len(self.set_string)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompatibilityResult:
|
||||
provider_package: str
|
||||
provider_member: str
|
||||
requirer_package: str
|
||||
requirer_member: str
|
||||
provider_labels: int
|
||||
required_labels: int
|
||||
cmp_result: int
|
||||
status: str
|
||||
|
||||
|
||||
def api_json(path: str, params: dict[str, object] | None = None) -> dict:
|
||||
url = API_BASE + path
|
||||
if params:
|
||||
url += "?" + urllib.parse.urlencode(params, doseq=True)
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "arsv-alt-set-compat/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=60) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def get_pkghash(package: str, branch: str, arch: str) -> str:
|
||||
data = api_json(
|
||||
"/site/pkghash_by_binary_name",
|
||||
{"branch": branch, "name": package, "arch": arch},
|
||||
)
|
||||
return str(data["pkghash"])
|
||||
|
||||
|
||||
def package_download_url(pkghash: str, branch: str, arch: str) -> str:
|
||||
data = api_json(
|
||||
f"/site/package_downloads_bin/{pkghash}",
|
||||
{"branch": branch, "arch": arch},
|
||||
)
|
||||
downloads = data.get("downloads") or []
|
||||
if not downloads or not downloads[0].get("packages"):
|
||||
raise RuntimeError(f"no download URL for pkghash={pkghash}")
|
||||
return downloads[0]["packages"][0]["url"]
|
||||
|
||||
|
||||
def run_text(command: list[str], cwd: Path | None = None, check: bool = True) -> str:
|
||||
proc = subprocess.run(command, cwd=cwd, text=True, capture_output=True)
|
||||
if check and proc.returncode != 0:
|
||||
raise RuntimeError(f"{' '.join(command)} failed with {proc.returncode}: {proc.stderr.strip()}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def download_file(url: str, destination: Path) -> None:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "arsv-alt-set-compat/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=120) as response, destination.open("wb") as out:
|
||||
shutil.copyfileobj(response, out)
|
||||
|
||||
|
||||
def rpm_members(rpm_path: Path) -> list[str]:
|
||||
return [line for line in run_text(["bsdtar", "-tf", str(rpm_path)]).splitlines() if line]
|
||||
|
||||
|
||||
def is_shared_library_member(member: str) -> bool:
|
||||
name = Path(member).name
|
||||
return not member.endswith("/") and re.search(r"(?:^|/)lib[^/]*\.so(?:\.|$)", member) is not None and ".debug" not in name
|
||||
|
||||
|
||||
def select_provider_members(members: Iterable[str]) -> list[str]:
|
||||
"""Files whose defined dynamic symbols are Provided labels."""
|
||||
return [member for member in members if is_shared_library_member(member)]
|
||||
|
||||
|
||||
def select_requirer_members(members: Iterable[str]) -> list[str]:
|
||||
"""Files whose undefined dynamic symbols are Required labels."""
|
||||
executable_prefixes = ("./bin/", "./usr/bin/", "./sbin/", "./usr/sbin/", "./usr/lib/systemd/")
|
||||
selected = []
|
||||
for member in members:
|
||||
if member.endswith("/"):
|
||||
continue
|
||||
if is_shared_library_member(member) or member.startswith(executable_prefixes):
|
||||
selected.append(member)
|
||||
return selected
|
||||
|
||||
|
||||
def extract_members(rpm_path: Path, extract_dir: Path, members: Iterable[str]) -> None:
|
||||
unique_members = sorted(set(members))
|
||||
if unique_members:
|
||||
run_text(["bsdtar", "-xf", str(rpm_path), "-C", str(extract_dir), *unique_members])
|
||||
|
||||
|
||||
def nm_symbols(path: Path, mode: str) -> list[str]:
|
||||
command = ["nm", "--dynamic", "-j", "-U", str(path)] if mode == "provided" else ["nm", "--dynamic", "-u", str(path)]
|
||||
proc = subprocess.run(command, text=True, capture_output=True)
|
||||
if proc.returncode != 0:
|
||||
return []
|
||||
if mode == "required":
|
||||
return parse_required_nm_output(proc.stdout)
|
||||
return [line.strip() for line in proc.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def parse_required_nm_output(output: str) -> list[str]:
|
||||
"""Parse `nm --dynamic -u` output, ignoring weak undefined references.
|
||||
|
||||
Plain `nm -j -u` discards the symbol type, but real ALT RPMs contain weak
|
||||
undefined hooks like `__gmon_start__` and `_ITM_*`. Those are optional ELF
|
||||
references, not hard Required labels, so keep only strong `U` entries.
|
||||
"""
|
||||
symbols = []
|
||||
for line in output.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
symbol_type, symbol = parts[-2], parts[-1]
|
||||
if symbol_type == "U":
|
||||
symbols.append(symbol)
|
||||
return symbols
|
||||
|
||||
|
||||
def normalize_required_symbol(symbol: str) -> str:
|
||||
"""Normalize `foo@VER` from undefined nm output to provider-like `foo@@VER`.
|
||||
|
||||
`nm -u` prints required versioned symbols with a single `@`, while defined
|
||||
default-version symbols commonly use `@@`. This normalization is for this
|
||||
script's compatibility model only; it is not a set.c compatibility shim.
|
||||
"""
|
||||
if "@@" in symbol or "@" not in symbol:
|
||||
return symbol
|
||||
name, version = symbol.split("@", 1)
|
||||
if not name or not version:
|
||||
return symbol
|
||||
return f"{name}@@{version}"
|
||||
|
||||
|
||||
def labels_to_set_string(labels: Iterable[str], bpp: int) -> str | None:
|
||||
item_set = rpmset.set_new()
|
||||
for label in labels:
|
||||
rpmset.set_add(item_set, label)
|
||||
# reimplement/set.py prints collision warnings to stdout; keep this script's
|
||||
# stdout machine-readable and route those warnings to stderr instead.
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
return rpmset.set_fini(item_set, bpp)
|
||||
|
||||
|
||||
def generate_label_set(role: str, member: str, labels: Iterable[str], bpp: int, package: str = "") -> LabelSet:
|
||||
unique_labels = tuple(sorted(set(label for label in labels if label)))
|
||||
set_string = labels_to_set_string(unique_labels, bpp)
|
||||
if set_string is None:
|
||||
raise ValueError(f"no labels for {role} {package}:{member}")
|
||||
return LabelSet(role=role, package=package, member=member, labels=unique_labels, set_string=set_string)
|
||||
|
||||
|
||||
def compare_status(cmp_result: int) -> str:
|
||||
# provider is first argument; compatible means provider is equal or superset.
|
||||
if cmp_result in (0, 1):
|
||||
return "compatible"
|
||||
return "incompatible"
|
||||
|
||||
|
||||
def compare_label_sets(provider: LabelSet, requirer: LabelSet) -> CompatibilityResult:
|
||||
cmp_result = rpmset.rpmsetcmp(provider.set_string, requirer.set_string)
|
||||
return CompatibilityResult(
|
||||
provider_package=provider.package,
|
||||
provider_member=provider.member,
|
||||
requirer_package=requirer.package,
|
||||
requirer_member=requirer.member,
|
||||
provider_labels=provider.label_count,
|
||||
required_labels=requirer.label_count,
|
||||
cmp_result=cmp_result,
|
||||
status=compare_status(cmp_result),
|
||||
)
|
||||
|
||||
|
||||
def build_dependency_results(provider_sets: list[LabelSet], requirer_sets: list[LabelSet], bpp: int) -> list[CompatibilityResult]:
|
||||
"""Compare each library only with the symbols actually required from it.
|
||||
|
||||
A package executable/library has one undefined-symbol list containing symbols
|
||||
required from all of its DT_NEEDED libraries. Comparing that whole list with
|
||||
one provider library gives false ``-2`` results: e.g. ``/usr/bin/curl`` needs
|
||||
symbols from libc, libssl, libcrypto, zlib, etc., and no single library is
|
||||
supposed to provide all of them.
|
||||
|
||||
For this local set.py check, keep the symbol-level ground truth around and
|
||||
split every requirer's labels by provider library: provider labels ∩ required
|
||||
labels. Each non-empty subset is then encoded as the requirement for exactly
|
||||
that provider and compared with ``rpmsetcmp(provider, required_subset)``.
|
||||
"""
|
||||
results: list[CompatibilityResult] = []
|
||||
for requirer in requirer_sets:
|
||||
required_labels = set(requirer.labels)
|
||||
for provider in provider_sets:
|
||||
required_from_provider = sorted(required_labels.intersection(provider.labels))
|
||||
if not required_from_provider:
|
||||
continue
|
||||
split_requirer = generate_label_set(
|
||||
"required",
|
||||
requirer.member,
|
||||
required_from_provider,
|
||||
bpp,
|
||||
package=requirer.package,
|
||||
)
|
||||
results.append(compare_label_sets(provider, split_requirer))
|
||||
return results
|
||||
|
||||
|
||||
def fetch_package_rpm(package: str, branch: str, arch: str, workdir: Path) -> PackageRPM:
|
||||
pkghash = get_pkghash(package, branch, arch)
|
||||
rpm_url = package_download_url(pkghash, branch, arch)
|
||||
rpm_path = workdir / Path(urllib.parse.urlparse(rpm_url).path).name
|
||||
if not rpm_path.exists():
|
||||
download_file(rpm_url, rpm_path)
|
||||
members = rpm_members(rpm_path)
|
||||
extract_dir = workdir / f"extract-{package.replace('/', '_').replace('+', '_')}"
|
||||
extract_dir.mkdir(parents=True, exist_ok=True)
|
||||
return PackageRPM(package, pkghash, rpm_path, extract_dir, members)
|
||||
|
||||
|
||||
def build_provider_sets(package_rpm: PackageRPM, bpp: int, max_files: int) -> list[LabelSet]:
|
||||
members = select_provider_members(package_rpm.members)[:max_files]
|
||||
extract_members(package_rpm.rpm_path, package_rpm.extract_dir, members)
|
||||
sets = []
|
||||
for member in members:
|
||||
labels = nm_symbols(package_rpm.extract_dir / member, "provided")
|
||||
if labels:
|
||||
sets.append(generate_label_set("provided", member, labels, bpp, package_rpm.name))
|
||||
return sets
|
||||
|
||||
|
||||
def build_requirer_sets(
|
||||
package_rpm: PackageRPM,
|
||||
bpp: int,
|
||||
max_files: int,
|
||||
normalize_versions: bool,
|
||||
) -> list[LabelSet]:
|
||||
members = select_requirer_members(package_rpm.members)[:max_files]
|
||||
extract_members(package_rpm.rpm_path, package_rpm.extract_dir, members)
|
||||
sets = []
|
||||
for member in members:
|
||||
labels = nm_symbols(package_rpm.extract_dir / member, "required")
|
||||
if normalize_versions:
|
||||
labels = [normalize_required_symbol(label) for label in labels]
|
||||
if labels:
|
||||
sets.append(generate_label_set("required", member, labels, bpp, package_rpm.name))
|
||||
return sets
|
||||
|
||||
|
||||
def aggregate_label_sets(role: str, package_names: list[str], label_sets: list[LabelSet], bpp: int) -> LabelSet:
|
||||
labels = [label for label_set in label_sets for label in label_set.labels]
|
||||
return generate_label_set(role, "+".join(package_names), labels, bpp, package="aggregate")
|
||||
|
||||
|
||||
def print_label_sets(title: str, label_sets: list[LabelSet]) -> None:
|
||||
print(f"\n# {title}")
|
||||
print("role\tpackage\tmember\tlabels\tset_len\tset_prefix")
|
||||
for label_set in label_sets:
|
||||
print(
|
||||
f"{label_set.role}\t{label_set.package}\t{label_set.member}\t"
|
||||
f"{label_set.label_count}\t{label_set.set_len}\t{label_set.set_string[:48]}"
|
||||
)
|
||||
|
||||
|
||||
def print_results(results: list[CompatibilityResult]) -> None:
|
||||
print("\n# compatibility")
|
||||
print("status\tcmp\tprovider_pkg\tprovider_member\tprovider_labels\trequirer_pkg\trequirer_member\trequired_labels")
|
||||
for result in results:
|
||||
print(
|
||||
f"{result.status}\t{result.cmp_result}\t{result.provider_package}\t{result.provider_member}\t"
|
||||
f"{result.provider_labels}\t{result.requirer_package}\t{result.requirer_member}\t{result.required_labels}"
|
||||
)
|
||||
summary = {status: sum(1 for result in results if result.status == status) for status in sorted({r.status for r in results})}
|
||||
print(f"summary: {summary}")
|
||||
|
||||
|
||||
def parse_package_list(values: list[str] | None) -> list[str]:
|
||||
packages = []
|
||||
for value in values or []:
|
||||
packages.extend(part for part in value.split(",") if part)
|
||||
return packages
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate Provided/Required set strings from ALT RPM labels with reimplement/set.py and compare them."
|
||||
)
|
||||
parser.add_argument("packages", nargs="*", help="packages used as both providers and requirers if explicit lists are omitted")
|
||||
parser.add_argument("--provider", action="append", help="provider package; can be repeated or comma-separated")
|
||||
parser.add_argument("--requirer", action="append", help="requirer package; can be repeated or comma-separated")
|
||||
parser.add_argument("--branch", default="sisyphus", help="ALT repository branch/packageset")
|
||||
parser.add_argument("--arch", default="x86_64", help="binary package architecture")
|
||||
parser.add_argument("--bpp", type=int, default=32, help="bits per hash used by local set.py")
|
||||
parser.add_argument("--max-provider-files", type=int, default=64, help="max provider ELF files per package")
|
||||
parser.add_argument("--max-requirer-files", type=int, default=64, help="max requirer ELF files per package")
|
||||
parser.add_argument("--all-pairs", action="store_true", help="compare every provider file set with every requirer file set")
|
||||
parser.add_argument(
|
||||
"--no-normalize-required-version",
|
||||
action="store_true",
|
||||
help="keep nm -u single-@ required symbols unchanged instead of converting foo@VER to foo@@VER",
|
||||
)
|
||||
parser.add_argument("--keep-workdir", action="store_true", help="keep downloaded RPMs/extracted files")
|
||||
parser.add_argument("--workdir", type=Path, help="directory for downloads/extraction")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
missing = [cmd for cmd in ("bsdtar", "nm") if shutil.which(cmd) is None]
|
||||
if missing:
|
||||
print(f"missing required command(s): {', '.join(missing)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
positional = args.packages or []
|
||||
providers = parse_package_list(args.provider) or positional or DEFAULT_PROVIDERS
|
||||
requirers = parse_package_list(args.requirer) or positional or DEFAULT_REQUIRERS
|
||||
|
||||
cleanup = False
|
||||
if args.workdir:
|
||||
workdir = args.workdir
|
||||
workdir.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
workdir = Path(tempfile.mkdtemp(prefix="arsv-alt-set-compat-"))
|
||||
cleanup = not args.keep_workdir
|
||||
|
||||
try:
|
||||
provider_sets: list[LabelSet] = []
|
||||
requirer_sets: list[LabelSet] = []
|
||||
for package in providers:
|
||||
provider_sets.extend(
|
||||
build_provider_sets(fetch_package_rpm(package, args.branch, args.arch, workdir), args.bpp, args.max_provider_files)
|
||||
)
|
||||
for package in requirers:
|
||||
requirer_sets.extend(
|
||||
build_requirer_sets(
|
||||
fetch_package_rpm(package, args.branch, args.arch, workdir),
|
||||
args.bpp,
|
||||
args.max_requirer_files,
|
||||
normalize_versions=not args.no_normalize_required_version,
|
||||
)
|
||||
)
|
||||
|
||||
print(f"branch: {args.branch}")
|
||||
print(f"arch: {args.arch}")
|
||||
print(f"bpp: {args.bpp}")
|
||||
print(f"providers: {', '.join(providers)}")
|
||||
print(f"requirers: {', '.join(requirers)}")
|
||||
print(f"required symbol version normalization: {not args.no_normalize_required_version}")
|
||||
print_label_sets("generated Provided sets", provider_sets)
|
||||
print_label_sets("generated Required sets", requirer_sets)
|
||||
|
||||
if not provider_sets or not requirer_sets:
|
||||
print("\nNo comparable sets generated.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.all_pairs:
|
||||
results = [compare_label_sets(provider, requirer) for provider in provider_sets for requirer in requirer_sets]
|
||||
else:
|
||||
results = build_dependency_results(provider_sets, requirer_sets, args.bpp)
|
||||
print_results(results)
|
||||
finally:
|
||||
if cleanup:
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
else:
|
||||
print(f"workdir: {workdir}")
|
||||
|
||||
return 0 if all(result.status == "compatible" for result in results) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -1,172 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check reimplement/set.py against ELF libraries installed on this Arch Linux system.
|
||||
|
||||
This is the local-system analogue of check_alt_set_impl.py. It treats shared
|
||||
libraries as providers (defined dynamic symbols) and executables/shared objects
|
||||
as requirers (undefined dynamic symbols). Required labels are split by provider
|
||||
library before comparing, so each library is checked only against symbols that
|
||||
it actually exports.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
import check_alt_set_impl as compat # noqa: E402
|
||||
|
||||
DEFAULT_PROVIDER_DIRS = [Path("/usr/lib")]
|
||||
DEFAULT_REQUIRER_DIRS = [Path("/usr/bin"), Path("/usr/lib")]
|
||||
|
||||
|
||||
def parse_path_list(values: list[str] | None, defaults: list[Path]) -> list[Path]:
|
||||
paths: list[Path] = []
|
||||
for value in values or []:
|
||||
paths.extend(Path(part) for part in value.split(",") if part)
|
||||
return paths or defaults
|
||||
|
||||
|
||||
def is_shared_library_path(path: Path) -> bool:
|
||||
name = path.name
|
||||
return ".debug" not in name and name.startswith("lib") and ".so" in name
|
||||
|
||||
|
||||
def iter_files(paths: list[Path], recursive: bool) -> list[Path]:
|
||||
files: list[Path] = []
|
||||
seen: set[Path] = set()
|
||||
for root in paths:
|
||||
if root.is_file():
|
||||
candidates = [root]
|
||||
elif recursive:
|
||||
candidates = (path for path in root.rglob("*") if path.is_file())
|
||||
else:
|
||||
candidates = (path for path in root.iterdir() if path.is_file()) if root.is_dir() else []
|
||||
for path in candidates:
|
||||
try:
|
||||
resolved = path.resolve()
|
||||
except OSError:
|
||||
continue
|
||||
if resolved in seen:
|
||||
continue
|
||||
seen.add(resolved)
|
||||
files.append(path)
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def executable_or_library(path: Path) -> bool:
|
||||
return is_shared_library_path(path) or os.access(path, os.X_OK)
|
||||
|
||||
|
||||
def build_local_provider_sets(paths: list[Path], bpp: int, recursive: bool, max_files: int) -> list[compat.LabelSet]:
|
||||
provider_files = [path for path in iter_files(paths, recursive) if is_shared_library_path(path)][:max_files]
|
||||
sets: list[compat.LabelSet] = []
|
||||
for path in provider_files:
|
||||
labels = compat.nm_symbols(path, "provided")
|
||||
if labels:
|
||||
sets.append(compat.generate_label_set("provided", str(path), labels, bpp, package="arch-local"))
|
||||
return sets
|
||||
|
||||
|
||||
def build_local_requirer_sets(
|
||||
paths: list[Path],
|
||||
bpp: int,
|
||||
recursive: bool,
|
||||
max_files: int,
|
||||
normalize_versions: bool,
|
||||
) -> list[compat.LabelSet]:
|
||||
requirer_files = [path for path in iter_files(paths, recursive) if executable_or_library(path)][:max_files]
|
||||
sets: list[compat.LabelSet] = []
|
||||
for path in requirer_files:
|
||||
labels = compat.nm_symbols(path, "required")
|
||||
if normalize_versions:
|
||||
labels = [compat.normalize_required_symbol(label) for label in labels]
|
||||
if labels:
|
||||
sets.append(compat.generate_label_set("required", str(path), labels, bpp, package="arch-local"))
|
||||
return sets
|
||||
|
||||
|
||||
def print_unmatched_required_labels(provider_sets: list[compat.LabelSet], requirer_sets: list[compat.LabelSet], limit: int) -> None:
|
||||
provided = {label for provider in provider_sets for label in provider.labels}
|
||||
rows: list[tuple[str, int, list[str]]] = []
|
||||
for requirer in requirer_sets:
|
||||
missing = sorted(set(requirer.labels).difference(provided))
|
||||
if missing:
|
||||
rows.append((requirer.member, len(missing), missing[:limit]))
|
||||
|
||||
print("\n# required labels not exported by scanned provider libraries")
|
||||
print("requirer\tmissing_labels\texamples")
|
||||
for member, count, examples in rows[:limit]:
|
||||
print(f"{member}\t{count}\t{', '.join(examples)}")
|
||||
if len(rows) > limit:
|
||||
print(f"... {len(rows) - limit} more requirer files with unmatched labels")
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate Provided/Required set strings from local Arch Linux ELF files and compare them with reimplement/set.py."
|
||||
)
|
||||
parser.add_argument("--provider-dir", action="append", help="directory/file containing provider libraries; repeat or comma-separate")
|
||||
parser.add_argument("--requirer-dir", action="append", help="directory/file containing requirer ELF files; repeat or comma-separate")
|
||||
parser.add_argument("--bpp", type=int, default=32, help="bits per hash used by local set.py")
|
||||
parser.add_argument("--max-provider-files", type=int, default=256, help="max provider libraries to inspect")
|
||||
parser.add_argument("--max-requirer-files", type=int, default=256, help="max requirer files to inspect")
|
||||
parser.add_argument("--recursive", action="store_true", help="scan directories recursively")
|
||||
parser.add_argument("--all-pairs", action="store_true", help="compare every provider set with every full requirer set")
|
||||
parser.add_argument(
|
||||
"--no-normalize-required-version",
|
||||
action="store_true",
|
||||
help="keep nm -u single-@ required symbols unchanged instead of converting foo@VER to foo@@VER",
|
||||
)
|
||||
parser.add_argument("--unmatched-limit", type=int, default=20, help="max unmatched-label rows/examples to print")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
if shutil.which("nm") is None:
|
||||
print("missing required command: nm", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
provider_paths = parse_path_list(args.provider_dir, DEFAULT_PROVIDER_DIRS)
|
||||
requirer_paths = parse_path_list(args.requirer_dir, DEFAULT_REQUIRER_DIRS)
|
||||
|
||||
provider_sets = build_local_provider_sets(provider_paths, args.bpp, args.recursive, args.max_provider_files)
|
||||
requirer_sets = build_local_requirer_sets(
|
||||
requirer_paths,
|
||||
args.bpp,
|
||||
args.recursive,
|
||||
args.max_requirer_files,
|
||||
normalize_versions=not args.no_normalize_required_version,
|
||||
)
|
||||
|
||||
print("system: arch-local")
|
||||
print(f"bpp: {args.bpp}")
|
||||
print(f"provider paths: {', '.join(str(path) for path in provider_paths)}")
|
||||
print(f"requirer paths: {', '.join(str(path) for path in requirer_paths)}")
|
||||
print(f"required symbol version normalization: {not args.no_normalize_required_version}")
|
||||
compat.print_label_sets("generated Provided sets", provider_sets)
|
||||
compat.print_label_sets("generated Required sets", requirer_sets)
|
||||
|
||||
if not provider_sets or not requirer_sets:
|
||||
print("\nNo comparable sets generated.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.all_pairs:
|
||||
results = [compat.compare_label_sets(provider, requirer) for provider in provider_sets for requirer in requirer_sets]
|
||||
else:
|
||||
results = compat.build_dependency_results(provider_sets, requirer_sets, args.bpp)
|
||||
compat.print_results(results)
|
||||
print_unmatched_required_labels(provider_sets, requirer_sets, args.unmatched_limit)
|
||||
|
||||
return 0 if results and all(result.status == "compatible" for result in results) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -1,142 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare one binary's required set string with a few provider libraries.
|
||||
|
||||
Usage:
|
||||
scripts/compare_binary_with_lib_sets.py [--bpp N] BINARY LIB.so [LIB.so ...]
|
||||
|
||||
For every library, this script compares:
|
||||
set(defined dynamic symbols from LIB) vs
|
||||
set(required dynamic symbols from BINARY that LIB provides)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from reimplement import set as rpmset # noqa: E402
|
||||
|
||||
|
||||
def run_nm(command: list[str]) -> str:
|
||||
proc = subprocess.run(command, text=True, capture_output=True)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"{' '.join(command)} failed with {proc.returncode}: {proc.stderr.strip()}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def normalize_required_symbol(symbol: str) -> str:
|
||||
"""Normalize nm's required `foo@VER` form to provider-like `foo@@VER`."""
|
||||
if "@@" in symbol or "@" not in symbol:
|
||||
return symbol
|
||||
name, version = symbol.split("@", 1)
|
||||
if not name or not version:
|
||||
return symbol
|
||||
return f"{name}@@{version}"
|
||||
|
||||
|
||||
def required_symbols(path: Path, normalize_versions: bool) -> set[str]:
|
||||
"""Return strong undefined dynamic symbols required by one ELF file."""
|
||||
output = run_nm(["nm", "--dynamic", "-u", str(path)])
|
||||
symbols: set[str] = set()
|
||||
for line in output.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 2 or parts[-2] != "U":
|
||||
continue
|
||||
symbol = parts[-1]
|
||||
symbols.add(normalize_required_symbol(symbol) if normalize_versions else symbol)
|
||||
return symbols
|
||||
|
||||
|
||||
def provided_symbols(path: Path) -> set[str]:
|
||||
"""Return defined dynamic symbols provided by one ELF shared library."""
|
||||
output = run_nm(["nm", "--dynamic", "-j", "-U", str(path)])
|
||||
return {line.strip() for line in output.splitlines() if line.strip()}
|
||||
|
||||
|
||||
def labels_to_set_string(labels: Iterable[str], bpp: int) -> str | None:
|
||||
item_set = rpmset.set_new()
|
||||
for label in sorted(set(labels)):
|
||||
rpmset.set_add(item_set, label)
|
||||
# set.py may print hash-collision warnings; keep stdout tabular.
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
return rpmset.set_fini(item_set, bpp)
|
||||
|
||||
|
||||
def compare_library(binary_required: set[str], library: Path, bpp: int) -> tuple[str, str, int, int, str, str]:
|
||||
provided = provided_symbols(library)
|
||||
required_from_library = binary_required.intersection(provided)
|
||||
provider_set = labels_to_set_string(provided, bpp)
|
||||
required_set = labels_to_set_string(required_from_library, bpp)
|
||||
|
||||
if not provided:
|
||||
return "no-provided-symbols", "", 0, 0, "-", "-"
|
||||
if not required_from_library:
|
||||
return "not-required", "", len(provided), 0, provider_set or "-", "-"
|
||||
|
||||
assert provider_set is not None
|
||||
assert required_set is not None
|
||||
cmp_result = rpmset.rpmsetcmp(provider_set, required_set)
|
||||
status = "compatible" if cmp_result in (0, 1) else "incompatible"
|
||||
return status, str(cmp_result), len(provided), len(required_from_library), provider_set, required_set
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Compare one binary against a few libraries using local set.py set strings.")
|
||||
parser.add_argument("binary", type=Path, help="ELF executable/shared object with required dynamic symbols")
|
||||
parser.add_argument("libraries", nargs="+", type=Path, help="provider shared libraries to compare against")
|
||||
parser.add_argument("--bpp", type=int, default=32, help="bits per hash used by local set.py")
|
||||
parser.add_argument(
|
||||
"--no-normalize-required-version",
|
||||
action="store_true",
|
||||
help="keep nm -u single-@ required symbols unchanged instead of converting foo@VER to foo@@VER",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
if shutil.which("nm") is None:
|
||||
print("missing required command: nm", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
required = required_symbols(args.binary, normalize_versions=not args.no_normalize_required_version)
|
||||
if not required:
|
||||
print(f"no strong dynamic required symbols found in {args.binary}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"binary\t{args.binary}")
|
||||
print(f"bpp\t{args.bpp}")
|
||||
print(f"required_symbols\t{len(required)}")
|
||||
print("status\tcmp\tlib\tprovided\trequired_from_lib\tprovider_set\trequired_set")
|
||||
|
||||
failed = False
|
||||
for library in args.libraries:
|
||||
try:
|
||||
status, cmp_result, provided_count, required_count, provider_set, required_set = compare_library(
|
||||
required, library, args.bpp
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
print(f"error\t\t{library}\t0\t0\t-\t-", flush=True)
|
||||
print(exc, file=sys.stderr)
|
||||
failed = True
|
||||
continue
|
||||
|
||||
print(
|
||||
f"{status}\t{cmp_result}\t{library}\t{provided_count}\t{required_count}\t{provider_set}\t{required_set}"
|
||||
)
|
||||
failed = failed or status in {"incompatible", "no-provided-symbols"}
|
||||
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -1,401 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Benchmark rpmsetcmp from a selected set.c on hardcoded ELF pairs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
import compare_realization as common
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
RUN_CMP_SOURCE = REPO_ROOT / "scripts" / "run_cmp.cpp"
|
||||
DEFAULT_RPM_ROOT = REPO_ROOT / "rpm-build"
|
||||
DEFAULT_RESULT_ROOT = REPO_ROOT / "res_cmp"
|
||||
|
||||
LIBC_CANDIDATES = (
|
||||
"/usr/lib/libc.so.6",
|
||||
"/lib64/libc.so.6",
|
||||
"/usr/lib64/libc.so.6",
|
||||
"/lib/libc.so.6",
|
||||
"/lib/x86_64-linux-gnu/libc.so.6",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ComparisonCase:
|
||||
name: str
|
||||
binary_candidates: tuple[str, ...]
|
||||
library_candidates: tuple[str, ...]
|
||||
|
||||
|
||||
# The benchmark corpus is intentionally fixed here rather than supplied via CLI.
|
||||
HARDCODED_CASES = (
|
||||
ComparisonCase("true-libc", ("/usr/bin/true", "/bin/true"), LIBC_CANDIDATES),
|
||||
ComparisonCase("ls-libc", ("/usr/bin/ls", "/bin/ls"), LIBC_CANDIDATES),
|
||||
ComparisonCase("bash-libc", ("/usr/bin/bash", "/bin/bash"), LIBC_CANDIDATES),
|
||||
ComparisonCase(
|
||||
"python-libc",
|
||||
("/usr/bin/python3", "/usr/local/bin/python3"),
|
||||
LIBC_CANDIDATES,
|
||||
),
|
||||
)
|
||||
|
||||
SUMMARY_FIELDS = (
|
||||
"case",
|
||||
"binary",
|
||||
"library",
|
||||
"runs",
|
||||
"cmp_result",
|
||||
"outputs_consistent",
|
||||
"median_rpmsetcmp_ns",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedCase:
|
||||
name: str
|
||||
binary: Path
|
||||
library: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreparedCase:
|
||||
case: ResolvedCase
|
||||
provider_set: str
|
||||
required_set: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ComparisonSummary:
|
||||
prepared: PreparedCase
|
||||
runs: int
|
||||
cmp_result: int
|
||||
outputs_consistent: bool
|
||||
median_ns: int | float
|
||||
|
||||
def as_row(self) -> dict[str, str]:
|
||||
return {
|
||||
"case": self.prepared.case.name,
|
||||
"binary": str(self.prepared.case.binary),
|
||||
"library": str(self.prepared.case.library),
|
||||
"runs": str(self.runs),
|
||||
"cmp_result": str(self.cmp_result),
|
||||
"outputs_consistent": "yes" if self.outputs_consistent else "no",
|
||||
"median_rpmsetcmp_ns": common.format_number(self.median_ns),
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv: Sequence[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Compile a selected set.c and benchmark only rpmsetcmp on the "
|
||||
"hardcoded binary/library corpus."
|
||||
)
|
||||
)
|
||||
parser.add_argument("set_c", type=Path, help="path to the set.c implementation")
|
||||
parser.add_argument(
|
||||
"-n",
|
||||
"--runs",
|
||||
type=common.positive_int,
|
||||
default=10,
|
||||
help="rpmsetcmp process runs per hardcoded case (default: 10)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rpm-root",
|
||||
type=Path,
|
||||
default=DEFAULT_RPM_ROOT,
|
||||
help="ALT rpm source root",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--res-dir",
|
||||
type=Path,
|
||||
default=DEFAULT_RESULT_ROOT,
|
||||
help="result root (default: repository res_cmp/)",
|
||||
)
|
||||
parser.add_argument("--name", help="implementation directory name under res_cmp/")
|
||||
parser.add_argument(
|
||||
"--bpp",
|
||||
type=common.bpp_value,
|
||||
help="force one bpp value while preparing provider/required sets",
|
||||
)
|
||||
parser.add_argument("--cc", default="cc", help="C compiler (default: cc)")
|
||||
parser.add_argument("--cxx", default="g++", help="C++ compiler (default: g++)")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def first_existing(candidates: Sequence[str]) -> Path | None:
|
||||
for candidate in candidates:
|
||||
path = Path(candidate)
|
||||
if path.is_file():
|
||||
return path.resolve()
|
||||
return None
|
||||
|
||||
|
||||
def resolve_cases(cases: Sequence[ComparisonCase]) -> list[ResolvedCase]:
|
||||
resolved: list[ResolvedCase] = []
|
||||
for case in cases:
|
||||
binary = first_existing(case.binary_candidates)
|
||||
library = first_existing(case.library_candidates)
|
||||
if binary is None or library is None:
|
||||
missing = "binary" if binary is None else "library"
|
||||
print(f"warning: skipping {case.name}: no hardcoded {missing} path exists", file=sys.stderr)
|
||||
continue
|
||||
resolved.append(ResolvedCase(common.safe_name(case.name), binary, library))
|
||||
if not resolved:
|
||||
raise RuntimeError("none of the hardcoded binary/library cases is available")
|
||||
return resolved
|
||||
|
||||
|
||||
def compile_cmp_runner(build_dir: Path, rpm_root: Path, cxx: str) -> Path:
|
||||
runner = build_dir / "run_cmp"
|
||||
command = [
|
||||
cxx,
|
||||
"-O2",
|
||||
"-std=c++17",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
str(RUN_CMP_SOURCE),
|
||||
str(build_dir / "set.o"),
|
||||
str(build_dir / "rpmmalloc.o"),
|
||||
"-o",
|
||||
str(runner),
|
||||
]
|
||||
log: list[str] = ["\n[rpmsetcmp runner]", "$ " + shlex.join(command)]
|
||||
completed = subprocess.run(command, cwd=rpm_root, text=True, capture_output=True)
|
||||
if completed.stdout:
|
||||
log.append(completed.stdout.rstrip())
|
||||
if completed.stderr:
|
||||
log.append(completed.stderr.rstrip())
|
||||
log.append(f"[exit {completed.returncode}]")
|
||||
build_log = build_dir / "build.log"
|
||||
with build_log.open("a", encoding="utf-8") as stream:
|
||||
stream.write("\n".join(log) + "\n")
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(f"run_cmp compilation failed; see {build_log}")
|
||||
return runner
|
||||
|
||||
|
||||
def run_set_once(
|
||||
runner: Path,
|
||||
target: Path,
|
||||
sets_dir: Path,
|
||||
slug: str,
|
||||
bpp: int | None,
|
||||
) -> common.ParsedRun:
|
||||
command = [str(runner)]
|
||||
if bpp is not None:
|
||||
command.extend(("--bpp", str(bpp)))
|
||||
command.append(str(target))
|
||||
completed = subprocess.run(command, text=True, capture_output=True)
|
||||
stdout_path = sets_dir / f"{slug}.tsv"
|
||||
stderr_path = sets_dir / f"{slug}.stderr"
|
||||
stdout_path.write_text(completed.stdout, encoding="utf-8")
|
||||
stderr_path.write_text(completed.stderr, encoding="utf-8")
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(f"run_set failed for {target}; see {stderr_path}")
|
||||
return common.parse_run_set_output(completed.stdout)
|
||||
|
||||
|
||||
def same_file(left: str, right: Path) -> bool:
|
||||
try:
|
||||
return os.path.samefile(left, right)
|
||||
except OSError:
|
||||
return Path(left).resolve() == right.resolve()
|
||||
|
||||
|
||||
def prepare_cases(
|
||||
cases: Sequence[ResolvedCase],
|
||||
run_set: Path,
|
||||
sets_dir: Path,
|
||||
bpp: int | None,
|
||||
) -> list[PreparedCase]:
|
||||
sets_dir.mkdir(parents=True)
|
||||
cache: dict[Path, common.ParsedRun] = {}
|
||||
used_slugs: set[str] = set()
|
||||
|
||||
def inspect(target: Path) -> common.ParsedRun:
|
||||
parsed = cache.get(target)
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
slug = common.target_slug(target, used_slugs)
|
||||
parsed = run_set_once(run_set, target, sets_dir, slug, bpp)
|
||||
cache[target] = parsed
|
||||
return parsed
|
||||
|
||||
prepared: list[PreparedCase] = []
|
||||
for case in cases:
|
||||
library_run = inspect(case.library)
|
||||
provider_rows = [row for row in library_run.rows if row.get("role") == "provided"]
|
||||
if len(provider_rows) != 1:
|
||||
raise RuntimeError(
|
||||
f"expected one provider set for {case.library}, got {len(provider_rows)}"
|
||||
)
|
||||
|
||||
binary_run = inspect(case.binary)
|
||||
required_rows = [
|
||||
row
|
||||
for row in binary_run.rows
|
||||
if row.get("role") == "required"
|
||||
and row.get("object")
|
||||
and same_file(row["object"], case.library)
|
||||
]
|
||||
if len(required_rows) != 1:
|
||||
raise RuntimeError(
|
||||
f"expected one {case.library.name} requirement from {case.binary}, "
|
||||
f"got {len(required_rows)}"
|
||||
)
|
||||
|
||||
provider_row = provider_rows[0]
|
||||
required_row = required_rows[0]
|
||||
if provider_row["bpp"] != required_row["bpp"]:
|
||||
raise RuntimeError(
|
||||
f"bpp mismatch for {case.name}: provider={provider_row['bpp']} "
|
||||
f"required={required_row['bpp']}"
|
||||
)
|
||||
prepared.append(
|
||||
PreparedCase(
|
||||
case=case,
|
||||
provider_set=provider_row["set"],
|
||||
required_set=required_row["set"],
|
||||
)
|
||||
)
|
||||
return prepared
|
||||
|
||||
|
||||
def parse_cmp_output(output: str) -> tuple[int, int]:
|
||||
values: dict[str, str] = {}
|
||||
for line in output.splitlines():
|
||||
if "\t" not in line:
|
||||
continue
|
||||
key, value = line.split("\t", 1)
|
||||
values[key] = value
|
||||
try:
|
||||
result = int(values["cmp_result"])
|
||||
elapsed = int(values["rpmsetcmp_ns"])
|
||||
except (KeyError, ValueError) as exception:
|
||||
raise RuntimeError("invalid run_cmp output") from exception
|
||||
if elapsed < 0:
|
||||
raise RuntimeError("run_cmp returned a negative duration")
|
||||
return result, elapsed
|
||||
|
||||
|
||||
def benchmark_case(
|
||||
runner: Path,
|
||||
prepared: PreparedCase,
|
||||
runs: int,
|
||||
output_dir: Path,
|
||||
) -> ComparisonSummary:
|
||||
output_dir.mkdir(parents=True)
|
||||
results: list[int] = []
|
||||
timings: list[int] = []
|
||||
for run_number in range(1, runs + 1):
|
||||
completed = subprocess.run(
|
||||
[str(runner), prepared.provider_set, prepared.required_set],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
stem = f"run-{run_number:03d}"
|
||||
stdout_path = output_dir / f"{stem}.tsv"
|
||||
stderr_path = output_dir / f"{stem}.stderr"
|
||||
stdout_path.write_text(completed.stdout, encoding="utf-8")
|
||||
stderr_path.write_text(completed.stderr, encoding="utf-8")
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"run_cmp failed for {prepared.case.name} on run {run_number}; "
|
||||
f"see {stderr_path}"
|
||||
)
|
||||
result, elapsed = parse_cmp_output(completed.stdout)
|
||||
results.append(result)
|
||||
timings.append(elapsed)
|
||||
|
||||
return ComparisonSummary(
|
||||
prepared=prepared,
|
||||
runs=runs,
|
||||
cmp_result=results[0],
|
||||
outputs_consistent=len(set(results)) == 1,
|
||||
median_ns=statistics.median(timings),
|
||||
)
|
||||
|
||||
|
||||
def write_summary(path: Path, summaries: Sequence[ComparisonSummary]) -> None:
|
||||
with path.open("w", newline="", encoding="utf-8") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(SUMMARY_FIELDS), delimiter="\t")
|
||||
writer.writeheader()
|
||||
for summary in summaries:
|
||||
writer.writerow(summary.as_row())
|
||||
|
||||
|
||||
def print_summary(summaries: Sequence[ComparisonSummary]) -> None:
|
||||
print("\t".join(SUMMARY_FIELDS))
|
||||
for summary in summaries:
|
||||
row = summary.as_row()
|
||||
print("\t".join(row[field] for field in SUMMARY_FIELDS))
|
||||
|
||||
|
||||
def write_metadata(
|
||||
path: Path,
|
||||
set_source: Path,
|
||||
rpm_root: Path,
|
||||
runs: int,
|
||||
cases: Sequence[ResolvedCase],
|
||||
) -> None:
|
||||
lines = [
|
||||
f"set_c\t{set_source}",
|
||||
f"rpm_root\t{rpm_root}",
|
||||
f"runs\t{runs}",
|
||||
]
|
||||
for case in cases:
|
||||
lines.append(f"case\t{case.name}\t{case.binary}\t{case.library}")
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main(argv: Sequence[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
try:
|
||||
common.require_commands((args.cc, args.cxx, "eu-readelf", "eu-elfclassify", "nm"))
|
||||
set_source = common.resolve_file(args.set_c, "set.c")
|
||||
rpm_root = common.resolve_directory(args.rpm_root, "rpm root")
|
||||
common.resolve_file(rpm_root / "system.h", "system.h")
|
||||
cases = resolve_cases(HARDCODED_CASES)
|
||||
name = common.safe_name(args.name) if args.name else common.default_name(set_source)
|
||||
result_dir = args.res_dir.expanduser().resolve() / name
|
||||
if result_dir.exists():
|
||||
shutil.rmtree(result_dir)
|
||||
build_dir = result_dir / "build"
|
||||
sets_dir = result_dir / "sets"
|
||||
runs_dir = result_dir / "runs"
|
||||
build_dir.mkdir(parents=True)
|
||||
runs_dir.mkdir()
|
||||
|
||||
run_set = common.compile_runner(set_source, rpm_root, build_dir, args.cc, args.cxx)
|
||||
run_cmp = compile_cmp_runner(build_dir, rpm_root, args.cxx)
|
||||
prepared = prepare_cases(cases, run_set, sets_dir, args.bpp)
|
||||
summaries = [
|
||||
benchmark_case(run_cmp, item, args.runs, runs_dir / item.case.name)
|
||||
for item in prepared
|
||||
]
|
||||
|
||||
write_summary(result_dir / "summary.tsv", summaries)
|
||||
write_metadata(result_dir / "metadata.tsv", set_source, rpm_root, args.runs, cases)
|
||||
print_summary(summaries)
|
||||
print(f"\nresults\t{result_dir}")
|
||||
return 0
|
||||
except (OSError, RuntimeError, ValueError) as exception:
|
||||
print(f"compare_cmp_realization: {exception}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -1,518 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build one set.c implementation and benchmark it through run_set.cpp.
|
||||
|
||||
The benchmark stores every raw run_set.cpp stdout/stderr stream and a TSV
|
||||
summary with median set.c API timings under res/<implementation>/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import shlex
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Sequence
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
RUN_SET_SOURCE = REPO_ROOT / "scripts" / "run_set.cpp"
|
||||
DEFAULT_RPM_ROOT = REPO_ROOT / "rpm-build"
|
||||
|
||||
TIMING_FIELDS = (
|
||||
"set_new_ns",
|
||||
"set_add_total_ns",
|
||||
"set_fini_ns",
|
||||
"set_free_ns",
|
||||
"set_api_total_ns",
|
||||
)
|
||||
SUMMARY_FIELDS = (
|
||||
"target",
|
||||
"kind",
|
||||
"runs",
|
||||
"sets_per_run",
|
||||
"outputs_consistent",
|
||||
*(f"median_{field}" for field in TIMING_FIELDS),
|
||||
)
|
||||
|
||||
# Each tuple describes one logical target. The first existing path is used.
|
||||
DEFAULT_TARGET_GROUPS = (
|
||||
("/usr/bin/true", "/bin/true"),
|
||||
("/usr/bin/ls", "/bin/ls"),
|
||||
("/usr/bin/bash", "/bin/bash"),
|
||||
("/usr/bin/python3", "/usr/local/bin/python3"),
|
||||
("/usr/lib/libc.so.6", "/lib64/libc.so.6", "/lib/x86_64-linux-gnu/libc.so.6"),
|
||||
("/usr/lib/libm.so.6", "/lib64/libm.so.6", "/lib/x86_64-linux-gnu/libm.so.6"),
|
||||
("/usr/lib/libz.so.1", "/usr/lib64/libz.so.1", "/lib/x86_64-linux-gnu/libz.so.1"),
|
||||
(
|
||||
"/usr/lib/libstdc++.so.6",
|
||||
"/usr/lib64/libstdc++.so.6",
|
||||
"/lib/x86_64-linux-gnu/libstdc++.so.6",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedRun:
|
||||
kind: str
|
||||
rows: tuple[dict[str, str], ...]
|
||||
totals: dict[str, int]
|
||||
output_signature: tuple[tuple[str, ...], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TargetSummary:
|
||||
target: Path
|
||||
kind: str
|
||||
runs: int
|
||||
sets_per_run: int
|
||||
outputs_consistent: bool
|
||||
medians: dict[str, int | float]
|
||||
|
||||
def as_row(self) -> dict[str, str]:
|
||||
row = {
|
||||
"target": str(self.target),
|
||||
"kind": self.kind,
|
||||
"runs": str(self.runs),
|
||||
"sets_per_run": str(self.sets_per_run),
|
||||
"outputs_consistent": "yes" if self.outputs_consistent else "no",
|
||||
}
|
||||
row.update(
|
||||
{
|
||||
f"median_{field}": format_number(value)
|
||||
for field, value in self.medians.items()
|
||||
}
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def positive_int(value: str) -> int:
|
||||
number = int(value)
|
||||
if number < 1:
|
||||
raise argparse.ArgumentTypeError("must be greater than zero")
|
||||
return number
|
||||
|
||||
|
||||
def bpp_value(value: str) -> int:
|
||||
number = int(value)
|
||||
if not 10 <= number <= 32:
|
||||
raise argparse.ArgumentTypeError("must be in [10, 32]")
|
||||
return number
|
||||
|
||||
|
||||
def safe_name(value: str) -> str:
|
||||
name = "".join(
|
||||
character if character.isalnum() or character in "._-" else "-"
|
||||
for character in value
|
||||
)
|
||||
name = name.strip(".-")
|
||||
if not name or name in {".", ".."}:
|
||||
raise ValueError(f"invalid result name: {value!r}")
|
||||
return name
|
||||
|
||||
|
||||
def default_name(set_source: Path) -> str:
|
||||
return safe_name(f"{set_source.parent.name}-{set_source.stem}")
|
||||
|
||||
|
||||
def parse_args(argv: Sequence[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Compile a selected set.c, run scripts/run_set.cpp repeatedly on ELF targets, "
|
||||
"and store raw outputs plus median set.c timings."
|
||||
)
|
||||
)
|
||||
parser.add_argument("set_c", type=Path, help="path to the set.c implementation")
|
||||
parser.add_argument(
|
||||
"-n",
|
||||
"--runs",
|
||||
type=positive_int,
|
||||
default=10,
|
||||
help="runs per ELF target (default: 10)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rpm-root", type=Path, default=DEFAULT_RPM_ROOT, help="ALT rpm source root"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--res-dir",
|
||||
type=Path,
|
||||
default=REPO_ROOT / "res",
|
||||
help="result root (default: repository res/)",
|
||||
)
|
||||
parser.add_argument("--name", help="implementation directory name under res/")
|
||||
parser.add_argument(
|
||||
"--target",
|
||||
action="append",
|
||||
type=Path,
|
||||
default=[],
|
||||
help="ELF target; repeat to override the built-in binary/library set",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bpp", type=bpp_value, help="force one bpp value for every run_set invocation"
|
||||
)
|
||||
parser.add_argument("--cc", default="cc", help="C compiler (default: cc)")
|
||||
parser.add_argument("--cxx", default="g++", help="C++ compiler (default: g++)")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def resolve_file(path: Path, description: str) -> Path:
|
||||
resolved = path.expanduser().resolve()
|
||||
if not resolved.is_file():
|
||||
raise RuntimeError(f"{description} is not a file: {path}")
|
||||
return resolved
|
||||
|
||||
|
||||
def resolve_directory(path: Path, description: str) -> Path:
|
||||
resolved = path.expanduser().resolve()
|
||||
if not resolved.is_dir():
|
||||
raise RuntimeError(f"{description} is not a directory: {path}")
|
||||
return resolved
|
||||
|
||||
|
||||
def select_default_targets() -> list[Path]:
|
||||
selected: list[Path] = []
|
||||
seen: set[Path] = set()
|
||||
for candidates in DEFAULT_TARGET_GROUPS:
|
||||
for candidate in candidates:
|
||||
path = Path(candidate)
|
||||
if not path.is_file():
|
||||
continue
|
||||
resolved = path.resolve()
|
||||
if resolved not in seen:
|
||||
selected.append(resolved)
|
||||
seen.add(resolved)
|
||||
break
|
||||
return selected
|
||||
|
||||
|
||||
def resolve_targets(explicit: Iterable[Path]) -> list[Path]:
|
||||
supplied = list(explicit)
|
||||
targets = (
|
||||
[resolve_file(path, "target") for path in supplied]
|
||||
if supplied
|
||||
else select_default_targets()
|
||||
)
|
||||
unique: list[Path] = []
|
||||
seen: set[Path] = set()
|
||||
for target in targets:
|
||||
if target not in seen:
|
||||
unique.append(target)
|
||||
seen.add(target)
|
||||
if not unique:
|
||||
raise RuntimeError("no ELF targets were found; pass at least one --target")
|
||||
return unique
|
||||
|
||||
|
||||
def require_commands(commands: Iterable[str]) -> None:
|
||||
missing = [command for command in commands if shutil.which(command) is None]
|
||||
if missing:
|
||||
raise RuntimeError("missing required commands: " + ", ".join(missing))
|
||||
|
||||
|
||||
def create_header_shim(rpm_root: Path, include_dir: Path) -> None:
|
||||
include_dir.mkdir(parents=True, exist_ok=True)
|
||||
by_name: dict[str, Path] = {}
|
||||
for header in sorted(rpm_root.glob("*/*.h")):
|
||||
existing = by_name.get(header.name)
|
||||
if existing is not None and existing.resolve() != header.resolve():
|
||||
raise RuntimeError(
|
||||
f"duplicate rpm header basename {header.name}: {existing} and {header}"
|
||||
)
|
||||
by_name[header.name] = header
|
||||
if not by_name:
|
||||
raise RuntimeError(f"no */*.h headers found below {rpm_root}")
|
||||
for name, source in by_name.items():
|
||||
(include_dir / name).symlink_to(source.resolve())
|
||||
|
||||
|
||||
def run_build_command(
|
||||
command: Sequence[str], cwd: Path, log: list[str]
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
log.append("$ " + shlex.join(str(part) for part in command))
|
||||
completed = subprocess.run(command, cwd=cwd, text=True, capture_output=True)
|
||||
if completed.stdout:
|
||||
log.append(completed.stdout.rstrip())
|
||||
if completed.stderr:
|
||||
log.append(completed.stderr.rstrip())
|
||||
log.append(f"[exit {completed.returncode}]")
|
||||
return completed
|
||||
|
||||
|
||||
def compile_runner(
|
||||
set_source: Path,
|
||||
rpm_root: Path,
|
||||
build_dir: Path,
|
||||
cc: str,
|
||||
cxx: str,
|
||||
) -> Path:
|
||||
include_dir = build_dir / "include" / "rpm"
|
||||
create_header_shim(rpm_root, include_dir)
|
||||
set_object = build_dir / "set.o"
|
||||
malloc_object = build_dir / "rpmmalloc.o"
|
||||
runner = build_dir / "run_set"
|
||||
rpmmalloc_source = resolve_file(rpm_root / "rpmio" / "rpmmalloc.c", "rpmmalloc.c")
|
||||
log: list[str] = [f"set_c={set_source}", f"rpm_root={rpm_root}"]
|
||||
|
||||
common_flags = [
|
||||
"-O3",
|
||||
"-std=gnu11",
|
||||
"-include",
|
||||
"stddef.h",
|
||||
f"-I{rpm_root}",
|
||||
f"-I{include_dir}",
|
||||
]
|
||||
compatibility_flags = [
|
||||
"-D_GNU_SOURCE=1",
|
||||
"-DSTDC_HEADERS=1",
|
||||
"-DHAVE_STRING_H=1",
|
||||
"-DHAVE_SETENV=1",
|
||||
"-DHAVE_STPCPY=1",
|
||||
"-DHAVE_STPNCPY=1",
|
||||
"-DHAVE_S_IFSOCK=1",
|
||||
"-DHAVE_S_ISLNK=1",
|
||||
"-DHAVE_S_ISSOCK=1",
|
||||
]
|
||||
|
||||
def attempt(extra_flags: Sequence[str], label: str) -> bool:
|
||||
log.append(f"\n[{label}]")
|
||||
set_command = [
|
||||
cc,
|
||||
*common_flags,
|
||||
*extra_flags,
|
||||
"-c",
|
||||
str(set_source),
|
||||
"-o",
|
||||
str(set_object),
|
||||
]
|
||||
malloc_command = [
|
||||
cc,
|
||||
*common_flags,
|
||||
*extra_flags,
|
||||
"-include",
|
||||
"stdarg.h",
|
||||
"-c",
|
||||
str(rpmmalloc_source),
|
||||
"-o",
|
||||
str(malloc_object),
|
||||
]
|
||||
link_command = [
|
||||
cxx,
|
||||
"-O3",
|
||||
"-std=c++17",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
str(RUN_SET_SOURCE),
|
||||
str(set_object),
|
||||
str(malloc_object),
|
||||
"-o",
|
||||
str(runner),
|
||||
]
|
||||
for command in (set_command, malloc_command, link_command):
|
||||
if run_build_command(command, rpm_root, log).returncode != 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
succeeded = attempt((), "minimal ALT-style build")
|
||||
if not succeeded:
|
||||
set_object.unlink(missing_ok=True)
|
||||
malloc_object.unlink(missing_ok=True)
|
||||
runner.unlink(missing_ok=True)
|
||||
succeeded = attempt(compatibility_flags, "modern libc compatibility retry")
|
||||
|
||||
build_log = build_dir / "build.log"
|
||||
build_log.write_text("\n".join(log) + "\n", encoding="utf-8")
|
||||
if not succeeded:
|
||||
raise RuntimeError(f"compilation failed; see {build_log}")
|
||||
return runner
|
||||
|
||||
|
||||
def parse_run_set_output(output: str) -> ParsedRun:
|
||||
lines = output.splitlines()
|
||||
try:
|
||||
table_start = next(
|
||||
index for index, line in enumerate(lines) if line.startswith("role\t")
|
||||
)
|
||||
except StopIteration as exception:
|
||||
raise RuntimeError("run_set output has no TSV result table") from exception
|
||||
|
||||
metadata: dict[str, str] = {}
|
||||
for line in lines[:table_start]:
|
||||
if "\t" not in line:
|
||||
continue
|
||||
key, value = line.split("\t", 1)
|
||||
metadata[key] = value
|
||||
rows = tuple(
|
||||
csv.DictReader(io.StringIO("\n".join(lines[table_start:])), delimiter="\t")
|
||||
)
|
||||
if not rows:
|
||||
raise RuntimeError("run_set output contains no result rows")
|
||||
|
||||
totals = {field: 0 for field in TIMING_FIELDS}
|
||||
for row in rows:
|
||||
for field in TIMING_FIELDS:
|
||||
try:
|
||||
totals[field] += int(row[field])
|
||||
except (KeyError, ValueError) as exception:
|
||||
raise RuntimeError(f"invalid {field} in run_set output") from exception
|
||||
signature_fields = ("role", "object", "labels", "bpp", "set")
|
||||
signature = tuple(tuple(row[field] for field in signature_fields) for row in rows)
|
||||
return ParsedRun(
|
||||
kind=metadata.get("kind", "unknown"),
|
||||
rows=rows,
|
||||
totals=totals,
|
||||
output_signature=signature,
|
||||
)
|
||||
|
||||
|
||||
def target_slug(target: Path, used: set[str]) -> str:
|
||||
base = safe_name(target.name)
|
||||
candidate = base
|
||||
if candidate in used:
|
||||
digest = hashlib.sha256(str(target).encode()).hexdigest()[:8]
|
||||
candidate = f"{base}-{digest}"
|
||||
used.add(candidate)
|
||||
return candidate
|
||||
|
||||
|
||||
def benchmark_target(
|
||||
runner: Path,
|
||||
target: Path,
|
||||
runs: int,
|
||||
output_dir: Path,
|
||||
bpp: int | None,
|
||||
) -> TargetSummary:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
parsed_runs: list[ParsedRun] = []
|
||||
for run_number in range(1, runs + 1):
|
||||
command = [str(runner)]
|
||||
if bpp is not None:
|
||||
command.extend(("--bpp", str(bpp)))
|
||||
command.append(str(target))
|
||||
completed = subprocess.run(command, text=True, capture_output=True)
|
||||
stem = f"run-{run_number:03d}"
|
||||
(output_dir / f"{stem}.tsv").write_text(completed.stdout, encoding="utf-8")
|
||||
(output_dir / f"{stem}.stderr").write_text(completed.stderr, encoding="utf-8")
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"run_set failed for {target} on run {run_number}; "
|
||||
f"see {output_dir / f'{stem}.stderr'}"
|
||||
)
|
||||
parsed_runs.append(parse_run_set_output(completed.stdout))
|
||||
|
||||
signatures = {parsed.output_signature for parsed in parsed_runs}
|
||||
kinds = {parsed.kind for parsed in parsed_runs}
|
||||
row_counts = {len(parsed.rows) for parsed in parsed_runs}
|
||||
medians = {
|
||||
field: statistics.median(parsed.totals[field] for parsed in parsed_runs)
|
||||
for field in TIMING_FIELDS
|
||||
}
|
||||
return TargetSummary(
|
||||
target=target,
|
||||
kind=parsed_runs[0].kind if len(kinds) == 1 else "inconsistent",
|
||||
runs=runs,
|
||||
sets_per_run=len(parsed_runs[0].rows) if len(row_counts) == 1 else -1,
|
||||
outputs_consistent=len(signatures) == 1
|
||||
and len(kinds) == 1
|
||||
and len(row_counts) == 1,
|
||||
medians=medians,
|
||||
)
|
||||
|
||||
|
||||
def format_number(value: int | float) -> str:
|
||||
number = float(value)
|
||||
return str(int(number)) if number.is_integer() else f"{number:.1f}"
|
||||
|
||||
|
||||
def write_summary(path: Path, summaries: Iterable[TargetSummary]) -> None:
|
||||
with path.open("w", newline="", encoding="utf-8") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=SUMMARY_FIELDS, delimiter="\t")
|
||||
writer.writeheader()
|
||||
for summary in summaries:
|
||||
writer.writerow(summary.as_row())
|
||||
|
||||
|
||||
def print_summary(summaries: Iterable[TargetSummary]) -> None:
|
||||
fields = (
|
||||
"target",
|
||||
"kind",
|
||||
"runs",
|
||||
"sets_per_run",
|
||||
"outputs_consistent",
|
||||
"median_set_api_total_ns",
|
||||
)
|
||||
print("\t".join(fields))
|
||||
for summary in summaries:
|
||||
row = summary.as_row()
|
||||
print("\t".join(row[field] for field in fields))
|
||||
|
||||
|
||||
def write_metadata(
|
||||
path: Path,
|
||||
set_source: Path,
|
||||
rpm_root: Path,
|
||||
runner: Path,
|
||||
runs: int,
|
||||
targets: Iterable[Path],
|
||||
) -> None:
|
||||
lines = [
|
||||
f"set_c\t{set_source}",
|
||||
f"rpm_root\t{rpm_root}",
|
||||
f"runner\t{runner}",
|
||||
f"runs\t{runs}",
|
||||
]
|
||||
lines.extend(f"target\t{target}" for target in targets)
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main(argv: Sequence[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
try:
|
||||
require_commands((args.cc, args.cxx, "eu-readelf", "eu-elfclassify", "nm"))
|
||||
set_source = resolve_file(args.set_c, "set.c")
|
||||
rpm_root = resolve_directory(args.rpm_root, "rpm root")
|
||||
resolve_file(rpm_root / "system.h", "system.h")
|
||||
targets = resolve_targets(args.target)
|
||||
name = safe_name(args.name) if args.name else default_name(set_source)
|
||||
result_root = args.res_dir.expanduser().resolve()
|
||||
result_dir = result_root / name
|
||||
if result_dir.exists():
|
||||
shutil.rmtree(result_dir)
|
||||
build_dir = result_dir / "build"
|
||||
runs_dir = result_dir / "runs"
|
||||
build_dir.mkdir(parents=True)
|
||||
runs_dir.mkdir()
|
||||
|
||||
runner = compile_runner(set_source, rpm_root, build_dir, args.cc, args.cxx)
|
||||
used_slugs: set[str] = set()
|
||||
summaries: list[TargetSummary] = []
|
||||
for target in targets:
|
||||
slug = target_slug(target, used_slugs)
|
||||
summaries.append(
|
||||
benchmark_target(runner, target, args.runs, runs_dir / slug, args.bpp)
|
||||
)
|
||||
|
||||
write_summary(result_dir / "summary.tsv", summaries)
|
||||
write_metadata(
|
||||
result_dir / "metadata.tsv",
|
||||
set_source,
|
||||
rpm_root,
|
||||
runner,
|
||||
args.runs,
|
||||
targets,
|
||||
)
|
||||
print_summary(summaries)
|
||||
print(f"\nresults\t{result_dir}")
|
||||
return 0
|
||||
except (OSError, RuntimeError, ValueError) as exception:
|
||||
print(f"compare_realization: {exception}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -1,611 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
||||
REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd)
|
||||
NEWSET_C="$REPO_ROOT/reimplement/newset.c"
|
||||
NEWSET_COMPAT="$SCRIPT_DIR/newset_compat.h"
|
||||
NEWSET_WRAPPER="$SCRIPT_DIR/newset_mkset.c"
|
||||
PACKAGE_PARSER="$SCRIPT_DIR/parse_sisyphus_packages.awk"
|
||||
RESUME_PARSER="$SCRIPT_DIR/completed_set_versions.awk"
|
||||
PAYLOAD_HELPERS="$SCRIPT_DIR/rpm_payload_safety.sh"
|
||||
|
||||
# shellcheck source=scripts/rpm_payload_safety.sh
|
||||
source "$PAYLOAD_HELPERS"
|
||||
|
||||
MIRROR=https://ftp.altlinux.org/pub/distributions/ALTLinux
|
||||
REPORT=sisyphus-set-compare.tsv
|
||||
LIMIT=
|
||||
ALL=0
|
||||
RESUME=0
|
||||
KEEP_WORK=0
|
||||
BUILD_MKSET=
|
||||
PACKAGES=()
|
||||
|
||||
usage()
|
||||
{
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
compare_sisyphus_set_versions.sh --all [OPTIONS]
|
||||
compare_sisyphus_set_versions.sh --limit N [OPTIONS]
|
||||
compare_sisyphus_set_versions.sh --package NAME [--package NAME ...] [OPTIONS]
|
||||
compare_sisyphus_set_versions.sh --build-mkset PATH
|
||||
|
||||
Scopes:
|
||||
--all Process every x86_64/noarch package record.
|
||||
--limit N Process only the first N records (testing).
|
||||
--package NAME Process one named package; may be repeated (testing).
|
||||
|
||||
Options:
|
||||
--report FILE Streaming TSV report (default: sisyphus-set-compare.tsv).
|
||||
--resume Skip completed package/architecture/version records.
|
||||
--keep-work Preserve the temporary working directory.
|
||||
--mirror URL ALT repository root.
|
||||
--build-mkset PATH Build only the mkset-compatible newset.c wrapper and exit.
|
||||
-h, --help Show this help.
|
||||
|
||||
The report gets a START row before processing and DEPENDENCY/SUMMARY rows are
|
||||
appended after every package, so it can be monitored while the script runs.
|
||||
No repository-wide run is possible without the explicit --all option.
|
||||
EOF
|
||||
}
|
||||
|
||||
die()
|
||||
{
|
||||
printf 'error: %s\n' "$*" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
--all) ALL=1; shift ;;
|
||||
--limit) (($# >= 2)) || die '--limit requires N'; LIMIT=$2; shift 2 ;;
|
||||
--package) (($# >= 2)) || die '--package requires NAME'; PACKAGES+=("$2"); shift 2 ;;
|
||||
--report) (($# >= 2)) || die '--report requires FILE'; REPORT=$2; shift 2 ;;
|
||||
--resume) RESUME=1; shift ;;
|
||||
--keep-work) KEEP_WORK=1; shift ;;
|
||||
--mirror) (($# >= 2)) || die '--mirror requires URL'; MIRROR=${2%/}; shift 2 ;;
|
||||
--build-mkset) (($# >= 2)) || die '--build-mkset requires PATH'; BUILD_MKSET=$2; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) die "unknown option: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
build_mkset()
|
||||
(
|
||||
{
|
||||
local output=$1 build
|
||||
build=$(mktemp -d "${TMPDIR:-/tmp}/arsv-newset-build.XXXXXX")
|
||||
trap 'rm -rf "$build"' EXIT
|
||||
: >"$build/rpmlib.h"
|
||||
: >"$build/system.h"
|
||||
mkdir -p "$(dirname -- "$output")"
|
||||
|
||||
"${CC:-cc}" -O2 -std=gnu11 -D_GNU_SOURCE \
|
||||
-I"$build" -include "$NEWSET_COMPAT" \
|
||||
-c "$NEWSET_C" -o "$build/newset.o"
|
||||
"${CC:-cc}" -O2 -std=gnu11 -D_GNU_SOURCE \
|
||||
"$NEWSET_WRAPPER" "$build/newset.o" -o "$output"
|
||||
chmod 755 "$output"
|
||||
}
|
||||
)
|
||||
|
||||
if [[ -n $BUILD_MKSET ]]; then
|
||||
build_mkset "$BUILD_MKSET"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ $MIRROR == http://* || $MIRROR == https://* ]] || die '--mirror must use http:// or https://'
|
||||
[[ $MIRROR != *[$'\t\r\n ']* ]] || die '--mirror must not contain whitespace'
|
||||
for package in "${PACKAGES[@]}"; do
|
||||
[[ $package =~ ^[A-Za-z0-9][A-Za-z0-9+_.-]*$ ]] || die "invalid package name: $package"
|
||||
done
|
||||
|
||||
scope_count=$ALL
|
||||
[[ -n $LIMIT ]] && scope_count=$((scope_count + 1))
|
||||
((${#PACKAGES[@]} > 0)) && scope_count=$((scope_count + 1))
|
||||
((scope_count == 1)) || die 'use --all, --limit, or --package (exactly one scope)'
|
||||
if [[ -n $LIMIT && ! $LIMIT =~ ^[1-9][0-9]*$ ]]; then
|
||||
die '--limit must be a positive integer'
|
||||
fi
|
||||
|
||||
for command in apt-get apt-cache rpm rpmquery rpm2cpio cpio curl md5sum awk sed sort find cp cc realpath; do
|
||||
command -v "$command" >/dev/null || die "required command not found: $command"
|
||||
done
|
||||
|
||||
rpmlibdir=$(rpm --eval '%_rpmlibdir')
|
||||
[[ -x $rpmlibdir/find-provides ]] || die "$rpmlibdir/find-provides is missing (install rpm-build)"
|
||||
[[ -x $rpmlibdir/find-requires ]] || die "$rpmlibdir/find-requires is missing (install rpm-build)"
|
||||
|
||||
case $REPORT in
|
||||
/*) ;;
|
||||
*) REPORT="$PWD/$REPORT" ;;
|
||||
esac
|
||||
mkdir -p "$(dirname -- "$REPORT")"
|
||||
if [[ ! -s $REPORT ]]; then
|
||||
printf 'timestamp\trecord\tpackage\tarchitecture\tstatus\tside\tcapability\toperator\texpected\tgenerated\tdetail\n' >"$REPORT"
|
||||
fi
|
||||
|
||||
clean_field()
|
||||
{
|
||||
local value=${1-}
|
||||
value=${value//$'\t'/ }
|
||||
value=${value//$'\r'/ }
|
||||
value=${value//$'\n'/ }
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
report_row()
|
||||
{
|
||||
local fields=() field row
|
||||
(($# == 11)) || {
|
||||
printf 'internal error: report row has %s fields, expected 11\n' "$#" >&2
|
||||
return 1
|
||||
}
|
||||
for field in "$@"; do
|
||||
fields+=("$(clean_field "$field")")
|
||||
done
|
||||
if [[ ${fields[1]} == SUMMARY ]]; then
|
||||
case ${fields[10]} in
|
||||
'') fields[10]='complete=1' ;;
|
||||
*'; ') fields[10]="${fields[10]}complete=1" ;;
|
||||
*) fields[10]="${fields[10]}; complete=1" ;;
|
||||
esac
|
||||
fi
|
||||
row=${fields[0]}
|
||||
for field in "${fields[@]:1}"; do
|
||||
printf -v row '%s\t%s' "$row" "$field"
|
||||
done
|
||||
printf '%s\n' "$row" >>"$REPORT"
|
||||
}
|
||||
|
||||
timestamp()
|
||||
{
|
||||
date -u '+%Y-%m-%dT%H:%M:%SZ'
|
||||
}
|
||||
|
||||
WORK=$(mktemp -d "${TMPDIR:-/tmp}/arsv-sisyphus-sets.XXXXXX")
|
||||
cleanup()
|
||||
{
|
||||
if ((KEEP_WORK)); then
|
||||
printf 'work directory preserved: %s\n' "$WORK" >&2
|
||||
else
|
||||
rm -rf "$WORK"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
mkdir -p "$WORK/apt/lists/partial" "$WORK/apt/archives/partial" "$WORK/tools"
|
||||
: >"$WORK/apt/apt.conf"
|
||||
: >"$WORK/apt/status"
|
||||
printf '%s\n' \
|
||||
"rpm [alt] $MIRROR Sisyphus/x86_64 classic" \
|
||||
"rpm [alt] $MIRROR Sisyphus/noarch classic" >"$WORK/apt/sources.list"
|
||||
|
||||
APT_OPTIONS=(
|
||||
-o 'Dir::Etc::main=-'
|
||||
-o 'Dir::Etc::parts=-'
|
||||
-o "Dir::Etc::sourcelist=$WORK/apt/sources.list"
|
||||
-o 'Dir::Etc::sourceparts=-'
|
||||
-o 'Dir::Etc::preferences=-'
|
||||
-o 'Dir::Etc::preferencesparts=-'
|
||||
-o "Dir::State::lists=$WORK/apt/lists"
|
||||
-o "Dir::State::status=$WORK/apt/status"
|
||||
-o "Dir::Cache::archives=$WORK/apt/archives"
|
||||
-o "Dir::Cache::pkgcache=$WORK/apt/pkgcache.bin"
|
||||
-o "Dir::Cache::srcpkgcache=$WORK/apt/srcpkgcache.bin"
|
||||
)
|
||||
|
||||
apt_get()
|
||||
{
|
||||
APT_CONFIG="$WORK/apt/apt.conf" apt-get "${APT_OPTIONS[@]}" "$@"
|
||||
}
|
||||
|
||||
apt_cache()
|
||||
{
|
||||
APT_CONFIG="$WORK/apt/apt.conf" apt-cache "${APT_OPTIONS[@]}" "$@"
|
||||
}
|
||||
|
||||
NEW_MKSET="$WORK/new-mkset"
|
||||
build_mkset "$NEW_MKSET"
|
||||
cp -as "$rpmlibdir"/. "$WORK/tools/"
|
||||
rm -f "$WORK/tools/mkset"
|
||||
ln -s "$NEW_MKSET" "$WORK/tools/mkset"
|
||||
|
||||
report_row "$(timestamp)" RUN - - start - - - - - "updating isolated Sisyphus indexes"
|
||||
if ! apt_get update >"$WORK/apt-update.log" 2>&1; then
|
||||
detail=$(sed -n '/./{p;q;}' "$WORK/apt-update.log")
|
||||
report_row "$(timestamp)" RUN - - apt_update_error - - - - - "$detail"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
QUEUE="$WORK/packages.tsv"
|
||||
if ! apt_cache dumpavail |
|
||||
awk -f "$PACKAGE_PARSER" |
|
||||
LC_ALL=C sort -t$'\034' -k1,1 -k2,2 >"$QUEUE"; then
|
||||
report_row "$(timestamp)" RUN - - metadata_error - - - - - 'unable to parse apt-cache dumpavail'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SELECTED="$WORK/selected.tsv"
|
||||
if ((${#PACKAGES[@]})); then
|
||||
: >"$SELECTED"
|
||||
for package in "${PACKAGES[@]}"; do
|
||||
if ! awk -F '\034' -v package="$package" '$1 == package { print; found=1 } END { exit !found }' \
|
||||
"$QUEUE" >>"$SELECTED"; then
|
||||
die "package not found in Sisyphus x86_64/noarch: $package"
|
||||
fi
|
||||
done
|
||||
elif [[ -n $LIMIT ]]; then
|
||||
sed -n "1,${LIMIT}p" "$QUEUE" >"$SELECTED"
|
||||
else
|
||||
cp "$QUEUE" "$SELECTED"
|
||||
fi
|
||||
|
||||
selected_count=$(wc -l <"$SELECTED")
|
||||
report_row "$(timestamp)" RUN - - ready - - - - - "selected packages: $selected_count"
|
||||
printf 'selected packages: %s; report: %s\n' "$selected_count" "$REPORT"
|
||||
|
||||
declare -A COMPLETED=()
|
||||
if ((RESUME)); then
|
||||
while IFS=$'\t' read -r package architecture version; do
|
||||
COMPLETED["$package"$'\t'"$architecture"$'\t'"$version"]=1
|
||||
done < <(awk -f "$RESUME_PARSER" "$REPORT")
|
||||
fi
|
||||
|
||||
first_log_line()
|
||||
{
|
||||
local file=$1
|
||||
[[ -s $file ]] || return 0
|
||||
sed -n '/./{p;q;}' "$file" | cut -c1-500
|
||||
}
|
||||
|
||||
last_log_line()
|
||||
{
|
||||
local file=$1
|
||||
[[ -s $file ]] || return 0
|
||||
awk 'NF { line=$0 } END { print line }' "$file" | cut -c1-500
|
||||
}
|
||||
|
||||
extract_rpm()
|
||||
{
|
||||
local package_file=$1 root=$2 log=$3 mode=$4
|
||||
local metadata="$root/../.arsv-file-metadata"
|
||||
local patterns="$root/../.arsv-nonlink-patterns"
|
||||
local symlinks="$root/../.arsv-symlinks"
|
||||
local field_separator=$'\034' record_separator=$'\035' record
|
||||
local filename link_target relative archive_name
|
||||
local query_format="[%{FILENAMES}${field_separator}%{FILELINKTOS}${record_separator}]"
|
||||
|
||||
if ! rpmquery -p --qf "$query_format" \
|
||||
"$package_file" >"$metadata" 2>>"$log"; then
|
||||
return 1
|
||||
fi
|
||||
case $mode in
|
||||
symlinks) : >"$symlinks" ;;
|
||||
files) : >"$patterns" ;;
|
||||
*) printf 'internal error: invalid RPM extraction mode: %s\n' "$mode" >>"$log"; return 1 ;;
|
||||
esac
|
||||
|
||||
while IFS= read -r -d "$record_separator" record; do
|
||||
if ! rpm_split_file_record "$record"; then
|
||||
printf 'unsupported RPM filename or symlink record\n' >>"$log"
|
||||
return 1
|
||||
fi
|
||||
filename=$RPM_FILENAME
|
||||
link_target=$RPM_LINK_TARGET
|
||||
[[ -n $filename && $filename == /* ]] || {
|
||||
printf 'unsafe RPM filename: %q\n' "$filename" >>"$log"
|
||||
return 1
|
||||
}
|
||||
relative=${filename#/}
|
||||
if [[ $relative == '..' || $relative == ../* || $relative == */../* || $relative == */.. ]]; then
|
||||
printf 'unsafe RPM filename: %q\n' "$filename" >>"$log"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n $link_target && $link_target != '(none)' ]]; then
|
||||
if [[ $mode == symlinks ]]; then
|
||||
printf '%s\t%s\n' "$relative" "$link_target" >>"$symlinks"
|
||||
fi
|
||||
elif [[ $mode == files ]]; then
|
||||
archive_name=./$relative
|
||||
if ! rpm_cpio_literal_pattern "$archive_name" >>"$patterns"; then
|
||||
printf 'unsupported RPM filename: %q\n' "$filename" >>"$log"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
done <"$metadata"
|
||||
|
||||
if [[ $mode == files ]]; then
|
||||
if ! rpm2cpio "$package_file" |
|
||||
cpio -it --quiet --no-absolute-filenames 2>>"$log" |
|
||||
while IFS= read -r member; do
|
||||
case $member in
|
||||
/*|../*|*/../*|*/..)
|
||||
printf 'unsafe cpio member: %q\n' "$member" >>"$log"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -s $patterns ]]; then
|
||||
if ! rpm2cpio "$package_file" |
|
||||
(cd "$root" && cpio -idm --quiet --no-absolute-filenames -E "$patterns") \
|
||||
2>>"$log"; then
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
cat "$symlinks"
|
||||
}
|
||||
|
||||
normalize_expected()
|
||||
{
|
||||
local side=$1 package_file=$2 output=$3
|
||||
if [[ $side == provides ]]; then
|
||||
rpmquery -p --qf \
|
||||
'[%{PROVIDENAME}\t%{PROVIDEFLAGS:depflags}\t%{PROVIDEVERSION}\n]' \
|
||||
"$package_file"
|
||||
else
|
||||
rpmquery -p --qf \
|
||||
'[%{REQUIRENAME}\t%{REQUIREFLAGS:depflags}\t%{REQUIREVERSION}\n]' \
|
||||
"$package_file"
|
||||
fi | awk -F '\t' '$3 ~ /^set:/ { print $1 "\t" $2 "\t" $3 }' |
|
||||
LC_ALL=C sort -u >"$output"
|
||||
}
|
||||
|
||||
normalize_generated()
|
||||
{
|
||||
local side=$1 input=$2 output=$3
|
||||
if [[ $side == provides ]]; then
|
||||
awk '$2 == "=" && $3 ~ /^set:/ { print $1 "\t" $2 "\t" $3 }' "$input"
|
||||
else
|
||||
awk '$2 == ">=" && $3 ~ /^set:/ { print $1 "\t" $2 "\t" $3 }' "$input"
|
||||
fi | LC_ALL=C sort -u >"$output"
|
||||
}
|
||||
|
||||
compare_side()
|
||||
{
|
||||
local side=$1 package=$2 architecture=$3 expected=$4 generated=$5 output=$6
|
||||
if ! awk -F '\t' -v OFS='\034' '
|
||||
FILENAME == ARGV[1] { key=$1 "\034" $2; expected[key]=$3; keys[key]=1; next }
|
||||
{ key=$1 "\034" $2; generated[key]=$3; keys[key]=1 }
|
||||
END {
|
||||
for (key in keys) {
|
||||
split(key, parts, "\034")
|
||||
e=expected[key]; g=generated[key]
|
||||
if (e == "") status="extra_generated"
|
||||
else if (g == "") status="missing_generated"
|
||||
else if (e == g) status="match"
|
||||
else status="mismatch"
|
||||
print status, parts[1], parts[2], e, g
|
||||
}
|
||||
}
|
||||
' "$expected" "$generated" |
|
||||
LC_ALL=C sort -t$'\034' -k2,2 -k3,3 >"$output"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# A non-whitespace separator preserves empty expected/generated fields.
|
||||
while IFS=$'\034' read -r status capability operator expected_set generated_set; do
|
||||
[[ -n $status ]] || continue
|
||||
report_row "$(timestamp)" DEPENDENCY "$package" "$architecture" "$status" \
|
||||
"$side" "$capability" "$operator" "$expected_set" "$generated_set" -
|
||||
done <"$output"
|
||||
}
|
||||
|
||||
process_package()
|
||||
{
|
||||
local package=$1 architecture=$2 version=$3 filename=$4 md5=$5
|
||||
local has_provides=$6 has_requires=$7
|
||||
local key=$package$'\t'$architecture$'\t'$version
|
||||
if [[ -n ${COMPLETED[$key]-} ]]; then
|
||||
printf 'skip completed: %s.%s\n' "$package" "$architecture"
|
||||
return 0
|
||||
fi
|
||||
|
||||
report_row "$(timestamp)" START "$package" "$architecture" processing - - - - - "$version"
|
||||
printf 'process: %s.%s\n' "$package" "$architecture"
|
||||
|
||||
if ((has_provides == 0 && has_requires == 0)); then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" no_set_metadata \
|
||||
- - - - - 'APT metadata contains no set: Provides/Requires'
|
||||
return 0
|
||||
fi
|
||||
|
||||
local package_work="$WORK/package"
|
||||
rm -rf "$package_work"
|
||||
mkdir -p "$package_work/root" "$package_work/archives/partial"
|
||||
local target="$package_work/target.rpm"
|
||||
local url="$MIRROR/Sisyphus/files/$architecture/RPMS/$filename"
|
||||
local log="$package_work/package.log"
|
||||
|
||||
if ! curl --silent --show-error -fL --retry 3 --retry-delay 2 \
|
||||
-o "$target" "$url" >"$log" 2>&1; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" download_error \
|
||||
- - - - - "$(first_log_line "$log")"
|
||||
return 0
|
||||
fi
|
||||
if [[ -n $md5 ]]; then
|
||||
local actual_md5
|
||||
if ! actual_md5=$(md5sum "$target" | awk '{print $1}'); then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" checksum_error \
|
||||
- - - "$md5" - "$filename"
|
||||
return 0
|
||||
fi
|
||||
if [[ $actual_md5 != "$md5" ]]; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" checksum_error \
|
||||
- - - "$md5" "$actual_md5" "$filename"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
local -a payload_rpms=("$target")
|
||||
if ((has_requires)); then
|
||||
mkdir -p "$package_work/root/var/lib/rpm"
|
||||
if ! rpm --root "$package_work/root" --initdb >>"$log" 2>&1; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" rpmdb_error \
|
||||
- - - - - "$(last_log_line "$log")"
|
||||
return 0
|
||||
fi
|
||||
if ! apt_get -y -d \
|
||||
-o "Dir::Cache::archives=$package_work/archives" \
|
||||
-o "RPM::RootDir=$package_work/root" \
|
||||
install "$target" >>"$log" 2>&1; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" dependency_download_error \
|
||||
- - - - - "$(last_log_line "$log")"
|
||||
return 0
|
||||
fi
|
||||
local dependency_rpm
|
||||
for dependency_rpm in "$package_work"/archives/*.rpm; do
|
||||
[[ -f $dependency_rpm ]] || continue
|
||||
payload_rpms+=("$dependency_rpm")
|
||||
done
|
||||
fi
|
||||
|
||||
local payload_rpm extraction_status
|
||||
local symlink_manifest="$package_work/symlinks.tsv"
|
||||
: >"$symlink_manifest"
|
||||
for payload_rpm in "${payload_rpms[@]}"; do
|
||||
if ! extract_rpm "$payload_rpm" "$package_work/root" "$log" symlinks \
|
||||
>>"$symlink_manifest"; then
|
||||
if [[ $payload_rpm == "$target" ]]; then
|
||||
extraction_status=extract_error
|
||||
else
|
||||
extraction_status=dependency_extract_error
|
||||
fi
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" "$extraction_status" \
|
||||
- - - - - "$(last_log_line "$log")"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
if ! rpm_install_symlink_manifest "$package_work/root" "$symlink_manifest" "$log"; then
|
||||
if ((has_requires)); then
|
||||
extraction_status=dependency_extract_error
|
||||
else
|
||||
extraction_status=extract_error
|
||||
fi
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" "$extraction_status" \
|
||||
- - - - - "$(last_log_line "$log")"
|
||||
return 0
|
||||
fi
|
||||
|
||||
for payload_rpm in "${payload_rpms[@]}"; do
|
||||
if ! extract_rpm "$payload_rpm" "$package_work/root" "$log" files; then
|
||||
if [[ $payload_rpm == "$target" ]]; then
|
||||
extraction_status=extract_error
|
||||
else
|
||||
extraction_status=dependency_extract_error
|
||||
fi
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" "$extraction_status" \
|
||||
- - - - - "$(last_log_line "$log")"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
if ! rpmquery -p --qf '[%{FILENAMES}\n]' "$target" |
|
||||
awk -v root="$package_work/root" \
|
||||
'{ if (substr($0,1,1)=="/") print root $0; else print root "/" $0 }' |
|
||||
while IFS= read -r path; do
|
||||
if [[ -e $path || -L $path ]]; then
|
||||
printf '%s\n' "$path"
|
||||
fi
|
||||
done >"$package_work/files"; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" file_list_error \
|
||||
- - - - - 'rpmquery failed while reading package filenames'
|
||||
return 0
|
||||
fi
|
||||
|
||||
local scanner_env=(
|
||||
"RPM_BUILD_ROOT=$package_work/root"
|
||||
"RPM_SUBPACKAGE_NAME=$package"
|
||||
"RPM_PACKAGE_NAME=$package"
|
||||
"RPMB_TOOLS_DIR=$WORK/tools"
|
||||
"RPMB_LIB_DIR=$rpmlibdir"
|
||||
"RPMB_AUTODEPS_DIR=$rpmlibdir"
|
||||
'RPM_FINDPROV_METHOD=none,lib'
|
||||
'RPM_FINDREQ_METHOD=none,lib'
|
||||
'RPM_SCRIPTS_DEBUG=0'
|
||||
)
|
||||
|
||||
local side generator expected generated comparison
|
||||
local total_mismatches=0 detail=''
|
||||
for side in provides requires; do
|
||||
[[ $side == provides && $has_provides == 1 ]] ||
|
||||
[[ $side == requires && $has_requires == 1 ]] || continue
|
||||
|
||||
generator="$package_work/generated.$side.raw"
|
||||
expected="$package_work/expected.$side.tsv"
|
||||
generated="$package_work/generated.$side.tsv"
|
||||
comparison="$package_work/comparison.$side.tsv"
|
||||
if ! normalize_expected "$side" "$target" "$expected"; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" metadata_error \
|
||||
"$side" - - - - 'rpmquery failed while reading set dependencies'
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ $side == provides ]]; then
|
||||
if ! env "${scanner_env[@]}" "$rpmlibdir/find-provides" \
|
||||
<"$package_work/files" >"$generator" 2>"$package_work/$side.log"; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" generator_error \
|
||||
"$side" - - - - "$(first_log_line "$package_work/$side.log")"
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
if ! env "${scanner_env[@]}" "$rpmlibdir/find-requires" \
|
||||
<"$package_work/files" >"$generator" 2>"$package_work/$side.log"; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" generator_error \
|
||||
"$side" - - - - "$(first_log_line "$package_work/$side.log")"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! normalize_generated "$side" "$generator" "$generated"; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" generator_output_error \
|
||||
"$side" - - - - 'unable to normalize generator output'
|
||||
return 0
|
||||
fi
|
||||
if ! compare_side "$side" "$package" "$architecture" "$expected" "$generated" "$comparison"; then
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" comparison_error \
|
||||
"$side" - - - - 'unable to compare generated dependencies'
|
||||
return 0
|
||||
fi
|
||||
|
||||
local expected_count generated_count mismatch_count extra_count
|
||||
expected_count=$(wc -l <"$expected")
|
||||
generated_count=$(wc -l <"$generated")
|
||||
mismatch_count=$(awk -F '\034' \
|
||||
'$1 == "mismatch" || $1 == "missing_generated" { count++ } END { print count+0 }' \
|
||||
"$comparison")
|
||||
extra_count=$(awk -F '\034' '$1 == "extra_generated" { count++ } END { print count+0 }' \
|
||||
"$comparison")
|
||||
total_mismatches=$((total_mismatches + mismatch_count + extra_count))
|
||||
detail+="$side expected=$expected_count generated=$generated_count differences=$mismatch_count extras=$extra_count; "
|
||||
done
|
||||
|
||||
local status=match
|
||||
((total_mismatches == 0)) || status=mismatch
|
||||
report_row "$(timestamp)" SUMMARY "$package" "$architecture" "$status" - - - - - "$detail"
|
||||
sync -f "$REPORT" 2>/dev/null || true
|
||||
}
|
||||
|
||||
while IFS=$'\034' read -r package architecture version filename md5 has_provides has_requires extra; do
|
||||
if [[ -n ${extra-} || -z $package || -z $architecture || -z $version || -z $filename ||
|
||||
! $has_provides =~ ^[01]$ || ! $has_requires =~ ^[01]$ ]]; then
|
||||
report_row "$(timestamp)" RUN "$package" "$architecture" queue_error - - - - - \
|
||||
'invalid package queue record'
|
||||
continue
|
||||
fi
|
||||
process_package "$package" "$architecture" "$version" "$filename" "$md5" \
|
||||
"$has_provides" "$has_requires"
|
||||
done <"$SELECTED"
|
||||
|
||||
report_row "$(timestamp)" RUN - - complete - - - - - "processed selection: $selected_count"
|
||||
printf 'complete; report: %s\n' "$REPORT"
|
||||
@@ -1,10 +0,0 @@
|
||||
BEGIN { FS = "\t" }
|
||||
|
||||
NF == 11 && $2 == "START" {
|
||||
version[$3 "\034" $4] = $11
|
||||
}
|
||||
|
||||
NF == 11 && $2 == "SUMMARY" && $11 ~ /(^|; )complete=1$/ && \
|
||||
version[$3 "\034" $4] != "" {
|
||||
print $3 "\t" $4 "\t" version[$3 "\034" $4]
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
/* mkset-compatible wrapper around reimplement/newset.c.
|
||||
* The stdin/argv contract matches rpm-build/tools/mkset.c. */
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
struct set;
|
||||
struct set *set_new(void);
|
||||
void set_add(struct set *set, const char *symbol);
|
||||
const char *set_fini(struct set *set, int bpp);
|
||||
struct set *set_free(struct set *set);
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
if (argc != 2) {
|
||||
fprintf(stderr, "usage: %s BPP\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
|
||||
char *end = NULL;
|
||||
errno = 0;
|
||||
long parsed_bpp = strtol(argv[1], &end, 10);
|
||||
if (errno || !end || *end || parsed_bpp < 10 || parsed_bpp > 32) {
|
||||
fprintf(stderr, "invalid BPP: %s (expected 10..32)\n", argv[1]);
|
||||
return 2;
|
||||
}
|
||||
int bpp = (int)parsed_bpp;
|
||||
|
||||
struct set *set = set_new();
|
||||
char *line = NULL;
|
||||
size_t allocated = 0;
|
||||
ssize_t length;
|
||||
int added = 0;
|
||||
|
||||
while ((length = getline(&line, &allocated, stdin)) >= 0) {
|
||||
if (length > 0 && line[length - 1] == '\n')
|
||||
line[--length] = '\0';
|
||||
if (length == 0)
|
||||
continue;
|
||||
set_add(set, line);
|
||||
++added;
|
||||
}
|
||||
|
||||
if (!added) {
|
||||
fputs("mkset: no symbols on standard input\n", stderr);
|
||||
free(line);
|
||||
set_free(set);
|
||||
return 2;
|
||||
}
|
||||
const char *encoded = set_fini(set, bpp);
|
||||
if (!encoded) {
|
||||
fputs("mkset: unable to encode set\n", stderr);
|
||||
free(line);
|
||||
set_free(set);
|
||||
return 1;
|
||||
}
|
||||
printf("set:%s\n", encoded);
|
||||
|
||||
free((void *)encoded);
|
||||
free(line);
|
||||
set_free(set);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
BEGIN {
|
||||
RS = ""
|
||||
FS = "\n"
|
||||
OFS = "\034"
|
||||
}
|
||||
|
||||
{
|
||||
package = architecture = version = filename = md5 = ""
|
||||
has_provides = has_requires = 0
|
||||
dependency_field = ""
|
||||
|
||||
for (i = 1; i <= NF; ++i) {
|
||||
if ($i ~ /^Package: /)
|
||||
package = substr($i, 10)
|
||||
else if ($i ~ /^Architecture: /)
|
||||
architecture = substr($i, 15)
|
||||
else if ($i ~ /^Version: /)
|
||||
version = substr($i, 10)
|
||||
else if ($i ~ /^Filename: /)
|
||||
filename = substr($i, 11)
|
||||
else if ($i ~ /^MD5Sum: /)
|
||||
md5 = substr($i, 9)
|
||||
|
||||
if ($i ~ /^Provides: /)
|
||||
dependency_field = "provides"
|
||||
else if ($i ~ /^(Pre-Depends|Depends): /)
|
||||
dependency_field = "requires"
|
||||
else if ($i !~ /^[[:space:]]/)
|
||||
dependency_field = ""
|
||||
|
||||
if (dependency_field == "provides" && index($i, "set:"))
|
||||
has_provides = 1
|
||||
if (dependency_field == "requires" && index($i, "set:"))
|
||||
has_requires = 1
|
||||
}
|
||||
|
||||
if (package != "" && architecture != "" && version != "" && filename != "")
|
||||
print package, architecture, version, filename, md5, has_provides, has_requires
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
rpm_safe_symlink_target()
|
||||
{
|
||||
local root=$1 relative_path=$2 link_target=$3
|
||||
local canonical_root link_directory resolved_target
|
||||
|
||||
canonical_root=$(realpath -m -- "$root") || return 1
|
||||
link_directory=$(realpath -m -- "$canonical_root/$(dirname -- "$relative_path")") || return 1
|
||||
|
||||
if [[ $link_target == /* ]]; then
|
||||
resolved_target=$(realpath -m -- "$canonical_root/${link_target#/}") || return 1
|
||||
else
|
||||
resolved_target=$(realpath -m -- "$link_directory/$link_target") || return 1
|
||||
fi
|
||||
|
||||
[[ $resolved_target == "$canonical_root" || $resolved_target == "$canonical_root"/* ]] || return 1
|
||||
|
||||
if [[ $link_target == /* ]]; then
|
||||
realpath -m --relative-to="$link_directory" -- "$resolved_target"
|
||||
else
|
||||
printf '%s\n' "$link_target"
|
||||
fi
|
||||
}
|
||||
|
||||
rpm_cpio_literal_pattern()
|
||||
{
|
||||
local input=$1 output= character index
|
||||
|
||||
[[ $input != *$'\n'* && $input != *$'\r'* ]] || return 1
|
||||
for ((index = 0; index < ${#input}; ++index)); do
|
||||
character=${input:index:1}
|
||||
case $character in
|
||||
'[') output+='[[]' ;;
|
||||
']') output+='[]]' ;;
|
||||
'*') output+='[*]' ;;
|
||||
'?') output+='[?]' ;;
|
||||
'\\') output+='[\\]' ;;
|
||||
*) output+=$character ;;
|
||||
esac
|
||||
done
|
||||
printf '%s\n' "$output"
|
||||
}
|
||||
|
||||
rpm_split_file_record()
|
||||
{
|
||||
local record=$1 separator=$'\034'
|
||||
|
||||
RPM_FILENAME=
|
||||
RPM_LINK_TARGET=
|
||||
[[ $record == *"$separator"* ]] || return 1
|
||||
RPM_FILENAME=${record%%"$separator"*}
|
||||
RPM_LINK_TARGET=${record#*"$separator"}
|
||||
[[ $RPM_LINK_TARGET != *"$separator"* ]] || return 1
|
||||
[[ $RPM_FILENAME != *$'\t'* && $RPM_FILENAME != *$'\n'* && $RPM_FILENAME != *$'\r'* ]] || return 1
|
||||
[[ $RPM_LINK_TARGET != *$'\t'* && $RPM_LINK_TARGET != *$'\n'* && $RPM_LINK_TARGET != *$'\r'* ]] || return 1
|
||||
}
|
||||
|
||||
rpm_install_symlink_manifest()
|
||||
{
|
||||
local root=$1 manifest=$2 log=$3 separator=$'\034'
|
||||
local canonical_root sorted row relative link_target safe_target destination parent existing
|
||||
|
||||
canonical_root=$(realpath -m -- "$root") || return 1
|
||||
|
||||
sorted=$(mktemp "${manifest}.sorted.XXXXXX") || return 1
|
||||
if ! awk -F '\t' -v separator="$separator" '
|
||||
NF >= 2 {
|
||||
path=$1
|
||||
depth=gsub(/\//, "/", path)
|
||||
printf "%08d%s%s\n", depth, separator, $0
|
||||
}
|
||||
' "$manifest" | LC_ALL=C sort -t "$separator" -k1,1n -k2,2 >"$sorted"; then
|
||||
rm -f -- "$sorted"
|
||||
return 1
|
||||
fi
|
||||
|
||||
while IFS=$separator read -r _depth row; do
|
||||
IFS=$'\t' read -r relative link_target <<<"$row"
|
||||
[[ -n $relative && -n $link_target ]] || continue
|
||||
destination=$canonical_root/$relative
|
||||
parent=$(realpath -m -- "$(dirname -- "$destination")") || {
|
||||
rm -f -- "$sorted"
|
||||
return 1
|
||||
}
|
||||
[[ $parent == "$canonical_root" || $parent == "$canonical_root"/* ]] || {
|
||||
printf 'escaping RPM symlink parent rejected: %q\n' "/$relative" >>"$log"
|
||||
rm -f -- "$sorted"
|
||||
return 1
|
||||
}
|
||||
if ! safe_target=$(rpm_safe_symlink_target "$canonical_root" "$relative" "$link_target"); then
|
||||
printf 'escaping RPM symlink rejected: %q -> %q\n' "/$relative" "$link_target" >>"$log"
|
||||
rm -f -- "$sorted"
|
||||
return 1
|
||||
fi
|
||||
mkdir -p -- "$parent" || {
|
||||
rm -f -- "$sorted"
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ -L $destination ]]; then
|
||||
existing=$(readlink -- "$destination") || {
|
||||
rm -f -- "$sorted"
|
||||
return 1
|
||||
}
|
||||
if [[ $existing == "$safe_target" ]]; then
|
||||
continue
|
||||
fi
|
||||
printf 'conflicting RPM symlinks: %q -> %q and %q\n' \
|
||||
"/$relative" "$existing" "$safe_target" >>"$log"
|
||||
rm -f -- "$sorted"
|
||||
return 1
|
||||
elif [[ -e $destination ]]; then
|
||||
printf 'RPM symlink conflicts with existing path: %q\n' "/$relative" >>"$log"
|
||||
rm -f -- "$sorted"
|
||||
return 1
|
||||
fi
|
||||
if ! ln -s -- "$safe_target" "$destination"; then
|
||||
printf 'unable to create RPM symlink: %q -> %q\n' "/$relative" "$safe_target" >>"$log"
|
||||
rm -f -- "$sorted"
|
||||
return 1
|
||||
fi
|
||||
done <"$sorted"
|
||||
|
||||
rm -f -- "$sorted"
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
extern "C" {
|
||||
#include "../rpm-build/lib/set.h"
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
using Clock = std::chrono::steady_clock;
|
||||
using Nanoseconds = std::chrono::nanoseconds;
|
||||
|
||||
void usage(const char* program)
|
||||
{
|
||||
std::cerr << "Usage: " << program << " PROVIDER_SET REQUIRED_SET\n";
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
if (argc != 3) {
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
|
||||
const std::string provider_set = argv[1];
|
||||
const std::string required_set = argv[2];
|
||||
|
||||
const auto start = Clock::now();
|
||||
const int result = rpmsetcmp(provider_set.c_str(), required_set.c_str());
|
||||
const auto finish = Clock::now();
|
||||
const std::int64_t elapsed =
|
||||
std::chrono::duration_cast<Nanoseconds>(finish - start).count();
|
||||
|
||||
std::cout << "provider_set\t" << provider_set << '\n';
|
||||
std::cout << "required_set\t" << required_set << '\n';
|
||||
std::cout << "cmp_result\t" << result << '\n';
|
||||
std::cout << "rpmsetcmp_ns\t" << elapsed << '\n';
|
||||
|
||||
if (result == -3 || result == -4) {
|
||||
std::cerr << "run_cmp: rpmsetcmp rejected an input set (result=" << result << ")\n";
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -1,585 +0,0 @@
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cerrno>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
extern "C" {
|
||||
#include "../rpm-build/lib/set.h"
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
using Clock = std::chrono::steady_clock;
|
||||
using Nanoseconds = std::chrono::nanoseconds;
|
||||
using LabelSet = std::set<std::string>;
|
||||
|
||||
struct CommandResult {
|
||||
int exit_code;
|
||||
std::string output;
|
||||
};
|
||||
|
||||
struct Timings {
|
||||
std::int64_t set_new_ns = 0;
|
||||
std::int64_t set_add_total_ns = 0;
|
||||
std::int64_t set_fini_ns = 0;
|
||||
std::int64_t set_free_ns = 0;
|
||||
|
||||
std::int64_t total_ns() const
|
||||
{
|
||||
return set_new_ns + set_add_total_ns + set_fini_ns + set_free_ns;
|
||||
}
|
||||
};
|
||||
|
||||
struct SetResult {
|
||||
std::string value;
|
||||
Timings timings;
|
||||
};
|
||||
|
||||
struct Options {
|
||||
std::filesystem::path input;
|
||||
std::optional<int> bpp;
|
||||
};
|
||||
|
||||
std::int64_t elapsed_ns(Clock::time_point start, Clock::time_point finish)
|
||||
{
|
||||
return std::chrono::duration_cast<Nanoseconds>(finish - start).count();
|
||||
}
|
||||
|
||||
std::string shortened(const std::string& value, std::size_t limit = 1200)
|
||||
{
|
||||
if (value.size() <= limit) {
|
||||
return value;
|
||||
}
|
||||
return value.substr(0, limit) + "\n... output truncated ...";
|
||||
}
|
||||
|
||||
CommandResult run_command(const std::vector<std::string>& command,
|
||||
const std::map<std::string, std::string>& environment = {})
|
||||
{
|
||||
if (command.empty()) {
|
||||
throw std::runtime_error("empty command");
|
||||
}
|
||||
|
||||
int output_pipe[2];
|
||||
if (pipe(output_pipe) != 0) {
|
||||
throw std::runtime_error("pipe failed: " + std::string(std::strerror(errno)));
|
||||
}
|
||||
|
||||
const pid_t child = fork();
|
||||
if (child < 0) {
|
||||
const int saved_errno = errno;
|
||||
close(output_pipe[0]);
|
||||
close(output_pipe[1]);
|
||||
throw std::runtime_error("fork failed: " + std::string(std::strerror(saved_errno)));
|
||||
}
|
||||
|
||||
if (child == 0) {
|
||||
close(output_pipe[0]);
|
||||
if (dup2(output_pipe[1], STDOUT_FILENO) < 0 || dup2(output_pipe[1], STDERR_FILENO) < 0) {
|
||||
_exit(126);
|
||||
}
|
||||
close(output_pipe[1]);
|
||||
|
||||
setenv("LC_ALL", "C", 1);
|
||||
for (const auto& [name, value] : environment) {
|
||||
setenv(name.c_str(), value.c_str(), 1);
|
||||
}
|
||||
|
||||
std::vector<char*> argv;
|
||||
argv.reserve(command.size() + 1);
|
||||
for (const std::string& argument : command) {
|
||||
argv.push_back(const_cast<char*>(argument.c_str()));
|
||||
}
|
||||
argv.push_back(nullptr);
|
||||
execvp(argv[0], argv.data());
|
||||
_exit(127);
|
||||
}
|
||||
|
||||
close(output_pipe[1]);
|
||||
std::string output;
|
||||
char buffer[16384];
|
||||
while (true) {
|
||||
const ssize_t count = read(output_pipe[0], buffer, sizeof(buffer));
|
||||
if (count > 0) {
|
||||
output.append(buffer, static_cast<std::size_t>(count));
|
||||
continue;
|
||||
}
|
||||
if (count < 0 && errno == EINTR) {
|
||||
continue;
|
||||
}
|
||||
if (count < 0) {
|
||||
const int saved_errno = errno;
|
||||
close(output_pipe[0]);
|
||||
waitpid(child, nullptr, 0);
|
||||
throw std::runtime_error("read from child failed: " + std::string(std::strerror(saved_errno)));
|
||||
}
|
||||
break;
|
||||
}
|
||||
close(output_pipe[0]);
|
||||
|
||||
int status = 0;
|
||||
while (waitpid(child, &status, 0) < 0) {
|
||||
if (errno != EINTR) {
|
||||
throw std::runtime_error("waitpid failed: " + std::string(std::strerror(errno)));
|
||||
}
|
||||
}
|
||||
|
||||
int exit_code = 128;
|
||||
if (WIFEXITED(status)) {
|
||||
exit_code = WEXITSTATUS(status);
|
||||
} else if (WIFSIGNALED(status)) {
|
||||
exit_code += WTERMSIG(status);
|
||||
}
|
||||
return {exit_code, std::move(output)};
|
||||
}
|
||||
|
||||
CommandResult checked_command(const std::vector<std::string>& command,
|
||||
const std::map<std::string, std::string>& environment = {})
|
||||
{
|
||||
CommandResult result = run_command(command, environment);
|
||||
if (result.exit_code != 0) {
|
||||
std::ostringstream message;
|
||||
message << command.front() << " exited with " << result.exit_code;
|
||||
if (!result.output.empty()) {
|
||||
message << ":\n" << shortened(result.output);
|
||||
}
|
||||
throw std::runtime_error(message.str());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string strip_symbol_version(std::string symbol)
|
||||
{
|
||||
const std::size_t at = symbol.find('@');
|
||||
if (at != std::string::npos) {
|
||||
symbol.resize(at);
|
||||
}
|
||||
return symbol;
|
||||
}
|
||||
|
||||
bool allowed_symbol_type(const std::string& type)
|
||||
{
|
||||
static const std::unordered_set<std::string> allowed = {
|
||||
"NOTYPE", "OBJECT", "FUNC", "COMMON", "TLS", "IFUNC", "GNU_IFUNC",
|
||||
};
|
||||
return allowed.count(type) != 0;
|
||||
}
|
||||
|
||||
bool allowed_symbol_binding(const std::string& binding)
|
||||
{
|
||||
static const std::unordered_set<std::string> allowed = {
|
||||
"GLOBAL", "WEAK", "UNIQUE", "GNU_UNIQUE",
|
||||
};
|
||||
return allowed.count(binding) != 0;
|
||||
}
|
||||
|
||||
bool allowed_symbol_visibility(const std::string& visibility)
|
||||
{
|
||||
return visibility == "DEFAULT" || visibility == "PROTECTED";
|
||||
}
|
||||
|
||||
bool special_symbol(const std::string& symbol)
|
||||
{
|
||||
static const std::unordered_set<std::string> ignored = {
|
||||
"__bss_start", "_edata", "_end", "_fini", "_init",
|
||||
};
|
||||
return ignored.count(symbol) != 0;
|
||||
}
|
||||
|
||||
LabelSet provided_labels(const std::filesystem::path& library)
|
||||
{
|
||||
const CommandResult result = checked_command(
|
||||
{"eu-readelf", "--wide", "--dyn-syms", library.string()});
|
||||
LabelSet labels;
|
||||
std::istringstream output(result.output);
|
||||
std::string line;
|
||||
while (std::getline(output, line)) {
|
||||
std::istringstream fields(line);
|
||||
std::string number;
|
||||
std::string value;
|
||||
std::string size;
|
||||
std::string type;
|
||||
std::string binding;
|
||||
std::string visibility;
|
||||
std::string section;
|
||||
std::string symbol;
|
||||
if (!(fields >> number >> value >> size >> type >> binding >> visibility >> section >> symbol)) {
|
||||
continue;
|
||||
}
|
||||
if (number.empty() || number.back() != ':') {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::uint64_t symbol_value = 0;
|
||||
try {
|
||||
symbol_value = std::stoull(value, nullptr, 16);
|
||||
} catch (const std::exception&) {
|
||||
continue;
|
||||
}
|
||||
if (symbol_value == 0 && type != "TLS") {
|
||||
continue;
|
||||
}
|
||||
if (!allowed_symbol_type(type) || !allowed_symbol_binding(binding) ||
|
||||
!allowed_symbol_visibility(visibility)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
symbol = strip_symbol_version(std::move(symbol));
|
||||
if (symbol.empty() || special_symbol(symbol) ||
|
||||
symbol.find_first_of("()@") != std::string::npos) {
|
||||
continue;
|
||||
}
|
||||
labels.insert(std::move(symbol));
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
|
||||
LabelSet weak_undefined_labels(const std::filesystem::path& executable)
|
||||
{
|
||||
const CommandResult result = checked_command({"nm", "--dynamic", executable.string()});
|
||||
LabelSet weak;
|
||||
std::istringstream output(result.output);
|
||||
std::string line;
|
||||
while (std::getline(output, line)) {
|
||||
std::istringstream fields(line);
|
||||
std::vector<std::string> parts;
|
||||
std::string part;
|
||||
while (fields >> part) {
|
||||
parts.push_back(part);
|
||||
}
|
||||
if (parts.size() != 2 || parts[0].size() != 1 ||
|
||||
std::string("wWvV").find(parts[0][0]) == std::string::npos) {
|
||||
continue;
|
||||
}
|
||||
weak.insert(strip_symbol_version(parts[1]));
|
||||
}
|
||||
return weak;
|
||||
}
|
||||
|
||||
std::string program_interpreter(const std::filesystem::path& executable)
|
||||
{
|
||||
const CommandResult result = checked_command(
|
||||
{"eu-readelf", "--program-headers", executable.string()});
|
||||
const std::string marker = "[Requesting program interpreter: ";
|
||||
const std::size_t start = result.output.find(marker);
|
||||
if (start == std::string::npos) {
|
||||
throw std::runtime_error("ELF executable has no PT_INTERP entry: " + executable.string());
|
||||
}
|
||||
const std::size_t value_start = start + marker.size();
|
||||
const std::size_t end = result.output.find(']', value_start);
|
||||
if (end == std::string::npos || end == value_start) {
|
||||
throw std::runtime_error("cannot parse PT_INTERP for " + executable.string());
|
||||
}
|
||||
return result.output.substr(value_start, end - value_start);
|
||||
}
|
||||
|
||||
bool same_path(const std::string& lhs, const std::filesystem::path& rhs)
|
||||
{
|
||||
std::error_code error;
|
||||
if (std::filesystem::equivalent(lhs, rhs, error)) {
|
||||
return true;
|
||||
}
|
||||
return std::filesystem::path(lhs).lexically_normal() == rhs.lexically_normal();
|
||||
}
|
||||
|
||||
std::map<std::string, LabelSet> required_labels_by_provider(
|
||||
const std::filesystem::path& executable)
|
||||
{
|
||||
const std::string interpreter = program_interpreter(executable);
|
||||
const std::map<std::string, std::string> environment = {
|
||||
{"LD_BIND_NOW", "1"},
|
||||
{"LD_DEBUG", "bindings"},
|
||||
{"LD_TRACE_LOADED_OBJECTS", "1"},
|
||||
{"LD_WARN", "1"},
|
||||
};
|
||||
const CommandResult result = checked_command(
|
||||
{interpreter, executable.string()}, environment);
|
||||
const LabelSet weak = weak_undefined_labels(executable);
|
||||
std::map<std::string, LabelSet> labels_by_provider;
|
||||
|
||||
std::istringstream output(result.output);
|
||||
std::string line;
|
||||
while (std::getline(output, line)) {
|
||||
const std::string binding_marker = "binding file ";
|
||||
const std::size_t binding = line.find(binding_marker);
|
||||
if (binding == std::string::npos) {
|
||||
continue;
|
||||
}
|
||||
const std::size_t source_start = binding + binding_marker.size();
|
||||
const std::size_t source_end = line.find(" [", source_start);
|
||||
const std::size_t to = source_end == std::string::npos
|
||||
? std::string::npos
|
||||
: line.find(" to ", source_end);
|
||||
const std::size_t provider_start = to == std::string::npos
|
||||
? std::string::npos
|
||||
: to + std::string(" to ").size();
|
||||
const std::size_t provider_end = provider_start == std::string::npos
|
||||
? std::string::npos
|
||||
: line.find(" [", provider_start);
|
||||
const std::size_t symbol_marker = provider_end == std::string::npos
|
||||
? std::string::npos
|
||||
: line.find(" symbol ", provider_end);
|
||||
if (source_end == std::string::npos || to == std::string::npos ||
|
||||
provider_start == std::string::npos || provider_end == std::string::npos ||
|
||||
symbol_marker == std::string::npos) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string source = line.substr(source_start, source_end - source_start);
|
||||
const std::string provider = line.substr(provider_start, provider_end - provider_start);
|
||||
if (!same_path(source, executable) || source == provider) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::size_t symbol_start = symbol_marker + std::string(" symbol ").size();
|
||||
if (symbol_start < line.size() && (line[symbol_start] == '`' || line[symbol_start] == '\'')) {
|
||||
++symbol_start;
|
||||
}
|
||||
std::size_t symbol_end = line.find('\'', symbol_start);
|
||||
if (symbol_end == std::string::npos) {
|
||||
symbol_end = line.find_first_of(" \t[", symbol_start);
|
||||
}
|
||||
if (symbol_end == std::string::npos) {
|
||||
symbol_end = line.size();
|
||||
}
|
||||
std::string symbol = strip_symbol_version(
|
||||
line.substr(symbol_start, symbol_end - symbol_start));
|
||||
if (symbol.empty() || weak.count(symbol) != 0) {
|
||||
continue;
|
||||
}
|
||||
labels_by_provider[provider].insert(std::move(symbol));
|
||||
}
|
||||
|
||||
for (auto it = labels_by_provider.begin(); it != labels_by_provider.end();) {
|
||||
if (it->second.empty()) {
|
||||
it = labels_by_provider.erase(it);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
return labels_by_provider;
|
||||
}
|
||||
|
||||
int suggested_bpp(std::size_t provider_label_count)
|
||||
{
|
||||
if (provider_label_count < 1) {
|
||||
provider_label_count = 1;
|
||||
}
|
||||
std::size_t value = provider_label_count - 1;
|
||||
int bits = 0;
|
||||
while (value != 0) {
|
||||
++bits;
|
||||
value >>= 1;
|
||||
}
|
||||
return std::min(32, bits + 10);
|
||||
}
|
||||
|
||||
SetResult build_set(const LabelSet& labels, int bpp)
|
||||
{
|
||||
if (labels.empty()) {
|
||||
throw std::runtime_error("cannot build a set from zero labels");
|
||||
}
|
||||
|
||||
Timings timings;
|
||||
auto start = Clock::now();
|
||||
struct set* value = set_new();
|
||||
auto finish = Clock::now();
|
||||
timings.set_new_ns = elapsed_ns(start, finish);
|
||||
if (value == nullptr) {
|
||||
throw std::runtime_error("set_new returned NULL");
|
||||
}
|
||||
|
||||
start = Clock::now();
|
||||
for (const std::string& label : labels) {
|
||||
set_add(value, label.c_str());
|
||||
}
|
||||
finish = Clock::now();
|
||||
timings.set_add_total_ns = elapsed_ns(start, finish);
|
||||
|
||||
start = Clock::now();
|
||||
const char* payload = set_fini(value, bpp);
|
||||
finish = Clock::now();
|
||||
timings.set_fini_ns = elapsed_ns(start, finish);
|
||||
if (payload == nullptr) {
|
||||
set_free(value);
|
||||
throw std::runtime_error("set_fini returned NULL");
|
||||
}
|
||||
std::string encoded = "set:" + std::string(payload);
|
||||
std::free(const_cast<char*>(payload));
|
||||
|
||||
start = Clock::now();
|
||||
value = set_free(value);
|
||||
finish = Clock::now();
|
||||
timings.set_free_ns = elapsed_ns(start, finish);
|
||||
(void)value;
|
||||
return {std::move(encoded), timings};
|
||||
}
|
||||
|
||||
enum class ElfKind {
|
||||
executable,
|
||||
shared_library,
|
||||
};
|
||||
|
||||
ElfKind classify_elf(const std::filesystem::path& input)
|
||||
{
|
||||
const CommandResult executable = run_command(
|
||||
{"eu-elfclassify", "--executable", input.string()});
|
||||
if (executable.exit_code == 0) {
|
||||
return ElfKind::executable;
|
||||
}
|
||||
if (executable.exit_code == 127) {
|
||||
throw std::runtime_error("eu-elfclassify is required but was not found");
|
||||
}
|
||||
|
||||
const CommandResult shared = run_command(
|
||||
{"eu-elfclassify", "--shared", input.string()});
|
||||
if (shared.exit_code == 0) {
|
||||
return ElfKind::shared_library;
|
||||
}
|
||||
throw std::runtime_error("input is not a supported dynamic ELF executable or shared library: " +
|
||||
input.string());
|
||||
}
|
||||
|
||||
int parse_bpp(const std::string& value)
|
||||
{
|
||||
std::size_t parsed = 0;
|
||||
int bpp = 0;
|
||||
try {
|
||||
bpp = std::stoi(value, &parsed);
|
||||
} catch (const std::exception&) {
|
||||
throw std::runtime_error("invalid bpp: " + value);
|
||||
}
|
||||
if (parsed != value.size() || bpp < 10 || bpp > 32) {
|
||||
throw std::runtime_error("bpp must be an integer in [10, 32]");
|
||||
}
|
||||
return bpp;
|
||||
}
|
||||
|
||||
void usage(const char* program)
|
||||
{
|
||||
std::cerr << "Usage: " << program << " [--bpp 10..32] ELF_PATH\n";
|
||||
}
|
||||
|
||||
Options parse_options(int argc, char** argv)
|
||||
{
|
||||
Options options;
|
||||
for (int index = 1; index < argc; ++index) {
|
||||
const std::string argument = argv[index];
|
||||
if (argument == "--help" || argument == "-h") {
|
||||
usage(argv[0]);
|
||||
std::exit(0);
|
||||
}
|
||||
if (argument == "--bpp") {
|
||||
if (++index >= argc) {
|
||||
throw std::runtime_error("--bpp requires a value");
|
||||
}
|
||||
options.bpp = parse_bpp(argv[index]);
|
||||
continue;
|
||||
}
|
||||
if (!argument.empty() && argument.front() == '-') {
|
||||
throw std::runtime_error("unknown option: " + argument);
|
||||
}
|
||||
if (!options.input.empty()) {
|
||||
throw std::runtime_error("exactly one ELF path is required");
|
||||
}
|
||||
options.input = argument;
|
||||
}
|
||||
if (options.input.empty()) {
|
||||
throw std::runtime_error("ELF path is required");
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
void print_header(const std::filesystem::path& input, ElfKind kind,
|
||||
const std::optional<int>& bpp)
|
||||
{
|
||||
std::cout << "input\t" << input.string() << '\n';
|
||||
std::cout << "kind\t"
|
||||
<< (kind == ElfKind::executable ? "executable" : "shared-library") << '\n';
|
||||
std::cout << "bpp_mode\t" << (bpp.has_value() ? "override" : "auto") << '\n';
|
||||
std::cout << "role\tobject\tlabels\tbpp\tset\tset_new_ns\tset_add_total_ns\t"
|
||||
"set_fini_ns\tset_free_ns\tset_api_total_ns\n";
|
||||
}
|
||||
|
||||
void print_row(const std::string& role, const std::string& object,
|
||||
std::size_t label_count, int bpp, const SetResult& result)
|
||||
{
|
||||
const Timings& timings = result.timings;
|
||||
std::cout << role << '\t' << object << '\t' << label_count << '\t' << bpp << '\t'
|
||||
<< result.value << '\t' << timings.set_new_ns << '\t'
|
||||
<< timings.set_add_total_ns << '\t' << timings.set_fini_ns << '\t'
|
||||
<< timings.set_free_ns << '\t' << timings.total_ns() << '\n';
|
||||
}
|
||||
|
||||
int run(const Options& options)
|
||||
{
|
||||
std::error_code error;
|
||||
const std::filesystem::path input = std::filesystem::canonical(options.input, error);
|
||||
if (error || !std::filesystem::is_regular_file(input)) {
|
||||
throw std::runtime_error("input is not a readable regular file: " + options.input.string());
|
||||
}
|
||||
if (input.string().find_first_of("\t\r\n") != std::string::npos) {
|
||||
throw std::runtime_error("input path contains characters unsupported by TSV output");
|
||||
}
|
||||
|
||||
const ElfKind kind = classify_elf(input);
|
||||
print_header(input, kind, options.bpp);
|
||||
|
||||
if (kind == ElfKind::shared_library) {
|
||||
const LabelSet labels = provided_labels(input);
|
||||
if (labels.empty()) {
|
||||
throw std::runtime_error("no provided labels found in " + input.string());
|
||||
}
|
||||
const int bpp = options.bpp.value_or(suggested_bpp(labels.size()));
|
||||
print_row("provided", input.string(), labels.size(), bpp, build_set(labels, bpp));
|
||||
return 0;
|
||||
}
|
||||
|
||||
const std::map<std::string, LabelSet> requirements = required_labels_by_provider(input);
|
||||
if (requirements.empty()) {
|
||||
throw std::runtime_error("no external dynamic symbol bindings found in " + input.string());
|
||||
}
|
||||
for (const auto& [provider, labels] : requirements) {
|
||||
std::size_t provider_count = 0;
|
||||
try {
|
||||
provider_count = provided_labels(provider).size();
|
||||
} catch (const std::exception& exception) {
|
||||
std::cerr << "warning: cannot inspect provider " << provider << ": "
|
||||
<< exception.what() << '\n';
|
||||
}
|
||||
const int bpp = options.bpp.value_or(
|
||||
suggested_bpp(provider_count == 0 ? labels.size() : provider_count));
|
||||
print_row("required", provider, labels.size(), bpp, build_set(labels, bpp));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
try {
|
||||
return run(parse_options(argc, argv));
|
||||
} catch (const std::exception& exception) {
|
||||
std::cerr << "run_set: " << exception.what() << '\n';
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ CPU=0
|
||||
RESET_WORK=1 # 0 — продолжить готовые сборки, 1 — начать всё заново.
|
||||
COLLECT_PERF=1
|
||||
PERF_RECORD=1 # Отдельный профильный прогон; не входит в среднее время.
|
||||
PERF_FREQUENCY=99
|
||||
PERF_FREQUENCY=499
|
||||
PERF_EVENTS='task-clock,context-switches,cpu-migrations,page-faults,minor-faults,major-faults,cycles,instructions,branches,branch-misses,cache-references,cache-misses,L1-dcache-loads,L1-dcache-load-misses,LLC-loads,LLC-load-misses'
|
||||
PACKAGER='krosh <gudovdo@my.msu.ru>'
|
||||
APT_SOURCE=/etc/apt/sources.list.d/alt.list
|
||||
@@ -147,7 +147,7 @@ append_perf_stat()
|
||||
|
||||
record_profile()
|
||||
{
|
||||
local operation=$1 variant=$2 perf_dir=$3 status data report
|
||||
local operation=$1 variant=$2 perf_dir=$3 dso_name=$4 status data report
|
||||
local libdir="$variant/lib/usr/lib64"
|
||||
local root="$COMMON/root"
|
||||
local -a command
|
||||
@@ -187,17 +187,39 @@ record_profile()
|
||||
--input "$data" \
|
||||
>"$report" \
|
||||
2>"$perf_dir/$operation.report.stderr" || true
|
||||
|
||||
perf --buildid-dir "$perf_dir/buildid-cache" report \
|
||||
--stdio \
|
||||
--no-children \
|
||||
--inline \
|
||||
--percent-limit 0 \
|
||||
--sort dso,symbol,srcline \
|
||||
--input "$data" \
|
||||
>"$perf_dir/$operation.lines.txt" \
|
||||
2>"$perf_dir/$operation.lines.stderr" || true
|
||||
|
||||
# Без TUI: perf 6.18 может упасть при выборе символа без self-samples.
|
||||
perf --buildid-dir "$perf_dir/buildid-cache" annotate \
|
||||
--stdio \
|
||||
--dsos "$dso_name" \
|
||||
--input "$data" \
|
||||
>"$perf_dir/$operation.annotate.txt" \
|
||||
2>"$perf_dir/$operation.annotate.stderr" || true
|
||||
fi
|
||||
}
|
||||
|
||||
prepare_perf_symbols()
|
||||
{
|
||||
local perf_dir=$1 debug_file=$2 debuginfo_rpm=$3
|
||||
local perf_dir=$1 runtime_file=$2 debug_file=$3 debuginfo_rpm=$4
|
||||
local unstripped="$perf_dir/librpm.unstripped"
|
||||
|
||||
mkdir -p "$perf_dir/buildid-cache"
|
||||
cp -a "$debuginfo_rpm" "$perf_dir/"
|
||||
# Split-debug ELF содержит DWARF, но не байты .text для annotate.
|
||||
# eu-unstrip объединяет runtime-код и matching debuginfo с тем же Build ID.
|
||||
eu-unstrip -o "$unstripped" "$runtime_file" "$debug_file"
|
||||
perf --buildid-dir "$perf_dir/buildid-cache" buildid-cache \
|
||||
--add "$debug_file" \
|
||||
--add "$unstripped" \
|
||||
>"$perf_dir/buildid-cache.stdout" \
|
||||
2>"$perf_dir/buildid-cache.stderr"
|
||||
}
|
||||
@@ -205,6 +227,7 @@ prepare_perf_symbols()
|
||||
benchmark_variant()
|
||||
{
|
||||
local variant=$1 result=$2 debug_file=$3 debuginfo_rpm=$4
|
||||
local runtime_file=$5 dso_name=$6
|
||||
local operation run average label status_text perf_result perf_dir
|
||||
local -a times statuses
|
||||
local -a operations=(
|
||||
@@ -228,7 +251,8 @@ benchmark_variant()
|
||||
if ((PERF_RECORD)); then
|
||||
rm -rf "$perf_dir"
|
||||
mkdir -p "$perf_dir"
|
||||
prepare_perf_symbols "$perf_dir" "$debug_file" "$debuginfo_rpm"
|
||||
prepare_perf_symbols "$perf_dir" "$runtime_file" \
|
||||
"$debug_file" "$debuginfo_rpm"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -267,13 +291,13 @@ benchmark_variant()
|
||||
|
||||
if ((COLLECT_PERF && PERF_RECORD)); then
|
||||
printf '%s: perf record\n' "$operation"
|
||||
record_profile "$operation" "$variant" "$perf_dir"
|
||||
record_profile "$operation" "$variant" "$perf_dir" "$dso_name"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
for command in git gear-hsh hsh rpm rpmquery rpm2cpio cpio apt-get apt-cache \
|
||||
taskset awk sed date sha256sum ldd readelf; do
|
||||
taskset awk sed date sha256sum ldd readelf readlink eu-unstrip; do
|
||||
command -v "$command" >/dev/null || fail "required command not found: $command"
|
||||
done
|
||||
if ((COLLECT_PERF)); then
|
||||
@@ -424,6 +448,8 @@ for setc in "${setc_files[@]}"; do
|
||||
libdir="$variant/lib/usr/lib64"
|
||||
[[ -e $libdir/librpm.so.7 && -e $libdir/librpmio.so.7 ]] || \
|
||||
fail "librpm libraries were not extracted for $filename"
|
||||
runtime_file=$(readlink -f "$libdir/librpm.so.7")
|
||||
dso_name=$(basename "$runtime_file")
|
||||
LD_LIBRARY_PATH="$libdir" ldd "$APT_GET" | \
|
||||
grep -F "$libdir/librpm.so.7" >/dev/null || \
|
||||
fail "apt-get does not load the built librpm for $filename"
|
||||
@@ -446,7 +472,7 @@ for setc in "${setc_files[@]}"; do
|
||||
fi
|
||||
|
||||
benchmark_variant "$variant" "$RESULT_DIR/$result_name" \
|
||||
"$debug_file" "$debuginfo_rpm"
|
||||
"$debug_file" "$debuginfo_rpm" "$runtime_file" "$dso_name"
|
||||
done
|
||||
|
||||
printf '\nResults:\n'
|
||||
|
||||
Reference in New Issue
Block a user