From 6c86bd664a8ea6f1ea443ee5e1d0ad893b50ffb3 Mon Sep 17 00:00:00 2001 From: Dmitrii Krosh Date: Wed, 29 Jul 2026 07:12:49 +0300 Subject: [PATCH] add franc between set9 and orig_set --- set_frank.c | 1136 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1136 insertions(+) create mode 100644 set_frank.c diff --git a/set_frank.c b/set_frank.c new file mode 100644 index 0000000..47da320 --- /dev/null +++ b/set_frank.c @@ -0,0 +1,1136 @@ +#include +#include +#include +#include +#include + +#include "rpmlib.h" +#ifdef SELF_TEST +#undef NDEBUG +#include +#endif +#include "set.h" +#include "system.h" + +#define CACHE_SIZE 512 +#define SENTINELS 8 + +struct set { + size_t cnt; + size_t symbols_cap; + size_t strings_len; + size_t strings_cap; + char* strings; + struct symbols { + size_t offset; + unsigned hash; + }* symbols_v; +}; + +struct set* set_new() { + struct set* set = xmalloc(sizeof *set); + set->cnt = 0; + set->symbols_cap = 0; + set->strings_len = 0; + set->strings_cap = 0; + set->strings = NULL; + set->symbols_v = NULL; + + return set; +} + +void set_add(struct set* set, const char* sym) { + if (set->cnt == set->symbols_cap) { + set->symbols_cap += 1024; + set->symbols_v = xrealloc(set->symbols_v, sizeof(*set->symbols_v) * set->symbols_cap); + } + + size_t length = strlen(sym) + 1; + size_t required = set->strings_len + length; + if (required > set->strings_cap) { + size_t capacity = set->strings_cap ? set->strings_cap : 4096; + while (capacity < required) capacity *= 2; + + set->strings = xrealloc(set->strings, capacity); + set->strings_cap = capacity; + } + + set->symbols_v[set->cnt].offset = set->strings_len; + set->symbols_v[set->cnt].hash = 0; + memcpy(set->strings + set->strings_len, sym, length); + set->strings_len = required; + set->cnt++; + + return; +} + +struct set* set_free(struct set* set) { + if (set) { + _free(set->strings); + _free(set->symbols_v); + set = _free(set); + } + + return NULL; +} + +// --- + +static unsigned hash(const char* str) { + unsigned hash = 0x9e3779b9; + const unsigned char* p = (const unsigned char*)str; + + while (*p) { + hash += *p++; + hash += (hash << 10); + hash ^= (hash >> 6); + } + + hash += (hash << 3); + hash ^= (hash >> 11); + hash += (hash << 15); + + return hash; +} + +int cmp(const void* arg1, const void* arg2) { + const struct symbols* s1 = arg1; + const struct symbols* s2 = arg2; + + if (s1->hash > s2->hash) return 1; + if (s2->hash > s1->hash) return -1; + + return 0; +} + +static void sort_symbols(struct symbols* values, size_t count, int bpp) { + if (count < 128) { + qsort(values, count, sizeof(*values), cmp); + return; + } + + struct symbols temporary[count]; + struct symbols* source = values; + struct symbols* destination = temporary; + unsigned passes = ((unsigned)bpp + 7) / 8; + + for (unsigned pass = 0; pass < passes; ++pass) { + size_t offsets[256] = {0}; + unsigned shift = pass * 8; + for (size_t i = 0; i < count; ++i) ++offsets[(source[i].hash >> shift) & 0xffu]; + + size_t position = 0; + for (size_t i = 0; i < 256; ++i) { + size_t bucket_count = offsets[i]; + offsets[i] = position; + position += bucket_count; + } + + for (size_t i = 0; i < count; ++i) { + unsigned bucket = (source[i].hash >> shift) & 0xffu; + destination[offsets[bucket]++] = source[i]; + } + + struct symbols* swap = source; + source = destination; + destination = swap; + } + + if (source != values) memcpy(values, source, count * sizeof(*values)); + + return; +} + +// --- + +static int log2i(int n) { + int m = 0; + while (n /= 2) m++; + + return m; +} + +// Calculate Mshift paramter for encoding. +static int encode_golomb_Mshift(int cnt, int bpp) { + // XXX Slightly better Mshift estimations are probably possible. + // Recheck "Compression and coding algorithms" by Moffat & Turpin. + int Mshift = bpp - log2i(cnt) - 1; + + // Adjust out-of-range values. + Mshift = (Mshift < 7) ? 7 : Mshift; + Mshift = (Mshift > 31) ? 31 : Mshift; + assert(Mshift < bpp); + + return Mshift; +} + +// Estimate how many bits can be filled up. +static inline int encode_golomb_size(int cnt, int Mshift) { + // XXX No precise estimation. However, we do not expect unary-encoded bits + // to take more than binary-encoded Mshift bits. + return Mshift * 2 * cnt + 16; +} + +// Estimate base62 buffer size required to encode a given number of bits. +static inline int encode_base62_size(int bit_cnt) { + // In the worst case, which is ZxZxZx..., five bits can make a character; + // the remaining bits can make a character, too. And the string must be + // null-terminated. + return bit_cnt / 5 + 2; +} + +static int encode_set_size(int cnt, int bpp) { + int Mshift = encode_golomb_Mshift(cnt, bpp); + int bit_cnt = encode_golomb_size(cnt, Mshift); + // two leading characters are special + return 2 + encode_base62_size(bit_cnt); +} + +// --- + +// Main base62 encoding routine: pack bit_arr into base62 string. +/* + * Base62 routines - encode bits with alnum characters. + * + * This is a base64-based base62 implementation. Values 0..61 are encoded + * with '0'..'9', 'a'..'z', and 'A'..'Z'. However, 'Z' is special: it will + * also encode 62 and 63. To achieve this, 'Z' will occupy two high bits in + * the next character. Thus 'Z' can be interpreted as an escape character + * (which indicates that the next character must be handled specially). + * Note that setting high bits to "00", "01" or "10" cannot contribute + * to another 'Z' (which would require high bits set to "11"). This is + * how multiple escapes are avoided. + */ + +static char* bits_to_char(int c, char* base62) { + assert(c >= 0 && c <= 61); + + if (c < 10) { + *base62++ = c + '0'; + } else if (c < 36) { + *base62++ = c - 10 + 'a'; + } else if (c < 62) { + *base62++ = c - 36 + 'A'; + } + + return base62; +} + +// --- + +static inline char encode_bpp(int bpp) { return bpp - 7 + 'a'; } + +struct encode_writer { + uint64_t bits; + unsigned filled; + unsigned escaped; + unsigned pending_high; + char* output; +}; + +static inline void encode_writer_digit(struct encode_writer* writer, unsigned value) { + assert(value < 62); + + if (value < 10) { + *writer->output++ = (char)('0' + value); + } else if (value < 36) { + *writer->output++ = (char)('a' + value - 10); + } else { + *writer->output++ = (char)('A' + value - 36); + } +} + +static inline void encode_writer_flush(struct encode_writer* writer) { + for (;;) { + unsigned width = writer->escaped ? 4u : 6u; + if (writer->filled < width) return; + + unsigned value = (unsigned)writer->bits & ((1u << width) - 1); + writer->bits >>= width; + writer->filled -= width; + + if (writer->escaped) { + encode_writer_digit(writer, writer->pending_high | value); + writer->escaped = 0; + } else if (value >= 61) { + encode_writer_digit(writer, 61); + writer->pending_high = (value - 61) << 4; + writer->escaped = 1; + } else { + encode_writer_digit(writer, value); + } + } +} + +static inline void encode_writer_zeros(struct encode_writer* writer, unsigned count) { + while (count) { + unsigned take = count > 56 ? 56 : count; + writer->filled += take; + count -= take; + + encode_writer_flush(writer); + } + + return; +} + +static inline void encode_writer_put(struct encode_writer* writer, uint64_t value, unsigned width) { + writer->bits |= value << writer->filled; + writer->filled += width; + encode_writer_flush(writer); + + return; +} + +static int encode_set(int cnt, const unsigned* hash_arr, int bpp, char* base62_str) { + const unsigned Mshift = (unsigned)encode_golomb_Mshift(cnt, bpp); + const unsigned mask = (1u << Mshift) - 1; + char* const start = base62_str; + unsigned previous = 0; + + *base62_str++ = encode_bpp(bpp); + *base62_str++ = encode_bpp((int)Mshift); + struct encode_writer writer = {.output = base62_str}; + + for (int i = 0; i < cnt; ++i) { + unsigned current = hash_arr[i]; + unsigned delta = current - previous; + previous = current; + + encode_writer_zeros(&writer, delta >> Mshift); + encode_writer_put(&writer, 1, 1); + encode_writer_put(&writer, delta & mask, Mshift); + } + + encode_writer_flush(&writer); + if (writer.filled || writer.escaped) { + unsigned value = (unsigned)writer.bits; + if (writer.escaped) value |= writer.pending_high; + encode_writer_digit(&writer, value); + } + + *writer.output = '\0'; + + return (int)(writer.output - start); +} + +const char* set_fini(struct set* set, int bpp) { + // Implementation for finalizing the set + + assert(set != NULL); + assert(set->cnt > 0); + assert(bpp >= 10 && bpp <= 32); + + unsigned mask = (bpp < 32) ? (1u << bpp) - 1 : ~0u; + + for (size_t i = 0; i < set->cnt; ++i) { + set->symbols_v[i].hash = hash(set->strings + set->symbols_v[i].offset) & mask; + } + + sort_symbols(set->symbols_v, set->cnt, bpp); + + // warn on hash collizions + for (size_t i = 0; i < set->cnt - 1; ++i) { + if (set->symbols_v[i].hash != set->symbols_v[i + 1].hash) continue; + const char* left = set->strings + set->symbols_v[i].offset; + const char* right = set->strings + set->symbols_v[i + 1].offset; + if (!strcmp(left, right)) continue; + + fprintf(stderr, "warning: hash collision: %s %s\n", left, right); + } + + unsigned unique_hash[set->cnt]; + size_t unique_cnt = 0; + + // delete duplicates + for (size_t i = 0; i < set->cnt; ++i) { + while (i + 1 < set->cnt && set->symbols_v[i].hash == set->symbols_v[i + 1].hash) { + ++i; + } + unique_hash[unique_cnt++] = set->symbols_v[i].hash; + } + + char base62_str[encode_set_size(unique_cnt, bpp)]; + encode_set(unique_cnt, unique_hash, bpp, base62_str); + + return xstrdup(base62_str); +} + +// --- + +struct set_meta { + const char* str; + const char* payload; + size_t len; + size_t payload_len; + int bpp; + int Mshift; + int bit_capacity; + int value_capacity; +}; + +static int set_meta_init(const char* str, struct set_meta* meta) { + // len >= 3 + if (!str[0] || !str[1] || !str[2]) return -4; + + int bpp = str[0] + 7 - 'a'; + if (bpp < 10 || bpp > 32) return -1; + + int Mshift = str[1] + 7 - 'a'; + if (Mshift < 7 || Mshift > 31) return -2; + if (Mshift >= bpp) return -3; + + *meta = (struct set_meta){ + .str = str, + .payload = str + 2, + .bpp = bpp, + .Mshift = Mshift, + }; + + return 0; +} + +static int set_meta_fini(struct set_meta* meta) { + size_t len = strlen(meta->str); + + size_t payload_len = len - 2; + + int bit_capacity = (int)payload_len * 6; + int value_capacity = bit_capacity / (meta->Mshift + 1); + + if (value_capacity < 1) return -4; + + meta->len = len; + meta->payload_len = payload_len; + meta->bit_capacity = bit_capacity; + meta->value_capacity = value_capacity; + + return 0; +} +// This table maps alnum characters to their numeric values. +static const int char_to_num[256] = { + [0 ... 255] = 0xee, [0] = 0xff, +#define C1(c, b) [c] = c - b +#define C2(c, b) C1(c, b), C1(c + 1, b) +#define C5(c, b) C1(c, b), C2(c + 1, b), C2(c + 3, b) +#define C10(c, b) C5(c, b), C5(c + 5, b) + C10('0', '0'), +#define C26(c, b) C1(c, b), C5(c + 1, b), C10(c + 6, b), C10(c + 16, b) + C26('a', 'a' + 10), C26('A', 'A' + 36), +}; + + +/* + * Combined base62+gololb decoding routine - implemented for efficiency. + * + * As Dmitry V. Levin once noticed, when it comes to speed, very few objections + * can be made against complicating the code. Which reminds me of Karl Marx, + * who said that there is not a crime at which a capitalist will scruple for + * the sake of 300 per cent profit, even at the chance of being hanged. Anyway, + * here Alexey Tourbin demonstrates that by using sophisticated - or should he + * say "ridiculously complicated" - techniques it is indeed possible to gain + * some profit, albeit of another kind. + */ + +// Word types (when two bytes from base62 string cast to unsigned short). +enum { + W_AA = 0x0000, + W_AZ = 0x1000, + W_ZA = 0x2000, + W_A0 = 0x3000, + W_0X = 0x4000, + W_EE = 0xeeee, +}; + +// Combine two characters into array index (with respect to endianness). +#include +#if BYTE_ORDER && BYTE_ORDER == LITTLE_ENDIAN +#define CCI(c1, c2) ((c1) | ((c2) << 8)) +#elif BYTE_ORDER && BYTE_ORDER == BIG_ENDIAN +#define CCI(c1, c2) ((c2) | ((c1) << 8)) +#else +#error "unknown byte order" +#endif + +// Maps base62 word into numeric value (decoded bits) ORed with word type. +static const unsigned short word_to_num[65536] = { + [0 ... 65535] = W_EE, +#define AA1(c1, c2, b1, b2) [CCI(c1, c2)] = (c1 - b1) | ((c2 - b2) << 6) +#define AA1x2(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA1(c1, c2 + 1, b1, b2) +#define AA1x3(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA1x2(c1, c2 + 1, b1, b2) +#define AA1x5(c1, c2, b1, b2) AA1x2(c1, c2, b1, b2), AA1x3(c1, c2 + 2, b1, b2) +#define AA1x10(c1, c2, b1, b2) AA1x5(c1, c2, b1, b2), AA1x5(c1, c2 + 5, b1, b2) +#define AA1x20(c1, c2, b1, b2) AA1x10(c1, c2, b1, b2), AA1x10(c1, c2 + 10, b1, b2) +#define AA1x25(c1, c2, b1, b2) AA1x5(c1, c2, b1, b2), AA1x20(c1, c2 + 5, b1, b2) +#define AA2x1(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA1(c1 + 1, c2, b1, b2) +#define AA3x1(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA2x1(c1 + 1, c2, b1, b2) +#define AA5x1(c1, c2, b1, b2) AA2x1(c1, c2, b1, b2), AA3x1(c1 + 2, c2, b1, b2) +#define AA10x1(c1, c2, b1, b2) AA5x1(c1, c2, b1, b2), AA5x1(c1 + 5, c2, b1, b2) +#define AA20x1(c1, c2, b1, b2) AA10x1(c1, c2, b1, b2), AA10x1(c1 + 10, c2, b1, b2) +#define AA25x1(c1, c2, b1, b2) AA5x1(c1, c2, b1, b2), AA20x1(c1 + 5, c2, b1, b2) +#define AA26x1(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA25x1(c1 + 1, c2, b1, b2) +#define AA2x5(c1, c2, b1, b2) AA1x5(c1, c2, b1, b2), AA1x5(c1 + 1, c2, b1, b2) +#define AA3x5(c1, c2, b1, b2) AA1x5(c1, c2, b1, b2), AA2x5(c1 + 1, c2, b1, b2) +#define AA5x5(c1, c2, b1, b2) AA2x5(c1, c2, b1, b2), AA3x5(c1 + 2, c2, b1, b2) +#define AA5x10(c1, c2, b1, b2) AA5x5(c1, c2, b1, b2), AA5x5(c1, c2 + 5, b1, b2) +#define AA10x5(c1, c2, b1, b2) AA5x5(c1, c2, b1, b2), AA5x5(c1 + 5, c2, b1, b2) +#define AA20x5(c1, c2, b1, b2) AA10x5(c1, c2, b1, b2), AA10x5(c1 + 10, c2, b1, b2) +#define AA25x5(c1, c2, b1, b2) AA5x5(c1, c2, b1, b2), AA20x5(c1 + 5, c2, b1, b2) +#define AA10x10(c1, c2, b1, b2) AA5x10(c1, c2, b1, b2), AA5x10(c1 + 5, c2, b1, b2) +#define AA10x20(c1, c2, b1, b2) AA10x10(c1, c2, b1, b2), AA10x10(c1, c2 + 10, b1, b2) +#define AA10x25(c1, c2, b1, b2) AA10x5(c1, c2, b1, b2), AA10x20(c1, c2 + 5, b1, b2) +#define AA10x26(c1, c2, b1, b2) AA10x1(c1, c2, b1, b2), AA10x25(c1, c2 + 1, b1, b2) +#define AA20x10(c1, c2, b1, b2) AA10x10(c1, c2, b1, b2), AA10x10(c1 + 10, c2, b1, b2) +#define AA25x10(c1, c2, b1, b2) AA5x10(c1, c2, b1, b2), AA20x10(c1 + 5, c2, b1, b2) +#define AA26x10(c1, c2, b1, b2) AA1x10(c1, c2, b1, b2), AA25x10(c1 + 1, c2, b1, b2) +#define AA25x20(c1, c2, b1, b2) AA25x10(c1, c2, b1, b2), AA25x10(c1, c2 + 10, b1, b2) +#define AA25x25(c1, c2, b1, b2) AA25x5(c1, c2, b1, b2), AA25x20(c1, c2 + 5, b1, b2) +#define AA25x26(c1, c2, b1, b2) AA25x1(c1, c2, b1, b2), AA25x25(c1, c2 + 1, b1, b2) +#define AA26x25(c1, c2, b1, b2) AA1x25(c1, c2, b1, b2), AA25x25(c1 + 1, c2, b1, b2) +#define AA26x26(c1, c2, b1, b2) AA26x1(c1, c2, b1, b2), AA26x25(c1, c2 + 1, b1, b2) + AA10x10('0', '0', '0', '0'), + AA10x26('0', 'a', '0', 'a' + 10), + AA10x25('0', 'A', '0', 'A' + 36), + AA26x10('a', '0', 'a' + 10, '0'), + AA25x10('A', '0', 'A' + 36, '0'), + AA26x26('a', 'a', 'a' + 10, 'a' + 10), + AA26x25('a', 'A', 'a' + 10, 'A' + 36), + AA25x26('A', 'a', 'A' + 36, 'a' + 10), + AA25x25('A', 'A', 'A' + 36, 'A' + 36), +#define AZ1(c, b) [CCI(c, 'Z')] = (c - b) | W_AZ +#define AZ2(c, b) AZ1(c, b), AZ1(c + 1, b) +#define AZ5(c, b) AZ1(c, b), AZ2(c + 1, b), AZ2(c + 3, b) +#define AZ10(c, b) AZ5(c, b), AZ5(c + 5, b) +#define AZ25(c, b) AZ5(c, b), AZ10(c + 5, b), AZ10(c + 15, b) +#define AZ26(c, b) AZ1(c, b), AZ25(c + 1, b) + AZ10('0', '0'), + AZ26('a', 'a' + 10), + AZ25('A', 'A' + 36), +#define ZA1(c, b) [CCI('Z', c)] = (61 + ((c - b) >> 4)) | (((c - b) & 0xf) << 6) | W_ZA +#define ZA2(c, b) ZA1(c, b), ZA1(c + 1, b) +#define ZA5(c, b) ZA1(c, b), ZA2(c + 1, b), ZA2(c + 3, b) +#define ZA10(c, b) ZA5(c, b), ZA5(c + 5, b) +#define ZA25(c, b) ZA5(c, b), ZA10(c + 5, b), ZA10(c + 15, b) +#define ZA26(c, b) ZA1(c, b), ZA25(c + 1, b) + ZA10('0', '0'), + ZA26('a', 'a' + 10), + ZA25('A', 'A' + 36), +#define A01(c, b) [CCI(c, 0)] = (c - b) | W_A0 +#define A02(c, b) A01(c, b), A01(c + 1, b) +#define A05(c, b) A01(c, b), A02(c + 1, b), A02(c + 3, b) +#define A010(c, b) A05(c, b), A05(c + 5, b) +#define A025(c, b) A05(c, b), A010(c + 5, b), A010(c + 15, b) +#define A026(c, b) A01(c, b), A025(c + 1, b) + A010('0', '0'), + A026('a', 'a' + 10), + A025('A', 'A' + 36), +#define OX(c) [CCI(0, c)] = W_0X +#define OX4(c) OX(c), OX(c + 1), OX(c + 2), OX(c + 3) +#define OX16(c) OX4(c), OX4(c + 4), OX4(c + 8), OX4(c + 12) +#define OX64(c) OX16(c), OX16(c + 16), OX16(c + 32), OX16(c + 48) +#define OX256(c) OX64(c), OX64(c + 64), OX64(c + 128), OX64(c + 192) + OX256('\0'), +}; + +// Combined base62+golomb decoding routine. +static int decode_base62_golomb(const char* base62, int Mshift, unsigned* v) { + unsigned* v_start = v; + unsigned mask = (1u << Mshift) - 1; + unsigned q = 0; + unsigned r = 0; + int rfill = 0; + long c, w; + int n, vbits, left; + unsigned bits, morebits; + // need align + if (1 & (long)base62) { + c = (unsigned char)*base62++; + bits = char_to_num[c]; + if (bits < 61) + goto put6q_align; + else { + if (bits == 0xff) goto eolq; + if (bits == 0xee) return -1; + assert(bits == 61); + goto esc1q; + } + } + // regular mode, process two-byte words +#define Get24(X) \ + w = *(unsigned short*)base62; \ + base62 += 2; \ + bits = word_to_num[w]; \ + if (bits >= 0x1000) goto gotNN##X; \ + w = *(unsigned short*)base62; \ + base62 += 2; \ + morebits = word_to_num[w]; \ + if (morebits >= 0x1000) goto put12##X; \ + bits |= (morebits << 12); \ + goto put24##X +#define Get12(X) bits = morebits +#define GotNN(X) \ + switch (bits & 0xf000) { \ + case W_AZ: \ + bits &= 0x0fff; \ + goto put6##X##_AZ; \ + case W_ZA: \ + bits &= 0x0fff; \ + goto put10##X##_ZA; \ + case W_A0: \ + bits &= 0x0fff; \ + goto put6##X##_A0; \ + case W_0X: \ + goto eol##X; \ + default: \ + return -2; \ + } +// make coroutines +get24q: + Get24(q); +get24r: + Get24(r); +get12q: + Get12(q); +gotNNq: + GotNN(q); +get12r: + Get12(r); +gotNNr: + GotNN(r); + // escape mode, handle 2 bytes one by one +#define Esc1(X) \ + bits = 61; \ + c = (unsigned char)*base62++; \ + morebits = char_to_num[c]; \ + if (morebits == 0xff) return -3; \ + if (morebits == 0xee) return -4; \ + switch (morebits & (16 + 32)) { \ + case 0: \ + break; \ + case 16: \ + bits = 62; \ + morebits &= ~16; \ + break; \ + case 32: \ + bits = 63; \ + morebits &= ~32; \ + break; \ + default: \ + return -5; \ + } \ + bits |= (morebits << 6); \ + goto put10##X##_esc1 +#define Esc2(X) \ + c = (unsigned char)*base62++; \ + bits = char_to_num[c]; \ + if (bits < 61) \ + goto put6##X##_esc2; \ + else { \ + if (bits == 0xff) goto eol##X; \ + if (bits == 0xee) return -6; \ + goto esc1##X; \ + } +// make coroutines +esc1q: + Esc1(q); +esc2q: + Esc2(q); +esc1r: + Esc1(r); +esc2r: + Esc2(r); + // golomb pieces +#define QInit(N) n = N +#define RInit(N) \ + n = N; \ + r |= (bits << rfill); \ + rfill += n +#define RMake(Get) \ + left = rfill - Mshift; \ + if (left < 0) goto Get##r; \ + r &= mask; \ + *v++ = (q << Mshift) | r; \ + q = 0; \ + bits >>= n - left; \ + n = left +#define QMake(Get) \ + if (bits == 0) { \ + q += n; \ + goto Get##q; \ + } \ + vbits = __builtin_ffs(bits); \ + n -= vbits; \ + bits >>= vbits; \ + q += vbits - 1; \ + r = bits; \ + rfill = n + // this assumes that minumum Mshift value is 7 +#define Put24Q(Get) \ + QInit(24); \ + QMake(Get); \ + RMake(Get); \ + QMake(Get); \ + RMake(Get); \ + QMake(Get); \ + RMake(Get); \ + goto Get##q +#define Put24R(Get) \ + RInit(24); \ + RMake(Get); \ + QMake(Get); \ + RMake(Get); \ + QMake(Get); \ + RMake(Get); \ + QMake(Get); \ + goto Get##r +#define Put12Q(Get) \ + QInit(12); \ + QMake(Get); \ + RMake(Get); \ + QMake(Get); \ + goto Get##r +#define Put12R(Get) \ + RInit(12); \ + RMake(Get); \ + QMake(Get); \ + RMake(Get); \ + QMake(Get); \ + goto Get##r +#define Put10Q(Get) \ + QInit(10); \ + QMake(Get); \ + RMake(Get); \ + QMake(Get); \ + goto Get##r +#define Put10R(Get) \ + RInit(10); \ + RMake(Get); \ + QMake(Get); \ + RMake(Get); \ + QMake(Get); \ + goto Get##r +#define Put6Q(Get) \ + QInit(6); \ + QMake(Get); \ + goto Get##r +#define Put6R(Get) \ + RInit(6); \ + RMake(Get); \ + QMake(Get); \ + goto Get##r +// make coroutines +put24q: + Put24Q(get24); +put24r: + Put24R(get24); +put12q: + Put12Q(get12); +put12r: + Put12R(get12); +put6q_align: +put6q_esc2: + Put6Q(get24); +put6r_esc2: + Put6R(get24); +put6q_AZ: + Put6Q(esc1); +put6r_AZ: + Put6R(esc1); +put10q_esc1: + Put10Q(esc2); +put10r_esc1: + Put10R(esc2); +put10q_ZA: + Put10Q(get24); +put10r_ZA: + Put10R(get24); +put6q_A0: + Put6Q(eol); +put6r_A0: + Put6R(eol); + // handle end of line and return +eolq: + if (q > 5) return -10; + return v - v_start; +eolr: + return -11; +} + + +static void decode_delta(int count, unsigned* values) { + assert(count > 0); + unsigned* end = values + count; + unsigned previous = *values++; + + while (values < end) { + *values += previous; + previous = *values++; + } +} + +static int decode_set(const struct set_meta* meta, unsigned* hash_arr) { + int count = decode_base62_golomb(meta->payload, meta->Mshift, hash_arr); + if (count < 0) return count; + + decode_delta(count, hash_arr); + + return count; +} + +// Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates. +static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp); + +static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigned** hash_pt, + unsigned cache_id) { + enum { CACHE_BUCKETS = 1024 }; + struct cache_ent { + struct cache_ent* bucket_next; + struct cache_ent* newer; + struct cache_ent* older; + char* str; + unsigned* hash_arr; + uint32_t fingerprint; + int len; + int cnt; + int target_bpp; + }; + + static unsigned cache_count[2]; + static struct cache_ent* buckets[2][CACHE_BUCKETS]; + static struct cache_ent* newest[2]; + static struct cache_ent* oldest[2]; + + assert(cache_id < 2); + + const unsigned char* str = (const unsigned char*)meta->str; + uint32_t fp = (uint32_t)str[0] | ((uint32_t)str[2] << 8) | ((uint32_t)str[3] << 16); + uint32_t mixed = fp ^ ((uint32_t)target_bpp * UINT32_C(0x85ebca6b)); + mixed ^= mixed >> 11; + mixed *= UINT32_C(0x9e3779b1); + mixed ^= mixed >> 16; + unsigned bucket = mixed & (CACHE_BUCKETS - 1); + + for (struct cache_ent* ent = buckets[cache_id][bucket]; ent; ent = ent->bucket_next) { + if (ent->fingerprint != fp || ent->target_bpp != target_bpp || strcmp(meta->str, ent->str) != 0) + continue; + + if (ent != newest[cache_id]) { + if (ent->newer) ent->newer->older = ent->older; + if (ent->older) ent->older->newer = ent->newer; + if (ent == oldest[cache_id]) oldest[cache_id] = ent->newer; + + ent->newer = NULL; + ent->older = newest[cache_id]; + newest[cache_id]->newer = ent; + newest[cache_id] = ent; + } + + *hash_pt = ent->hash_arr; + + return ent->cnt; + } + + if (set_meta_fini(meta) < 0) return -4; + + int len = (int)meta->len; + int capacity = meta->value_capacity; + struct cache_ent* ent = + xmalloc(sizeof(*ent) + ((size_t)capacity + SENTINELS) * sizeof(unsigned) + len + 2); + ent->hash_arr = (unsigned*)(ent + 1); + ent->str = (char*)(ent->hash_arr + capacity + SENTINELS); + memcpy(ent->str, meta->str, (size_t)len + 1); + ent->str[len + 1] = '\0'; + + struct set_meta decode_meta = *meta; + decode_meta.str = ent->str; + decode_meta.payload = ent->str + 2; + int cnt = decode_set(&decode_meta, ent->hash_arr); + if (cnt <= 0) { + _free(ent); + return cnt; + } + + if (target_bpp < meta->bpp) { + unsigned temporary[capacity]; + unsigned* current = ent->hash_arr; + unsigned* destination = temporary; + + for (int bpp = meta->bpp - 1; bpp >= target_bpp; --bpp) { + cnt = downsample_set(cnt, current, destination, bpp); + unsigned* swap = current; + current = destination; + destination = swap; + } + + if (current != ent->hash_arr) { + memcpy(ent->hash_arr, current, (size_t)cnt * sizeof(*current)); + } + } + + for (int i = 0; i < SENTINELS; ++i) ent->hash_arr[cnt + i] = ~0u; + + ent->fingerprint = fp; + ent->len = len; + ent->cnt = cnt; + ent->target_bpp = target_bpp; + + if (cache_count[cache_id] == CACHE_SIZE) { + struct cache_ent* victim = oldest[cache_id]; + oldest[cache_id] = victim->newer; + if (oldest[cache_id]) oldest[cache_id]->older = NULL; + if (victim == newest[cache_id]) newest[cache_id] = NULL; + + uint32_t victim_mixed = + victim->fingerprint ^ ((uint32_t)victim->target_bpp * UINT32_C(0x85ebca6b)); + victim_mixed ^= victim_mixed >> 11; + victim_mixed *= UINT32_C(0x9e3779b1); + victim_mixed ^= victim_mixed >> 16; + unsigned victim_bucket = victim_mixed & (CACHE_BUCKETS - 1); + struct cache_ent** link = &buckets[cache_id][victim_bucket]; + while (*link != victim) link = &(*link)->bucket_next; + *link = victim->bucket_next; + _free(victim); + } else { + ++cache_count[cache_id]; + } + + ent->bucket_next = buckets[cache_id][bucket]; + buckets[cache_id][bucket] = ent; + ent->newer = NULL; + ent->older = newest[cache_id]; + if (newest[cache_id]) { + newest[cache_id]->newer = ent; + } else { + oldest[cache_id] = ent; + } + newest[cache_id] = ent; + + *hash_pt = ent->hash_arr; + + return cnt; +} + +// Reduce a set of (bpp + 1) values to a set of bpp values. +static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp) { + unsigned mask = (1u << bpp) - 1; + + // find the first element with high bit set + int l = 0; + int u = cnt; + while (l < u) { + int i = (l + u) / 2; + + if (hash_pt[i] <= mask) { + l = i + 1; + } else { + u = i; + } + } + + // initialize parts + const unsigned* ds_start = ds_pt; + const unsigned *v1 = hash_pt + 0, *v1_end = hash_pt + u; + const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + cnt; + + // merge v1 and v2 into w + if (v1 < v1_end && v2 < v2_end) { + unsigned v1_val = *v1; + unsigned v2_val = *v2 & mask; + + while (1) { + if (v1_val < v2_val) { + *ds_pt++ = v1_val; + v1++; + + if (v1 == v1_end) break; + + v1_val = *v1; + } else if (v2_val < v1_val) { + *ds_pt++ = v2_val; + v2++; + + if (v2 == v2_end) break; + + v2_val = *v2 & mask; + } else { + *ds_pt++ = v1_val; + v1++; + v2++; + + if (v1 == v1_end) break; + if (v2 == v2_end) break; + + v1_val = *v1; + v2_val = *v2 & mask; + } + } + } + + // append what's left + while (v1 < v1_end) *ds_pt++ = *v1++; + while (v2 < v2_end) *ds_pt++ = *v2++ & mask; + + return ds_pt - ds_start; +} + +static int sorted_subset(const unsigned* small, size_t small_count, const unsigned* large, + size_t large_count) { + const unsigned* const small_end = small + small_count; + const unsigned* const large_end = large + large_count; + int subset = 1; + + for (int i = 0; i < SENTINELS; ++i) assert(large_end[i] == ~0u); + unsigned small_value = *small; + + // Loop pieces from the original set.c traversal. The sentinel tail lets + // the search advance in groups of four or eight without boundary checks. +#define IFLT4 \ + if (*large < small_value) { \ + large += 4; \ + while (*large < small_value) large += 4; \ + large -= 2; \ + if (*large < small_value) \ + large++; \ + else \ + large--; \ + if (*large < small_value) large++; \ + if (large == large_end) break; \ + } +#define IFLT8 \ + if (*large < small_value) { \ + large += 8; \ + while (*large < small_value) large += 8; \ + large -= 4; \ + if (*large < small_value) \ + large += 2; \ + else \ + large -= 2; \ + if (*large < small_value) \ + large++; \ + else \ + large--; \ + if (*large < small_value) large++; \ + if (large == large_end) break; \ + } +#define IFGE \ + if (*large == small_value) { \ + large++; \ + small++; \ + if (large == large_end) break; \ + if (small == small_end) break; \ + small_value = *small; \ + } else { \ + subset = 0; \ + small++; \ + if (small == small_end) break; \ + small_value = *small; \ + } + + if (large_count / small_count >= 16) { + while (1) { + IFLT8; + IFGE; + } + } else { + while (1) { + IFLT4; + IFGE; + } + } + +#undef IFGE +#undef IFLT8 +#undef IFLT4 + + if (small < small_end) subset = 0; + + return subset; +} + +// main API routine +int rpmsetcmp(const char* str1, const char* str2) { + if (strncmp(str1, "set:", 4) == 0) str1 += 4; + if (strncmp(str2, "set:", 4) == 0) str2 += 4; + + struct set_meta meta1; + struct set_meta meta2; + + if (set_meta_init(str1, &meta1) < 0) return -3; + if (set_meta_init(str2, &meta2) < 0) return -4; + + int target_bpp = meta1.bpp < meta2.bpp ? meta1.bpp : meta2.bpp; + + // Decode and cache the first operand at the comparison precision. + const unsigned* hash_arr1 = NULL; + int cnt1 = cache_decode_set(&meta1, target_bpp, &hash_arr1, 0); + if (cnt1 < 0) return -3; + + // Metadata for both operands has already been validated, and set1 has been + // decoded, so this preserves set8's malformed-input error precedence. + if (str1 == str2 || strcmp(str1, str2) == 0) return 0; + + // Requirement sets are frequently reused by dependency solvers too. + const unsigned* hash_arr2 = NULL; + int cnt2 = cache_decode_set(&meta2, target_bpp, &hash_arr2, 1); + if (cnt2 < 0) return -4; + + // Cardinality determines which strict-inclusion result is even possible. + // For equal cardinalities, sorted unique sets are equal iff their bytes match. + if (cnt1 == cnt2) { + return memcmp(hash_arr1, hash_arr2, (size_t)cnt1 * sizeof(*hash_arr1)) == 0 ? 0 : -2; + } + if (cnt1 > cnt2) { + return sorted_subset(hash_arr2, (size_t)cnt2, hash_arr1, (size_t)cnt1) ? 1 : -2; + } + return sorted_subset(hash_arr1, (size_t)cnt1, hash_arr2, (size_t)cnt2) ? -1 : -2; +} + +// --- + +#ifdef SELF_TEST +int main(void) { + struct set* set1 = set_new(); + set_add(set1, "mama"); + set_add(set1, "myla"); + set_add(set1, "ramu"); + const char* str10 = set_fini(set1, 16); + fprintf(stderr, "set10=%s\n", str10); + + int cmp; + struct set* set2 = set_new(); + set_add(set2, "myla"); + set_add(set2, "mama"); + const char* str20 = set_fini(set2, 16); + fprintf(stderr, "set20=%s\n", str20); + cmp = rpmsetcmp(str10, str20); + assert(cmp == 1); + + set_add(set2, "ramu"); + const char* str21 = set_fini(set2, 16); + fprintf(stderr, "set21=%s\n", str21); + cmp = rpmsetcmp(str10, str21); + assert(cmp == 0); + + set_add(set2, "baba"); + const char* str22 = set_fini(set2, 16); + cmp = rpmsetcmp(str10, str22); + assert(cmp == -1); + + set_add(set1, "deda"); + const char* str11 = set_fini(set1, 16); + cmp = rpmsetcmp(str11, str22); + assert(cmp == -2); + + set1 = set_free(set1); + set2 = set_free(set2); + str10 = _free(str10); + str11 = _free(str11); + str20 = _free(str20); + str21 = _free(str21); + str22 = _free(str22); + + fprintf(stderr, "%s: api test OK\n", __FILE__); + + return 0; +} +#endif