#include <assert.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>

#include "rpmlib.h"
#ifdef SELF_TEST
#undef NDEBUG
#include <stdio.h>
#endif
#include "set.h"
#include "system.h"

#define CACHE_SIZE 512
#define SENTINELS 8

struct set {
  size_t cnt;
  size_t symbols_cap;
  size_t strings_len;
  size_t strings_cap;
  char* strings;
  struct symbols {
    size_t offset;
    unsigned hash;
  }* symbols_v;
};

struct set* set_new() {
  struct set* set = xmalloc(sizeof *set);
  set->cnt = 0;
  set->symbols_cap = 0;
  set->strings_len = 0;
  set->strings_cap = 0;
  set->strings = NULL;
  set->symbols_v = NULL;

  return set;
}

void set_add(struct set* set, const char* sym) {
  if (set->cnt == set->symbols_cap) {
    set->symbols_cap += 1024;
    set->symbols_v = xrealloc(set->symbols_v, sizeof(*set->symbols_v) * set->symbols_cap);
  }

  size_t length = strlen(sym) + 1;
  size_t required = set->strings_len + length;
  if (required > set->strings_cap) {
    size_t capacity = set->strings_cap ? set->strings_cap : 4096;
    while (capacity < required) capacity *= 2;

    set->strings = xrealloc(set->strings, capacity);
    set->strings_cap = capacity;
  }

  set->symbols_v[set->cnt].offset = set->strings_len;
  set->symbols_v[set->cnt].hash = 0;
  memcpy(set->strings + set->strings_len, sym, length);
  set->strings_len = required;
  set->cnt++;

  return;
}

struct set* set_free(struct set* set) {
  if (set) {
    _free(set->strings);
    _free(set->symbols_v);
    set = _free(set);
  }

  return NULL;
}

// ---

static unsigned hash(const char* str) {
  unsigned hash = 0x9e3779b9;
  const unsigned char* p = (const unsigned char*)str;

  while (*p) {
    hash += *p++;
    hash += (hash << 10);
    hash ^= (hash >> 6);
  }

  hash += (hash << 3);
  hash ^= (hash >> 11);
  hash += (hash << 15);

  return hash;
}

int cmp(const void* arg1, const void* arg2) {
  const struct symbols* s1 = arg1;
  const struct symbols* s2 = arg2;

  if (s1->hash > s2->hash) return 1;
  if (s2->hash > s1->hash) return -1;

  return 0;
}

static void sort_symbols(struct symbols* values, size_t count, int bpp) {
  if (count < 128) {
    qsort(values, count, sizeof(*values), cmp);
    return;
  }

  struct symbols temporary[count];
  struct symbols* source = values;
  struct symbols* destination = temporary;
  unsigned passes = ((unsigned)bpp + 7) / 8;

  for (unsigned pass = 0; pass < passes; ++pass) {
    size_t offsets[256] = {0};
    unsigned shift = pass * 8;
    for (size_t i = 0; i < count; ++i) ++offsets[(source[i].hash >> shift) & 0xffu];

    size_t position = 0;
    for (size_t i = 0; i < 256; ++i) {
      size_t bucket_count = offsets[i];
      offsets[i] = position;
      position += bucket_count;
    }

    for (size_t i = 0; i < count; ++i) {
      unsigned bucket = (source[i].hash >> shift) & 0xffu;
      destination[offsets[bucket]++] = source[i];
    }

    struct symbols* swap = source;
    source = destination;
    destination = swap;
  }

  if (source != values) memcpy(values, source, count * sizeof(*values));

  return;
}

// ---

static int log2i(int n) {
  int m = 0;
  while (n /= 2) m++;

  return m;
}

// Calculate Mshift paramter for encoding.
static int encode_golomb_Mshift(int cnt, int bpp) {
  // XXX Slightly better Mshift estimations are probably possible.
  // Recheck "Compression and coding algorithms" by Moffat & Turpin.
  int Mshift = bpp - log2i(cnt) - 1;

  // Adjust out-of-range values.
  Mshift = (Mshift < 7) ? 7 : Mshift;
  Mshift = (Mshift > 31) ? 31 : Mshift;
  assert(Mshift < bpp);

  return Mshift;
}

// Estimate how many bits can be filled up.
static inline int encode_golomb_size(int cnt, int Mshift) {
  // XXX No precise estimation.  However, we do not expect unary-encoded bits
  // to take more than binary-encoded Mshift bits.
  return Mshift * 2 * cnt + 16;
}

// Estimate base62 buffer size required to encode a given number of bits.
static inline int encode_base62_size(int bit_cnt) {
  // In the worst case, which is ZxZxZx..., five bits can make a character;
  // the remaining bits can make a character, too.  And the string must be
  // null-terminated.
  return bit_cnt / 5 + 2;
}

static int encode_set_size(int cnt, int bpp) {
  int Mshift = encode_golomb_Mshift(cnt, bpp);
  int bit_cnt = encode_golomb_size(cnt, Mshift);
  // two leading characters are special
  return 2 + encode_base62_size(bit_cnt);
}

// ---

// Main base62 encoding routine: pack bit_arr into base62 string.
/*
 * Base62 routines - encode bits with alnum characters.
 *
 * This is a base64-based base62 implementation.  Values 0..61 are encoded
 * with '0'..'9', 'a'..'z', and 'A'..'Z'.  However, 'Z' is special: it will
 * also encode 62 and 63.  To achieve this, 'Z' will occupy two high bits in
 * the next character.  Thus 'Z' can be interpreted as an escape character
 * (which indicates that the next character must be handled specially).
 * Note that setting high bits to "00", "01" or "10" cannot contribute
 * to another 'Z' (which would require high bits set to "11").  This is
 * how multiple escapes are avoided.
 */

static char* bits_to_char(int c, char* base62) {
  assert(c >= 0 && c <= 61);

  if (c < 10) {
    *base62++ = c + '0';
  } else if (c < 36) {
    *base62++ = c - 10 + 'a';
  } else if (c < 62) {
    *base62++ = c - 36 + 'A';
  }

  return base62;
}

// ---

static inline char encode_bpp(int bpp) { return bpp - 7 + 'a'; }

struct encode_writer {
  uint64_t bits;
  unsigned filled;
  unsigned escaped;
  unsigned pending_high;
  char* output;
};

static inline void encode_writer_digit(struct encode_writer* writer, unsigned value) {
  assert(value < 62);

  if (value < 10) {
    *writer->output++ = (char)('0' + value);
  } else if (value < 36) {
    *writer->output++ = (char)('a' + value - 10);
  } else {
    *writer->output++ = (char)('A' + value - 36);
  }
}

static inline void encode_writer_flush(struct encode_writer* writer) {
  for (;;) {
    unsigned width = writer->escaped ? 4u : 6u;
    if (writer->filled < width) return;

    unsigned value = (unsigned)writer->bits & ((1u << width) - 1);
    writer->bits >>= width;
    writer->filled -= width;

    if (writer->escaped) {
      encode_writer_digit(writer, writer->pending_high | value);
      writer->escaped = 0;
    } else if (value >= 61) {
      encode_writer_digit(writer, 61);
      writer->pending_high = (value - 61) << 4;
      writer->escaped = 1;
    } else {
      encode_writer_digit(writer, value);
    }
  }
}

static inline void encode_writer_zeros(struct encode_writer* writer, unsigned count) {
  while (count) {
    unsigned take = count > 56 ? 56 : count;
    writer->filled += take;
    count -= take;

    encode_writer_flush(writer);
  }

  return;
}

static inline void encode_writer_put(struct encode_writer* writer, uint64_t value, unsigned width) {
  writer->bits |= value << writer->filled;
  writer->filled += width;
  encode_writer_flush(writer);

  return;
}

static int encode_set(int cnt, const unsigned* hash_arr, int bpp, char* base62_str) {
  const unsigned Mshift = (unsigned)encode_golomb_Mshift(cnt, bpp);
  const unsigned mask = (1u << Mshift) - 1;
  char* const start = base62_str;
  unsigned previous = 0;

  *base62_str++ = encode_bpp(bpp);
  *base62_str++ = encode_bpp((int)Mshift);
  struct encode_writer writer = {.output = base62_str};

  for (int i = 0; i < cnt; ++i) {
    unsigned current = hash_arr[i];
    unsigned delta = current - previous;
    previous = current;

    encode_writer_zeros(&writer, delta >> Mshift);
    encode_writer_put(&writer, 1, 1);
    encode_writer_put(&writer, delta & mask, Mshift);
  }

  encode_writer_flush(&writer);
  if (writer.filled || writer.escaped) {
    unsigned value = (unsigned)writer.bits;
    if (writer.escaped) value |= writer.pending_high;
    encode_writer_digit(&writer, value);
  }

  *writer.output = '\0';

  return (int)(writer.output - start);
}

const char* set_fini(struct set* set, int bpp) {
  // Implementation for finalizing the set

  assert(set != NULL);
  assert(set->cnt > 0);
  assert(bpp >= 10 && bpp <= 32);

  unsigned mask = (bpp < 32) ? (1u << bpp) - 1 : ~0u;

  for (size_t i = 0; i < set->cnt; ++i) {
    set->symbols_v[i].hash = hash(set->strings + set->symbols_v[i].offset) & mask;
  }

  sort_symbols(set->symbols_v, set->cnt, bpp);

  // warn on hash collizions
  for (size_t i = 0; i < set->cnt - 1; ++i) {
    if (set->symbols_v[i].hash != set->symbols_v[i + 1].hash) continue;
    const char* left = set->strings + set->symbols_v[i].offset;
    const char* right = set->strings + set->symbols_v[i + 1].offset;
    if (!strcmp(left, right)) continue;

    fprintf(stderr, "warning: hash collision: %s %s\n", left, right);
  }

  unsigned unique_hash[set->cnt];
  size_t unique_cnt = 0;

  // delete duplicates
  for (size_t i = 0; i < set->cnt; ++i) {
    while (i + 1 < set->cnt && set->symbols_v[i].hash == set->symbols_v[i + 1].hash) {
      ++i;
    }
    unique_hash[unique_cnt++] = set->symbols_v[i].hash;
  }

  char base62_str[encode_set_size(unique_cnt, bpp)];
  encode_set(unique_cnt, unique_hash, bpp, base62_str);

  return xstrdup(base62_str);
}

// ---

struct set_meta {
  const char* str;
  const char* payload;
  size_t len;
  size_t payload_len;
  int bpp;
  int Mshift;
  int bit_capacity;
  int value_capacity;
};

static int set_meta_init(const char* str, struct set_meta* meta) {
  // len >= 3
  if (!str[0] || !str[1] || !str[2]) return -4;

  int bpp = str[0] + 7 - 'a';
  if (bpp < 10 || bpp > 32) return -1;

  int Mshift = str[1] + 7 - 'a';
  if (Mshift < 7 || Mshift > 31) return -2;
  if (Mshift >= bpp) return -3;

  *meta = (struct set_meta){
      .str = str,
      .payload = str + 2,
      .bpp = bpp,
      .Mshift = Mshift,
  };

  return 0;
}

static int set_meta_fini(struct set_meta* meta) {
  size_t len = strlen(meta->str);

  size_t payload_len = len - 2;

  int bit_capacity = (int)payload_len * 6;
  int value_capacity = bit_capacity / (meta->Mshift + 1);

  if (value_capacity < 1) return -4;

  meta->len = len;
  meta->payload_len = payload_len;
  meta->bit_capacity = bit_capacity;
  meta->value_capacity = value_capacity;

  return 0;
}
// This table maps alnum characters to their numeric values.
static const int char_to_num[256] = {
    [0 ... 255] = 0xee, [0] = 0xff,
#define C1(c, b) [c] = c - b
#define C2(c, b) C1(c, b), C1(c + 1, b)
#define C5(c, b) C1(c, b), C2(c + 1, b), C2(c + 3, b)
#define C10(c, b) C5(c, b), C5(c + 5, b)
    C10('0', '0'),
#define C26(c, b) C1(c, b), C5(c + 1, b), C10(c + 6, b), C10(c + 16, b)
    C26('a', 'a' + 10), C26('A', 'A' + 36),
};


/*
 * Combined base62+gololb decoding routine - implemented for efficiency.
 *
 * As Dmitry V. Levin once noticed, when it comes to speed, very few objections
 * can be made against complicating the code.  Which reminds me of Karl Marx,
 * who said that there is not a crime at which a capitalist will scruple for
 * the sake of 300 per cent profit, even at the chance of being hanged.  Anyway,
 * here Alexey Tourbin demonstrates that by using sophisticated - or should he
 * say "ridiculously complicated" - techniques it is indeed possible to gain
 * some profit, albeit of another kind.
 */

// Word types (when two bytes from base62 string cast to unsigned short).
enum {
  W_AA = 0x0000,
  W_AZ = 0x1000,
  W_ZA = 0x2000,
  W_A0 = 0x3000,
  W_0X = 0x4000,
  W_EE = 0xeeee,
};

// Combine two characters into array index (with respect to endianness).
#include <sys/types.h>
#if BYTE_ORDER && BYTE_ORDER == LITTLE_ENDIAN
#define CCI(c1, c2) ((c1) | ((c2) << 8))
#elif BYTE_ORDER && BYTE_ORDER == BIG_ENDIAN
#define CCI(c1, c2) ((c2) | ((c1) << 8))
#else
#error "unknown byte order"
#endif

// Maps base62 word into numeric value (decoded bits) ORed with word type.
static const unsigned short word_to_num[65536] = {
    [0 ... 65535] = W_EE,
#define AA1(c1, c2, b1, b2) [CCI(c1, c2)] = (c1 - b1) | ((c2 - b2) << 6)
#define AA1x2(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA1(c1, c2 + 1, b1, b2)
#define AA1x3(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA1x2(c1, c2 + 1, b1, b2)
#define AA1x5(c1, c2, b1, b2) AA1x2(c1, c2, b1, b2), AA1x3(c1, c2 + 2, b1, b2)
#define AA1x10(c1, c2, b1, b2) AA1x5(c1, c2, b1, b2), AA1x5(c1, c2 + 5, b1, b2)
#define AA1x20(c1, c2, b1, b2) AA1x10(c1, c2, b1, b2), AA1x10(c1, c2 + 10, b1, b2)
#define AA1x25(c1, c2, b1, b2) AA1x5(c1, c2, b1, b2), AA1x20(c1, c2 + 5, b1, b2)
#define AA2x1(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA1(c1 + 1, c2, b1, b2)
#define AA3x1(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA2x1(c1 + 1, c2, b1, b2)
#define AA5x1(c1, c2, b1, b2) AA2x1(c1, c2, b1, b2), AA3x1(c1 + 2, c2, b1, b2)
#define AA10x1(c1, c2, b1, b2) AA5x1(c1, c2, b1, b2), AA5x1(c1 + 5, c2, b1, b2)
#define AA20x1(c1, c2, b1, b2) AA10x1(c1, c2, b1, b2), AA10x1(c1 + 10, c2, b1, b2)
#define AA25x1(c1, c2, b1, b2) AA5x1(c1, c2, b1, b2), AA20x1(c1 + 5, c2, b1, b2)
#define AA26x1(c1, c2, b1, b2) AA1(c1, c2, b1, b2), AA25x1(c1 + 1, c2, b1, b2)
#define AA2x5(c1, c2, b1, b2) AA1x5(c1, c2, b1, b2), AA1x5(c1 + 1, c2, b1, b2)
#define AA3x5(c1, c2, b1, b2) AA1x5(c1, c2, b1, b2), AA2x5(c1 + 1, c2, b1, b2)
#define AA5x5(c1, c2, b1, b2) AA2x5(c1, c2, b1, b2), AA3x5(c1 + 2, c2, b1, b2)
#define AA5x10(c1, c2, b1, b2) AA5x5(c1, c2, b1, b2), AA5x5(c1, c2 + 5, b1, b2)
#define AA10x5(c1, c2, b1, b2) AA5x5(c1, c2, b1, b2), AA5x5(c1 + 5, c2, b1, b2)
#define AA20x5(c1, c2, b1, b2) AA10x5(c1, c2, b1, b2), AA10x5(c1 + 10, c2, b1, b2)
#define AA25x5(c1, c2, b1, b2) AA5x5(c1, c2, b1, b2), AA20x5(c1 + 5, c2, b1, b2)
#define AA10x10(c1, c2, b1, b2) AA5x10(c1, c2, b1, b2), AA5x10(c1 + 5, c2, b1, b2)
#define AA10x20(c1, c2, b1, b2) AA10x10(c1, c2, b1, b2), AA10x10(c1, c2 + 10, b1, b2)
#define AA10x25(c1, c2, b1, b2) AA10x5(c1, c2, b1, b2), AA10x20(c1, c2 + 5, b1, b2)
#define AA10x26(c1, c2, b1, b2) AA10x1(c1, c2, b1, b2), AA10x25(c1, c2 + 1, b1, b2)
#define AA20x10(c1, c2, b1, b2) AA10x10(c1, c2, b1, b2), AA10x10(c1 + 10, c2, b1, b2)
#define AA25x10(c1, c2, b1, b2) AA5x10(c1, c2, b1, b2), AA20x10(c1 + 5, c2, b1, b2)
#define AA26x10(c1, c2, b1, b2) AA1x10(c1, c2, b1, b2), AA25x10(c1 + 1, c2, b1, b2)
#define AA25x20(c1, c2, b1, b2) AA25x10(c1, c2, b1, b2), AA25x10(c1, c2 + 10, b1, b2)
#define AA25x25(c1, c2, b1, b2) AA25x5(c1, c2, b1, b2), AA25x20(c1, c2 + 5, b1, b2)
#define AA25x26(c1, c2, b1, b2) AA25x1(c1, c2, b1, b2), AA25x25(c1, c2 + 1, b1, b2)
#define AA26x25(c1, c2, b1, b2) AA1x25(c1, c2, b1, b2), AA25x25(c1 + 1, c2, b1, b2)
#define AA26x26(c1, c2, b1, b2) AA26x1(c1, c2, b1, b2), AA26x25(c1, c2 + 1, b1, b2)
    AA10x10('0', '0', '0', '0'),
    AA10x26('0', 'a', '0', 'a' + 10),
    AA10x25('0', 'A', '0', 'A' + 36),
    AA26x10('a', '0', 'a' + 10, '0'),
    AA25x10('A', '0', 'A' + 36, '0'),
    AA26x26('a', 'a', 'a' + 10, 'a' + 10),
    AA26x25('a', 'A', 'a' + 10, 'A' + 36),
    AA25x26('A', 'a', 'A' + 36, 'a' + 10),
    AA25x25('A', 'A', 'A' + 36, 'A' + 36),
#define AZ1(c, b) [CCI(c, 'Z')] = (c - b) | W_AZ
#define AZ2(c, b) AZ1(c, b), AZ1(c + 1, b)
#define AZ5(c, b) AZ1(c, b), AZ2(c + 1, b), AZ2(c + 3, b)
#define AZ10(c, b) AZ5(c, b), AZ5(c + 5, b)
#define AZ25(c, b) AZ5(c, b), AZ10(c + 5, b), AZ10(c + 15, b)
#define AZ26(c, b) AZ1(c, b), AZ25(c + 1, b)
    AZ10('0', '0'),
    AZ26('a', 'a' + 10),
    AZ25('A', 'A' + 36),
#define ZA1(c, b) [CCI('Z', c)] = (61 + ((c - b) >> 4)) | (((c - b) & 0xf) << 6) | W_ZA
#define ZA2(c, b) ZA1(c, b), ZA1(c + 1, b)
#define ZA5(c, b) ZA1(c, b), ZA2(c + 1, b), ZA2(c + 3, b)
#define ZA10(c, b) ZA5(c, b), ZA5(c + 5, b)
#define ZA25(c, b) ZA5(c, b), ZA10(c + 5, b), ZA10(c + 15, b)
#define ZA26(c, b) ZA1(c, b), ZA25(c + 1, b)
    ZA10('0', '0'),
    ZA26('a', 'a' + 10),
    ZA25('A', 'A' + 36),
#define A01(c, b) [CCI(c, 0)] = (c - b) | W_A0
#define A02(c, b) A01(c, b), A01(c + 1, b)
#define A05(c, b) A01(c, b), A02(c + 1, b), A02(c + 3, b)
#define A010(c, b) A05(c, b), A05(c + 5, b)
#define A025(c, b) A05(c, b), A010(c + 5, b), A010(c + 15, b)
#define A026(c, b) A01(c, b), A025(c + 1, b)
    A010('0', '0'),
    A026('a', 'a' + 10),
    A025('A', 'A' + 36),
#define OX(c) [CCI(0, c)] = W_0X
#define OX4(c) OX(c), OX(c + 1), OX(c + 2), OX(c + 3)
#define OX16(c) OX4(c), OX4(c + 4), OX4(c + 8), OX4(c + 12)
#define OX64(c) OX16(c), OX16(c + 16), OX16(c + 32), OX16(c + 48)
#define OX256(c) OX64(c), OX64(c + 64), OX64(c + 128), OX64(c + 192)
    OX256('\0'),
};

// Combined base62+golomb decoding routine.
static int decode_base62_golomb(const char* base62, int Mshift, unsigned* v) {
  unsigned* v_start = v;
  unsigned mask = (1u << Mshift) - 1;
  unsigned q = 0;
  unsigned r = 0;
  int rfill = 0;
  long c, w;
  int n, vbits, left;
  unsigned bits, morebits;
  // need align
  if (1 & (long)base62) {
    c = (unsigned char)*base62++;
    bits = char_to_num[c];
    if (bits < 61)
      goto put6q_align;
    else {
      if (bits == 0xff) goto eolq;
      if (bits == 0xee) return -1;
      assert(bits == 61);
      goto esc1q;
    }
  }
  // regular mode, process two-byte words
#define Get24(X)                         \
  w = *(unsigned short*)base62;          \
  base62 += 2;                           \
  bits = word_to_num[w];                 \
  if (bits >= 0x1000) goto gotNN##X;     \
  w = *(unsigned short*)base62;          \
  base62 += 2;                           \
  morebits = word_to_num[w];             \
  if (morebits >= 0x1000) goto put12##X; \
  bits |= (morebits << 12);              \
  goto put24##X
#define Get12(X) bits = morebits
#define GotNN(X)           \
  switch (bits & 0xf000) { \
    case W_AZ:             \
      bits &= 0x0fff;      \
      goto put6##X##_AZ;   \
    case W_ZA:             \
      bits &= 0x0fff;      \
      goto put10##X##_ZA;  \
    case W_A0:             \
      bits &= 0x0fff;      \
      goto put6##X##_A0;   \
    case W_0X:             \
      goto eol##X;         \
    default:               \
      return -2;           \
  }
// make coroutines
get24q:
  Get24(q);
get24r:
  Get24(r);
get12q:
  Get12(q);
gotNNq:
  GotNN(q);
get12r:
  Get12(r);
gotNNr:
  GotNN(r);
  // escape mode, handle 2 bytes one by one
#define Esc1(X)                    \
  bits = 61;                       \
  c = (unsigned char)*base62++;    \
  morebits = char_to_num[c];       \
  if (morebits == 0xff) return -3; \
  if (morebits == 0xee) return -4; \
  switch (morebits & (16 + 32)) {  \
    case 0:                        \
      break;                       \
    case 16:                       \
      bits = 62;                   \
      morebits &= ~16;             \
      break;                       \
    case 32:                       \
      bits = 63;                   \
      morebits &= ~32;             \
      break;                       \
    default:                       \
      return -5;                   \
  }                                \
  bits |= (morebits << 6);         \
  goto put10##X##_esc1
#define Esc2(X)                    \
  c = (unsigned char)*base62++;    \
  bits = char_to_num[c];           \
  if (bits < 61)                   \
    goto put6##X##_esc2;           \
  else {                           \
    if (bits == 0xff) goto eol##X; \
    if (bits == 0xee) return -6;   \
    goto esc1##X;                  \
  }
// make coroutines
esc1q:
  Esc1(q);
esc2q:
  Esc2(q);
esc1r:
  Esc1(r);
esc2r:
  Esc2(r);
  // golomb pieces
#define QInit(N) n = N
#define RInit(N)        \
  n = N;                \
  r |= (bits << rfill); \
  rfill += n
#define RMake(Get)           \
  left = rfill - Mshift;     \
  if (left < 0) goto Get##r; \
  r &= mask;                 \
  *v++ = (q << Mshift) | r;  \
  q = 0;                     \
  bits >>= n - left;         \
  n = left
#define QMake(Get)             \
  if (bits == 0) {             \
    q += n;                    \
    goto Get##q;               \
  }                            \
  vbits = __builtin_ffs(bits); \
  n -= vbits;                  \
  bits >>= vbits;              \
  q += vbits - 1;              \
  r = bits;                    \
  rfill = n
  // this assumes that minumum Mshift value is 7
#define Put24Q(Get) \
  QInit(24);        \
  QMake(Get);       \
  RMake(Get);       \
  QMake(Get);       \
  RMake(Get);       \
  QMake(Get);       \
  RMake(Get);       \
  goto Get##q
#define Put24R(Get) \
  RInit(24);        \
  RMake(Get);       \
  QMake(Get);       \
  RMake(Get);       \
  QMake(Get);       \
  RMake(Get);       \
  QMake(Get);       \
  goto Get##r
#define Put12Q(Get) \
  QInit(12);        \
  QMake(Get);       \
  RMake(Get);       \
  QMake(Get);       \
  goto Get##r
#define Put12R(Get) \
  RInit(12);        \
  RMake(Get);       \
  QMake(Get);       \
  RMake(Get);       \
  QMake(Get);       \
  goto Get##r
#define Put10Q(Get) \
  QInit(10);        \
  QMake(Get);       \
  RMake(Get);       \
  QMake(Get);       \
  goto Get##r
#define Put10R(Get) \
  RInit(10);        \
  RMake(Get);       \
  QMake(Get);       \
  RMake(Get);       \
  QMake(Get);       \
  goto Get##r
#define Put6Q(Get) \
  QInit(6);        \
  QMake(Get);      \
  goto Get##r
#define Put6R(Get) \
  RInit(6);        \
  RMake(Get);      \
  QMake(Get);      \
  goto Get##r
// make coroutines
put24q:
  Put24Q(get24);
put24r:
  Put24R(get24);
put12q:
  Put12Q(get12);
put12r:
  Put12R(get12);
put6q_align:
put6q_esc2:
  Put6Q(get24);
put6r_esc2:
  Put6R(get24);
put6q_AZ:
  Put6Q(esc1);
put6r_AZ:
  Put6R(esc1);
put10q_esc1:
  Put10Q(esc2);
put10r_esc1:
  Put10R(esc2);
put10q_ZA:
  Put10Q(get24);
put10r_ZA:
  Put10R(get24);
put6q_A0:
  Put6Q(eol);
put6r_A0:
  Put6R(eol);
  // handle end of line and return
eolq:
  if (q > 5) return -10;
  return v - v_start;
eolr:
  return -11;
}


static void decode_delta(int count, unsigned* values) {
  assert(count > 0);
  unsigned* end = values + count;
  unsigned previous = *values++;

  while (values < end) {
    *values += previous;
    previous = *values++;
  }
}

static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
  int count = decode_base62_golomb(meta->payload, meta->Mshift, hash_arr);
  if (count < 0) return count;

  decode_delta(count, hash_arr);

  return count;
}

// Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates.
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp);

static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigned** hash_pt,
                            unsigned cache_id) {
  enum { CACHE_BUCKETS = 1024 };
  struct cache_ent {
    struct cache_ent* bucket_next;
    struct cache_ent* newer;
    struct cache_ent* older;
    char* str;
    unsigned* hash_arr;
    uint32_t fingerprint;
    int len;
    int cnt;
    int target_bpp;
  };

  static unsigned cache_count[2];
  static struct cache_ent* buckets[2][CACHE_BUCKETS];
  static struct cache_ent* newest[2];
  static struct cache_ent* oldest[2];

  assert(cache_id < 2);

  const unsigned char* str = (const unsigned char*)meta->str;
  uint32_t fp = (uint32_t)str[0] | ((uint32_t)str[2] << 8) | ((uint32_t)str[3] << 16);
  uint32_t mixed = fp ^ ((uint32_t)target_bpp * UINT32_C(0x85ebca6b));
  mixed ^= mixed >> 11;
  mixed *= UINT32_C(0x9e3779b1);
  mixed ^= mixed >> 16;
  unsigned bucket = mixed & (CACHE_BUCKETS - 1);

  for (struct cache_ent* ent = buckets[cache_id][bucket]; ent; ent = ent->bucket_next) {
    if (ent->fingerprint != fp || ent->target_bpp != target_bpp || strcmp(meta->str, ent->str) != 0)
      continue;

    if (ent != newest[cache_id]) {
      if (ent->newer) ent->newer->older = ent->older;
      if (ent->older) ent->older->newer = ent->newer;
      if (ent == oldest[cache_id]) oldest[cache_id] = ent->newer;

      ent->newer = NULL;
      ent->older = newest[cache_id];
      newest[cache_id]->newer = ent;
      newest[cache_id] = ent;
    }

    *hash_pt = ent->hash_arr;

    return ent->cnt;
  }

  if (set_meta_fini(meta) < 0) return -4;

  int len = (int)meta->len;
  int capacity = meta->value_capacity;
  struct cache_ent* ent =
      xmalloc(sizeof(*ent) + ((size_t)capacity + SENTINELS) * sizeof(unsigned) + len + 2);
  ent->hash_arr = (unsigned*)(ent + 1);
  ent->str = (char*)(ent->hash_arr + capacity + SENTINELS);
  memcpy(ent->str, meta->str, (size_t)len + 1);
  ent->str[len + 1] = '\0';

  struct set_meta decode_meta = *meta;
  decode_meta.str = ent->str;
  decode_meta.payload = ent->str + 2;
  int cnt = decode_set(&decode_meta, ent->hash_arr);
  if (cnt <= 0) {
    _free(ent);
    return cnt;
  }

  if (target_bpp < meta->bpp) {
    unsigned temporary[capacity];
    unsigned* current = ent->hash_arr;
    unsigned* destination = temporary;

    for (int bpp = meta->bpp - 1; bpp >= target_bpp; --bpp) {
      cnt = downsample_set(cnt, current, destination, bpp);
      unsigned* swap = current;
      current = destination;
      destination = swap;
    }

    if (current != ent->hash_arr) {
      memcpy(ent->hash_arr, current, (size_t)cnt * sizeof(*current));
    }
  }

  for (int i = 0; i < SENTINELS; ++i) ent->hash_arr[cnt + i] = ~0u;

  ent->fingerprint = fp;
  ent->len = len;
  ent->cnt = cnt;
  ent->target_bpp = target_bpp;

  if (cache_count[cache_id] == CACHE_SIZE) {
    struct cache_ent* victim = oldest[cache_id];
    oldest[cache_id] = victim->newer;
    if (oldest[cache_id]) oldest[cache_id]->older = NULL;
    if (victim == newest[cache_id]) newest[cache_id] = NULL;

    uint32_t victim_mixed =
        victim->fingerprint ^ ((uint32_t)victim->target_bpp * UINT32_C(0x85ebca6b));
    victim_mixed ^= victim_mixed >> 11;
    victim_mixed *= UINT32_C(0x9e3779b1);
    victim_mixed ^= victim_mixed >> 16;
    unsigned victim_bucket = victim_mixed & (CACHE_BUCKETS - 1);
    struct cache_ent** link = &buckets[cache_id][victim_bucket];
    while (*link != victim) link = &(*link)->bucket_next;
    *link = victim->bucket_next;
    _free(victim);
  } else {
    ++cache_count[cache_id];
  }

  ent->bucket_next = buckets[cache_id][bucket];
  buckets[cache_id][bucket] = ent;
  ent->newer = NULL;
  ent->older = newest[cache_id];
  if (newest[cache_id]) {
    newest[cache_id]->newer = ent;
  } else {
    oldest[cache_id] = ent;
  }
  newest[cache_id] = ent;

  *hash_pt = ent->hash_arr;

  return cnt;
}

// Reduce a set of (bpp + 1) values to a set of bpp values.
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp) {
  unsigned mask = (1u << bpp) - 1;

  // find the first element with high bit set
  int l = 0;
  int u = cnt;
  while (l < u) {
    int i = (l + u) / 2;

    if (hash_pt[i] <= mask) {
      l = i + 1;
    } else {
      u = i;
    }
  }

  // initialize parts
  const unsigned* ds_start = ds_pt;
  const unsigned *v1 = hash_pt + 0, *v1_end = hash_pt + u;
  const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + cnt;

  // merge v1 and v2 into w
  if (v1 < v1_end && v2 < v2_end) {
    unsigned v1_val = *v1;
    unsigned v2_val = *v2 & mask;

    while (1) {
      if (v1_val < v2_val) {
        *ds_pt++ = v1_val;
        v1++;

        if (v1 == v1_end) break;

        v1_val = *v1;
      } else if (v2_val < v1_val) {
        *ds_pt++ = v2_val;
        v2++;

        if (v2 == v2_end) break;

        v2_val = *v2 & mask;
      } else {
        *ds_pt++ = v1_val;
        v1++;
        v2++;

        if (v1 == v1_end) break;
        if (v2 == v2_end) break;

        v1_val = *v1;
        v2_val = *v2 & mask;
      }
    }
  }

  // append what's left
  while (v1 < v1_end) *ds_pt++ = *v1++;
  while (v2 < v2_end) *ds_pt++ = *v2++ & mask;

  return ds_pt - ds_start;
}

static int sorted_subset(const unsigned* small, size_t small_count, const unsigned* large,
                         size_t large_count) {
  const unsigned* const small_end = small + small_count;
  const unsigned* const large_end = large + large_count;
  int subset = 1;

  for (int i = 0; i < SENTINELS; ++i) assert(large_end[i] == ~0u);
  unsigned small_value = *small;

  // Loop pieces from the original set.c traversal.  The sentinel tail lets
  // the search advance in groups of four or eight without boundary checks.
#define IFLT4                                \
  if (*large < small_value) {                \
    large += 4;                              \
    while (*large < small_value) large += 4; \
    large -= 2;                              \
    if (*large < small_value)                \
      large++;                               \
    else                                     \
      large--;                               \
    if (*large < small_value) large++;       \
    if (large == large_end) break;           \
  }
#define IFLT8                                \
  if (*large < small_value) {                \
    large += 8;                              \
    while (*large < small_value) large += 8; \
    large -= 4;                              \
    if (*large < small_value)                \
      large += 2;                            \
    else                                     \
      large -= 2;                            \
    if (*large < small_value)                \
      large++;                               \
    else                                     \
      large--;                               \
    if (*large < small_value) large++;       \
    if (large == large_end) break;           \
  }
#define IFGE                         \
  if (*large == small_value) {       \
    large++;                         \
    small++;                         \
    if (large == large_end) break;   \
    if (small == small_end) break;   \
    small_value = *small;            \
  } else {                           \
    subset = 0;                      \
    small++;                         \
    if (small == small_end) break;   \
    small_value = *small;            \
  }

  if (large_count / small_count >= 16) {
    while (1) {
      IFLT8;
      IFGE;
    }
  } else {
    while (1) {
      IFLT4;
      IFGE;
    }
  }

#undef IFGE
#undef IFLT8
#undef IFLT4

  if (small < small_end) subset = 0;

  return subset;
}

// main API routine
int rpmsetcmp(const char* str1, const char* str2) {
  if (strncmp(str1, "set:", 4) == 0) str1 += 4;
  if (strncmp(str2, "set:", 4) == 0) str2 += 4;

  struct set_meta meta1;
  struct set_meta meta2;

  if (set_meta_init(str1, &meta1) < 0) return -3;
  if (set_meta_init(str2, &meta2) < 0) return -4;

  int target_bpp = meta1.bpp < meta2.bpp ? meta1.bpp : meta2.bpp;

  // Decode and cache the first operand at the comparison precision.
  const unsigned* hash_arr1 = NULL;
  int cnt1 = cache_decode_set(&meta1, target_bpp, &hash_arr1, 0);
  if (cnt1 < 0) return -3;

  // Metadata for both operands has already been validated, and set1 has been
  // decoded, so this preserves set8's malformed-input error precedence.
  if (str1 == str2 || strcmp(str1, str2) == 0) return 0;

  // Requirement sets are frequently reused by dependency solvers too.
  const unsigned* hash_arr2 = NULL;
  int cnt2 = cache_decode_set(&meta2, target_bpp, &hash_arr2, 1);
  if (cnt2 < 0) return -4;

  // Cardinality determines which strict-inclusion result is even possible.
  // For equal cardinalities, sorted unique sets are equal iff their bytes match.
  if (cnt1 == cnt2) {
    return memcmp(hash_arr1, hash_arr2, (size_t)cnt1 * sizeof(*hash_arr1)) == 0 ? 0 : -2;
  }
  if (cnt1 > cnt2) {
    return sorted_subset(hash_arr2, (size_t)cnt2, hash_arr1, (size_t)cnt1) ? 1 : -2;
  }
  return sorted_subset(hash_arr1, (size_t)cnt1, hash_arr2, (size_t)cnt2) ? -1 : -2;
}

// ---

#ifdef SELF_TEST
int main(void) {
  struct set* set1 = set_new();
  set_add(set1, "mama");
  set_add(set1, "myla");
  set_add(set1, "ramu");
  const char* str10 = set_fini(set1, 16);
  fprintf(stderr, "set10=%s\n", str10);

  int cmp;
  struct set* set2 = set_new();
  set_add(set2, "myla");
  set_add(set2, "mama");
  const char* str20 = set_fini(set2, 16);
  fprintf(stderr, "set20=%s\n", str20);
  cmp = rpmsetcmp(str10, str20);
  assert(cmp == 1);

  set_add(set2, "ramu");
  const char* str21 = set_fini(set2, 16);
  fprintf(stderr, "set21=%s\n", str21);
  cmp = rpmsetcmp(str10, str21);
  assert(cmp == 0);

  set_add(set2, "baba");
  const char* str22 = set_fini(set2, 16);
  cmp = rpmsetcmp(str10, str22);
  assert(cmp == -1);

  set_add(set1, "deda");
  const char* str11 = set_fini(set1, 16);
  cmp = rpmsetcmp(str11, str22);
  assert(cmp == -2);

  set1 = set_free(set1);
  set2 = set_free(set2);
  str10 = _free(str10);
  str11 = _free(str11);
  str20 = _free(str20);
  str21 = _free(str21);
  str22 = _free(str22);

  fprintf(stderr, "%s: api test OK\n", __FILE__);

  return 0;
}
#endif