add decode & rpmsetcmp part

reimplement/newset.c

@@ -198,12 +198,10 @@ static char* bits_to_char(int c, char* base62) {
   return base62;
 }
 
-// заполняет с младших, в случае Z - ставит в старшие
+// filling from the least significant bits, in case of Z - put in the most significant bits
 static int encode_base62(int bit_cnt, const char* bit_pt, char* base62_str_pt) {
   char* base62_start = base62_str_pt;
 
-  // из-за этого заполнения я не знаю, как написать лучше...
-
   int bits2 = 0;  // number of high bits set
   int bits6 = 0;  // number of regular bits set
   int num6b = 0;  // pending 6-bit number
@@ -239,14 +237,16 @@ static int encode_base62(int bit_cnt, const char* bit_pt, char* base62_str_pt) {
 
 // ---
 
+static inline char encode_bpp(int bpp) { return bpp - 7 + 'a'; }
+
 static int encode_set(int cnt, unsigned* hash_arr, int bpp, char* base62_str) {
   int Mshift = encode_golomb_Mshift(cnt, bpp);
   int bit_cnt = encode_golomb_size(cnt, Mshift);
 
   char bit_arr[bit_cnt];
 
-  *base62_str++ = bpp - 7 + 'a';
-  *base62_str++ = Mshift - 7 + 'a';
+  *base62_str++ = encode_bpp(bpp);
+  *base62_str++ = encode_bpp(Mshift);
 
   // hash_arr -> delta_arr
   encode_delta(cnt, hash_arr);
@@ -300,6 +300,322 @@ const char* set_fini(struct set* set, int bpp) {
   return xstrdup(base62_str);
 }
 
+// ---
+
+// decode bpp or Mshift value
+static inline int decode_bpp(const char* str) { return *str++ + 7 - 'a'; }
+
+static int decode_set_check(const char* str) {
+  // 7..32 values encoded with 'a'..'z'
+  int bpp = decode_bpp(str);
+  if (bpp < 10 || bpp > 32) return -1;
+
+  // golomb parameter
+  int Mshift = decode_bpp(str + 1);
+  if (Mshift < 7 || Mshift > 31) return -2;
+  if (Mshift >= bpp) return -3;
+
+  // no empty sets for now
+  if (*str == '\0') return -4;
+
+  return 0;
+}
+
+static int decode_set_size(const char* str) {
+  int bit_cnt = 6 * (strlen(str) - 2);  // each base62 char can encode up to 6 bits
+
+  return bit_cnt / (decode_bpp(str + 1) + 1);  // estimate number of values based on Mshift
+}
+
+static int char_to_num(char c) {
+  if (c == '\0') return 0xff;  // end of string
+
+  if (c >= '0' && c <= '9') return c - '0';
+  if (c >= 'a' && c <= 'z') return c - 'a' + 10;
+  if (c >= 'A' && c <= 'Z') return c - 'A' + 36;
+
+  return 0xee;  // invalid character
+}
+
+// надо посмотреть, насколько в действительности это делает хуже
+static char* putnbits(int n, int c, char* bit_pt) {
+  for (int i = 0; i < n; ++i) {
+    *bit_pt++ = (c >> i) & 1;
+  }
+
+  return bit_pt;
+}
+
+// Main base62 decoding routine: unpack base62 string into bitv[].
+static int decode_base62(const char* base62_str, char* bit_pt) {
+  char* bit_start = bit_pt;
+
+  unsigned num6b = char_to_num(*base62_str++);  // pending 6-bit number
+  while (num6b != 0xff) {
+    if (num6b == 0xee) return -1;
+
+    if (num6b < 61) {
+      bit_pt = putnbits(6, num6b, bit_pt);
+    } else {
+      assert(num6b == 61);
+      // 61 62 63 cases
+
+      unsigned mask = (1 << 4) | (1 << 5);  // high bits mask
+      int num4b = char_to_num(*base62_str++);
+      if (num4b == 0xff) return -2;
+      if (num4b == 0xee) return -3;
+
+      int num2b = num4b & mask;  // high bits
+      num4b &= ~mask;            // low bits
+      assert(num2b != mask);     // not both bits set
+
+      bit_pt = putnbits(6, 61 + num2b >> 4, bit_pt);  // 61 + (0|1|2) in high bits
+      bit_pt = putnbits(4, num4b, bit_pt);
+    }
+
+    num6b = char_to_num(*base62_str++);
+  }
+
+  return bit_pt - bit_start;
+}
+
+// Main golomb decoding routine: unpackage bits into values.
+static int decode_golomb(int bit_cnt, const char* bit_pt, int Mshift, unsigned* golomb_pt) {
+  unsigned* golomb_start = golomb_pt;
+
+  // next value
+  while (bit_cnt > 0) {
+    // first part
+    unsigned q = 0;
+    char bit = 0;
+    while (bit_cnt > 0) {
+      bit_cnt--;
+      bit = *bit_pt++;
+
+      if (bit == 0) {
+        q++;
+      } else {
+        break;
+      }
+    }
+
+    // trailing zero bits in the input are okay
+    if (bit_cnt == 0 && bit == 0) {
+      // up to 5 bits can be used to complete last character
+      if (q > 5) {
+        return -10;
+      }
+
+      break;
+    }
+
+    // otherwise, incomplete value is not okay
+    if (bit_cnt < Mshift) {
+      return -11;
+    }
+
+    // second part
+    unsigned r = 0;
+    int i;
+    for (i = 0; i < Mshift; i++) {
+      bit_cnt--;
+      if (*bit_pt++) {
+        r |= (1 << i);
+      }
+    }
+
+    // the value
+    *golomb_pt++ = (q << Mshift) | r;
+  }
+
+  return golomb_pt - golomb_start;
+}
+
+static void decode_delta(int cnt, unsigned* delta_pt) {
+  assert(cnt > 0);
+  unsigned* delta_end = delta_pt + cnt;
+  unsigned prev = *delta_pt++;
+
+  while (delta_pt < delta_end) {
+    *delta_pt += prev;
+    prev = *delta_pt++;
+  }
+
+  return;
+}
+
+static int decode_set(const char* str, unsigned* hash_arr) {
+  int Mshift = decode_bpp(str + 1);
+  const char* base62_str = str + 2;
+
+  // base62
+  char bit_arr[6 * strlen(base62_str)];  // each base62 char can encode up to 6 bits
+  int bit_cnt = decode_base62(base62_str, bit_arr);
+  if (bit_cnt < 0) return bit_cnt;
+
+  // golomb
+  int cnt = decode_golomb(bit_cnt, bit_arr, Mshift, hash_arr);
+  if (cnt < 0) return cnt;
+
+  // delta
+  decode_delta(cnt, hash_arr);
+
+  return cnt;
+}
+
+// Reduce a set of (bpp + 1) values to a set of bpp values.
+static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp) {
+  unsigned mask = (1 << bpp) - 1;
+
+  // find the first element with high bit set
+  int l = 0;
+  int u = cnt;
+  while (l < u) {
+    int i = (l + u) / 2;
+
+    if (hash_pt[i] <= mask) {
+      l = i + 1;
+    } else {
+      u = i;
+    }
+  }
+
+  // initialize parts
+  const unsigned* ds_start = ds_pt;
+  const unsigned *v1 = hash_pt + 0, *v1_end = hash_pt + u;
+  const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + cnt;
+
+  // merge v1 and v2 into w
+  if (v1 < v1_end && v2 < v2_end) {
+    unsigned v1_val = *v1;
+    unsigned v2_val = *v2 & mask;
+
+    while (1) {
+      if (v1_val < v2_val) {
+        *ds_pt++ = v1_val;
+        v1++;
+
+        if (v1 == v1_end) break;
+
+        v1_val = *v1;
+      } else if (v2_val < v1_val) {
+        *ds_pt++ = v2_val;
+        v2++;
+
+        if (v2 == v2_end) break;
+
+        v2_val = *v2 & mask;
+      } else {
+        *ds_pt++ = v1_val;
+        v1++;
+        v2++;
+
+        if (v1 == v1_end) break;
+        if (v2 == v2_end) break;
+
+        v1_val = *v1;
+        v2_val = *v2 & mask;
+      }
+    }
+  }
+
+  // append what's left
+  while (v1 < v1_end) *ds_pt++ = *v1++;
+  while (v2 < v2_end) *ds_pt++ = *v2++ & mask;
+
+  return ds_pt - ds_start;
+}
+
+// main API routine
+int rpmsetcmp(const char* str1, const char* str2) {
+  if (strncmp(str1, "set:", 4) == 0) str1 += 4;
+  if (strncmp(str2, "set:", 4) == 0) str2 += 4;
+
+  if (decode_set_check(str1) < 0) return -3;
+  if (decode_set_check(str2) < 0) return -4;
+
+  // decode set1
+  int cnt1 = decode_set_size(str1);
+  unsigned bufA1[cnt1];
+  unsigned bufB1[cnt1];
+  unsigned* hash_arr1 = bufA1;
+  cnt1 = decode_set(str1, hash_arr1);
+  if (cnt1 < 0) return -3;
+
+  // decode set2
+  int cnt2 = decode_set_size(str2);
+  unsigned bufA2[cnt2];
+  unsigned bufB2[cnt2];
+  unsigned* hash_arr2 = bufA2;
+  cnt2 = decode_set(str2, hash_arr2);
+  if (cnt2 < 0) return -4;
+
+  int bpp1 = decode_bpp(str1);
+  int bpp2 = decode_bpp(str2);
+  int min_bpp = (bpp1 < bpp2) ? bpp1 : bpp2;
+
+  while (bpp1 > min_bpp) {
+    unsigned* pt1 = bufA1;
+    if (hash_arr1 == pt1) {
+      pt1 = bufB1;
+    }
+
+    bpp1--;
+    cnt1 = downsample_set(cnt1, hash_arr1, pt1, bpp1);
+    hash_arr1 = pt1;
+  }
+
+  while (bpp2 > min_bpp) {
+    unsigned* pt2 = bufA2;
+    if (hash_arr2 == pt2) {
+      pt2 = bufB2;
+    }
+
+    bpp2--;
+    cnt2 = downsample_set(cnt2, hash_arr2, pt2, bpp2);
+    hash_arr2 = pt2;
+  }
+
+  // compare
+  int ge = 1;
+  int le = 1;
+  const unsigned* end1 = hash_arr1 + cnt1;
+  const unsigned* end2 = hash_arr2 + cnt2;
+
+  while (hash_arr1 < end1 && hash_arr2 < end2) {
+    if (*hash_arr1 < *hash_arr2) {
+      le = 0;
+      hash_arr1++;
+    } else if (*hash_arr2 < *hash_arr1) {
+      ge = 0;
+      hash_arr2++;
+    } else {
+      hash_arr1++;
+      hash_arr2++;
+    }
+  }
+
+  if (hash_arr1 < end1) {
+    le = 0;
+  }
+  if (hash_arr2 < end2) {
+    ge = 0;
+  }
+
+  if (ge && le) {
+    return 0;
+  } else if (ge) {
+    return 1;
+  } else if (le) {
+    return -1;
+  }
+
+  return -2;
+}
+
+// ---
+
+#ifdef SELF_TEST
 int main(void) {
   struct set* set1 = set_new();
   set_add(set1, "mama");
@@ -308,5 +624,41 @@ int main(void) {
   const char* str10 = set_fini(set1, 16);
   fprintf(stderr, "set10=%s\n", str10);
 
+  int cmp;
+  struct set* set2 = set_new();
+  set_add(set2, "myla");
+  set_add(set2, "mama");
+  const char* str20 = set_fini(set2, 16);
+  fprintf(stderr, "set20=%s\n", str20);
+  cmp = rpmsetcmp(str10, str20);
+  assert(cmp == 1);
+
+  set_add(set2, "ramu");
+  const char* str21 = set_fini(set2, 16);
+  fprintf(stderr, "set21=%s\n", str21);
+  cmp = rpmsetcmp(str10, str21);
+  assert(cmp == 0);
+
+  set_add(set2, "baba");
+  const char* str22 = set_fini(set2, 16);
+  cmp = rpmsetcmp(str10, str22);
+  assert(cmp == -1);
+
+  set_add(set1, "deda");
+  const char* str11 = set_fini(set1, 16);
+  cmp = rpmsetcmp(str11, str22);
+  assert(cmp == -2);
+
+  set1 = set_free(set1);
+  set2 = set_free(set2);
+  str10 = _free(str10);
+  str11 = _free(str11);
+  str20 = _free(str20);
+  str21 = _free(str21);
+  str22 = _free(str22);
+
+  fprintf(stderr, "%s: api test OK\n", __FILE__);
+
   return 0;
 }
+#endif