add more self-tests

reimplement/set9.c

@@ -1,3 +1,6 @@
+#ifdef SELF_TEST
+#undef NDEBUG
+#endif
 #include <assert.h>
 #include <stdint.h>
 #include <stdlib.h>
@@ -6,7 +9,6 @@
 
 #include "rpmlib.h"
 #ifdef SELF_TEST
-#undef NDEBUG
 #include <stdio.h>
 #endif
 #include "set.h"
@@ -853,7 +855,310 @@ int rpmsetcmp(const char* str1, const char* str2) {
 // ---
 
 #ifdef SELF_TEST
-int main(void) {
+static void test_hash(void) {
+  assert(hash("") == UINT32_C(0xecd739e9));
+  assert(hash("mama") == UINT32_C(0xd6707329));
+  assert(hash("myla") == UINT32_C(0x29171f6c));
+  assert(hash("ramu") == UINT32_C(0x41196985));
+
+  fprintf(stderr, "%s: hash test OK\n", __FILE__);
+}
+
+static void test_sort(void) {
+  struct symbols small[] = {
+      {.offset = 0, .hash = 9}, {.offset = 1, .hash = 1}, {.offset = 2, .hash = 7},
+      {.offset = 3, .hash = 3}, {.offset = 4, .hash = 5},
+  };
+  const unsigned small_expected[] = {1, 3, 5, 7, 9};
+  const size_t offset_expected[] = {1, 3, 4, 2, 0};
+
+  sort_symbols(small, sizeof(small) / sizeof(*small), 16);
+
+  for (size_t i = 0; i < sizeof(small) / sizeof(*small); ++i) {
+    assert(small[i].hash == small_expected[i]);
+    assert(small[i].offset == offset_expected[i]);
+  }
+
+  enum { LARGE_COUNT = 257 };
+  const int bpps[] = {10, 16, 24, 32};
+  for (size_t bpp_i = 0; bpp_i < sizeof(bpps) / sizeof(*bpps); ++bpp_i) {
+    int bpp = bpps[bpp_i];
+    unsigned mask = bpp < 32 ? (1u << bpp) - 1 : ~0u;
+    struct symbols values[LARGE_COUNT];
+    struct symbols expected[LARGE_COUNT];
+
+    for (size_t i = 0; i < LARGE_COUNT; ++i) {
+      values[i].offset = i;
+      values[i].hash = ((unsigned)i * UINT32_C(0x9e3779b1) ^ UINT32_C(0x85ebca6b)) & mask;
+    }
+    memcpy(expected, values, sizeof(values));
+    qsort(expected, LARGE_COUNT, sizeof(*expected), cmp);
+
+    sort_symbols(values, LARGE_COUNT, bpp);
+    for (size_t i = 0; i < LARGE_COUNT; ++i) {
+      assert(values[i].offset == expected[i].offset);
+      assert(values[i].hash == expected[i].hash);
+    }
+  }
+
+  fprintf(stderr, "%s: sort test OK\n", __FILE__);
+}
+
+static void test_encode_decode(void) {
+  const unsigned original_values[] = {
+      0x020a, 0x07e5, 0x3305, 0x35f5, 0x4980, 0x4c4f, 0x74ef, 0x7739,
+      0x82ae, 0x8415, 0xa3e7, 0xb07e, 0xb584, 0xb89f, 0xbb40, 0xf39e,
+  };
+
+  const int original_count = (int)(sizeof(original_values) / sizeof(*original_values));
+  char encoded[encode_set_size(original_count, 16)];
+  int len = encode_set(original_count, original_values, 16, encoded);
+
+  assert(len == (int)strlen(encoded));
+  assert(strcmp(encoded, "jelgTKwwIMbKUZs24kk9ptXp1BZuBI1Z6Ixa0Z20") == 0);
+
+  struct set_meta meta;
+  assert(set_meta_init(encoded, &meta) == 0);
+  assert(meta.bpp == 16);
+  assert(meta.Mshift == 11);
+  assert(set_meta_fini(&meta) == 0);
+
+  unsigned decoded[meta.value_capacity];
+  int count = decode_set(&meta, decoded);
+  assert(count == original_count);
+  assert(memcmp(decoded, original_values, sizeof(original_values)) == 0);
+
+  const int bpps[] = {10, 16, 24, 32};
+  enum { VALUE_COUNT = 32 };
+  for (size_t bpp_i = 0; bpp_i < sizeof(bpps) / sizeof(*bpps); ++bpp_i) {
+    int bpp = bpps[bpp_i];
+    uint64_t mask = bpp < 32 ? (UINT64_C(1) << bpp) - 1 : UINT32_MAX;
+    unsigned values[VALUE_COUNT];
+
+    // uniform distribution of values
+    for (int i = 0; i < VALUE_COUNT; ++i) {
+      values[i] = (unsigned)(((uint64_t)(i + 1) * mask) / (VALUE_COUNT + 1));
+    }
+
+    char buf[encode_set_size(VALUE_COUNT, bpp)];
+    assert(encode_set(VALUE_COUNT, values, bpp, buf) > 0);
+    assert(set_meta_init(buf, &meta) == 0);
+    assert(set_meta_fini(&meta) == 0);
+    unsigned result[meta.value_capacity];
+    count = decode_set(&meta, result);
+    assert(count == VALUE_COUNT);
+    assert(memcmp(result, values, sizeof(values)) == 0);
+  }
+
+  fprintf(stderr, "%s: encode/decode test OK\n", __FILE__);
+}
+
+static void test_metadata_and_chunks(void) {
+  for (int c = 0; c <= 255; ++c) {
+    unsigned char expected = 0xee;
+    if (c == 0) {
+      expected = 0xff;
+    } else if (c >= '0' && c <= '9') {
+      expected = (unsigned char)(c - '0');
+    } else if (c >= 'a' && c <= 'z') {
+      expected = (unsigned char)(c - 'a' + 10);
+    } else if (c >= 'A' && c <= 'Z') {
+      expected = (unsigned char)(c - 'A' + 36);
+    }
+    assert(char_to_num[c] == expected);
+  }
+
+  struct chunk_case {
+    const char* input;
+    int rc;
+    uint64_t chunk;
+    unsigned width;
+  };
+
+  const struct chunk_case cases[] = {
+      {.input = "0", .rc = 1, .chunk = 0, .width = 6},
+      {.input = "Y", .rc = 1, .chunk = 60, .width = 6},
+      {.input = "Z0", .rc = 1, .chunk = 61, .width = 10},
+      {.input = "Zg", .rc = 1, .chunk = 62, .width = 10},
+      {.input = "Zw", .rc = 1, .chunk = 63, .width = 10},
+      {.input = "", .rc = 0, .chunk = 0, .width = 0},
+      {.input = "!", .rc = -1, .chunk = 0, .width = 0},
+      {.input = "Z", .rc = -2, .chunk = 0, .width = 0},
+      {.input = "Z!", .rc = -3, .chunk = 0, .width = 0},
+      {.input = "ZM", .rc = -4, .chunk = 0, .width = 0},
+  };
+
+  for (size_t i = 0; i < sizeof(cases) / sizeof(*cases); ++i) {
+    const unsigned char* input = (const unsigned char*)cases[i].input;
+    uint64_t chunk = 0;
+    unsigned width = 0;
+    int rc = decode_chunk(&input, &chunk, &width);
+    assert(rc == cases[i].rc);
+
+    if (rc > 0) {
+      assert(chunk == cases[i].chunk);
+      assert(width == cases[i].width);
+    }
+  }
+
+  struct set_meta meta;
+  assert(set_meta_init("", &meta) == -4);  // too short
+  assert(set_meta_init("da", &meta) == -4);
+  assert(set_meta_init("ca0", &meta) == -1);  // incorrect bpp
+  assert(set_meta_init("{a0", &meta) == -1);
+  assert(set_meta_init("d`0", &meta) == -2);  // incorrect Mshift
+  assert(set_meta_init("dz0", &meta) == -2);
+  assert(set_meta_init("dd0", &meta) == -3);  // Mshift == bpp
+  assert(set_meta_init("da0", &meta) == 0);
+  assert(set_meta_fini(&meta) == -4);  // not enough data
+  assert(set_meta_init("da00", &meta) == 0);
+  assert(set_meta_fini(&meta) == 0);  // ok
+  assert(meta.len == 4);
+  assert(meta.payload_len == 2);
+  assert(meta.bit_capacity == 12);
+  assert(meta.value_capacity == 1);
+
+  fprintf(stderr, "%s: metadata/chunk test OK\n", __FILE__);
+}
+
+static void test_downsample(void) {
+  const unsigned mixed[] = {0, 2, 5, 8, 10, 13, 15};
+  const unsigned mixed_expected[] = {0, 2, 5, 7};
+
+  unsigned result[sizeof(mixed) / sizeof(*mixed)];
+  int count = downsample_set((int)(sizeof(mixed) / sizeof(*mixed)), mixed, result, 3);
+  assert(count == (int)(sizeof(mixed_expected) / sizeof(*mixed_expected)));
+  assert(memcmp(result, mixed_expected, sizeof(mixed_expected)) == 0);
+
+  const unsigned low[] = {1, 2, 3};
+  count = downsample_set((int)(sizeof(low) / sizeof(*low)), low, result,
+                         3);  // sizeof(result) >= sizeof(low)
+  assert(count == (int)(sizeof(low) / sizeof(*low)));
+  assert(memcmp(result, low, sizeof(low)) == 0);
+
+  const unsigned high[] = {8, 9};
+  const unsigned high_expected[] = {0, 1};
+  count = downsample_set((int)(sizeof(high) / sizeof(*high)), high, result, 3);
+  assert(count == (int)(sizeof(high_expected) / sizeof(*high_expected)));
+  assert(memcmp(result, high_expected, sizeof(high_expected)) == 0);
+
+  fprintf(stderr, "%s: downsample test OK\n", __FILE__);
+}
+
+static void test_subset(void) {
+  const unsigned dense_large[] = {1, 2, 3, 4, 5, 6, 7};
+  const unsigned dense_small[] = {2, 4, 6};
+  const unsigned dense_missing[] = {2, 4, 8};
+  assert(sorted_subset(dense_small, 3, dense_large, 7) == 1);
+  assert(sorted_subset(dense_missing, 3, dense_large, 7) == 0);  // 0 - incompatible
+
+  unsigned sparse_large[64];
+  for (size_t i = 0; i < sizeof(sparse_large) / sizeof(*sparse_large); ++i) {
+    sparse_large[i] = (unsigned)i;
+  }
+  const unsigned sparse_small[] = {0, 17, 63};
+  const unsigned sparse_missing[] = {0, 17, 64};
+  assert(sorted_subset(sparse_small, 3, sparse_large, 64) == 1);
+  assert(sorted_subset(sparse_missing, 3, sparse_large, 64) == 0);
+
+  assert(step_lower_bound(sparse_large, sparse_large, 1, 8) == sparse_large);
+  assert(step_lower_bound(sparse_large, sparse_large + 64, 0, 8) == sparse_large);
+  assert(step_lower_bound(sparse_large, sparse_large + 64, 18, 8) == sparse_large + 18);
+  assert(step_lower_bound(sparse_large, sparse_large + 64, 64, 8) == sparse_large + 64);
+
+  fprintf(stderr, "%s: subset test OK\n", __FILE__);
+}
+
+static void test_cache(void) {
+  const unsigned values[] = {0x020a, 0x3305, 0x4980, 0x82ae, 0xb584, 0xf39e};
+  const int value_count = (int)(sizeof(values) / sizeof(*values));
+  char encoded[encode_set_size(value_count, 16)];
+  assert(encode_set(value_count, values, 16, encoded) > 0);
+
+  struct set_meta meta;
+  const unsigned* first = NULL;
+  const unsigned* second = NULL;
+  assert(set_meta_init(encoded, &meta) == 0);
+  int count = cache_decode_set(&meta, 16, &first, 0);
+  assert(count == value_count);
+  assert(memcmp(first, values, sizeof(values)) == 0);
+  assert(set_meta_init(encoded, &meta) == 0);
+  assert(cache_decode_set(&meta, 16, &second, 0) == value_count);
+  assert(second == first);
+
+  unsigned downsampled[value_count];
+  int downsampled_count = downsample_set(value_count, values, downsampled, 15);
+  assert(set_meta_init(encoded, &meta) == 0);
+  assert(cache_decode_set(&meta, 15, &second, 0) == downsampled_count);
+  assert(memcmp(second, downsampled, (size_t)downsampled_count * sizeof(*downsampled)) == 0);
+
+  char last_encoded[encode_set_size(1, 16)];
+  for (unsigned i = 1; i <= CACHE_SIZE + 8; ++i) {
+    char item_encoded[encode_set_size(1, 16)];
+    assert(encode_set(1, &i, 16, item_encoded) > 0);
+    assert(set_meta_init(item_encoded, &meta) == 0);
+    assert(cache_decode_set(&meta, 16, &second, 0) == 1);
+    assert(second[0] == i);
+    if (i == CACHE_SIZE + 8) memcpy(last_encoded, item_encoded, sizeof(last_encoded));
+  }
+
+  assert(set_meta_init(last_encoded, &meta) == 0);
+  assert(cache_decode_set(&meta, 16, &first, 0) == 1);
+  assert(set_meta_init(last_encoded, &meta) == 0);
+  assert(cache_decode_set(&meta, 16, &second, 0) == 1);
+  assert(first == second);
+
+  fprintf(stderr, "%s: cache test OK\n", __FILE__);
+}
+
+static void test_builder(void) {
+  enum { SYMBOL_COUNT = 1100 };
+  struct set* set = set_new();
+  struct symbols expected_symbols[SYMBOL_COUNT];
+
+  for (int i = 0; i < SYMBOL_COUNT; ++i) {
+    char symbol[32];
+    int written = snprintf(symbol, sizeof(symbol), "symbol-%04d", i);
+    assert(written > 0 && (size_t)written < sizeof(symbol));
+    set_add(set, symbol);
+    expected_symbols[i].offset = (size_t)i;
+    expected_symbols[i].hash = hash(symbol);
+  }
+
+  assert(set->cnt == SYMBOL_COUNT);
+  assert(set->symbols_cap >= SYMBOL_COUNT);
+  assert(set->strings_len > 4096);
+  assert(set->strings_cap >= set->strings_len);
+
+  qsort(expected_symbols, SYMBOL_COUNT, sizeof(*expected_symbols), cmp);
+  unsigned expected_hashes[SYMBOL_COUNT];
+  int expected_count = 0;
+  for (int i = 0; i < SYMBOL_COUNT; ++i) {
+    if (i == 0 || expected_symbols[i].hash != expected_symbols[i - 1].hash) {
+      expected_hashes[expected_count++] = expected_symbols[i].hash;
+    }
+  }
+
+  const char* encoded = set_fini(set, 32);
+  struct set_meta meta;
+  assert(set_meta_init(encoded, &meta) == 0);
+  assert(set_meta_fini(&meta) == 0);
+  unsigned decoded[meta.value_capacity];
+  int count = decode_set(&meta, decoded);
+  assert(count == expected_count);
+  assert(memcmp(decoded, expected_hashes, (size_t)expected_count * sizeof(*decoded)) == 0);
+  for (int i = 1; i < count; ++i) assert(decoded[i - 1] < decoded[i]);
+
+  set = set_free(set);
+  encoded = _free((void*)encoded);
+  assert(set == NULL);
+  assert(encoded == NULL);
+  assert(set_free(NULL) == NULL);
+
+  fprintf(stderr, "%s: builder test OK\n", __FILE__);
+}
+
+static void test_api(void) {
   struct set* set1 = set_new();
   set_add(set1, "mama");
   set_add(set1, "myla");
@@ -894,7 +1199,22 @@ int main(void) {
   str21 = _free((void*)str21);
   str22 = _free((void*)str22);
 
+  assert(rpmsetcmp("bad", "bad") == -3);
+  assert(rpmsetcmp("da00", "bad") == -4);
+
   fprintf(stderr, "%s: api test OK\n", __FILE__);
+}
+
+int main(void) {
+  test_hash();
+  test_sort();
+  test_encode_decode();
+  test_metadata_and_chunks();
+  test_downsample();
+  test_subset();
+  test_cache();
+  test_builder();
+  test_api();
 
   return 0;
 }