fused decoder for hash_set

new_version/direct_hash/about.md

@@ -5,3 +5,17 @@
 - позволить индексироваться по элементам хэша
 - кратно сократить время на дешифровку
 - кэшировать можно соответствия индекс - значение, чтобы не работать с битами (может и не стоит того)
+  - стоит этого
+
+```text
+symbols=1000 required=500 bpp=32
+implementation  set_chars  format
+set9                3994  golomb/base62
+direct              5340  D1/base64
+
+operation                 set9       direct   direct/set9
+set_fini only           150.44 us   99.28 us         0.66x
+new+add+fini           1034.22 us  949.62 us         0.92x
+rpmsetcmp cold          292.86 us  210.30 us         0.72x
+rpmsetcmp warm            5.89 us   23.02 us         3.91x
+```

new_version/direct_hash/hash_set.c

@@ -245,56 +245,45 @@ const char* set_fini(struct set* set, int bpp) {
   return output;
 }
 
-static int base64_value(unsigned char c) {
+static const unsigned char base64_values[256] = {
-  if (c >= 'A' && c <= 'Z') return c - 'A';
+    ['A'] = 1,  ['B'] = 2,  ['C'] = 3,  ['D'] = 4,  ['E'] = 5,  ['F'] = 6,  ['G'] = 7,  ['H'] = 8,
-  if (c >= 'a' && c <= 'z') return c - 'a' + 26;
+    ['I'] = 9,  ['J'] = 10, ['K'] = 11, ['L'] = 12, ['M'] = 13, ['N'] = 14, ['O'] = 15, ['P'] = 16,
-  if (c >= '0' && c <= '9') return c - '0' + 52;
+    ['Q'] = 17, ['R'] = 18, ['S'] = 19, ['T'] = 20, ['U'] = 21, ['V'] = 22, ['W'] = 23, ['X'] = 24,
-  if (c == '+') return 62;
+    ['Y'] = 25, ['Z'] = 26, ['a'] = 27, ['b'] = 28, ['c'] = 29, ['d'] = 30, ['e'] = 31, ['f'] = 32,
-  if (c == '/') return 63;
+    ['g'] = 33, ['h'] = 34, ['i'] = 35, ['j'] = 36, ['k'] = 37, ['l'] = 38, ['m'] = 39, ['n'] = 40,
-  return -1;
+    ['o'] = 41, ['p'] = 42, ['q'] = 43, ['r'] = 44, ['s'] = 45, ['t'] = 46, ['u'] = 47, ['v'] = 48,
+    ['w'] = 49, ['x'] = 50, ['y'] = 51, ['z'] = 52, ['0'] = 53, ['1'] = 54, ['2'] = 55, ['3'] = 56,
+    ['4'] = 57, ['5'] = 58, ['6'] = 59, ['7'] = 60, ['8'] = 61, ['9'] = 62, ['+'] = 63, ['/'] = 64,
+};
+
+static inline int base64_value(unsigned char c) { return (int)base64_values[c] - 1; }
+
+struct decode_writer {
+  unsigned* hashes;
+  size_t capacity;
+  size_t written;
+  uint64_t bits;
+  uint64_t mask;
+  unsigned filled;
+  unsigned bpp;
+};
+
+static inline int decode_writer_put(struct decode_writer* writer, uint32_t bytes,
+                                    unsigned byte_count) {
+  writer->bits |= (uint64_t)bytes << writer->filled;
+  writer->filled += byte_count * 8;
+
+  while (writer->filled >= writer->bpp) {
+    if (writer->written == writer->capacity) return -1;
+
+    unsigned current = (unsigned)(writer->bits & writer->mask);
+    writer->bits >>= writer->bpp;
+    writer->filled -= writer->bpp;
+    if (writer->written > 0 && writer->hashes[writer->written - 1] >= current) return -1;
+    writer->hashes[writer->written++] = current;
   }
 
-static int base64_decode(const char* input, size_t input_len, unsigned char** bytes,
-                         size_t* byte_count) {
-  if (input_len == 0 || input_len % 4 != 0) return -1;
-  if (input_len / 4 > SIZE_MAX / 3) return -1;
-
-  size_t capacity = input_len / 4 * 3;
-  unsigned char* output = xmalloc(capacity);
-  unsigned char* destination = output;
-
-  for (size_t offset = 0; offset < input_len; offset += 4) {
-    int v0 = base64_value((unsigned char)input[offset]);
-    int v1 = base64_value((unsigned char)input[offset + 1]);
-    int last = offset + 4 == input_len;
-    if (v0 < 0 || v1 < 0) goto invalid;
-
-    *destination++ = (unsigned char)((v0 << 2) | (v1 >> 4));
-    if (input[offset + 2] == '=') {
-      if (!last || input[offset + 3] != '=' || (v1 & 0x0f) != 0) goto invalid;
-      continue;
-    }
-
-    int v2 = base64_value((unsigned char)input[offset + 2]);
-    if (v2 < 0) goto invalid;
-    *destination++ = (unsigned char)((v1 << 4) | (v2 >> 2));
-    if (input[offset + 3] == '=') {
-      if (!last || (v2 & 0x03) != 0) goto invalid;
-      continue;
-    }
-
-    int v3 = base64_value((unsigned char)input[offset + 3]);
-    if (v3 < 0) goto invalid;
-    *destination++ = (unsigned char)((v2 << 6) | v3);
-  }
-
-  *bytes = output;
-  *byte_count = (size_t)(destination - output);
   return 0;
-
-invalid:
-  _free(output);
-  return -1;
 }
 
 static int decode_set(const char* str, struct decoded_set* decoded) {
@@ -307,62 +296,67 @@ static int decode_set(const char* str, struct decoded_set* decoded) {
   unsigned bpp = (unsigned)(str[2] - '0') * 10 + (unsigned)(str[3] - '0');
   if (bpp < 10 || bpp > 32) return -1;
 
-  unsigned char* bytes;
+  const char* input = str + FORMAT_HEADER_LEN;
-  size_t byte_count;
+  size_t input_len = str_len - FORMAT_HEADER_LEN;
-  if (base64_decode(str + FORMAT_HEADER_LEN, str_len - FORMAT_HEADER_LEN, &bytes, &byte_count) < 0)
+  if (input_len == 0 || input_len % 4 != 0 || input_len / 4 > SIZE_MAX / 3) return -1;
-    return -1;
+
-  if (byte_count > SIZE_MAX / 8) {
+  size_t byte_count = input_len / 4 * 3;
-    _free(bytes);
+  if (input[input_len - 1] == '=') --byte_count;
-    return -1;
+  if (input[input_len - 2] == '=') --byte_count;
-  }
+  if (byte_count > SIZE_MAX / 8) return -1;
 
   size_t count = byte_count * 8 / bpp;
-  if (count == 0 || count > (SIZE_MAX - 7) / bpp || (count * bpp + 7) / 8 != byte_count) {
+  if (count == 0 || count > (SIZE_MAX - 7) / bpp || count > SIZE_MAX / sizeof(unsigned) ||
-    _free(bytes);
+      (count * bpp + 7) / 8 != byte_count) {
     return -1;
   }
 
   unsigned* hashes = xmalloc(count * sizeof(*hashes));
-  const unsigned char* input = bytes;
+  struct decode_writer writer = {
-  const unsigned char* input_end = bytes + byte_count;
+      .hashes = hashes,
-  uint64_t bits = 0;
+      .capacity = count,
-  unsigned filled = 0;
+      .mask = bpp < 32 ? (UINT64_C(1) << bpp) - 1 : UINT32_MAX,
-  uint64_t mask = bpp < 32 ? (UINT64_C(1) << bpp) - 1 : UINT32_MAX;
+      .bpp = bpp,
+  };
 
-  for (size_t i = 0; i < count; ++i) {
+  for (size_t offset = 0; offset < input_len; offset += 4) {
-    while (filled < bpp) {
+    int v0 = base64_value((unsigned char)input[offset]);
-      if (input == input_end) {
+    int v1 = base64_value((unsigned char)input[offset + 1]);
-        _free(hashes);
+    int last = offset + 4 == input_len;
-        _free(bytes);
+    if (v0 < 0 || v1 < 0) goto invalid;
-        return -1;
+
-      }
+    uint32_t bytes = (uint32_t)((v0 << 2) | (v1 >> 4));
-      bits |= (uint64_t)*input++ << filled;
+    if (input[offset + 2] == '=') {
-      filled += 8;
+      if (!last || input[offset + 3] != '=' || (v1 & 0x0f) != 0 ||
-    }
+          decode_writer_put(&writer, bytes, 1) < 0)
-    hashes[i] = (unsigned)(bits & mask);
+        goto invalid;
-    bits >>= bpp;
+      continue;
-    filled -= bpp;
-    if (i > 0 && hashes[i - 1] >= hashes[i]) {
-      _free(hashes);
-      _free(bytes);
-      return -1;
-    }
     }
 
-  while (input < input_end) {
+    int v2 = base64_value((unsigned char)input[offset + 2]);
-    bits |= (uint64_t)*input++ << filled;
+    if (v2 < 0) goto invalid;
-    filled += 8;
+    bytes |= (uint32_t)(((v1 & 0x0f) << 4) | (v2 >> 2)) << 8;
+    if (input[offset + 3] == '=') {
+      if (!last || (v2 & 0x03) != 0 || decode_writer_put(&writer, bytes, 2) < 0) goto invalid;
+      continue;
     }
-  _free(bytes);
+
-  if (bits != 0) {
+    int v3 = base64_value((unsigned char)input[offset + 3]);
-    _free(hashes);
+    if (v3 < 0) goto invalid;
-    return -1;
+    bytes |= (uint32_t)(((v2 & 0x03) << 6) | v3) << 16;
+    if (decode_writer_put(&writer, bytes, 3) < 0) goto invalid;
   }
 
+  if (writer.written != count || writer.bits != 0) goto invalid;
+
   decoded->hashes = hashes;
   decoded->count = count;
   decoded->bpp = bpp;
   return 0;
+
+invalid:
+  _free(hashes);
+  return -1;
 }
 
 /* Reduce a sorted set of (bpp + 1)-bit values to a sorted set of bpp-bit values. */