Clean up .gitignore comments

.gitignore

@@ -1,21 +1,22 @@
 *.o
 
 tt
+setc/
 
-# Generated benchmark results
+#Generated benchmark results
 /res/
 /res_cmp/
 /res_old/
 
-# Byte-compiled / optimized / DLL files
+#Byte - compiled / optimized / DLL files
 __pycache__/
 *.py[codz]
 *$py.class
 
-# C extensions
+#C extensions
 *.so
 
-# Distribution / packaging
+#Distribution / packaging
 .Python
 build/
 develop-eggs/
@@ -35,17 +36,17 @@ share/python-wheels/
 *.egg
 MANIFEST
 
-# PyInstaller
-#   Usually these files are written by a python script from a template
-#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+#PyInstaller
+#Usually these files are written by a python script from a template
+#before PyInstaller builds the exe, so as to inject date / other infos into it.
 *.manifest
 *.spec
 
-# Installer logs
+#Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
 
-# Unit test / coverage reports
+#Unit test / coverage reports
 htmlcov/
 .tox/
 .nox/
@@ -60,103 +61,103 @@ coverage.xml
 .pytest_cache/
 cover/
 
-# Translations
+#Translations
 *.mo
 *.pot
 
-# Django stuff:
+#Django stuff:
 *.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
 
-# Flask stuff:
+#Flask stuff:
 instance/
 .webassets-cache
 
-# Scrapy stuff:
+#Scrapy stuff:
 .scrapy
 
-# Sphinx documentation
+#Sphinx documentation
 docs/_build/
 
-# PyBuilder
+#PyBuilder
 .pybuilder/
 target/
 
-# Jupyter Notebook
+#Jupyter Notebook
 .ipynb_checkpoints
 
-# IPython
+#IPython
 profile_default/
 ipython_config.py
 
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
+#pyenv
+#For a library or package, you might want to ignore these files since the code is
+#intended to run in multiple environments; otherwise, check them in:
+#.python - version
 
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-# Pipfile.lock
+#pipenv
+#According to pypa / pipenv #598, it is recommended to include Pipfile.lock in version control.
+#However, in case of collaboration, if having platform - specific dependencies or dependencies
+#having no cross - platform support, pipenv may install dependencies that don't work, or not
+#install all needed dependencies.
+#Pipfile.lock
 
-# UV
-#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-# uv.lock
+#UV
+#Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#This is especially recommended for binary packages to ensure reproducibility, and is more
+#commonly ignored for libraries.
+#uv.lock
 
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-# poetry.lock
-# poetry.toml
+#poetry
+#Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#This is especially recommended for binary packages to ensure reproducibility, and is more
+#commonly ignored for libraries.
+#https:  // python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
 
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
-#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
-# pdm.lock
-# pdm.toml
+#pdm
+#Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm recommends including project - wide configuration in pdm.toml, but excluding.pdm - python.
+#https:  // pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
 .pdm-python
 .pdm-build/
 
-# pixi
-#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
-# pixi.lock
-#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
-#   in the .venv directory. It is recommended not to include this directory in version control.
+#pixi
+#Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#Pixi creates a virtual environment in the.pixi directory, just like venv module creates one
+#in the.venv directory.It is recommended not to include this directory in version control.
 .pixi
 
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+#PEP 582; used by e.g.github.com / David - OConnor / pyflow and github.com / pdm - project / pdm
 __pypackages__/
 
-# Celery stuff
+#Celery stuff
 celerybeat-schedule
 celerybeat.pid
 
-# Redis
+#Redis
 *.rdb
 *.aof
 *.pid
 
-# RabbitMQ
+#RabbitMQ
 mnesia/
 rabbitmq/
 rabbitmq-data/
 
-# ActiveMQ
+#ActiveMQ
 activemq-data/
 
-# SageMath parsed files
+#SageMath parsed files
 *.sage.py
 
-# Environments
+#Environments
 .env
 .envrc
 .venv
@@ -166,64 +167,65 @@ ENV/
 env.bak/
 venv.bak/
 
-# Spyder project settings
+#Spyder project settings
 .spyderproject
 .spyproject
 
-# Rope project settings
+#Rope project settings
 .ropeproject
 
-# mkdocs documentation
+#mkdocs documentation
 /site
 
-# mypy
+#mypy
 .mypy_cache/
 .dmypy.json
 dmypy.json
 
-# Pyre type checker
+#Pyre type checker
 .pyre/
 
-# pytype static type analyzer
+#pytype static type analyzer
 .pytype/
 
-# Cython debug symbols
+#Cython debug symbols
 cython_debug/
 
-# PyCharm
-#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#   and can be added to the global gitignore or merged into this file.  For a more nuclear
-#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
-# .idea/
+#PyCharm
+#JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#be found at https:  // github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#and can be added to the global gitignore or merged into this file.For a more nuclear
+#option(not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea /
 
-# Abstra
-#   Abstra is an AI-powered process automation framework.
-#   Ignore directories containing user credentials, local state, and settings.
-#   Learn more at https://abstra.io/docs
+#Abstra
+#Abstra is an AI - powered process automation framework.
+#Ignore directories containing user credentials, local state, and settings.
+#Learn more at https:  // abstra.io/docs
 .abstra/
 
-# Visual Studio Code
-#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
-#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
-#   and can be added to the global gitignore or merged into this file. However, if you prefer,
-#   you could uncomment the following to ignore the entire vscode folder
-# .vscode/
-# Temporary file for partial code execution
+#Visual Studio Code
+#Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
+#that can be found at \
+    https:  // github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#and can be added to the global gitignore or merged into this file.However, if you prefer,
+#you could uncomment the following to ignore the entire vscode folder
+#.vscode /
+#Temporary file for partial code execution
 tempCodeRunnerFile.py
 
-# Ruff stuff:
+#Ruff stuff:
 .ruff_cache/
 
-# PyPI configuration file
+#PyPI configuration file
 .pypirc
 
-# Marimo
+#Marimo
 marimo/_static/
 marimo/_lsp/
 __marimo__/
 
-# Streamlit
+#Streamlit
 .streamlit/secrets.toml
 
 alt-rpm

Docs/set:version.md

@@ -169,41 +169,6 @@ The encoding alphabet is `0-9,a-z,A-Z`, but the `Z` character encodes 61, 62, an
 
 Note that this encoding cannot produce a `ZZ` sequence, because the `Z` character requires the two high bits to be set to `11`.
 
-!!! Example is incorrect.
-
-Transformation example:
-
-```
-Mshift = 5
-bits = 1 01111 00 1 00010
-```
-
-Or, equivalently:
-
-```
-bits = 101111 001000 10
-```
-
-Bits are read least-significant bit first. For the first six bits:
-
-```
-101111 = 1*1 + 0*2 + 1*4 + 1*8 + 1*16 + 1*32 = 61 = Z
-```
-
-Two bits, `00`, are then inserted into the sequence immediately after the bits encoding Z.
-
-```
-bits = 101111 000010 0010
-000010 = 16 = g
-0010(00) = 4 = 4
-```
-
-The resulting `base62` string is:
-
-```
-10111100100010 = Zg4
-```
-
 ## Comparing set-strings
 
 When set-strings are compared, the transformation process is reversed until the hash values are obtained:

Docs/set:version_ru.md

@@ -169,41 +169,6 @@ bits = 00 1 00101010
 
 Заметим, что данным образом невозможно получить последовательность `ZZ`, т.к. символ `Z` требует двух старших бит, выставленных в `11`
 
-!!! Пример некорректный, кодирование с младшего идёт, Z закидывает в старшие
-
-Пример преобразования:
-
-```
-Mshift = 5
-bits = 1 01111 00 1 00010
-```
-
-Или же
-
-```
-bits = 101111 001000 10
-```
-
-Биты читаются младшим битом вперёд. Первые 6 бит:
-
-```
-101111 = 1*1 + 0*2 + 1*4 + 1*8 + 1*16 + 1*32 = 61 = Z
-```
-
-Тогда в последовательность добавляется два бита `00` сразу после битов, кодирующих Z.
-
-```
-bits = 101111 000010 0010
-000010 = 16 = g
-0010(00) = 4 = 4
-```
-
-Итоговая строка в `base62`:
-
-```
-10111100100010 = Zg4
-```
-
 ## Сравнение set-строк
 
 При сравнении set-строк происходит обратный процесс преобразования до получения хэш-значений

reimplement/newset.c

@@ -1,17 +1,25 @@
 #include <assert.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/types.h>
 
 #include "rpmlib.h"
-#include "stdint.h"
-#include "stdio.h"
+#ifdef SELF_TEST
+#undef NDEBUG
+#include <stdio.h>
+#endif
+#include "set.h"
 #include "system.h"
 
+#define CACHE_SIZE 256
+#define PIVOT_SIZE 243
+#define SENTINELS 0
+
 struct set {
   size_t cnt;
   struct symbols {
     const char* str;
-    int hash;
+    unsigned hash;
   }* symbols_v;
 };
 
@@ -142,9 +150,9 @@ static void encode_delta(int cnt, unsigned* hash_pt) {
 
 // Main golomb encoding routine: package integers into bits.
 // http://algo2.iti.uni-karlsruhe.de/singler/publications/cacheefficientbloomfilters-wea2007.pdf
-// The first integer is then stored in unary coding (which is a variable-length sequence of '0'
-// followed by a terminating '1'); the second part is stored in normal binary coding (using Mshift
-// bits).
+// The first integer is then stored in unary coding (which is a variable-length
+// sequence of '0' followed by a terminating '1'); the second part is stored in
+// normal binary coding (using Mshift bits).
 static int encode_golomb(int cnt, const unsigned* delta_pt, int Mshift, char* bit_pt) {
   char* start_pt = bit_pt;
   const unsigned mask = (1 << Mshift) - 1;
@@ -199,7 +207,8 @@ static char* bits_to_char(int c, char* base62) {
   return base62;
 }
 
-// filling from the least significant bits, in case of Z - put in the most significant bits
+// filling from the least significant bits, in case of Z - put in the most
+// significant bits
 static int encode_base62(int bit_cnt, const char* bit_pt, char* base62_str_pt) {
   char* base62_start = base62_str_pt;
 
@@ -267,7 +276,7 @@ const char* set_fini(struct set* set, int bpp) {
   assert(set->cnt > 0);
   assert(bpp >= 10 && bpp <= 32);
 
-  int mask = (bpp < 32) ? (1u << bpp) - 1 : ~0u;
+  unsigned mask = (bpp < 32) ? (1u << bpp) - 1 : ~0u;
 
   for (size_t i = 0; i < set->cnt; ++i) {
     set->symbols_v[i].hash = hash(set->symbols_v[i].str) & mask;
@@ -284,7 +293,7 @@ const char* set_fini(struct set* set, int bpp) {
             set->symbols_v[i + 1].str);
   }
 
-  int unique_hash[set->cnt];
+  unsigned unique_hash[set->cnt];
   size_t unique_cnt = 0;
 
   // delete duplicates
@@ -317,7 +326,7 @@ static int decode_set_check(const char* str) {
   if (Mshift >= bpp) return -3;
 
   // no empty sets for now
-  if (*str == '\0') return -4;
+  if (*(str + 2) == '\0') return -4;
 
   return 0;
 }
@@ -338,16 +347,27 @@ static int char_to_num(char c) {
   return 0xee;  // invalid character
 }
 
-// надо посмотреть, насколько в действительности это делает хуже
-static char* putnbits(int n, int c, char* bit_pt) {
-  for (int i = 0; i < n; ++i) {
-    *bit_pt++ = (c >> i) & 1;
-  }
+static char* put6bits(int c, char* bit_pt) {
+  *bit_pt++ = (c >> 0) & 1;
+  *bit_pt++ = (c >> 1) & 1;
+  *bit_pt++ = (c >> 2) & 1;
+  *bit_pt++ = (c >> 3) & 1;
+  *bit_pt++ = (c >> 4) & 1;
+  *bit_pt++ = (c >> 5) & 1;
 
   return bit_pt;
 }
 
-// Main base62 decoding routine: unpack base62 string into bitv[].
+static char* put4bits(int c, char* bit_pt) {
+  *bit_pt++ = (c >> 0) & 1;
+  *bit_pt++ = (c >> 1) & 1;
+  *bit_pt++ = (c >> 2) & 1;
+  *bit_pt++ = (c >> 3) & 1;
+
+  return bit_pt;
+}
+
+// Main base62 decoding routine: unpack base62 string into bit_pt[].
 static int decode_base62(const char* base62_str, char* bit_pt) {
   char* bit_start = bit_pt;
 
@@ -356,7 +376,7 @@ static int decode_base62(const char* base62_str, char* bit_pt) {
     if (num6b == 0xee) return -1;
 
     if (num6b < 61) {
-      bit_pt = putnbits(6, num6b, bit_pt);
+      bit_pt = put6bits(num6b, bit_pt);
     } else {
       assert(num6b == 61);
       // 61 62 63 cases
@@ -370,8 +390,8 @@ static int decode_base62(const char* base62_str, char* bit_pt) {
       num4b &= ~mask;            // low bits
       assert(num2b != mask);     // not both bits set
 
-      bit_pt = putnbits(6, 61 + (num2b >> 4), bit_pt);  // 61 + (0|1|2) in high bits
-      bit_pt = putnbits(4, num4b, bit_pt);
+      bit_pt = put6bits(61 + (num2b >> 4), bit_pt);  // 61 + (0|1|2) in high bits
+      bit_pt = put4bits(num4b, bit_pt);
     }
 
     num6b = char_to_num(*base62_str++);
@@ -464,6 +484,84 @@ static int decode_set(const char* str, unsigned* hash_arr) {
   return cnt;
 }
 
+// Special decode_set version with LRU caching.
+static int cache_decode_set(const char* str, const unsigned** hash_pt) {
+  struct cache_ent {
+    char* str;
+    int len;
+    int cnt;
+    unsigned* hash_arr;
+  };
+
+  static int cache_cnt;
+  static unsigned cache_arr[CACHE_SIZE];
+  static struct cache_ent* ent_arr[CACHE_SIZE];
+
+  struct cache_ent* ent;
+  unsigned fp = str[0] | (str[2] << 8) | (str[3] << 16);
+
+  int i = 0;
+  for (unsigned* cache_pt = cache_arr; cache_pt < cache_arr + cache_cnt; ++cache_pt, ++i) {
+    if (fp == *cache_pt) {
+      ent = ent_arr[i];
+
+      if (memcmp(str, ent->str, ent->len + 1) == 0) {
+        // hit, move to front
+        if (i) {
+          memmove(cache_arr + 1, cache_arr, i * sizeof(cache_arr[0]));
+          memmove(ent_arr + 1, ent_arr, i * sizeof(ent_arr[0]));
+
+          cache_arr[0] = fp;
+          ent_arr[0] = ent;
+        }
+
+        *hash_pt = ent->hash_arr;
+
+        return ent->cnt;
+      }
+    }
+  }
+
+  // decode
+  int len = strlen(str);
+  int cnt = decode_set_size(str);
+  ent = xmalloc(sizeof(*ent) + len + 1 + (cnt + SENTINELS) * sizeof(unsigned));
+  ent->hash_arr = (unsigned*)(ent + 1);
+  ent->str = (char*)(ent->hash_arr + cnt + SENTINELS);
+
+  cnt = ent->cnt = decode_set(str, ent->hash_arr);
+  if (cnt <= 0) {
+    _free(ent);
+    return cnt;
+  }
+
+  for (i = 0; i < SENTINELS; ++i) {
+    ent->hash_arr[cnt + i] = ~0u;
+  }
+
+  memcpy(ent->str, str, len + 1);
+  ent->len = len;
+
+  // insert
+  if (cache_cnt < CACHE_SIZE) {
+    i = cache_cnt++;
+  } else {
+    // free last entry
+    free(ent_arr[CACHE_SIZE - 1]);
+
+    // position at midpoint
+    i = PIVOT_SIZE;
+    memmove(cache_arr + i + 1, cache_arr + i, (CACHE_SIZE - i - 1) * sizeof(cache_arr[0]));
+    memmove(ent_arr + i + 1, ent_arr + i, (CACHE_SIZE - i - 1) * sizeof(ent_arr[0]));
+  }
+
+  cache_arr[i] = fp;
+  ent_arr[i] = ent;
+  *hash_pt = ent->hash_arr;
+
+  return cnt;
+}
+
 // Reduce a set of (bpp + 1) values to a set of bpp values.
 static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp) {
   unsigned mask = (1 << bpp) - 1;
@@ -527,6 +625,42 @@ static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int
   return ds_pt - ds_start;
 }
 
+static unsigned* gallop_lower_bound(unsigned* first, const unsigned* last, unsigned value) {
+  size_t n = (size_t)(last - first);
+
+  if (n == 0 || first[0] >= value) {
+    return first;
+  }
+
+  size_t lo = 0;
+  size_t hi = 1;
+
+  while (hi < n && first[hi] < value) {
+    lo = hi;
+
+    if (hi > n / 2) {
+      hi = n;
+      break;
+    }
+
+    hi *= 2;
+  }
+
+  size_t left = lo + 1;
+  size_t right = hi < n ? hi + 1 : n;
+
+  while (left < right) {
+    size_t mid = left + (right - left) / 2;
+
+    if (first[mid] < value)
+      left = mid + 1;
+    else
+      right = mid;
+  }
+
+  return first + left;
+}
+
 // main API routine
 int rpmsetcmp(const char* str1, const char* str2) {
   if (strncmp(str1, "set:", 4) == 0) str1 += 4;
@@ -536,12 +670,11 @@ int rpmsetcmp(const char* str1, const char* str2) {
   if (decode_set_check(str2) < 0) return -4;
 
   // decode set1
-  int cnt1 = decode_set_size(str1);
-  unsigned bufA1[cnt1];
-  unsigned bufB1[cnt1];
-  unsigned* hash_arr1 = bufA1;
-  cnt1 = decode_set(str1, hash_arr1);
+  const unsigned* hash_arr1 = NULL;
+  int cnt1 = cache_decode_set(str1, &hash_arr1);
   if (cnt1 < 0) return -3;
+  unsigned bufA1[cnt1 + SENTINELS];
+  unsigned bufB1[cnt1 + SENTINELS];
 
   // decode set2
   int cnt2 = decode_set_size(str2);
@@ -584,16 +717,29 @@ int rpmsetcmp(const char* str1, const char* str2) {
   const unsigned* end2 = hash_arr2 + cnt2;
 
   while (hash_arr1 < end1 && hash_arr2 < end2) {
-    if (*hash_arr1 < *hash_arr2) {
-      le = 0;
-      hash_arr1++;
-    } else if (*hash_arr2 < *hash_arr1) {
+    if (*hash_arr2 < *hash_arr1) {
       ge = 0;
-      hash_arr2++;
+      ++hash_arr2;
+    } else if (*hash_arr1 == *hash_arr2) {
+      ++hash_arr1;
+      ++hash_arr2;
     } else {
-      hash_arr1++;
-      hash_arr2++;
+      le = 0;
+
+      hash_arr1 = gallop_lower_bound(hash_arr1, end1, *hash_arr2);
+
+      if (hash_arr1 == end1) break;
+
+      if (*hash_arr1 == *hash_arr2) {
+        ++hash_arr1;
+        ++hash_arr2;
+      } else {
+        ge = 0;
+        ++hash_arr2;
+      }
     }
+
+    if (!ge && !le) break;
   }
 
   if (hash_arr1 < end1) {

reimplement/newset_f.c

@@ -1,108 +0,0 @@
-#include "stdint.h"
-#include "stdio.h"
-#include "system.h"
-
-struct set {
-  size_t cnt;
-  struct symbols {
-    const char* str;
-    uint64_t hash;
-  }* symbols_v;
-};
-
-struct set* set_new() {
-  // should we use x___ funcs?
-  struct set* set = xmalloc(sizeof *set);
-  set->cnt = 0;
-  set->symbols_v = NULL;
-
-  return set;
-}
-
-void set_add(struct set* set, const char* sym) {
-  const int delta = 1024;
-
-  if (set->cnt % delta == 0) {
-    set->symbols_v = xrealloc(set->symbols_v, sizeof(*set->symbols_v) * (set->cnt + delta));
-  }
-
-  set->symbols_v[set->cnt].str = xstrdup(sym);
-  set->symbols_v[set->cnt].hash = 0;
-  set->cnt++;
-
-  return;
-}
-
-struct set* set_free(struct set* set) {
-  if (set) {
-    for (size_t i = 0; i < set->cnt; ++i) {
-      _free((char*)set->symbols_v[i].str);
-    }
-
-    _free(set->symbols_v);
-    set = _free(set);
-  }
-
-  return NULL;
-}
-
-// ---
-
-uint64_t hash(const char* str) {
-  uint64_t h = 0xcbf29ce484222325ULL;
-
-  while (*str) {
-    h ^= (uint64_t)(unsigned char)(*str++);
-    h *= 0x100000001b3ULL;
-  }
-
-  return h;
-}
-
-int cmp(const void* arg1, const void* arg2) {
-  const struct symbols* s1 = (const struct symbols*)arg1;
-  const struct symbols* s2 = (const struct symbols*)arg2;
-
-  if (s1->hash > s2->hash) return 1;
-  if (s2->hash > s1->hash) return -1;
-
-  return 0;
-}
-
-const char* set_fini(struct set* set, int bpp) {
-  // Implementation for finalizing the set
-
-  assert(set != NULL);
-  assert(set->cnt > 0);
-  assert(bpp >= 10 && bpp <= 63);
-
-  uint64_t mask = (1ULL << bpp) - 1;
-
-  for (size_t i = 0; i < set->cnt; ++i) {
-    set->symbols_v[i].hash = hash(set->symbols_v[i].str) & mask;
-  }
-
-  qsort(set->symbols_v, set->cnt, sizeof *set->symbols_v, cmp);
-
-  // warn on hash collizions
-  for (size_t i = 0; i < set->cnt - 1; ++i) {
-    if (set->symbols_v[i].hash != set->symbols_v[i + 1].hash) continue;
-    if (!strcmp(set->symbols_v[i].str, set->symbols_v[i + 1].str)) continue;
-
-    fprintf(stderr, "warning: hash collision: %s %s\n", set->symbols_v[i].str,
-            set->symbols_v[i + 1].str);
-  }
-
-  uint64_t unique_hash[set->cnt];
-  size_t unique_cnt = 0;
-
-  // delete duplicates
-  for (size_t i = 0; i < set->cnt; ++i) {
-    while (i + 1 < set->cnt && set->symbols_v[i].hash == set->symbols_v[i + 1].hash) {
-      ++i;
-    }
-    unique_hash[unique_cnt++] = set->symbols_v[i].hash;
-  }
-
-  return NULL;
-}

scripts/rpmsetcmp/compare_random_sets.py

@@ -0,0 +1,414 @@
+#!/usr/bin/env python3
+"""Continuously compare rpmsetcmp results from set.c and newset.c."""
+
+from __future__ import annotations
+
+import random
+import shlex
+import subprocess
+import tempfile
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+MAX_N = 1000  # max words in file
+MAX_M = 100  # max len for word
+MIN_BPP = 10
+MAX_BPP = 32
+ALPHABET = ".0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
+LARGE_ALPHABET = "".join(chr(code) for code in range(33, 127))
+CASES = (1, 0, -2, -3)
+CC = "cc"
+CFLAGS = ("-O2", "-std=gnu11", "-D_GNU_SOURCE")
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+REPO_ROOT = SCRIPT_DIR.parent.parent
+OLD_SET_SOURCE = REPO_ROOT / "set.c"
+NEW_SET_SOURCE = REPO_ROOT / "reimplement" / "newset.c"
+MKSET_SOURCE = SCRIPT_DIR / "mkset.c"
+SETCMP_SOURCE = SCRIPT_DIR / "setcmp.c"
+COMPAT_HEADER = SCRIPT_DIR / "newset_compat.h"
+ERROR_DIR = SCRIPT_DIR / "error"
+
+SET_HEADER = """\
+#ifndef ARSV_SET_H
+#define ARSV_SET_H
+
+struct set;
+struct set *set_new(void);
+void set_add(struct set *set, const char *symbol);
+const char *set_fini(struct set *set, int bpp);
+int rpmsetcmp(const char *set1, const char *set2);
+
+#endif
+"""
+
+
+@dataclass(frozen=True)
+class Outcome:
+    result: int | None
+    returncode: int
+    stdout: str
+    stderr: str
+
+    def comparison_key(self) -> tuple[object, ...]:
+        if self.result is not None:
+            return ("result", self.result)
+        return ("process", self.returncode, self.stdout, self.stderr)
+
+    def describe(self) -> str:
+        if self.result is not None:
+            return str(self.result)
+        return (
+            f"exit={self.returncode}, stdout={self.stdout!r}, "
+            f"stderr={self.stderr!r}"
+        )
+
+
+def build_binary(
+    source: Path, wrapper: Path, output: Path, build_dir: Path
+) -> None:
+    command = [
+        CC,
+        *CFLAGS,
+        f"-I{build_dir}",
+        "-include",
+        str(COMPAT_HEADER),
+        str(source),
+        str(wrapper),
+        "-o",
+        str(output),
+    ]
+    completed = subprocess.run(command, text=True, capture_output=True)
+    if completed.returncode != 0:
+        raise RuntimeError(
+            f"Compilation failed: {shlex.join(command)}\n{completed.stderr}"
+        )
+
+
+def run_mkset(binary: Path, input_path: Path, bpp: int) -> str:
+    with input_path.open("r", encoding="ascii") as input_file:
+        completed = subprocess.run(
+            [str(binary), str(bpp)],
+            stdin=input_file,
+            text=True,
+            capture_output=True,
+        )
+    if completed.returncode != 0:
+        raise RuntimeError(
+            f"{binary.name} failed with code {completed.returncode}:\n"
+            f"{completed.stderr}"
+        )
+    return completed.stdout.strip()
+
+
+def run_setcmp(binary: Path, set1: str, set2: str) -> Outcome:
+    completed = subprocess.run(
+        [str(binary), set1, set2],
+        text=True,
+        capture_output=True,
+    )
+    stdout = completed.stdout.strip()
+    stderr = completed.stderr.strip()
+    result: int | None = None
+
+    if completed.returncode == 0:
+        try:
+            result = int(stdout)
+        except ValueError:
+            pass
+    elif completed.returncode == 1:
+        if "set1 error" in stderr:
+            result = -3
+        elif "set2 error" in stderr:
+            result = -4
+
+    return Outcome(result, completed.returncode, stdout, stderr)
+
+
+def random_word(alphabet: str = ALPHABET) -> str:
+    length = random.randint(1, MAX_M)
+    return "".join(random.choices(alphabet, k=length))
+
+
+def generate_words(n: int) -> list[str]:
+    words: list[str] = []
+    used: set[str] = set()
+    while len(words) < n:
+        word = random_word()
+        if word not in used:
+            used.add(word)
+            words.append(word)
+    return words
+
+
+def write_words(path: Path, words: list[str]) -> None:
+    path.write_text("".join(f"{word}\n" for word in words), encoding="ascii")
+
+
+def generate_second_words(case: int, words: list[str]) -> list[str]:
+    if case == 0:
+        return words.copy()
+
+    remove_count = random.randint(1, len(words) - 1)
+    removed_indices = set(random.sample(range(len(words)), remove_count))
+    second_words = [
+        word for index, word in enumerate(words) if index not in removed_indices
+    ]
+
+    if case == -2:
+        used = set(words)
+        add_count = random.randint(1, remove_count)
+        while add_count > 0:
+            word = random_word()
+            if word not in used:
+                used.add(word)
+                second_words.append(word)
+                add_count -= 1
+        random.shuffle(second_words)
+
+    return second_words
+
+
+def generate_invalid_set(alphabet: str) -> str:
+    invalid_first_chars = [character for character in alphabet if character not in "cdefghijklmnopqrstuvwxyz"]
+    first = random.choice(invalid_first_chars)
+    length = random.randint(1, MAX_M)
+    tail = "".join(random.choices(alphabet, k=length - 1))
+    return f"set:{first}{tail}"
+
+
+def compare_pair(
+    label: str,
+    setcmp: Path,
+    setcmp_new: Path,
+    old_set1: str,
+    old_set2: str,
+    new_set1: str,
+    new_set2: str,
+) -> tuple[str, Outcome, Outcome]:
+    old_outcome = run_setcmp(setcmp, old_set1, old_set2)
+    new_outcome = run_setcmp(setcmp_new, new_set1, new_set2)
+    return label, old_outcome, new_outcome
+
+
+def save_error(
+    test_number: int,
+    case: int,
+    bpp: int,
+    artifacts: dict[str, bytes],
+    comparisons: list[tuple[str, Outcome, Outcome]],
+) -> None:
+    timestamp = time.time_ns()
+    case_name = str(case).replace("-", "minus")
+    stem = f"test_{test_number}_case_{case_name}_bpp{bpp}_{timestamp}"
+
+    for suffix, content in artifacts.items():
+        (ERROR_DIR / f"{stem}_{suffix}").write_bytes(content)
+
+    details = []
+    for label, old_outcome, new_outcome in comparisons:
+        details.append(
+            f"{label}: setcmp={old_outcome.describe()} "
+            f"setcmp_new={new_outcome.describe()}"
+        )
+    (ERROR_DIR / f"{stem}_results.txt").write_text(
+        "\n".join(details) + "\n", encoding="utf-8"
+    )
+    print(f"mismatch: saved inputs as {ERROR_DIR / stem}_*", flush=True)
+
+
+def run_relation_case(
+    case: int,
+    test_number: int,
+    n: int,
+    bpp: int,
+    input1_path: Path,
+    input2_path: Path,
+    mkset: Path,
+    mkset_new: Path,
+    setcmp: Path,
+    setcmp_new: Path,
+) -> None:
+    words1 = generate_words(n)
+    words2 = generate_second_words(case, words1)
+    write_words(input1_path, words1)
+    write_words(input2_path, words2)
+
+    old_set1 = run_mkset(mkset, input1_path, bpp)
+    new_set1 = run_mkset(mkset_new, input1_path, bpp)
+    old_set2 = run_mkset(mkset, input2_path, bpp)
+    new_set2 = run_mkset(mkset_new, input2_path, bpp)
+
+    comparisons = [
+        compare_pair(
+            str(case),
+            setcmp,
+            setcmp_new,
+            old_set1,
+            old_set2,
+            new_set1,
+            new_set2,
+        )
+    ]
+    if case == 1:
+        comparisons.append(
+            compare_pair(
+                "-1 (swapped)",
+                setcmp,
+                setcmp_new,
+                old_set2,
+                old_set1,
+                new_set2,
+                new_set1,
+            )
+        )
+
+    mismatches = [
+        comparison
+        for comparison in comparisons
+        if comparison[1].comparison_key() != comparison[2].comparison_key()
+    ]
+    if mismatches:
+        save_error(
+            test_number,
+            case,
+            bpp,
+            {
+                "input1.txt": input1_path.read_bytes(),
+                "input2.txt": input2_path.read_bytes(),
+            },
+            comparisons,
+        )
+
+
+def run_invalid_case(
+    test_number: int,
+    n: int,
+    bpp: int,
+    input_path: Path,
+    mkset: Path,
+    mkset_new: Path,
+    setcmp: Path,
+    setcmp_new: Path,
+    invalid_alphabet: str | None = None,
+) -> None:
+    words = generate_words(n)
+    write_words(input_path, words)
+    old_valid_set = run_mkset(mkset, input_path, bpp)
+    new_valid_set = run_mkset(mkset_new, input_path, bpp)
+
+    if invalid_alphabet is None:
+        invalid_alphabet = random.choice((ALPHABET, LARGE_ALPHABET))
+    invalid_set = generate_invalid_set(invalid_alphabet)
+    alphabet_name = "same" if invalid_alphabet == ALPHABET else "large"
+
+    comparisons = [
+        compare_pair(
+            f"-3 ({alphabet_name} alphabet)",
+            setcmp,
+            setcmp_new,
+            invalid_set,
+            old_valid_set,
+            invalid_set,
+            new_valid_set,
+        ),
+        compare_pair(
+            f"-4 ({alphabet_name} alphabet, swapped)",
+            setcmp,
+            setcmp_new,
+            old_valid_set,
+            invalid_set,
+            new_valid_set,
+            invalid_set,
+        ),
+    ]
+
+    mismatches = [
+        comparison
+        for comparison in comparisons
+        if comparison[1].comparison_key() != comparison[2].comparison_key()
+    ]
+    if mismatches:
+        save_error(
+            test_number,
+            -3,
+            bpp,
+            {
+                "input.txt": input_path.read_bytes(),
+                "invalid_set.txt": f"{invalid_set}\n".encode("ascii"),
+            },
+            comparisons,
+        )
+
+
+def main() -> None:
+    if MAX_N < 2 or MAX_M < 1:
+        raise ValueError("MAX_N must be at least 2 and MAX_M at least 1")
+    if not 10 <= MIN_BPP <= MAX_BPP <= 32:
+        raise ValueError("MIN_BPP and MAX_BPP must be in the range 10..32")
+
+    ERROR_DIR.mkdir(parents=True, exist_ok=True)
+
+    with tempfile.TemporaryDirectory(prefix="arsv-rpmsetcmp-") as temporary:
+        build_dir = Path(temporary)
+        (build_dir / "rpmlib.h").touch()
+        (build_dir / "system.h").touch()
+        (build_dir / "set.h").write_text(SET_HEADER, encoding="ascii")
+
+        mkset = build_dir / "mkset"
+        mkset_new = build_dir / "mkset_new"
+        setcmp = build_dir / "setcmp"
+        setcmp_new = build_dir / "setcmp_new"
+        input1_path = build_dir / "input1.txt"
+        input2_path = build_dir / "input2.txt"
+
+        build_binary(OLD_SET_SOURCE, MKSET_SOURCE, mkset, build_dir)
+        build_binary(NEW_SET_SOURCE, MKSET_SOURCE, mkset_new, build_dir)
+        build_binary(OLD_SET_SOURCE, SETCMP_SOURCE, setcmp, build_dir)
+        build_binary(NEW_SET_SOURCE, SETCMP_SOURCE, setcmp_new, build_dir)
+
+        tests_completed = 0
+        while True:
+            case = random.choice(CASES)
+            min_n = 2 if case in (1, -2) else 1
+            n = random.randint(min_n, MAX_N)
+            bpp = random.randint(MIN_BPP, MAX_BPP)
+            current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            print(
+                f"{current_time} tests={tests_completed} case={case} "
+                f"bpp={bpp} n={n}",
+                flush=True,
+            )
+
+            if case == -3:
+                run_invalid_case(
+                    tests_completed + 1,
+                    n,
+                    bpp,
+                    input1_path,
+                    mkset,
+                    mkset_new,
+                    setcmp,
+                    setcmp_new,
+                )
+            else:
+                run_relation_case(
+                    case,
+                    tests_completed + 1,
+                    n,
+                    bpp,
+                    input1_path,
+                    input2_path,
+                    mkset,
+                    mkset_new,
+                    setcmp,
+                    setcmp_new,
+                )
+
+            tests_completed += 1
+
+
+if __name__ == "__main__":
+    main()

scripts/rpmsetcmp/mkset.c

@@ -0,0 +1,30 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "set.h"
+
+int main(int argc, const char **argv)
+{
+    assert(argc == 2);
+    int bpp = atoi(argv[1]);
+    assert(bpp >= 10);
+    assert(bpp <= 32);
+    struct set *set = set_new();
+    char *line = NULL;
+    size_t alloc_size = 0;
+    ssize_t len;
+    int added = 0;
+    while ((len = getline(&line, &alloc_size, stdin)) >= 0) {
+	if (len > 0 && line[len-1] == '\n')
+	    line[--len] = '\0';
+	if (len == 0)
+	    continue;
+	set_add(set, line);
+	added++;
+    }
+    assert(added > 0);
+    const char *str = set_fini(set, bpp);
+    assert(str);
+    printf("set:%s\n", str);
+    return 0;
+}

scripts/rpmsetcmp/setcmp.c

@@ -0,0 +1,52 @@
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "set.h"
+
+static
+int setcmp(const char *s1, const char *s2)
+{
+    int cmp = rpmsetcmp(s1, s2);
+    switch (cmp) {
+    case 1:
+    case 0:
+    case -1:
+    case -2:
+	printf("%d\n", cmp);
+	return 0;
+    case -3:
+	fprintf(stderr, "%s: set1 error\n", __FILE__);
+	break;
+    case -4:
+	fprintf(stderr, "%s: set2 error\n", __FILE__);
+	break;
+    default:
+	fprintf(stderr, "%s: unknown error\n", __FILE__);
+	break;
+    }
+    return 1;
+}
+
+int main(int argc, const char **argv)
+{
+    assert(argc == 1 || argc == 3);
+    if (argc == 3)
+	return setcmp(argv[1], argv[2]);
+    int rc = 0;
+    while (1) {
+	char *s1 = NULL, *s2 = NULL;
+	int n = scanf("%ms %ms", &s1, &s2);
+	if (n == EOF)
+	    break;
+	assert(n == 2);
+	assert(s1 && s2);
+	rc |= setcmp(s1, s2);
+	free(s1);
+	free(s2);
+    }
+    return rc;
+}

scripts/rpmsetinit/compare_random_sets.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Continuously compare set strings produced by set.c and newset.c."""
+
+from __future__ import annotations
+
+import random
+import shlex
+import subprocess
+import tempfile
+import time
+from datetime import datetime
+from pathlib import Path
+
+MAX_N = 1000  # max words in file
+MAX_M = 100  # max len for word
+MIN_BPP = 10
+MAX_BPP = 32
+ALPHABET = ".0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
+CC = "cc"
+CFLAGS = ("-O2", "-std=gnu11", "-D_GNU_SOURCE")
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+REPO_ROOT = SCRIPT_DIR.parent.parent
+OLD_SET_SOURCE = REPO_ROOT / "set.c"
+NEW_SET_SOURCE = REPO_ROOT / "reimplement" / "newset.c"
+MKSET_SOURCE = SCRIPT_DIR / "mkset.c"
+COMPAT_HEADER = SCRIPT_DIR / "newset_compat.h"
+ERROR_DIR = SCRIPT_DIR / "error"
+
+SET_HEADER = """\
+#ifndef ARSV_SET_H
+#define ARSV_SET_H
+
+struct set;
+struct set *set_new(void);
+void set_add(struct set *set, const char *symbol);
+const char *set_fini(struct set *set, int bpp);
+
+#endif
+"""
+
+
+def build_mkset(source: Path, output: Path, build_dir: Path) -> None:
+    command = [
+        CC,
+        *CFLAGS,
+        f"-I{build_dir}",
+        "-include",
+        str(COMPAT_HEADER),
+        str(source),
+        str(MKSET_SOURCE),
+        "-o",
+        str(output),
+    ]
+    completed = subprocess.run(command, text=True, capture_output=True)
+    if completed.returncode != 0:
+        raise RuntimeError(
+            f"Compilation failed: {shlex.join(command)}\n{completed.stderr}"
+        )
+
+
+def run_mkset(binary: Path, input_path: Path, bpp: int) -> str:
+    with input_path.open("r", encoding="ascii") as input_file:
+        completed = subprocess.run(
+            [str(binary), str(bpp)],
+            stdin=input_file,
+            text=True,
+            capture_output=True,
+        )
+    if completed.returncode != 0:
+        raise RuntimeError(
+            f"{binary.name} failed with code {completed.returncode}:\n"
+            f"{completed.stderr}"
+        )
+    return completed.stdout.strip()
+
+
+def generate_input(path: Path, n: int) -> None:
+    with path.open("w", encoding="ascii") as output:
+        for _ in range(n):
+            m = random.randint(1, MAX_M)
+            output.write("".join(random.choices(ALPHABET, k=m)))
+            output.write("\n")
+
+
+def main() -> None:
+    if MAX_N < 1 or MAX_M < 1:
+        raise ValueError("MAX_N and MAX_M must be at least 1")
+    if not 10 <= MIN_BPP <= MAX_BPP <= 32:
+        raise ValueError("MIN_BPP and MAX_BPP must be in the range 10..32")
+
+    ERROR_DIR.mkdir(parents=True, exist_ok=True)
+
+    with tempfile.TemporaryDirectory(prefix="arsv-rpmsetinit-") as temporary:
+        build_dir = Path(temporary)
+        (build_dir / "rpmlib.h").touch()
+        (build_dir / "system.h").touch()
+        (build_dir / "set.h").write_text(SET_HEADER, encoding="ascii")
+
+        mkset = build_dir / "mkset"
+        mkset_new = build_dir / "mkset_new"
+        input_path = build_dir / "input.txt"
+
+        build_mkset(OLD_SET_SOURCE, mkset, build_dir)
+        build_mkset(NEW_SET_SOURCE, mkset_new, build_dir)
+
+        tests_completed = 0
+        while True:
+            n = random.randint(1, MAX_N)
+            bpp = random.randint(MIN_BPP, MAX_BPP)
+            current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            print(
+                f"{current_time} tests={tests_completed} bpp={bpp} n={n}",
+                flush=True,
+            )
+
+            generate_input(input_path, n)
+            new_result = run_mkset(mkset_new, input_path, bpp)
+            old_result = run_mkset(mkset, input_path, bpp)
+
+            tests_completed += 1
+            if new_result != old_result:
+                error_path = ERROR_DIR / (
+                    f"test_{tests_completed}_bpp{bpp}_n{n}_{time.time_ns()}.txt"
+                )
+                error_path.write_bytes(input_path.read_bytes())
+                print(f"mismatch: saved input to {error_path}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/rpmsetinit/mkset.c

@@ -0,0 +1,30 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "set.h"
+
+int main(int argc, const char **argv)
+{
+    assert(argc == 2);
+    int bpp = atoi(argv[1]);
+    assert(bpp >= 10);
+    assert(bpp <= 32);
+    struct set *set = set_new();
+    char *line = NULL;
+    size_t alloc_size = 0;
+    ssize_t len;
+    int added = 0;
+    while ((len = getline(&line, &alloc_size, stdin)) >= 0) {
+	if (len > 0 && line[len-1] == '\n')
+	    line[--len] = '\0';
+	if (len == 0)
+	    continue;
+	set_add(set, line);
+	added++;
+    }
+    assert(added > 0);
+    const char *str = set_fini(set, bpp);
+    assert(str);
+    printf("set:%s\n", str);
+    return 0;
+}