add t1ha2 hash and test results

hash_testing/new_testing/generate_input.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""Generate unique words similar to a given word by applying random mutations."""
+
+from __future__ import annotations
+
+import argparse
+import random
+import string
+import sys
+from collections.abc import Callable, Sequence
+from pathlib import Path
+
+DEFAULT_ALPHABET = string.ascii_letters + string.digits + "_"
+
+OPERATION_ALIASES = {
+    "1": "replace",
+    "replace": "replace",
+    "2": "delete",
+    "delete": "delete",
+    "3": "add",
+    "add": "add",
+    "4": "swap",
+    "swap": "swap",
+    "5": "case",
+    "case": "case",
+    "6": "first",
+    "first": "first",
+    "7": "last",
+    "last": "last",
+}
+
+OPERATION_HELP = """операция изменения:
+  1, replace  заменить случайный символ
+  2, delete   удалить случайный символ
+  3, add      добавить символ в случайную позицию
+  4, swap     переставить два соседних символа
+  5, case     сменить регистр случайного символа
+  6, first    изменить первый символ
+  7, last     изменить последний символ"""
+
+
+class MutationError(ValueError):
+    """Raised when the selected mutation cannot be applied."""
+
+
+def different_character(current: str, alphabet: str, rng: random.Random) -> str:
+    choices = [character for character in alphabet if character != current]
+    if not choices:
+        raise MutationError("алфавит не содержит символа, отличного от заменяемого")
+    return rng.choice(choices)
+
+
+def replace_character(word: str, alphabet: str, rng: random.Random) -> str:
+    if not word:
+        raise MutationError("нельзя заменить символ в пустом слове")
+    index = rng.randrange(len(word))
+    replacement = different_character(word[index], alphabet, rng)
+    return word[:index] + replacement + word[index + 1 :]
+
+
+def delete_character(word: str, _alphabet: str, rng: random.Random) -> str:
+    if not word:
+        raise MutationError("нельзя удалить символ из пустого слова")
+    index = rng.randrange(len(word))
+    return word[:index] + word[index + 1 :]
+
+
+def add_character(word: str, alphabet: str, rng: random.Random) -> str:
+    index = rng.randrange(len(word) + 1)
+    return word[:index] + rng.choice(alphabet) + word[index:]
+
+
+def swap_adjacent(word: str, _alphabet: str, rng: random.Random) -> str:
+    indexes = [
+        index for index in range(len(word) - 1) if word[index] != word[index + 1]
+    ]
+    if not indexes:
+        raise MutationError(
+            "для перестановки нужны хотя бы два соседних различных символа"
+        )
+    index = rng.choice(indexes)
+    return word[:index] + word[index + 1] + word[index] + word[index + 2 :]
+
+
+def change_case(word: str, _alphabet: str, rng: random.Random) -> str:
+    indexes = []
+    replacements: dict[int, str] = {}
+    for index, character in enumerate(word):
+        swapped = character.swapcase()
+        if swapped != character and len(swapped) == 1:
+            indexes.append(index)
+            replacements[index] = swapped
+    if not indexes:
+        raise MutationError("в слове нет символов, у которых можно сменить регистр")
+    index = rng.choice(indexes)
+    return word[:index] + replacements[index] + word[index + 1 :]
+
+
+def change_first(word: str, alphabet: str, rng: random.Random) -> str:
+    if not word:
+        raise MutationError("нельзя изменить первый символ пустого слова")
+    replacement = different_character(word[0], alphabet, rng)
+    return replacement + word[1:]
+
+
+def change_last(word: str, alphabet: str, rng: random.Random) -> str:
+    if not word:
+        raise MutationError("нельзя изменить последний символ пустого слова")
+    replacement = different_character(word[-1], alphabet, rng)
+    return word[:-1] + replacement
+
+
+Mutation = Callable[[str, str, random.Random], str]
+MUTATIONS: dict[str, Mutation] = {
+    "replace": replace_character,
+    "delete": delete_character,
+    "add": add_character,
+    "swap": swap_adjacent,
+    "case": change_case,
+    "first": change_first,
+    "last": change_last,
+}
+
+
+def parse_operation(value: str) -> str:
+    try:
+        return OPERATION_ALIASES[value.lower()]
+    except KeyError as error:
+        valid = ", ".join(OPERATION_ALIASES)
+        raise argparse.ArgumentTypeError(
+            f"неизвестная операция {value!r}; допустимы: {valid}"
+        ) from error
+
+
+def generate_words(
+    source: str,
+    count: int,
+    operation: str,
+    operation_count: int,
+    alphabet: str = DEFAULT_ALPHABET,
+    seed: int | None = None,
+    max_attempts: int | None = None,
+) -> list[str]:
+    """Generate up to ``count`` unique mutations made by exactly N operations."""
+    if count < 1:
+        raise ValueError("количество выходных слов должно быть положительным")
+    if operation_count < 1:
+        raise ValueError("количество операций должно быть положительным")
+    if not alphabet:
+        raise ValueError("алфавит не должен быть пустым")
+
+    mutation = MUTATIONS[operation]
+    rng = random.Random(seed)
+    attempt_limit = max_attempts or max(10_000, count * 1_000)
+    words: list[str] = []
+    seen: set[str] = set()
+
+    for _attempt in range(attempt_limit):
+        candidate = source
+        try:
+            for _ in range(operation_count):
+                candidate = mutation(candidate, alphabet, rng)
+        except MutationError:
+            continue
+
+        if candidate != source and candidate not in seen:
+            seen.add(candidate)
+            words.append(candidate)
+            if len(words) == count:
+                return words
+
+    return words
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Создаёт список уникальных слов, похожих на исходное. "
+            "Каждое слово получается независимо от исходного ровно заданным "
+            "числом случайных операций."
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=(
+            f"{OPERATION_HELP}\n\n"
+            "Пример:\n"
+            "  python3 nexus.py example -o swap -n 20 -k 2 --seed 42"
+        ),
+    )
+    parser.add_argument("word", help="исходное слово")
+    parser.add_argument(
+        "-o", "--operation", required=True, type=parse_operation, help=OPERATION_HELP
+    )
+    parser.add_argument(
+        "-n",
+        "--count",
+        type=int,
+        default=10,
+        help="верхняя граница числа уникальных выходных слов (по умолчанию: 10)",
+    )
+    parser.add_argument(
+        "-k",
+        "--operations",
+        type=int,
+        default=1,
+        help="число операций над каждым словом (по умолчанию: 1)",
+    )
+    parser.add_argument(
+        "--alphabet",
+        default=DEFAULT_ALPHABET,
+        help=("символы для добавления и замены " f"(по умолчанию: {DEFAULT_ALPHABET})"),
+    )
+    parser.add_argument(
+        "--seed", type=int, help="seed генератора для воспроизводимого результата"
+    )
+    parser.add_argument(
+        "--max-attempts",
+        type=int,
+        help="предельное число попыток собрать уникальные слова",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        help="записать слова в файл вместо стандартного вывода",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.max_attempts is not None and args.max_attempts < 1:
+        parser.error("--max-attempts должен быть положительным")
+
+    try:
+        words = generate_words(
+            source=args.word,
+            count=args.count,
+            operation=args.operation,
+            operation_count=args.operations,
+            alphabet=args.alphabet,
+            seed=args.seed,
+            max_attempts=args.max_attempts,
+        )
+    except (MutationError, ValueError) as error:
+        parser.error(str(error))
+
+    if len(words) < args.count:
+        print(
+            f"warning: operation={args.operation}: generated {len(words)} "
+            f"of at most {args.count} unique words",
+            file=sys.stderr,
+        )
+
+    output = "".join(f"{word}\n" for word in words)
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(output, encoding="utf-8")
+    else:
+        sys.stdout.write(output)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

hash_testing/new_testing/hash_funcs/jenkinsOAAT/bin_hash.c

@@ -0,0 +1,109 @@
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define JOAAT_SEED UINT32_C(0x9e3779b9)
+
+static uint32_t jenkins_oaat(const unsigned char *data, size_t length)
+{
+    uint32_t hash = JOAAT_SEED;
+
+    for (size_t index = 0; index < length; ++index) {
+        hash += data[index];
+        hash += hash << 10;
+        hash ^= hash >> 6;
+    }
+
+    hash += hash << 3;
+    hash ^= hash >> 11;
+    hash += hash << 15;
+    return hash;
+}
+
+static int is_ascii_trailing_space(unsigned char character)
+{
+    return character == ' ' || character == '\t' || character == '\n' ||
+           character == '\r' || character == '\v' || character == '\f';
+}
+
+static int read_stdin(unsigned char **data, size_t *length)
+{
+    size_t capacity = 256;
+    unsigned char *buffer = malloc(capacity);
+
+    if (buffer == NULL) {
+        return -1;
+    }
+
+    *length = 0;
+    for (;;) {
+        size_t available = capacity - *length;
+        size_t bytes_read = fread(buffer + *length, 1, available, stdin);
+        *length += bytes_read;
+
+        if (bytes_read < available) {
+            if (ferror(stdin)) {
+                free(buffer);
+                return -1;
+            }
+            break;
+        }
+
+        if (capacity > SIZE_MAX / 2) {
+            free(buffer);
+            return -1;
+        }
+        capacity *= 2;
+
+        unsigned char *larger_buffer = realloc(buffer, capacity);
+        if (larger_buffer == NULL) {
+            free(buffer);
+            return -1;
+        }
+        buffer = larger_buffer;
+    }
+
+    *data = buffer;
+    return 0;
+}
+
+int main(int argc, char **argv)
+{
+    const unsigned char *word;
+    unsigned char *stdin_buffer = NULL;
+    size_t length;
+
+    if (argc > 2) {
+        fprintf(stderr, "usage: %s [ASCII_WORD]\n", argv[0]);
+        return EXIT_FAILURE;
+    }
+
+    if (argc == 2) {
+        word = (const unsigned char *)argv[1];
+        length = strlen(argv[1]);
+    } else {
+        if (read_stdin(&stdin_buffer, &length) != 0) {
+            fprintf(stderr, "failed to read input\n");
+            return EXIT_FAILURE;
+        }
+        word = stdin_buffer;
+    }
+
+    while (length > 0 && is_ascii_trailing_space(word[length - 1])) {
+        --length;
+    }
+
+    for (size_t index = 0; index < length; ++index) {
+        if (word[index] > 0x7f) {
+            fprintf(stderr, "input must contain ASCII characters only\n");
+            free(stdin_buffer);
+            return EXIT_FAILURE;
+        }
+    }
+
+    printf("%08" PRIx32 "\n", jenkins_oaat(word, length));
+    free(stdin_buffer);
+    return EXIT_SUCCESS;
+}

hash_testing/new_testing/hash_funcs/jenkinsOAAT/hash.c

@@ -0,0 +1,13 @@
+unsigned int hash(const char* str) {
+  unsigned int hash = 0x9e3779b9;
+  const unsigned char* p = (const unsigned char*)str;
+  while (*p) {
+    hash += *p++;
+    hash += (hash << 10);
+    hash ^= (hash >> 6);
+  }
+  hash += (hash << 3);
+  hash ^= (hash >> 11);
+  hash += (hash << 15);
+  return hash;
+}

hash_testing/new_testing/hash_funcs/t1ha2/bin_hash.c

@@ -0,0 +1,113 @@
+/*
+ * Standalone t1ha2_atonce command-line wrapper for avalanche testing.
+ *
+ * Upstream: https://gitflic.ru/project/erthink/t1ha
+ * Commit: 00eb779b6c042ccd831ec2f1ae757409c73f39f6
+ * Algorithm: t1ha2_atonce(data, length, seed=0), stable portable 64-bit mode.
+ *
+ * The vendored upstream implementation is licensed under the zlib License;
+ * see upstream/LICENSE. This wrapper is an altered integration file and is not
+ * represented as an original upstream source file.
+ */
+
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define T1HA0_DISABLED
+#define T1HA1_DISABLED
+#define T1HA_SYS_UNALIGNED_ACCESS 0
+#define T1HA_USE_FAST_ONESHOT_READ 0
+#include "upstream/src/t1ha2.c"
+
+#define T1HA2_SEED UINT64_C(0)
+
+static int is_ascii_trailing_space(unsigned char character)
+{
+    return character == ' ' || character == '\t' || character == '\n' ||
+           character == '\r' || character == '\v' || character == '\f';
+}
+
+static int read_stdin(unsigned char **data, size_t *length)
+{
+    size_t capacity = 256;
+    unsigned char *buffer = malloc(capacity);
+
+    if (buffer == NULL) {
+        return -1;
+    }
+
+    *length = 0;
+    for (;;) {
+        size_t available = capacity - *length;
+        size_t bytes_read = fread(buffer + *length, 1, available, stdin);
+        *length += bytes_read;
+
+        if (bytes_read < available) {
+            if (ferror(stdin)) {
+                free(buffer);
+                return -1;
+            }
+            break;
+        }
+
+        if (capacity > SIZE_MAX / 2U) {
+            free(buffer);
+            return -1;
+        }
+        capacity *= 2U;
+
+        {
+            unsigned char *larger_buffer = realloc(buffer, capacity);
+            if (larger_buffer == NULL) {
+                free(buffer);
+                return -1;
+            }
+            buffer = larger_buffer;
+        }
+    }
+
+    *data = buffer;
+    return 0;
+}
+
+int main(int argc, char **argv)
+{
+    const unsigned char *word;
+    unsigned char *stdin_buffer = NULL;
+    size_t length;
+
+    if (argc > 2) {
+        fprintf(stderr, "usage: %s [ASCII_WORD]\n", argv[0]);
+        return EXIT_FAILURE;
+    }
+
+    if (argc == 2) {
+        word = (const unsigned char *)argv[1];
+        length = strlen(argv[1]);
+    } else {
+        if (read_stdin(&stdin_buffer, &length) != 0) {
+            fprintf(stderr, "failed to read input\n");
+            return EXIT_FAILURE;
+        }
+        word = stdin_buffer;
+    }
+
+    while (length > 0U && is_ascii_trailing_space(word[length - 1U])) {
+        --length;
+    }
+
+    for (size_t index = 0; index < length; ++index) {
+        if (word[index] > 0x7fU) {
+            fprintf(stderr, "input must contain ASCII characters only\n");
+            free(stdin_buffer);
+            return EXIT_FAILURE;
+        }
+    }
+
+    printf("%016" PRIx64 "\n", t1ha2_atonce(word, length, T1HA2_SEED));
+    free(stdin_buffer);
+    return EXIT_SUCCESS;
+}

hash_testing/new_testing/hash_funcs/t1ha2/upstream/LICENSE

@@ -0,0 +1,23 @@
+ zlib License, see https://en.wikipedia.org/wiki/Zlib_License
+
+ Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com,
+ Fast Positive Hash.
+
+ Portions Copyright (c) 2010-2013 Leonid Yuriev <leo@yuriev.ru>,
+ The 1Hippeus project (t1h).
+
+ This software is provided 'as-is', without any express or implied
+ warranty. In no event will the authors be held liable for any damages
+ arising from the use of this software.
+
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+
+ 1. The origin of this software must not be misrepresented; you must not
+    claim that you wrote the original software. If you use this software
+    in a product, an acknowledgement in the product documentation would be
+    appreciated but is not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+    misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.

hash_testing/new_testing/hash_funcs/t1ha2/upstream/src/t1ha2.c

@@ -0,0 +1,383 @@
+/*
+ *  Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com,
+ *  Fast Positive Hash.
+ *
+ *  Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>,
+ *  The 1Hippeus project (t1h).
+ *
+ *  This software is provided 'as-is', without any express or implied
+ *  warranty. In no event will the authors be held liable for any damages
+ *  arising from the use of this software.
+ *
+ *  Permission is granted to anyone to use this software for any purpose,
+ *  including commercial applications, and to alter it and redistribute it
+ *  freely, subject to the following restrictions:
+ *
+ *  1. The origin of this software must not be misrepresented; you must not
+ *     claim that you wrote the original software. If you use this software
+ *     in a product, an acknowledgement in the product documentation would be
+ *     appreciated but is not required.
+ *  2. Altered source versions must be plainly marked as such, and must not be
+ *     misrepresented as being the original software.
+ *  3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ *  1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ *     but portable and without penalties it can run on any 64-bit CPU.
+ *  2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ *     and all others portable hash-functions (which do not use specific
+ *     hardware tricks).
+ *  3. Not suitable for cryptography.
+ *
+ * The Future will (be) Positive. Всё будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#ifndef T1HA2_DISABLED
+#include "t1ha_bits.h"
+#include "t1ha_selfcheck.h"
+
+static __always_inline void init_ab(t1ha_state256_t *s, uint64_t x,
+                                    uint64_t y) {
+  s->n.a = x;
+  s->n.b = y;
+}
+
+static __always_inline void init_cd(t1ha_state256_t *s, uint64_t x,
+                                    uint64_t y) {
+  s->n.c = rot64(y, 23) + ~x;
+  s->n.d = ~y + rot64(x, 19);
+}
+
+/* TODO: C++ template in the next version */
+#define T1HA2_UPDATE(ENDIANNES, ALIGNESS, state, v)                            \
+  do {                                                                         \
+    t1ha_state256_t *const s = state;                                          \
+    const uint64_t w0 = fetch64_##ENDIANNES##_##ALIGNESS(v + 0);               \
+    const uint64_t w1 = fetch64_##ENDIANNES##_##ALIGNESS(v + 1);               \
+    const uint64_t w2 = fetch64_##ENDIANNES##_##ALIGNESS(v + 2);               \
+    const uint64_t w3 = fetch64_##ENDIANNES##_##ALIGNESS(v + 3);               \
+                                                                               \
+    const uint64_t d02 = w0 + rot64(w2 + s->n.d, 56);                          \
+    const uint64_t c13 = w1 + rot64(w3 + s->n.c, 19);                          \
+    s->n.d ^= s->n.b + rot64(w1, 38);                                          \
+    s->n.c ^= s->n.a + rot64(w0, 57);                                          \
+    s->n.b ^= prime_6 * (c13 + w2);                                            \
+    s->n.a ^= prime_5 * (d02 + w3);                                            \
+  } while (0)
+
+static __always_inline void squash(t1ha_state256_t *s) {
+  s->n.a ^= prime_6 * (s->n.c + rot64(s->n.d, 23));
+  s->n.b ^= prime_5 * (rot64(s->n.c, 19) + s->n.d);
+}
+
+/* TODO: C++ template in the next version */
+#define T1HA2_LOOP(ENDIANNES, ALIGNESS, state, data, len)                      \
+  do {                                                                         \
+    const void *detent = (const uint8_t *)data + len - 31;                     \
+    do {                                                                       \
+      const uint64_t *v = (const uint64_t *)data;                              \
+      data = (const uint64_t *)data + 4;                                       \
+      prefetch(data);                                                          \
+      T1HA2_UPDATE(le, ALIGNESS, state, v);                                    \
+    } while (likely(data < detent));                                           \
+  } while (0)
+
+/* TODO: C++ template in the next version */
+#define T1HA2_TAIL_AB(ENDIANNES, ALIGNESS, state, data, len)                   \
+  do {                                                                         \
+    t1ha_state256_t *const s = state;                                          \
+    const uint64_t *v = (const uint64_t *)data;                                \
+    switch (len) {                                                             \
+    default:                                                                   \
+      mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++),         \
+              prime_4);                                                        \
+    /* fall through */                                                         \
+    case 24:                                                                   \
+    case 23:                                                                   \
+    case 22:                                                                   \
+    case 21:                                                                   \
+    case 20:                                                                   \
+    case 19:                                                                   \
+    case 18:                                                                   \
+    case 17:                                                                   \
+      mixup64(&s->n.b, &s->n.a, fetch64_##ENDIANNES##_##ALIGNESS(v++),         \
+              prime_3);                                                        \
+    /* fall through */                                                         \
+    case 16:                                                                   \
+    case 15:                                                                   \
+    case 14:                                                                   \
+    case 13:                                                                   \
+    case 12:                                                                   \
+    case 11:                                                                   \
+    case 10:                                                                   \
+    case 9:                                                                    \
+      mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++),         \
+              prime_2);                                                        \
+    /* fall through */                                                         \
+    case 8:                                                                    \
+    case 7:                                                                    \
+    case 6:                                                                    \
+    case 5:                                                                    \
+    case 4:                                                                    \
+    case 3:                                                                    \
+    case 2:                                                                    \
+    case 1:                                                                    \
+      mixup64(&s->n.b, &s->n.a, tail64_##ENDIANNES##_##ALIGNESS(v, len),       \
+              prime_1);                                                        \
+    /* fall through */                                                         \
+    case 0:                                                                    \
+      return final64(s->n.a, s->n.b);                                          \
+    }                                                                          \
+  } while (0)
+
+/* TODO: C++ template in the next version */
+#define T1HA2_TAIL_ABCD(ENDIANNES, ALIGNESS, state, data, len)                 \
+  do {                                                                         \
+    t1ha_state256_t *const s = state;                                          \
+    const uint64_t *v = (const uint64_t *)data;                                \
+    switch (len) {                                                             \
+    default:                                                                   \
+      mixup64(&s->n.a, &s->n.d, fetch64_##ENDIANNES##_##ALIGNESS(v++),         \
+              prime_4);                                                        \
+    /* fall through */                                                         \
+    case 24:                                                                   \
+    case 23:                                                                   \
+    case 22:                                                                   \
+    case 21:                                                                   \
+    case 20:                                                                   \
+    case 19:                                                                   \
+    case 18:                                                                   \
+    case 17:                                                                   \
+      mixup64(&s->n.b, &s->n.a, fetch64_##ENDIANNES##_##ALIGNESS(v++),         \
+              prime_3);                                                        \
+    /* fall through */                                                         \
+    case 16:                                                                   \
+    case 15:                                                                   \
+    case 14:                                                                   \
+    case 13:                                                                   \
+    case 12:                                                                   \
+    case 11:                                                                   \
+    case 10:                                                                   \
+    case 9:                                                                    \
+      mixup64(&s->n.c, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++),         \
+              prime_2);                                                        \
+    /* fall through */                                                         \
+    case 8:                                                                    \
+    case 7:                                                                    \
+    case 6:                                                                    \
+    case 5:                                                                    \
+    case 4:                                                                    \
+    case 3:                                                                    \
+    case 2:                                                                    \
+    case 1:                                                                    \
+      mixup64(&s->n.d, &s->n.c, tail64_##ENDIANNES##_##ALIGNESS(v, len),       \
+              prime_1);                                                        \
+    /* fall through */                                                         \
+    case 0:                                                                    \
+      return final128(s->n.a, s->n.b, s->n.c, s->n.d, extra_result);           \
+    }                                                                          \
+  } while (0)
+
+static __always_inline uint64_t final128(uint64_t a, uint64_t b, uint64_t c,
+                                         uint64_t d, uint64_t *h) {
+  mixup64(&a, &b, rot64(c, 41) ^ d, prime_0);
+  mixup64(&b, &c, rot64(d, 23) ^ a, prime_6);
+  mixup64(&c, &d, rot64(a, 19) ^ b, prime_5);
+  mixup64(&d, &a, rot64(b, 31) ^ c, prime_4);
+  *h = c + d;
+  return a ^ b;
+}
+
+//------------------------------------------------------------------------------
+
+uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed) {
+  t1ha_state256_t state;
+  init_ab(&state, seed, length);
+
+#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT
+  if (unlikely(length > 32)) {
+    init_cd(&state, seed, length);
+#if defined(__LCC__) && __LCC__ > 123
+/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
+ * в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
+ * говорят, что это нецелесообразно */
+#pragma comb_oper
+#endif /* E2K LCC > 1.23 */
+    T1HA2_LOOP(le, unaligned, &state, data, length);
+    squash(&state);
+    length &= 31;
+  }
+  T1HA2_TAIL_AB(le, unaligned, &state, data, length);
+#else
+  const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+  if (misaligned) {
+    if (unlikely(length > 32)) {
+      init_cd(&state, seed, length);
+#if defined(__LCC__) && __LCC__ > 123
+/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
+ * в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
+ * говорят, что это нецелесообразно */
+#pragma comb_oper
+#endif /* E2K LCC > 1.23 */
+      T1HA2_LOOP(le, unaligned, &state, data, length);
+      squash(&state);
+      length &= 31;
+    }
+    T1HA2_TAIL_AB(le, unaligned, &state, data, length);
+  } else {
+    if (unlikely(length > 32)) {
+      init_cd(&state, seed, length);
+#if defined(__LCC__) && __LCC__ > 123
+/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
+ * в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
+ * говорят, что это нецелесообразно */
+#pragma comb_oper
+#endif /* E2K LCC > 1.23 */
+      T1HA2_LOOP(le, aligned, &state, data, length);
+      squash(&state);
+      length &= 31;
+    }
+    T1HA2_TAIL_AB(le, aligned, &state, data, length);
+  }
+#endif
+}
+
+uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
+                         const void *__restrict data, size_t length,
+                         uint64_t seed) {
+  t1ha_state256_t state;
+  init_ab(&state, seed, length);
+  init_cd(&state, seed, length);
+
+#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT
+  if (unlikely(length > 32)) {
+#if defined(__LCC__) && __LCC__ > 123
+/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
+ * в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
+ * говорят, что это нецелесообразно */
+#pragma comb_oper
+#endif /* E2K LCC > 1.23 */
+    T1HA2_LOOP(le, unaligned, &state, data, length);
+    length &= 31;
+  }
+  T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
+#else
+  const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+  if (misaligned) {
+    if (unlikely(length > 32)) {
+#if defined(__LCC__) && __LCC__ > 123
+/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
+ * в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
+ * говорят, что это нецелесообразно */
+#pragma comb_oper
+#endif /* E2K LCC > 1.23 */
+      T1HA2_LOOP(le, unaligned, &state, data, length);
+      length &= 31;
+    }
+    T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
+  } else {
+    if (unlikely(length > 32)) {
+#if defined(__LCC__) && __LCC__ > 123
+/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
+ * в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
+ * говорят, что это нецелесообразно */
+#pragma comb_oper
+#endif /* E2K LCC > 1.23 */
+      T1HA2_LOOP(le, aligned, &state, data, length);
+      length &= 31;
+    }
+    T1HA2_TAIL_ABCD(le, aligned, &state, data, length);
+  }
+#endif
+}
+
+//------------------------------------------------------------------------------
+
+void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y) {
+  init_ab(&ctx->state, seed_x, seed_y);
+  init_cd(&ctx->state, seed_x, seed_y);
+  ctx->partial = 0;
+  ctx->total = 0;
+}
+
+void t1ha2_update(t1ha_context_t *__restrict ctx, const void *__restrict data,
+                  size_t length) {
+  ctx->total += length;
+
+  if (ctx->partial) {
+    const size_t left = 32 - ctx->partial;
+    const size_t chunk = (length >= left) ? left : length;
+    memcpy(ctx->buffer.bytes + ctx->partial, data, chunk);
+    ctx->partial += chunk;
+    if (ctx->partial < 32) {
+      assert(left >= length);
+      return;
+    }
+    ctx->partial = 0;
+    data = (const uint8_t *)data + chunk;
+    length -= chunk;
+    T1HA2_UPDATE(le, aligned, &ctx->state, ctx->buffer.u64);
+  }
+
+  if (length >= 32) {
+#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT
+#if defined(__LCC__) && __LCC__ > 123
+/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
+ * в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
+ * говорят, что это нецелесообразно */
+#pragma comb_oper
+#endif /* E2K LCC > 1.23 */
+    T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
+#else
+    const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
+    if (misaligned) {
+#if defined(__LCC__) && __LCC__ > 123
+/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
+ * в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
+ * говорят, что это нецелесообразно */
+#pragma comb_oper
+#endif /* E2K LCC > 1.23 */
+      T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
+    } else {
+#if defined(__LCC__) && __LCC__ > 123
+/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
+ * в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
+ * говорят, что это нецелесообразно */
+#pragma comb_oper
+#endif /* E2K LCC > 1.23 */
+      T1HA2_LOOP(le, aligned, &ctx->state, data, length);
+    }
+#endif
+    length &= 31;
+  }
+
+  if (length)
+    memcpy(ctx->buffer.bytes, data, ctx->partial = length);
+}
+
+uint64_t t1ha2_final(t1ha_context_t *__restrict ctx,
+                     uint64_t *__restrict extra_result) {
+  uint64_t bits = (ctx->total << 3) ^ (UINT64_C(1) << 63);
+#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
+  bits = bswap64(bits);
+#endif
+  t1ha2_update(ctx, &bits, 8);
+
+  if (likely(!extra_result)) {
+    squash(&ctx->state);
+    T1HA2_TAIL_AB(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
+  }
+
+  T1HA2_TAIL_ABCD(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
+}
+
+#endif /* T1HA2_DISABLED */

hash_testing/new_testing/hash_funcs/t1ha2/upstream/src/t1ha_selfcheck.h

@@ -0,0 +1,76 @@
+/*
+ *  Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com,
+ *  Fast Positive Hash.
+ *
+ *  Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>,
+ *  The 1Hippeus project (t1h).
+ *
+ *  This software is provided 'as-is', without any express or implied
+ *  warranty. In no event will the authors be held liable for any damages
+ *  arising from the use of this software.
+ *
+ *  Permission is granted to anyone to use this software for any purpose,
+ *  including commercial applications, and to alter it and redistribute it
+ *  freely, subject to the following restrictions:
+ *
+ *  1. The origin of this software must not be misrepresented; you must not
+ *     claim that you wrote the original software. If you use this software
+ *     in a product, an acknowledgement in the product documentation would be
+ *     appreciated but is not required.
+ *  2. Altered source versions must be plainly marked as such, and must not be
+ *     misrepresented as being the original software.
+ *  3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ *  1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ *     but portable and without penalties it can run on any 64-bit CPU.
+ *  2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ *     and all others portable hash-functions (which do not use specific
+ *     hardware tricks).
+ *  3. Not suitable for cryptography.
+ *
+ * The Future will (be) Positive. Всё будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#pragma once
+#if defined(_MSC_VER) && _MSC_VER > 1800
+#pragma warning(disable : 4464) /* relative include path contains '..' */
+#endif                          /* MSVC */
+#include "../t1ha.h"
+
+/***************************************************************************/
+/* Self-checking */
+
+extern const uint8_t t1ha_test_pattern[64];
+int t1ha_selfcheck(uint64_t (*hash)(const void *, size_t, uint64_t),
+                   const uint64_t *reference_values);
+
+#ifndef T1HA2_DISABLED
+extern const uint64_t t1ha_refval_2atonce[81];
+extern const uint64_t t1ha_refval_2atonce128[81];
+extern const uint64_t t1ha_refval_2stream[81];
+extern const uint64_t t1ha_refval_2stream128[81];
+#endif /* T1HA2_DISABLED */
+
+#ifndef T1HA1_DISABLED
+extern const uint64_t t1ha_refval_64le[81];
+extern const uint64_t t1ha_refval_64be[81];
+#endif /* T1HA1_DISABLED */
+
+#ifndef T1HA0_DISABLED
+extern const uint64_t t1ha_refval_32le[81];
+extern const uint64_t t1ha_refval_32be[81];
+#if T1HA0_AESNI_AVAILABLE
+extern const uint64_t t1ha_refval_ia32aes_a[81];
+extern const uint64_t t1ha_refval_ia32aes_b[81];
+#endif /* T1HA0_AESNI_AVAILABLE */
+#endif /* T1HA0_DISABLED */

hash_testing/new_testing/hash_funcs/t1ha2/upstream/t1ha.h

@@ -0,0 +1,719 @@
+/*
+ *  Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com,
+ *  Fast Positive Hash.
+ *
+ *  Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>,
+ *  The 1Hippeus project (t1h).
+ *
+ *  This software is provided 'as-is', without any express or implied
+ *  warranty. In no event will the authors be held liable for any damages
+ *  arising from the use of this software.
+ *
+ *  Permission is granted to anyone to use this software for any purpose,
+ *  including commercial applications, and to alter it and redistribute it
+ *  freely, subject to the following restrictions:
+ *
+ *  1. The origin of this software must not be misrepresented; you must not
+ *     claim that you wrote the original software. If you use this software
+ *     in a product, an acknowledgement in the product documentation would be
+ *     appreciated but is not required.
+ *  2. Altered source versions must be plainly marked as such, and must not be
+ *     misrepresented as being the original software.
+ *  3. This notice may not be removed or altered from any source distribution.
+ */
+
+/*
+ * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
+ * by [Positive Technologies](https://www.ptsecurity.ru)
+ *
+ * Briefly, it is a 64-bit Hash Function:
+ *  1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
+ *     but portable and without penalties it can run on any 64-bit CPU.
+ *  2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
+ *     and all others portable hash-functions (which do not use specific
+ *     hardware tricks).
+ *  3. Not suitable for cryptography.
+ *
+ * The Future will (be) Positive. Всё будет хорошо.
+ *
+ * ACKNOWLEDGEMENT:
+ * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
+ * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
+ */
+
+#pragma once
+
+/*****************************************************************************
+ *
+ * PLEASE PAY ATTENTION TO THE FOLLOWING NOTES
+ * about macros definitions which controls t1ha behaviour and/or performance.
+ *
+ *
+ * 1) T1HA_SYS_UNALIGNED_ACCESS = Defines the system/platform/CPU/architecture
+ *                                abilities for unaligned data access.
+ *
+ *    By default, when the T1HA_SYS_UNALIGNED_ACCESS not defined,
+ *    it will defined on the basis hardcoded knowledge about of capabilities
+ *    of most common CPU architectures. But you could override this
+ *    default behavior when build t1ha library itself:
+ *
+ *      // To disable unaligned access at all.
+ *      #define T1HA_SYS_UNALIGNED_ACCESS 0
+ *
+ *      // To enable unaligned access, but indicate that it significantly slow.
+ *      #define T1HA_SYS_UNALIGNED_ACCESS 1
+ *
+ *      // To enable unaligned access, and indicate that it effecient.
+ *      #define T1HA_SYS_UNALIGNED_ACCESS 2
+ *
+ *
+ * 2) T1HA_USE_FAST_ONESHOT_READ = Controls the data reads at the end of buffer.
+ *
+ *    When defined to non-zero, t1ha will use 'one shot' method for reading
+ *    up to 8 bytes at the end of data. In this case just the one 64-bit read
+ *    will be performed even when the available less than 8 bytes.
+ *
+ *    This is little bit faster that switching by length of data tail.
+ *    Unfortunately this will triggering a false-positive alarms from Valgrind,
+ *    AddressSanitizer and other similar tool.
+ *
+ *    By default, t1ha defines it to 1, but you could override this
+ *    default behavior when build t1ha library itself:
+ *
+ *      // For little bit faster and small code.
+ *      #define T1HA_USE_FAST_ONESHOT_READ 1
+ *
+ *      // For calmness if doubt.
+ *      #define T1HA_USE_FAST_ONESHOT_READ 0
+ *
+ *
+ * 3) T1HA0_RUNTIME_SELECT = Controls choice fastest function in runtime.
+ *
+ *    t1ha library offers the t1ha0() function as the fastest for current CPU.
+ *    But actual CPU's features/capabilities and may be significantly different,
+ *    especially on x86 platform. Therefore, internally, t1ha0() may require
+ *    dynamic dispatching for choice best implementation.
+ *
+ *    By default, t1ha enables such runtime choice and (may be) corresponding
+ *    indirect calls if it reasonable, but you could override this default
+ *    behavior when build t1ha library itself:
+ *
+ *      // To enable runtime choice of fastest implementation.
+ *      #define T1HA0_RUNTIME_SELECT 1
+ *
+ *      // To disable runtime choice of fastest implementation.
+ *      #define T1HA0_RUNTIME_SELECT 0
+ *
+ *    When T1HA0_RUNTIME_SELECT is nonzero the t1ha0_resolve() function could
+ *    be used to get actual t1ha0() implementation address at runtime. This is
+ *    useful for two cases:
+ *      - calling by local pointer-to-function usually is little
+ *        bit faster (less overhead) than via a PLT thru the DSO boundary.
+ *      - GNU Indirect functions (see below) don't supported by environment
+ *        and calling by t1ha0_funcptr is not available and/or expensive.
+ *
+ * 4) T1HA_USE_INDIRECT_FUNCTIONS = Controls usage of GNU Indirect functions.
+ *
+ *    In continue of T1HA0_RUNTIME_SELECT the T1HA_USE_INDIRECT_FUNCTIONS
+ *    controls usage of ELF indirect functions feature. In general, when
+ *    available, this reduces overhead of indirect function's calls though
+ *    a DSO-bundary (https://sourceware.org/glibc/wiki/GNU_IFUNC).
+ *
+ *    By default, t1ha engage GNU Indirect functions when it available
+ *    and useful, but you could override this default behavior when build
+ *    t1ha library itself:
+ *
+ *      // To enable use of GNU ELF Indirect functions.
+ *      #define T1HA_USE_INDIRECT_FUNCTIONS 1
+ *
+ *      // To disable use of GNU ELF Indirect functions. This may be useful
+ *      // if the actual toolchain or the system's loader don't support ones.
+ *      #define T1HA_USE_INDIRECT_FUNCTIONS 0
+ *
+ * 5) T1HA0_AESNI_AVAILABLE = Controls AES-NI detection and dispatching on x86.
+ *
+ *    In continue of T1HA0_RUNTIME_SELECT the T1HA0_AESNI_AVAILABLE controls
+ *    detection and usage of AES-NI CPU's feature. On the other hand, this
+ *    requires compiling parts of t1ha library with certain properly options,
+ *    and could be difficult or inconvenient in some cases.
+ *
+ *    By default, t1ha engade AES-NI for t1ha0() on the x86 platform, but
+ *    you could override this default behavior when build t1ha library itself:
+ *
+ *      // To disable detection and usage of AES-NI instructions for t1ha0().
+ *      // This may be useful when you unable to build t1ha library properly
+ *      // or known that AES-NI will be unavailable at the deploy.
+ *      #define T1HA0_AESNI_AVAILABLE 0
+ *
+ *      // To force detection and usage of AES-NI instructions for t1ha0(),
+ *      // but I don't known reasons to anybody would need this.
+ *      #define T1HA0_AESNI_AVAILABLE 1
+ *
+ * 6) T1HA0_DISABLED, T1HA1_DISABLED, T1HA2_DISABLED = Controls availability of
+ *    t1ha functions.
+ *
+ *    In some cases could be useful to import/use only few of t1ha functions
+ *    or just the one. So, this definitions allows disable corresponding parts
+ *    of t1ha library.
+ *
+ *      // To disable t1ha0(), t1ha0_32le(), t1ha0_32be() and all AES-NI.
+ *      #define T1HA0_DISABLED
+ *
+ *      // To disable t1ha1_le() and t1ha1_be().
+ *      #define T1HA1_DISABLED
+ *
+ *      // To disable t1ha2_atonce(), t1ha2_atonce128() and so on.
+ *      #define T1HA2_DISABLED
+ *
+ *****************************************************************************/
+
+#define T1HA_VERSION_MAJOR 2
+#define T1HA_VERSION_MINOR 1
+#define T1HA_VERSION_RELEASE 1
+
+#ifndef __has_attribute
+#define __has_attribute(x) (0)
+#endif
+
+#ifndef __has_include
+#define __has_include(x) (0)
+#endif
+
+#ifndef __GNUC_PREREQ
+#if defined(__GNUC__) && defined(__GNUC_MINOR__)
+#define __GNUC_PREREQ(maj, min)                                                \
+  ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+#else
+#define __GNUC_PREREQ(maj, min) 0
+#endif
+#endif /* __GNUC_PREREQ */
+
+#ifndef __CLANG_PREREQ
+#ifdef __clang__
+#define __CLANG_PREREQ(maj, min)                                               \
+  ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
+#else
+#define __CLANG_PREREQ(maj, min) (0)
+#endif
+#endif /* __CLANG_PREREQ */
+
+#ifndef __LCC_PREREQ
+#ifdef __LCC__
+#define __LCC_PREREQ(maj, min)                                                 \
+  ((__LCC__ << 16) + __LCC_MINOR__ >= ((maj) << 16) + (min))
+#else
+#define __LCC_PREREQ(maj, min) (0)
+#endif
+#endif /* __LCC_PREREQ */
+
+/*****************************************************************************/
+
+#ifdef _MSC_VER
+/* Avoid '16' bytes padding added after data member 't1ha_context::total'
+ * and other warnings from std-headers if warning-level > 3. */
+#pragma warning(push, 3)
+#endif
+
+#if defined(__cplusplus) && __cplusplus >= 201103L
+#include <climits>
+#include <cstddef>
+#include <cstdint>
+#else
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#endif
+
+/*****************************************************************************/
+
+#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
+    defined(i486) || defined(__i486) || defined(__i486__) ||                   \
+    defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) ||   \
+    defined(__i686) || defined(__i686__) || defined(_M_IX86) ||                \
+    defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) ||            \
+    defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) ||          \
+    defined(__amd64__) || defined(__amd64) || defined(_M_X64) ||               \
+    defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
+#ifndef __ia32__
+/* LY: define neutral __ia32__ for x86 and x86-64 archs */
+#define __ia32__ 1
+#endif /* __ia32__ */
+#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) ||        \
+                            defined(__amd64) || defined(_M_X64))
+/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
+#define __amd64__ 1
+#endif /* __amd64__ */
+#endif /* all x86 */
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) ||           \
+    !defined(__ORDER_BIG_ENDIAN__)
+
+/* *INDENT-OFF* */
+/* clang-format off */
+
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) ||  \
+    defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
+#include <endian.h>
+#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) ||       \
+    defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
+#include <machine/endian.h>
+#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
+#include <sys/isa_defs.h>
+#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) ||             \
+    (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
+#include <sys/endian.h>
+#include <sys/types.h>
+#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) ||   \
+    defined(__NETBSD__) || defined(__NetBSD__) ||                              \
+    defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif /* OS */
+
+/* *INDENT-ON* */
+/* clang-format on */
+
+#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
+#define __BYTE_ORDER__ __BYTE_ORDER
+#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
+#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
+#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
+#define __BYTE_ORDER__ _BYTE_ORDER
+#else
+#define __ORDER_LITTLE_ENDIAN__ 1234
+#define __ORDER_BIG_ENDIAN__ 4321
+
+#if defined(__LITTLE_ENDIAN__) ||                                              \
+    (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) ||                      \
+    defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) ||    \
+    defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) ||            \
+    defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) ||                \
+    defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) ||   \
+    defined(__BFIN__) || defined(__ia64__) || defined(_IA64) ||                \
+    defined(__IA64__) || defined(__ia64) || defined(_M_IA64) ||                \
+    defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) ||        \
+    defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) ||              \
+    defined(__WINDOWS__)
+#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
+
+#elif defined(__BIG_ENDIAN__) ||                                               \
+    (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) ||                      \
+    defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) ||    \
+    defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) ||            \
+    defined(__m68k__) || defined(M68000) || defined(__hppa__) ||               \
+    defined(__hppa) || defined(__HPPA__) || defined(__sparc__) ||              \
+    defined(__sparc) || defined(__370__) || defined(__THW_370__) ||            \
+    defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
+#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
+
+#else
+#error __BYTE_ORDER__ should be defined.
+#endif /* Arch */
+
+#endif
+#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
+
+/*****************************************************************************/
+
+#ifndef __dll_export
+#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__)
+#if defined(__GNUC__) || __has_attribute(dllexport)
+#define __dll_export __attribute__((dllexport))
+#else
+#define __dll_export __declspec(dllexport)
+#endif
+#elif defined(__GNUC__) || __has_attribute(__visibility__)
+#define __dll_export __attribute__((__visibility__("default")))
+#else
+#define __dll_export
+#endif
+#endif /* __dll_export */
+
+#ifndef __dll_import
+#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__)
+#if defined(__GNUC__) || __has_attribute(dllimport)
+#define __dll_import __attribute__((dllimport))
+#else
+#define __dll_import __declspec(dllimport)
+#endif
+#elif defined(__GNUC__) || __has_attribute(__visibility__)
+#define __dll_import __attribute__((__visibility__("default")))
+#else
+#define __dll_import
+#endif
+#endif /* __dll_import */
+
+#ifndef __force_inline
+#ifdef _MSC_VER
+#define __force_inline __forceinline
+#elif __GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__)
+#define __force_inline __inline __attribute__((__always_inline__))
+#else
+#define __force_inline __inline
+#endif
+#endif /* __force_inline */
+
+#ifndef T1HA_API
+#if defined(t1ha_EXPORTS)
+#define T1HA_API __dll_export
+#elif defined(t1ha_IMPORTS)
+#define T1HA_API __dll_import
+#else
+#define T1HA_API
+#endif
+#endif /* T1HA_API */
+
+#if defined(_MSC_VER) && defined(__ia32__)
+#define T1HA_ALIGN_PREFIX __declspec(align(32)) /* required only for SIMD */
+#else
+#define T1HA_ALIGN_PREFIX
+#endif /* _MSC_VER */
+
+#if defined(__GNUC__) && defined(__ia32__)
+#define T1HA_ALIGN_SUFFIX                                                      \
+  __attribute__((__aligned__(32))) /* required only for SIMD */
+#else
+#define T1HA_ALIGN_SUFFIX
+#endif /* GCC x86 */
+
+#ifndef T1HA_USE_INDIRECT_FUNCTIONS
+/* GNU ELF indirect functions usage control. For more info please see
+ * https://en.wikipedia.org/wiki/Executable_and_Linkable_Format
+ * and https://sourceware.org/glibc/wiki/GNU_IFUNC */
+#if defined(__ELF__) && defined(__amd64__) &&                                  \
+    (__has_attribute(__ifunc__) ||                                             \
+     (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 &&             \
+      !defined(__SANITIZE_ADDRESS__) && !defined(__SSP_ALL__)))
+/* Enable gnu_indirect_function by default if :
+ *  - ELF AND x86_64
+ *  - attribute(__ifunc__) is available OR
+ *    GCC >= 4 WITHOUT -fsanitize=address NOR -fstack-protector-all */
+#define T1HA_USE_INDIRECT_FUNCTIONS 1
+#else
+#define T1HA_USE_INDIRECT_FUNCTIONS 0
+#endif
+#endif /* T1HA_USE_INDIRECT_FUNCTIONS */
+
+#if __GNUC_PREREQ(4, 0)
+#pragma GCC visibility push(hidden)
+#endif /* __GNUC_PREREQ(4,0) */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef union T1HA_ALIGN_PREFIX t1ha_state256 {
+  uint8_t bytes[32];
+  uint32_t u32[8];
+  uint64_t u64[4];
+  struct {
+    uint64_t a, b, c, d;
+  } n;
+} t1ha_state256_t T1HA_ALIGN_SUFFIX;
+
+typedef struct t1ha_context {
+  t1ha_state256_t state;
+  t1ha_state256_t buffer;
+  size_t partial;
+  uint64_t total;
+} t1ha_context_t;
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+/******************************************************************************
+ *
+ * Self-testing API.
+ *
+ * Unfortunately, some compilers (exactly only Microsoft Visual C/C++) has
+ * a bugs which leads t1ha-functions to produce wrong results. This API allows
+ * check the correctness of the actual code in runtime.
+ *
+ * All check-functions returns 0 on success, or -1 in case the corresponding
+ * hash-function failed verification. PLEASE, always perform such checking at
+ * initialization of your code, if you using MSVC or other troubleful compilers.
+ */
+
+T1HA_API int t1ha_selfcheck__all_enabled(void);
+
+#ifndef T1HA2_DISABLED
+T1HA_API int t1ha_selfcheck__t1ha2_atonce(void);
+T1HA_API int t1ha_selfcheck__t1ha2_atonce128(void);
+T1HA_API int t1ha_selfcheck__t1ha2_stream(void);
+T1HA_API int t1ha_selfcheck__t1ha2(void);
+#endif /* T1HA2_DISABLED */
+
+#ifndef T1HA1_DISABLED
+T1HA_API int t1ha_selfcheck__t1ha1_le(void);
+T1HA_API int t1ha_selfcheck__t1ha1_be(void);
+T1HA_API int t1ha_selfcheck__t1ha1(void);
+#endif /* T1HA1_DISABLED */
+
+#ifndef T1HA0_DISABLED
+T1HA_API int t1ha_selfcheck__t1ha0_32le(void);
+T1HA_API int t1ha_selfcheck__t1ha0_32be(void);
+T1HA_API int t1ha_selfcheck__t1ha0(void);
+
+/* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */
+#ifndef T1HA0_AESNI_AVAILABLE
+#if defined(__e2k__) ||                                                        \
+    (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800))
+#define T1HA0_AESNI_AVAILABLE 1
+#else
+#define T1HA0_AESNI_AVAILABLE 0
+#endif
+#endif /* ifndef T1HA0_AESNI_AVAILABLE */
+
+#if T1HA0_AESNI_AVAILABLE
+T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_noavx(void);
+T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx(void);
+#ifndef __e2k__
+T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx2(void);
+#endif
+#endif /* if T1HA0_AESNI_AVAILABLE */
+#endif /* T1HA0_DISABLED */
+
+/******************************************************************************
+ *
+ *  t1ha2 = 64 and 128-bit, SLIGHTLY MORE ATTENTION FOR QUALITY AND STRENGTH.
+ *
+ *    - The recommended version of "Fast Positive Hash" with good quality
+ *      for checksum, hash tables and fingerprinting.
+ *    - Portable and extremely efficiency on modern 64-bit CPUs.
+ *      Designed for 64-bit little-endian platforms,
+ *      in other cases will runs slowly.
+ *    - Great quality of hashing and still faster than other non-t1ha hashes.
+ *      Provides streaming mode and 128-bit result.
+ *
+ * Note: Due performance reason 64- and 128-bit results are completely
+ *       different each other, i.e. 64-bit result is NOT any part of 128-bit.
+ */
+#ifndef T1HA2_DISABLED
+
+/* The at-once variant with 64-bit result */
+T1HA_API uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed);
+
+/* The at-once variant with 128-bit result.
+ * Argument `extra_result` is NOT optional and MUST be valid.
+ * The high 64-bit part of 128-bit hash will be always unconditionally
+ * stored to the address given by `extra_result` argument. */
+T1HA_API uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
+                                  const void *__restrict data, size_t length,
+                                  uint64_t seed);
+
+/* The init/update/final trinity for streaming.
+ * Return 64 or 128-bit result depentently from `extra_result` argument. */
+T1HA_API void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y);
+T1HA_API void t1ha2_update(t1ha_context_t *__restrict ctx,
+                           const void *__restrict data, size_t length);
+
+/* Argument `extra_result` is optional and MAY be NULL.
+ *  - If `extra_result` is NOT NULL then the 128-bit hash will be calculated,
+ *    and high 64-bit part of it will be stored to the address given
+ *    by `extra_result` argument.
+ *  - Otherwise the 64-bit hash will be calculated
+ *    and returned from function directly.
+ *
+ * Note: Due performance reason 64- and 128-bit results are completely
+ *       different each other, i.e. 64-bit result is NOT any part of 128-bit. */
+T1HA_API uint64_t t1ha2_final(t1ha_context_t *__restrict ctx,
+                              uint64_t *__restrict extra_result /* optional */);
+
+#endif /* T1HA2_DISABLED */
+
+/******************************************************************************
+ *
+ *  t1ha1 = 64-bit, BASELINE FAST PORTABLE HASH:
+ *
+ *    - Runs faster on 64-bit platforms in other cases may runs slowly.
+ *    - Portable and stable, returns same 64-bit result
+ *      on all architectures and CPUs.
+ *    - Unfortunately it fails the "strict avalanche criteria",
+ *      see test results at https://github.com/demerphq/smhasher.
+ *
+ *      This flaw is insignificant for the t1ha1() purposes and imperceptible
+ *      from a practical point of view.
+ *      However, nowadays this issue has resolved in the next t1ha2(),
+ *      that was initially planned to providing a bit more quality.
+ */
+#ifndef T1HA1_DISABLED
+
+/* The little-endian variant. */
+T1HA_API uint64_t t1ha1_le(const void *data, size_t length, uint64_t seed);
+
+/* The big-endian variant. */
+T1HA_API uint64_t t1ha1_be(const void *data, size_t length, uint64_t seed);
+
+#endif /* T1HA1_DISABLED */
+
+/******************************************************************************
+ *
+ *  t1ha0 = 64-bit, JUST ONLY FASTER:
+ *
+ *    - Provides fast-as-possible hashing for current CPU, including
+ *      32-bit systems and engaging the available hardware acceleration.
+ *    - It is a facade that selects most quick-and-dirty hash
+ *      for the current processor. For instance, on IA32 (x86) actual function
+ *      will be selected in runtime, depending on current CPU capabilities
+ *
+ * BE CAREFUL!!!  THIS IS MEANS:
+ *
+ *   1. The quality of hash is a subject for tradeoffs with performance.
+ *      So, the quality and strength of t1ha0() may be lower than t1ha1(),
+ *      especially on 32-bit targets, but then much faster.
+ *      However, guaranteed that it passes all SMHasher tests.
+ *
+ *   2. No warranty that the hash result will be same for particular
+ *      key on another machine or another version of libt1ha.
+ *
+ *      Briefly, such hash-results and their derivatives, should be
+ *      used only in runtime, but should not be persist or transferred
+ *      over a network.
+ *
+ *
+ *  When T1HA0_RUNTIME_SELECT is nonzero the t1ha0_resolve() function could
+ *  be used to get actual t1ha0() implementation address at runtime. This is
+ *  useful for two cases:
+ *    - calling by local pointer-to-function usually is little
+ *      bit faster (less overhead) than via a PLT thru the DSO boundary.
+ *    - GNU Indirect functions (see below) don't supported by environment
+ *      and calling by t1ha0_funcptr is not available and/or expensive.
+ */
+
+#ifndef T1HA0_DISABLED
+
+/* The little-endian variant for 32-bit CPU. */
+uint64_t t1ha0_32le(const void *data, size_t length, uint64_t seed);
+/* The big-endian variant for 32-bit CPU. */
+uint64_t t1ha0_32be(const void *data, size_t length, uint64_t seed);
+
+/* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */
+#ifndef T1HA0_AESNI_AVAILABLE
+#if defined(__e2k__) ||                                                        \
+    (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800))
+#define T1HA0_AESNI_AVAILABLE 1
+#else
+#define T1HA0_AESNI_AVAILABLE 0
+#endif
+#endif /* T1HA0_AESNI_AVAILABLE */
+
+/* Define T1HA0_RUNTIME_SELECT to 0 for disable dispatching t1ha0 at runtime. */
+#ifndef T1HA0_RUNTIME_SELECT
+#if T1HA0_AESNI_AVAILABLE && !defined(__e2k__)
+#define T1HA0_RUNTIME_SELECT 1
+#else
+#define T1HA0_RUNTIME_SELECT 0
+#endif
+#endif /* T1HA0_RUNTIME_SELECT */
+
+#if !T1HA0_RUNTIME_SELECT && !defined(T1HA0_USE_DEFINE)
+#if defined(__LCC__)
+#define T1HA0_USE_DEFINE 1
+#else
+#define T1HA0_USE_DEFINE 0
+#endif
+#endif /* T1HA0_USE_DEFINE */
+
+#if T1HA0_AESNI_AVAILABLE
+uint64_t t1ha0_ia32aes_noavx(const void *data, size_t length, uint64_t seed);
+uint64_t t1ha0_ia32aes_avx(const void *data, size_t length, uint64_t seed);
+#ifndef __e2k__
+uint64_t t1ha0_ia32aes_avx2(const void *data, size_t length, uint64_t seed);
+#endif
+#endif /* T1HA0_AESNI_AVAILABLE */
+
+#if T1HA0_RUNTIME_SELECT
+typedef uint64_t (*t1ha0_function_t)(const void *, size_t, uint64_t);
+T1HA_API t1ha0_function_t t1ha0_resolve(void);
+#if T1HA_USE_INDIRECT_FUNCTIONS
+T1HA_API uint64_t t1ha0(const void *data, size_t length, uint64_t seed);
+#else
+/* Otherwise function pointer will be used.
+ * Unfortunately this may cause some overhead calling. */
+T1HA_API extern uint64_t (*t1ha0_funcptr)(const void *data, size_t length,
+                                          uint64_t seed);
+static __force_inline uint64_t t1ha0(const void *data, size_t length,
+                                     uint64_t seed) {
+  return t1ha0_funcptr(data, length, seed);
+}
+#endif /* T1HA_USE_INDIRECT_FUNCTIONS */
+
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+
+#if T1HA0_USE_DEFINE
+
+#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) &&                \
+    (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED))
+#if defined(T1HA1_DISABLED)
+#define t1ha0 t1ha2_atonce
+#else
+#define t1ha0 t1ha1_be
+#endif /* T1HA1_DISABLED */
+#else  /* 32/64 */
+#define t1ha0 t1ha0_32be
+#endif /* 32/64 */
+
+#else /* T1HA0_USE_DEFINE */
+
+static __force_inline uint64_t t1ha0(const void *data, size_t length,
+                                     uint64_t seed) {
+#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) &&                \
+    (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED))
+#if defined(T1HA1_DISABLED)
+  return t1ha2_atonce(data, length, seed);
+#else
+  return t1ha1_be(data, length, seed);
+#endif /* T1HA1_DISABLED */
+#else  /* 32/64 */
+  return t1ha0_32be(data, length, seed);
+#endif /* 32/64 */
+}
+
+#endif /* !T1HA0_USE_DEFINE */
+
+#else /* !T1HA0_RUNTIME_SELECT && __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */
+
+#if T1HA0_USE_DEFINE
+
+#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) &&                \
+    (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED))
+#if defined(T1HA1_DISABLED)
+#define t1ha0 t1ha2_atonce
+#else
+#define t1ha0 t1ha1_le
+#endif /* T1HA1_DISABLED */
+#else  /* 32/64 */
+#define t1ha0 t1ha0_32le
+#endif /* 32/64 */
+
+#else
+
+static __force_inline uint64_t t1ha0(const void *data, size_t length,
+                                     uint64_t seed) {
+#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) &&                \
+    (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED))
+#if defined(T1HA1_DISABLED)
+  return t1ha2_atonce(data, length, seed);
+#else
+  return t1ha1_le(data, length, seed);
+#endif /* T1HA1_DISABLED */
+#else  /* 32/64 */
+  return t1ha0_32le(data, length, seed);
+#endif /* 32/64 */
+}
+
+#endif /* !T1HA0_USE_DEFINE */
+
+#endif /* !T1HA0_RUNTIME_SELECT */
+
+#endif /* T1HA0_DISABLED */
+
+#ifdef __cplusplus
+}
+#endif
+
+#if __GNUC_PREREQ(4, 0)
+#pragma GCC visibility pop
+#endif /* __GNUC_PREREQ(4,0) */

hash_testing/new_testing/hash_funcs/xxh64/bin_hash.c

@@ -0,0 +1,224 @@
+/*
+ * Standalone XXH64 command-line wrapper for avalanche testing.
+ *
+ * XXH64 algorithm derived from xxHash by Yann Collet:
+ * https://github.com/Cyan4973/xxHash
+ *
+ * Copyright (C) 2012-2023 Yann Collet
+ *
+ * BSD 2-Clause License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DAMAGES ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define XXH64_SEED UINT64_C(0)
+#define XXH_PRIME64_1 UINT64_C(11400714785074694791)
+#define XXH_PRIME64_2 UINT64_C(14029467366897019727)
+#define XXH_PRIME64_3 UINT64_C(1609587929392839161)
+#define XXH_PRIME64_4 UINT64_C(9650029242287828579)
+#define XXH_PRIME64_5 UINT64_C(2870177450012600261)
+
+static uint64_t rotate_left64(uint64_t value, unsigned int count)
+{
+    return (value << count) | (value >> (64U - count));
+}
+
+static uint32_t read_little_endian32(const unsigned char *data)
+{
+    return (uint32_t)data[0] | ((uint32_t)data[1] << 8U) |
+           ((uint32_t)data[2] << 16U) | ((uint32_t)data[3] << 24U);
+}
+
+static uint64_t read_little_endian64(const unsigned char *data)
+{
+    return (uint64_t)read_little_endian32(data) |
+           ((uint64_t)read_little_endian32(data + 4) << 32U);
+}
+
+static uint64_t xxh64_round(uint64_t accumulator, uint64_t input)
+{
+    accumulator += input * XXH_PRIME64_2;
+    accumulator = rotate_left64(accumulator, 31U);
+    accumulator *= XXH_PRIME64_1;
+    return accumulator;
+}
+
+static uint64_t xxh64_merge_round(uint64_t accumulator, uint64_t value)
+{
+    value = xxh64_round(UINT64_C(0), value);
+    accumulator ^= value;
+    accumulator = accumulator * XXH_PRIME64_1 + XXH_PRIME64_4;
+    return accumulator;
+}
+
+static uint64_t xxh64(const unsigned char *data, size_t length, uint64_t seed)
+{
+    const unsigned char *position = data;
+    const unsigned char *const end = data + length;
+    uint64_t hash;
+
+    if (length >= 32U) {
+        const unsigned char *const block_end = end - 32U;
+        uint64_t accumulator1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+        uint64_t accumulator2 = seed + XXH_PRIME64_2;
+        uint64_t accumulator3 = seed;
+        uint64_t accumulator4 = seed - XXH_PRIME64_1;
+
+        do {
+            accumulator1 = xxh64_round(accumulator1, read_little_endian64(position));
+            position += 8;
+            accumulator2 = xxh64_round(accumulator2, read_little_endian64(position));
+            position += 8;
+            accumulator3 = xxh64_round(accumulator3, read_little_endian64(position));
+            position += 8;
+            accumulator4 = xxh64_round(accumulator4, read_little_endian64(position));
+            position += 8;
+        } while (position <= block_end);
+
+        hash = rotate_left64(accumulator1, 1U) +
+               rotate_left64(accumulator2, 7U) +
+               rotate_left64(accumulator3, 12U) +
+               rotate_left64(accumulator4, 18U);
+        hash = xxh64_merge_round(hash, accumulator1);
+        hash = xxh64_merge_round(hash, accumulator2);
+        hash = xxh64_merge_round(hash, accumulator3);
+        hash = xxh64_merge_round(hash, accumulator4);
+    } else {
+        hash = seed + XXH_PRIME64_5;
+    }
+
+    hash += (uint64_t)length;
+
+    while ((size_t)(end - position) >= 8U) {
+        uint64_t value = xxh64_round(UINT64_C(0), read_little_endian64(position));
+        hash ^= value;
+        hash = rotate_left64(hash, 27U) * XXH_PRIME64_1 + XXH_PRIME64_4;
+        position += 8;
+    }
+
+    if ((size_t)(end - position) >= 4U) {
+        hash ^= (uint64_t)read_little_endian32(position) * XXH_PRIME64_1;
+        hash = rotate_left64(hash, 23U) * XXH_PRIME64_2 + XXH_PRIME64_3;
+        position += 4;
+    }
+
+    while (position < end) {
+        hash ^= (uint64_t)(*position) * XXH_PRIME64_5;
+        hash = rotate_left64(hash, 11U) * XXH_PRIME64_1;
+        ++position;
+    }
+
+    hash ^= hash >> 33U;
+    hash *= XXH_PRIME64_2;
+    hash ^= hash >> 29U;
+    hash *= XXH_PRIME64_3;
+    hash ^= hash >> 32U;
+    return hash;
+}
+
+static int is_ascii_trailing_space(unsigned char character)
+{
+    return character == ' ' || character == '\t' || character == '\n' ||
+           character == '\r' || character == '\v' || character == '\f';
+}
+
+static int read_stdin(unsigned char **data, size_t *length)
+{
+    size_t capacity = 256;
+    unsigned char *buffer = malloc(capacity);
+
+    if (buffer == NULL) {
+        return -1;
+    }
+
+    *length = 0;
+    for (;;) {
+        size_t available = capacity - *length;
+        size_t bytes_read = fread(buffer + *length, 1, available, stdin);
+        *length += bytes_read;
+
+        if (bytes_read < available) {
+            if (ferror(stdin)) {
+                free(buffer);
+                return -1;
+            }
+            break;
+        }
+
+        if (capacity > SIZE_MAX / 2U) {
+            free(buffer);
+            return -1;
+        }
+        capacity *= 2U;
+
+        {
+            unsigned char *larger_buffer = realloc(buffer, capacity);
+            if (larger_buffer == NULL) {
+                free(buffer);
+                return -1;
+            }
+            buffer = larger_buffer;
+        }
+    }
+
+    *data = buffer;
+    return 0;
+}
+
+int main(int argc, char **argv)
+{
+    const unsigned char *word;
+    unsigned char *stdin_buffer = NULL;
+    size_t length;
+
+    if (argc > 2) {
+        fprintf(stderr, "usage: %s [ASCII_WORD]\n", argv[0]);
+        return EXIT_FAILURE;
+    }
+
+    if (argc == 2) {
+        word = (const unsigned char *)argv[1];
+        length = strlen(argv[1]);
+    } else {
+        if (read_stdin(&stdin_buffer, &length) != 0) {
+            fprintf(stderr, "failed to read input\n");
+            return EXIT_FAILURE;
+        }
+        word = stdin_buffer;
+    }
+
+    while (length > 0U && is_ascii_trailing_space(word[length - 1U])) {
+        --length;
+    }
+
+    for (size_t index = 0; index < length; ++index) {
+        if (word[index] > 0x7fU) {
+            fprintf(stderr, "input must contain ASCII characters only\n");
+            free(stdin_buffer);
+            return EXIT_FAILURE;
+        }
+    }
+
+    printf("%016" PRIx64 "\n", xxh64(word, length, XXH64_SEED));
+    free(stdin_buffer);
+    return EXIT_SUCCESS;
+}

hash_testing/new_testing/plot_probability_map.py

@@ -0,0 +1,445 @@
+#!/usr/bin/env python3
+"""Render avalanche bit-probability CSV files as two PNG bar charts."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import math
+from dataclasses import dataclass
+from pathlib import Path
+from statistics import fmean
+from typing import Sequence
+
+from PIL import Image, ImageDraw, ImageFont
+
+BACKGROUND = "#f7f8fa"
+PANEL = "#ffffff"
+GRID = "#d9dee7"
+TEXT = "#172033"
+MUTED = "#637083"
+REFERENCE = "#d24b4b"
+COLORS = (
+    "#377eb8",
+    "#4daf4a",
+    "#984ea3",
+    "#ff7f00",
+    "#e41a1c",
+    "#00a6a6",
+    "#a65628",
+)
+
+
+@dataclass(frozen=True)
+class ProbabilityRow:
+    operation: str
+    pairs: int
+    probabilities: list[float]
+
+
+@dataclass(frozen=True)
+class ChartScale:
+    minimum: float
+    maximum: float
+    ticks: tuple[float, ...]
+
+
+def nice_step(value: float) -> float:
+    exponent = math.floor(math.log10(value))
+    fraction = value / 10**exponent
+    nice_fraction = min((1.0, 2.0, 2.5, 5.0, 10.0), key=lambda item: abs(item - fraction))
+    return nice_fraction * 10**exponent
+
+
+def make_scale(
+    values: Sequence[float],
+    *,
+    hard_limits: tuple[float, float],
+    reference: float | None = None,
+) -> ChartScale:
+    """Build a padded shared scale constrained to hard limits."""
+    hard_minimum, hard_maximum = hard_limits
+    finite_values = [value for value in values if math.isfinite(value)]
+    if reference is not None:
+        finite_values.append(reference)
+    if not finite_values:
+        finite_values = [hard_minimum, hard_maximum]
+
+    minimum = min(finite_values)
+    maximum = max(finite_values)
+    if minimum == maximum:
+        expansion = (hard_maximum - hard_minimum) * 0.1
+        minimum -= expansion / 2
+        maximum += expansion / 2
+
+    span = maximum - minimum
+    padded_minimum = max(hard_minimum, minimum - span * 0.1)
+    padded_maximum = min(hard_maximum, maximum + span * 0.1)
+    step = nice_step(max((padded_maximum - padded_minimum) / 5, 1e-12))
+    scaled_minimum = max(hard_minimum, math.floor(padded_minimum / step) * step)
+    scaled_maximum = min(hard_maximum, math.ceil(padded_maximum / step) * step)
+    if scaled_minimum == scaled_maximum:
+        scaled_minimum, scaled_maximum = hard_minimum, hard_maximum
+
+    tick_count = round((scaled_maximum - scaled_minimum) / step)
+    ticks = [scaled_minimum + index * step for index in range(tick_count + 1)]
+    if reference is not None and scaled_minimum <= reference <= scaled_maximum:
+        ticks.append(reference)
+    normalized_ticks = tuple(
+        sorted({round(value, 12) for value in ticks if scaled_minimum <= value <= scaled_maximum})
+    )
+    return ChartScale(scaled_minimum, scaled_maximum, normalized_ticks)
+
+
+def load_font(size: int, bold: bool = False) -> ImageFont.FreeTypeFont | ImageFont.ImageFont:
+    names = (
+        "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf",
+        "/usr/share/fonts/TTF/DejaVuSans-Bold.ttf"
+        if bold
+        else "/usr/share/fonts/TTF/DejaVuSans.ttf",
+        "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf"
+        if bold
+        else "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
+    )
+    for name in names:
+        try:
+            return ImageFont.truetype(name, size)
+        except OSError:
+            continue
+    return ImageFont.load_default()
+
+
+def read_probability_map(path: Path) -> list[ProbabilityRow]:
+    """Read operation rows and numerically ordered bit columns from a CSV file."""
+    try:
+        stream = path.open(encoding="utf-8", newline="")
+    except OSError as error:
+        raise ValueError(f"не удалось открыть {path}: {error}") from error
+
+    with stream:
+        reader = csv.DictReader(stream)
+        fields = reader.fieldnames
+        if not fields or "operation" not in fields or "pairs" not in fields:
+            raise ValueError("CSV должен содержать колонки operation и pairs")
+
+        bit_fields: list[tuple[int, str]] = []
+        for field in fields:
+            if not field.startswith("bit_"):
+                continue
+            try:
+                bit_fields.append((int(field.removeprefix("bit_")), field))
+            except ValueError as error:
+                raise ValueError(f"некорректная битовая колонка: {field}") from error
+        bit_fields.sort()
+        if not bit_fields:
+            raise ValueError("CSV не содержит колонок bit_N")
+        expected_bits = list(range(len(bit_fields)))
+        actual_bits = [bit for bit, _field in bit_fields]
+        if actual_bits != expected_bits:
+            raise ValueError("битовые колонки должны непрерывно идти от bit_0")
+
+        rows: list[ProbabilityRow] = []
+        for line_number, row in enumerate(reader, start=2):
+            operation = (row.get("operation") or "").strip()
+            if not operation:
+                raise ValueError(f"строка {line_number}: пустая операция")
+            try:
+                pairs = int(row["pairs"] or "")
+            except (TypeError, ValueError) as error:
+                raise ValueError(
+                    f"строка {line_number}: некорректное число пар"
+                ) from error
+            if pairs < 0:
+                raise ValueError(f"строка {line_number}: число пар меньше нуля")
+
+            probabilities: list[float] = []
+            for _bit, field in bit_fields:
+                try:
+                    value = float(row[field] or "")
+                except (TypeError, ValueError) as error:
+                    raise ValueError(
+                        f"строка {line_number}: некорректное значение {field}"
+                    ) from error
+                if not math.isnan(value) and not 0.0 <= value <= 1.0:
+                    raise ValueError(
+                        f"строка {line_number}: {field} должен быть от 0 до 1"
+                    )
+                probabilities.append(value)
+            rows.append(ProbabilityRow(operation, pairs, probabilities))
+
+    if not rows:
+        raise ValueError("CSV не содержит строк с операциями")
+    return rows
+
+
+def mean_absolute_deviation(probabilities: Sequence[float]) -> float:
+    """Return the mean |p - 0.5| over finite bit probabilities."""
+    deviations = [
+        abs(probability - 0.5)
+        for probability in probabilities
+        if math.isfinite(probability)
+    ]
+    return fmean(deviations) if deviations else float("nan")
+
+
+def text_width(
+    draw: ImageDraw.ImageDraw,
+    text: str,
+    font: ImageFont.FreeTypeFont | ImageFont.ImageFont,
+) -> int:
+    box = draw.textbbox((0, 0), text, font=font)
+    return round(box[2] - box[0])
+
+
+def draw_centered_text(
+    draw: ImageDraw.ImageDraw,
+    center_x: float,
+    y: float,
+    text: str,
+    font: ImageFont.FreeTypeFont | ImageFont.ImageFont,
+    fill: str = TEXT,
+) -> None:
+    draw.text(
+        (center_x - text_width(draw, text, font) / 2, y),
+        text,
+        font=font,
+        fill=fill,
+    )
+
+
+def draw_bit_panel(
+    draw: ImageDraw.ImageDraw,
+    bounds: tuple[int, int, int, int],
+    row: ProbabilityRow,
+    color: str,
+    scale: ChartScale,
+) -> None:
+    left, top, right, bottom = bounds
+    title_font = load_font(22, bold=True)
+    label_font = load_font(14)
+    tick_font = load_font(12)
+
+    draw.rounded_rectangle(bounds, radius=12, fill=PANEL, outline=GRID, width=1)
+    draw.text((left + 18, top + 13), row.operation, font=title_font, fill=TEXT)
+    pairs_text = f"pairs: {row.pairs}"
+    draw.text(
+        (right - 18 - text_width(draw, pairs_text, label_font), top + 17),
+        pairs_text,
+        font=label_font,
+        fill=MUTED,
+    )
+
+    plot_left = left + 54
+    plot_right = right - 18
+    plot_top = top + 55
+    plot_bottom = bottom - 42
+    plot_height = plot_bottom - plot_top
+
+    scale_span = scale.maximum - scale.minimum
+    for probability in scale.ticks:
+        y = round(plot_bottom - (probability - scale.minimum) / scale_span * plot_height)
+        line_color = REFERENCE if probability == 0.5 else GRID
+        line_width = 2 if probability == 0.5 else 1
+        draw.line((plot_left, y, plot_right, y), fill=line_color, width=line_width)
+        label = f"{probability:.3g}"
+        draw.text(
+            (plot_left - 8 - text_width(draw, label, tick_font), y - 7),
+            label,
+            font=tick_font,
+            fill=MUTED,
+        )
+
+    bit_count = len(row.probabilities)
+    slot_width = (plot_right - plot_left) / bit_count
+    bar_width = max(1, int(slot_width * 0.72))
+    for bit, probability in enumerate(row.probabilities):
+        if not math.isfinite(probability):
+            continue
+        center = plot_left + (bit + 0.5) * slot_width
+        x0 = round(center - bar_width / 2)
+        x1 = round(center + bar_width / 2)
+        y = round(
+            plot_bottom
+            - (probability - scale.minimum) / scale_span * plot_height
+        )
+        draw.rectangle((x0, y, x1, plot_bottom), fill=color)
+
+    tick_step = max(1, math.ceil(bit_count / 16))
+    for bit in range(0, bit_count, tick_step):
+        center = plot_left + (bit + 0.5) * slot_width
+        label = str(bit)
+        draw.text(
+            (center - text_width(draw, label, tick_font) / 2, plot_bottom + 7),
+            label,
+            font=tick_font,
+            fill=MUTED,
+        )
+    draw_centered_text(
+        draw,
+        (plot_left + plot_right) / 2,
+        bottom - 21,
+        "output bit (0 = LSB)",
+        tick_font,
+        MUTED,
+    )
+
+
+def render_bit_probabilities(
+    rows: Sequence[ProbabilityRow], output: Path, title: str
+) -> None:
+    columns = 2 if len(rows) > 1 else 1
+    panel_width = 760
+    panel_height = 330
+    gap = 18
+    margin = 24
+    title_height = 70
+    row_count = math.ceil(len(rows) / columns)
+    width = margin * 2 + columns * panel_width + (columns - 1) * gap
+    height = title_height + margin + row_count * panel_height + (row_count - 1) * gap
+
+    image = Image.new("RGB", (width, height), BACKGROUND)
+    draw = ImageDraw.Draw(image)
+    scale = make_scale(
+        [probability for row in rows for probability in row.probabilities],
+        reference=0.5,
+        hard_limits=(0.0, 1.0),
+    )
+    draw_centered_text(draw, width / 2, 18, title, load_font(30, bold=True))
+    draw_centered_text(
+        draw,
+        width / 2,
+        52,
+        f"Shared scale {scale.minimum:.3g}–{scale.maximum:.3g}; red line = ideal p=0.5",
+        load_font(14),
+        MUTED,
+    )
+
+    for index, row in enumerate(rows):
+        column = index % columns
+        grid_row = index // columns
+        left = margin + column * (panel_width + gap)
+        top = title_height + grid_row * (panel_height + gap)
+        draw_bit_panel(
+            draw,
+            (left, top, left + panel_width, top + panel_height),
+            row,
+            COLORS[index % len(COLORS)],
+            scale,
+        )
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    image.save(output, "PNG", optimize=True)
+
+
+def render_mean_deviations(
+    rows: Sequence[ProbabilityRow], output: Path, title: str
+) -> None:
+    width, height = 1200, 720
+    image = Image.new("RGB", (width, height), BACKGROUND)
+    draw = ImageDraw.Draw(image)
+    draw_centered_text(draw, width / 2, 22, title, load_font(30, bold=True))
+    draw_centered_text(
+        draw,
+        width / 2,
+        58,
+        "Mean absolute deviation from ideal avalanche probability: mean(|p - 0.5|)",
+        load_font(15),
+        MUTED,
+    )
+
+    plot_left, plot_right = 90, width - 40
+    plot_top, plot_bottom = 110, height - 120
+    plot_height = plot_bottom - plot_top
+    deviations = [mean_absolute_deviation(row.probabilities) for row in rows]
+    scale = make_scale(deviations, hard_limits=(0.0, 0.5))
+    scale_span = scale.maximum - scale.minimum
+    tick_font = load_font(13)
+    label_font = load_font(15)
+    value_font = load_font(14, bold=True)
+
+    for value in scale.ticks:
+        y = round(plot_bottom - (value - scale.minimum) / scale_span * plot_height)
+        draw.line((plot_left, y, plot_right, y), fill=GRID, width=1)
+        label = f"{value:.3g}"
+        draw.text(
+            (plot_left - 10 - text_width(draw, label, tick_font), y - 7),
+            label,
+            font=tick_font,
+            fill=MUTED,
+        )
+
+    slot_width = (plot_right - plot_left) / len(rows)
+    bar_width = min(105, max(20, int(slot_width * 0.62)))
+    for index, (row, deviation) in enumerate(zip(rows, deviations, strict=True)):
+        center = plot_left + (index + 0.5) * slot_width
+        x0 = round(center - bar_width / 2)
+        x1 = round(center + bar_width / 2)
+        if math.isfinite(deviation):
+            y = round(
+                plot_bottom
+                - (deviation - scale.minimum) / scale_span * plot_height
+            )
+            draw.rectangle((x0, y, x1, plot_bottom), fill=COLORS[index % len(COLORS)])
+            value_label = f"{deviation:.4f}"
+        else:
+            y = plot_bottom
+            value_label = "n/a"
+        draw_centered_text(draw, center, max(plot_top, y - 22), value_label, value_font)
+        draw_centered_text(draw, center, plot_bottom + 12, row.operation, label_font)
+        draw_centered_text(
+            draw, center, plot_bottom + 36, f"pairs: {row.pairs}", tick_font, MUTED
+        )
+
+    draw.line((plot_left, plot_top, plot_left, plot_bottom), fill=TEXT, width=2)
+    draw.line((plot_left, plot_bottom, plot_right, plot_bottom), fill=TEXT, width=2)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    image.save(output, "PNG", optimize=True)
+
+
+def generate_plots(source: Path, output_directory: Path | None = None) -> tuple[Path, Path]:
+    rows = read_probability_map(source)
+    destination = output_directory or source.parent
+    bit_output = destination / f"{source.stem}_bits.png"
+    deviation_output = destination / f"{source.stem}_deviation.png"
+    display_name = source.stem
+
+    render_bit_probabilities(rows, bit_output, f"{display_name}: per-bit avalanche map")
+    render_mean_deviations(
+        rows,
+        deviation_output,
+        f"{display_name}: deviation from p=0.5",
+    )
+    return bit_output, deviation_output
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Создаёт две PNG-гистограммы из CSV вероятностной карты: "
+            "вероятности по битам для каждой операции и среднее |p-0.5|."
+        )
+    )
+    parser.add_argument("csv_file", type=Path, help="CSV из probability_map.py")
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        type=Path,
+        help="каталог PNG (по умолчанию каталог исходного CSV)",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        outputs = generate_plots(args.csv_file, args.output_dir)
+    except ValueError as error:
+        raise SystemExit(f"error: {error}") from error
+
+    for output in outputs:
+        print(f"wrote {output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

hash_testing/new_testing/probability_map.py

@@ -0,0 +1,354 @@
+#!/usr/bin/env python3
+"""Build bit-probability maps for hash functions and word mutation types.
+
+For every hash listed in HASHES, the script compares each source word hash with
+hashes of generated similar words.  A table cell contains the probability that
+the corresponding output bit changed (XOR with its source word hash).
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import re
+import stat
+import subprocess
+import sys
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import TextIO
+
+from generate_input import (
+    DEFAULT_ALPHABET,
+    MUTATIONS,
+    MutationError,
+    generate_words,
+    parse_operation,
+)
+
+ROOT = Path(__file__).resolve().parent
+HASH_FUNCS_DIR = ROOT / "hash_funcs"
+DEFAULT_OUTPUT_DIR = ROOT / "probability_maps"
+
+# Add directory names from hash_funcs here to include more implementations.
+HASHES = [
+    "jenkinsOAAT",
+    "xxh64",
+    "t1ha2",
+]
+
+HEX_HASH = re.compile(r"(?:0[xX])?([0-9a-fA-F]+)")
+ProbabilityRow = tuple[int, list[float]]
+
+
+class HashToolError(RuntimeError):
+    """Raised when a hash executable cannot be prepared or invoked."""
+
+
+def prepare_hash(hash_name: str, hash_funcs_dir: Path = HASH_FUNCS_DIR) -> Path:
+    """Return an executable hash tool, building or preparing it if necessary."""
+    if not hash_name or Path(hash_name).name != hash_name:
+        raise HashToolError(f"некорректное имя хэша: {hash_name!r}")
+
+    hash_directory = hash_funcs_dir / hash_name
+    if not hash_directory.is_dir():
+        raise HashToolError(f"не найдена папка хэша: {hash_directory}")
+
+    binary = hash_directory / "bin_hash"
+    if binary.is_file():
+        binary.chmod(binary.stat().st_mode | stat.S_IXUSR)
+        return binary
+
+    sources = (
+        (hash_directory / "bin_hash.c", "cc"),
+        (hash_directory / "bin_hash.cpp", "c++"),
+        (hash_directory / "bin_hash.cc", "c++"),
+        (hash_directory / "bin_hash.cxx", "c++"),
+    )
+    for source, compiler in sources:
+        if not source.is_file():
+            continue
+        command = [
+            compiler,
+            "-O2",
+            "-Wall",
+            "-Wextra",
+            "-Wpedantic",
+            "-Werror",
+            str(source),
+            "-o",
+            str(binary),
+        ]
+        if compiler == "cc":
+            command[1:1] = ["-std=c11"]
+        else:
+            command[1:1] = ["-std=c++17"]
+        result = subprocess.run(command, text=True, capture_output=True)
+        if result.returncode != 0:
+            details = result.stderr.strip() or result.stdout.strip()
+            raise HashToolError(f"не удалось скомпилировать {source}: {details}")
+        return binary
+
+    python_source = hash_directory / "bin_hash.py"
+    if python_source.is_file():
+        content = python_source.read_text(encoding="utf-8")
+        if not content.startswith("#!"):
+            python_source.write_text(
+                "#!/usr/bin/env python3\n" + content,
+                encoding="utf-8",
+            )
+        python_source.chmod(
+            python_source.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+        )
+        return python_source
+
+    raise HashToolError(
+        f"для {hash_name} не найден bin_hash, bin_hash.c/cpp/cc/cxx или bin_hash.py"
+    )
+
+
+def run_hash(executable: Path, word: str) -> tuple[int, int]:
+    """Run a hash tool and return its integer value and explicit output width."""
+    try:
+        result = subprocess.run(
+            [str(executable), word],
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+    except OSError as error:
+        raise HashToolError(f"не удалось запустить {executable}: {error}") from error
+
+    if result.returncode != 0:
+        details = result.stderr.strip() or result.stdout.strip()
+        raise HashToolError(
+            f"{executable} завершился с кодом {result.returncode}: {details}"
+        )
+
+    output = result.stdout.strip()
+    match = HEX_HASH.fullmatch(output)
+    if match is None:
+        raise HashToolError(f"{executable} вернул не шестнадцатеричный хэш: {output!r}")
+
+    digits = match.group(1)
+    return int(digits, 16), len(digits) * 4
+
+
+def bit_probabilities(
+    source_hash: int, changed_hashes: Sequence[int], bits: int
+) -> list[float]:
+    """Calculate per-bit change probabilities, ordered from LSB to MSB."""
+    if bits < 1:
+        raise ValueError("число бит должно быть положительным")
+    if not changed_hashes:
+        raise ValueError("список изменённых хэшей не должен быть пустым")
+
+    changed_counts = [0] * bits
+    for changed_hash in changed_hashes:
+        difference = source_hash ^ changed_hash
+        for bit in range(bits):
+            changed_counts[bit] += (difference >> bit) & 1
+
+    sample_count = len(changed_hashes)
+    return [count / sample_count for count in changed_counts]
+
+
+def write_csv_table(stream: TextIO, rows: Mapping[str, ProbabilityRow]) -> None:
+    """Write one operation-by-bit probability table as CSV."""
+    if not rows:
+        raise ValueError("таблица вероятностей не должна быть пустой")
+
+    widths = {len(probabilities) for _pair_count, probabilities in rows.values()}
+    if len(widths) != 1:
+        raise ValueError("все строки таблицы должны иметь одинаковое число бит")
+    bits = widths.pop()
+
+    writer = csv.writer(stream, lineterminator="\n")
+    writer.writerow(["operation", "pairs", *(f"bit_{bit}" for bit in range(bits))])
+    for operation, (pair_count, probabilities) in rows.items():
+        writer.writerow(
+            [
+                operation,
+                pair_count,
+                *(f"{probability:.6f}" for probability in probabilities),
+            ]
+        )
+
+
+def build_probability_table(
+    executable: Path,
+    sources: Sequence[str],
+    operations: Sequence[str],
+    count: int,
+    operation_count: int,
+    alphabet: str,
+    seed: int | None,
+    max_attempts: int | None,
+) -> dict[str, ProbabilityRow]:
+    """Aggregate avalanche probabilities across all source words."""
+    if not sources:
+        raise ValueError("нужно указать хотя бы одно исходное слово")
+
+    source_hashes: list[int] = []
+    bits: int | None = None
+    for source in sources:
+        source_hash, source_bits = run_hash(executable, source)
+        if bits is None:
+            bits = source_bits
+        elif source_bits != bits:
+            raise HashToolError(
+                f"{executable} вернул хэши разной ширины: "
+                f"{bits} и {source_bits} бит"
+            )
+        source_hashes.append(source_hash)
+
+    assert bits is not None
+    table: dict[str, ProbabilityRow] = {}
+
+    for operation in operations:
+        differences: list[int] = []
+        for source, source_hash in zip(sources, source_hashes, strict=True):
+            words = generate_words(
+                source=source,
+                count=count,
+                operation=operation,
+                operation_count=operation_count,
+                alphabet=alphabet,
+                seed=seed,
+                max_attempts=max_attempts,
+            )
+            if len(words) < count:
+                print(
+                    f"warning: operation={operation} word={source!r}: "
+                    f"generated {len(words)} of at most {count} unique words",
+                    file=sys.stderr,
+                )
+            for word in words:
+                changed_hash, changed_bits = run_hash(executable, word)
+                if changed_bits != bits:
+                    raise HashToolError(
+                        f"{executable} вернул хэши разной ширины: "
+                        f"{bits} и {changed_bits} бит"
+                    )
+                differences.append(source_hash ^ changed_hash)
+
+        if differences:
+            table[operation] = (
+                len(differences),
+                bit_probabilities(0, differences, bits),
+            )
+        else:
+            table[operation] = (0, [float("nan")] * bits)
+
+    return table
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Строит для каждого хэша CSV-таблицу вероятностей изменения "
+            "выходных битов. bit_0 — младший бит."
+        )
+    )
+    parser.add_argument(
+        "words",
+        nargs="+",
+        help="одно или несколько исходных ASCII-слов",
+    )
+    parser.add_argument(
+        "-o",
+        "--operation",
+        action="append",
+        type=parse_operation,
+        help=(
+            "тип изменения (номер или имя как в generate_input.py); "
+            "можно повторять, по умолчанию используются все типы"
+        ),
+    )
+    parser.add_argument(
+        "-n",
+        "--count",
+        type=int,
+        default=10,
+        help=(
+            "верхняя граница числа уникальных изменённых слов для каждого "
+            "исходного слова и типа (по умолчанию: 10)"
+        ),
+    )
+    parser.add_argument(
+        "-k",
+        "--operations",
+        type=int,
+        default=1,
+        help="число операций над каждым словом (по умолчанию: 1)",
+    )
+    parser.add_argument(
+        "--alphabet",
+        default=DEFAULT_ALPHABET,
+        help="алфавит для добавления и замены",
+    )
+    parser.add_argument(
+        "--seed", type=int, help="seed генератора для воспроизводимого результата"
+    )
+    parser.add_argument(
+        "--max-attempts",
+        type=int,
+        help="предельное число попыток собрать уникальные слова",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=DEFAULT_OUTPUT_DIR,
+        help=("каталог для CSV-таблиц " f"(по умолчанию: {DEFAULT_OUTPUT_DIR})"),
+    )
+    parser.add_argument(
+        "--hash",
+        dest="hashes",
+        action="append",
+        help="проверить только указанный хэш; можно повторять (по умолчанию HASHES)",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.count < 1:
+        parser.error("--count должен быть положительным")
+    if args.operations < 1:
+        parser.error("--operations должен быть положительным")
+    if args.max_attempts is not None and args.max_attempts < 1:
+        parser.error("--max-attempts должен быть положительным")
+
+    operations = list(dict.fromkeys(args.operation or MUTATIONS.keys()))
+    hashes = list(dict.fromkeys(args.hashes or HASHES))
+    if not hashes:
+        parser.error("массив HASHES не должен быть пустым")
+
+    args.output.mkdir(parents=True, exist_ok=True)
+    try:
+        for hash_name in hashes:
+            executable = prepare_hash(hash_name)
+            table = build_probability_table(
+                executable=executable,
+                sources=args.words,
+                operations=operations,
+                count=args.count,
+                operation_count=args.operations,
+                alphabet=args.alphabet,
+                seed=args.seed,
+                max_attempts=args.max_attempts,
+            )
+            output_path = args.output / f"{hash_name}.csv"
+            with output_path.open("w", encoding="utf-8", newline="") as stream:
+                write_csv_table(stream, table)
+            print(f"wrote {output_path}")
+    except (HashToolError, MutationError, ValueError) as error:
+        parser.error(str(error))
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

hash_testing/new_testing/probability_maps/jenkinsOAAT.csv

@@ -0,0 +1,8 @@
+operation,pairs,bit_0,bit_1,bit_2,bit_3,bit_4,bit_5,bit_6,bit_7,bit_8,bit_9,bit_10,bit_11,bit_12,bit_13,bit_14,bit_15,bit_16,bit_17,bit_18,bit_19,bit_20,bit_21,bit_22,bit_23,bit_24,bit_25,bit_26,bit_27,bit_28,bit_29,bit_30,bit_31
+replace,1984,0.484375,0.499496,0.494960,0.497984,0.506048,0.490423,0.501512,0.511593,0.496472,0.499496,0.515121,0.504536,0.485383,0.500504,0.496976,0.477319,0.510081,0.490423,0.492440,0.501512,0.502016,0.484879,0.490927,0.490927,0.511593,0.494960,0.512097,0.523185,0.498992,0.505544,0.496472,0.500504
+delete,31,0.612903,0.451613,0.612903,0.516129,0.419355,0.548387,0.451613,0.612903,0.419355,0.548387,0.677419,0.419355,0.580645,0.516129,0.451613,0.516129,0.516129,0.645161,0.548387,0.516129,0.548387,0.419355,0.516129,0.419355,0.645161,0.612903,0.516129,0.580645,0.516129,0.419355,0.580645,0.354839
+add,2173,0.501611,0.492867,0.500690,0.505292,0.515877,0.505292,0.505292,0.505752,0.497929,0.502991,0.485044,0.503451,0.496088,0.524160,0.517257,0.507593,0.489646,0.476760,0.496549,0.521859,0.514956,0.514036,0.513116,0.508053,0.492867,0.503451,0.518638,0.506673,0.495628,0.506213,0.518178,0.489185
+swap,28,0.607143,0.500000,0.428571,0.535714,0.392857,0.535714,0.500000,0.357143,0.571429,0.428571,0.678571,0.750000,0.428571,0.464286,0.392857,0.464286,0.500000,0.428571,0.428571,0.428571,0.464286,0.428571,0.500000,0.678571,0.571429,0.464286,0.607143,0.571429,0.571429,0.535714,0.428571,0.607143
+case,29,0.344828,0.551724,0.517241,0.448276,0.448276,0.310345,0.482759,0.344828,0.379310,0.413793,0.413793,0.448276,0.620690,0.620690,0.448276,0.586207,0.517241,0.551724,0.482759,0.448276,0.413793,0.517241,0.275862,0.655172,0.551724,0.586207,0.551724,0.482759,0.448276,0.551724,0.517241,0.655172
+first,186,0.500000,0.500000,0.494624,0.467742,0.521505,0.451613,0.489247,0.478495,0.521505,0.543011,0.516129,0.510753,0.478495,0.510753,0.510753,0.510753,0.456989,0.473118,0.456989,0.543011,0.489247,0.478495,0.456989,0.467742,0.500000,0.483871,0.483871,0.489247,0.559140,0.559140,0.462366,0.483871
+last,186,0.510753,0.516129,0.510753,0.467742,0.510753,0.505376,0.473118,0.516129,0.500000,0.516129,0.516129,0.521505,0.483871,0.526882,0.510753,0.451613,0.548387,0.489247,0.505376,0.430108,0.526882,0.456989,0.478495,0.505376,0.516129,0.537634,0.526882,0.478495,0.467742,0.494624,0.446237,0.548387

hash_testing/new_testing/probability_maps/t1ha2.csv

@@ -0,0 +1,8 @@
+operation,pairs,bit_0,bit_1,bit_2,bit_3,bit_4,bit_5,bit_6,bit_7,bit_8,bit_9,bit_10,bit_11,bit_12,bit_13,bit_14,bit_15,bit_16,bit_17,bit_18,bit_19,bit_20,bit_21,bit_22,bit_23,bit_24,bit_25,bit_26,bit_27,bit_28,bit_29,bit_30,bit_31,bit_32,bit_33,bit_34,bit_35,bit_36,bit_37,bit_38,bit_39,bit_40,bit_41,bit_42,bit_43,bit_44,bit_45,bit_46,bit_47,bit_48,bit_49,bit_50,bit_51,bit_52,bit_53,bit_54,bit_55,bit_56,bit_57,bit_58,bit_59,bit_60,bit_61,bit_62,bit_63
+replace,1984,0.511593,0.515121,0.510081,0.525706,0.499496,0.504032,0.495464,0.494960,0.522681,0.521169,0.507056,0.484879,0.502016,0.482359,0.513609,0.513105,0.500504,0.503024,0.496472,0.509073,0.491431,0.488911,0.495464,0.491431,0.494960,0.491431,0.496976,0.496976,0.502016,0.511593,0.502016,0.496976,0.504536,0.499496,0.494456,0.496976,0.496976,0.490927,0.523690,0.489919,0.495968,0.478327,0.492944,0.529234,0.495968,0.496472,0.500000,0.493448,0.519657,0.510585,0.493448,0.496976,0.495464,0.501512,0.512097,0.500504,0.494960,0.494456,0.512097,0.494456,0.494960,0.495464,0.494456,0.515121
+delete,31,0.741935,0.387097,0.548387,0.354839,0.483871,0.451613,0.483871,0.612903,0.451613,0.645161,0.516129,0.419355,0.419355,0.483871,0.516129,0.483871,0.548387,0.741935,0.354839,0.483871,0.645161,0.516129,0.516129,0.548387,0.516129,0.354839,0.387097,0.387097,0.354839,0.548387,0.419355,0.516129,0.548387,0.516129,0.322581,0.419355,0.387097,0.483871,0.483871,0.419355,0.451613,0.612903,0.451613,0.419355,0.516129,0.580645,0.548387,0.483871,0.387097,0.612903,0.419355,0.516129,0.483871,0.548387,0.516129,0.516129,0.677419,0.451613,0.483871,0.451613,0.516129,0.612903,0.516129,0.483871
+add,2173,0.496088,0.508514,0.489646,0.488725,0.500230,0.514496,0.491486,0.479521,0.499770,0.484123,0.488725,0.495168,0.499310,0.484584,0.498850,0.530143,0.488265,0.484123,0.519098,0.494248,0.497469,0.505752,0.501611,0.520018,0.497929,0.512195,0.498389,0.474919,0.498389,0.504372,0.491026,0.514496,0.496549,0.479521,0.492867,0.499310,0.499770,0.498389,0.480902,0.493787,0.492407,0.505292,0.492867,0.499310,0.494708,0.511735,0.524620,0.506213,0.518638,0.518178,0.492867,0.498850,0.498389,0.502531,0.509434,0.498850,0.492867,0.499310,0.501611,0.506213,0.482743,0.503451,0.511275,0.493787
+swap,28,0.571429,0.571429,0.607143,0.535714,0.428571,0.392857,0.428571,0.535714,0.571429,0.535714,0.571429,0.464286,0.285714,0.714286,0.607143,0.500000,0.500000,0.357143,0.392857,0.571429,0.678571,0.464286,0.428571,0.392857,0.357143,0.392857,0.607143,0.571429,0.392857,0.642857,0.464286,0.678571,0.750000,0.428571,0.607143,0.428571,0.535714,0.500000,0.571429,0.642857,0.428571,0.357143,0.535714,0.571429,0.535714,0.571429,0.535714,0.464286,0.607143,0.642857,0.642857,0.678571,0.285714,0.500000,0.642857,0.571429,0.500000,0.357143,0.464286,0.428571,0.571429,0.571429,0.678571,0.535714
+case,29,0.413793,0.413793,0.620690,0.586207,0.379310,0.586207,0.482759,0.448276,0.551724,0.448276,0.655172,0.586207,0.655172,0.586207,0.379310,0.551724,0.517241,0.310345,0.517241,0.517241,0.310345,0.620690,0.482759,0.655172,0.551724,0.379310,0.413793,0.482759,0.517241,0.517241,0.482759,0.448276,0.586207,0.448276,0.620690,0.448276,0.448276,0.448276,0.551724,0.586207,0.551724,0.413793,0.413793,0.586207,0.517241,0.482759,0.551724,0.482759,0.586207,0.586207,0.379310,0.482759,0.586207,0.379310,0.482759,0.344828,0.620690,0.482759,0.448276,0.551724,0.482759,0.551724,0.344828,0.482759
+first,186,0.500000,0.500000,0.526882,0.526882,0.489247,0.483871,0.456989,0.532258,0.510753,0.483871,0.537634,0.456989,0.500000,0.424731,0.500000,0.435484,0.532258,0.456989,0.478495,0.532258,0.430108,0.489247,0.521505,0.478495,0.478495,0.532258,0.478495,0.532258,0.500000,0.473118,0.473118,0.537634,0.575269,0.526882,0.505376,0.526882,0.424731,0.462366,0.548387,0.478495,0.462366,0.532258,0.500000,0.473118,0.473118,0.559140,0.510753,0.526882,0.564516,0.521505,0.532258,0.521505,0.564516,0.537634,0.467742,0.473118,0.500000,0.424731,0.424731,0.462366,0.494624,0.489247,0.537634,0.559140
+last,186,0.510753,0.543011,0.580645,0.516129,0.516129,0.564516,0.500000,0.478495,0.489247,0.494624,0.494624,0.478495,0.521505,0.462366,0.526882,0.521505,0.564516,0.451613,0.489247,0.543011,0.537634,0.467742,0.467742,0.526882,0.500000,0.526882,0.478495,0.532258,0.462366,0.473118,0.569892,0.505376,0.516129,0.494624,0.456989,0.478495,0.537634,0.473118,0.516129,0.494624,0.440860,0.505376,0.537634,0.548387,0.408602,0.569892,0.537634,0.559140,0.494624,0.559140,0.526882,0.559140,0.489247,0.569892,0.537634,0.543011,0.543011,0.494624,0.537634,0.505376,0.473118,0.494624,0.532258,0.483871

hash_testing/new_testing/probability_maps/xxh64.csv

@@ -0,0 +1,8 @@
+operation,pairs,bit_0,bit_1,bit_2,bit_3,bit_4,bit_5,bit_6,bit_7,bit_8,bit_9,bit_10,bit_11,bit_12,bit_13,bit_14,bit_15,bit_16,bit_17,bit_18,bit_19,bit_20,bit_21,bit_22,bit_23,bit_24,bit_25,bit_26,bit_27,bit_28,bit_29,bit_30,bit_31,bit_32,bit_33,bit_34,bit_35,bit_36,bit_37,bit_38,bit_39,bit_40,bit_41,bit_42,bit_43,bit_44,bit_45,bit_46,bit_47,bit_48,bit_49,bit_50,bit_51,bit_52,bit_53,bit_54,bit_55,bit_56,bit_57,bit_58,bit_59,bit_60,bit_61,bit_62,bit_63
+replace,1984,0.491431,0.497984,0.532258,0.504536,0.494456,0.519657,0.516129,0.493448,0.501008,0.519657,0.490423,0.495968,0.516129,0.498488,0.491935,0.509073,0.520665,0.487903,0.497984,0.487903,0.502016,0.504536,0.502520,0.510081,0.512097,0.492440,0.505544,0.492944,0.491431,0.500000,0.509577,0.511593,0.491431,0.485383,0.481351,0.493448,0.498992,0.498992,0.499496,0.504032,0.495968,0.483367,0.495464,0.498488,0.512097,0.489415,0.502520,0.496472,0.501008,0.495464,0.503528,0.507056,0.510081,0.504032,0.487399,0.523690,0.511089,0.472278,0.496976,0.501008,0.503528,0.504032,0.494456,0.491935
+delete,31,0.548387,0.516129,0.451613,0.580645,0.548387,0.483871,0.548387,0.387097,0.451613,0.451613,0.451613,0.387097,0.677419,0.419355,0.354839,0.612903,0.387097,0.419355,0.548387,0.645161,0.451613,0.548387,0.451613,0.322581,0.516129,0.483871,0.516129,0.387097,0.483871,0.483871,0.483871,0.451613,0.483871,0.516129,0.645161,0.483871,0.580645,0.612903,0.677419,0.516129,0.516129,0.516129,0.516129,0.516129,0.677419,0.419355,0.516129,0.612903,0.612903,0.451613,0.483871,0.419355,0.516129,0.419355,0.580645,0.483871,0.645161,0.419355,0.419355,0.677419,0.451613,0.483871,0.419355,0.516129
+add,2173,0.506213,0.502531,0.484584,0.506213,0.505292,0.469397,0.497929,0.499310,0.487345,0.516797,0.502071,0.504832,0.515416,0.498389,0.486884,0.492867,0.505752,0.484123,0.513116,0.503451,0.508974,0.494248,0.501150,0.509894,0.485964,0.482743,0.490106,0.501150,0.487345,0.491486,0.515416,0.502071,0.497009,0.480902,0.499310,0.510815,0.509894,0.499310,0.511735,0.510354,0.498389,0.485504,0.497009,0.511275,0.504832,0.490566,0.495628,0.495628,0.498850,0.497009,0.485504,0.498850,0.506673,0.488265,0.492407,0.509434,0.495168,0.489185,0.500230,0.489646,0.498850,0.493327,0.493327,0.504832
+swap,28,0.535714,0.607143,0.428571,0.321429,0.535714,0.464286,0.500000,0.357143,0.464286,0.500000,0.464286,0.607143,0.535714,0.357143,0.607143,0.321429,0.428571,0.571429,0.464286,0.357143,0.357143,0.464286,0.464286,0.607143,0.500000,0.642857,0.642857,0.535714,0.428571,0.464286,0.535714,0.535714,0.285714,0.607143,0.500000,0.571429,0.428571,0.642857,0.500000,0.607143,0.571429,0.392857,0.500000,0.535714,0.535714,0.392857,0.571429,0.285714,0.642857,0.500000,0.535714,0.321429,0.428571,0.428571,0.464286,0.678571,0.642857,0.571429,0.607143,0.535714,0.535714,0.571429,0.285714,0.357143
+case,29,0.448276,0.551724,0.586207,0.620690,0.482759,0.655172,0.655172,0.586207,0.517241,0.379310,0.551724,0.448276,0.482759,0.482759,0.586207,0.448276,0.517241,0.448276,0.448276,0.517241,0.551724,0.620690,0.655172,0.655172,0.551724,0.586207,0.655172,0.586207,0.482759,0.379310,0.586207,0.517241,0.517241,0.551724,0.517241,0.655172,0.413793,0.551724,0.551724,0.448276,0.517241,0.379310,0.517241,0.517241,0.586207,0.310345,0.620690,0.379310,0.413793,0.448276,0.586207,0.689655,0.482759,0.310345,0.551724,0.413793,0.448276,0.517241,0.586207,0.551724,0.517241,0.448276,0.448276,0.448276
+first,186,0.521505,0.456989,0.543011,0.516129,0.505376,0.494624,0.569892,0.516129,0.564516,0.500000,0.500000,0.500000,0.516129,0.446237,0.580645,0.532258,0.521505,0.521505,0.473118,0.462366,0.462366,0.473118,0.526882,0.500000,0.500000,0.516129,0.532258,0.532258,0.489247,0.559140,0.462366,0.564516,0.521505,0.467742,0.456989,0.516129,0.537634,0.483871,0.548387,0.500000,0.532258,0.516129,0.478495,0.478495,0.462366,0.483871,0.494624,0.526882,0.537634,0.478495,0.446237,0.473118,0.478495,0.564516,0.435484,0.467742,0.526882,0.494624,0.494624,0.494624,0.543011,0.478495,0.430108,0.494624
+last,186,0.526882,0.446237,0.526882,0.521505,0.494624,0.521505,0.521505,0.483871,0.516129,0.500000,0.478495,0.500000,0.564516,0.500000,0.381720,0.500000,0.537634,0.473118,0.532258,0.543011,0.505376,0.510753,0.483871,0.456989,0.564516,0.489247,0.596774,0.526882,0.510753,0.462366,0.494624,0.494624,0.408602,0.548387,0.505376,0.543011,0.494624,0.467742,0.462366,0.440860,0.548387,0.505376,0.516129,0.510753,0.543011,0.462366,0.478495,0.505376,0.478495,0.532258,0.510753,0.586022,0.526882,0.478495,0.467742,0.516129,0.543011,0.478495,0.510753,0.478495,0.494624,0.505376,0.548387,0.478495

hash_testing/new_testing/tests/test_plot_probability_map.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Behavior tests for plot_probability_map.py."""
+
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+from PIL import Image
+
+import plot_probability_map
+
+
+class ProbabilityMapPlotTests(unittest.TestCase):
+    def test_reads_bit_columns_and_ignores_operation_and_pairs(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            path = Path(temporary_directory) / "hash.csv"
+            path.write_text(
+                "operation,pairs,bit_0,bit_1\n"
+                "replace,4,0.25,0.75\n"
+                "delete,2,0.5,nan\n",
+                encoding="utf-8",
+            )
+
+            rows = plot_probability_map.read_probability_map(path)
+
+            self.assertEqual(rows[0].operation, "replace")
+            self.assertEqual(rows[0].pairs, 4)
+            self.assertEqual(rows[0].probabilities, [0.25, 0.75])
+            self.assertEqual(rows[1].operation, "delete")
+            self.assertEqual(rows[1].pairs, 2)
+            self.assertEqual(len(rows[1].probabilities), 2)
+
+    def test_mean_absolute_deviation_from_half_ignores_nan(self) -> None:
+        result = plot_probability_map.mean_absolute_deviation(
+            [0.25, 0.5, 0.75, float("nan")]
+        )
+
+        self.assertEqual(result, 1 / 6)
+
+    def test_shared_scale_zooms_to_all_values_and_reference(self) -> None:
+        scale = plot_probability_map.make_scale(
+            [0.45, 0.48, 0.52, 0.55],
+            reference=0.5,
+            hard_limits=(0.0, 1.0),
+        )
+
+        self.assertGreater(scale.minimum, 0.0)
+        self.assertLess(scale.maximum, 1.0)
+        self.assertLessEqual(scale.minimum, 0.45)
+        self.assertGreaterEqual(scale.maximum, 0.55)
+        self.assertIn(0.5, scale.ticks)
+
+    def test_deviation_scale_uses_data_range_instead_of_fixed_half(self) -> None:
+        scale = plot_probability_map.make_scale(
+            [0.05, 0.08], hard_limits=(0.0, 0.5)
+        )
+
+        self.assertGreater(scale.minimum, 0.0)
+        self.assertLess(scale.maximum, 0.5)
+        self.assertLessEqual(scale.minimum, 0.05)
+        self.assertGreaterEqual(scale.maximum, 0.08)
+
+    def test_generates_two_nonempty_png_files(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            root = Path(temporary_directory)
+            source = root / "sample.csv"
+            source.write_text(
+                "operation,pairs,bit_0,bit_1,bit_2,bit_3\n"
+                "replace,4,0.25,0.50,0.75,1.0\n"
+                "delete,2,0.10,0.20,0.30,0.40\n",
+                encoding="utf-8",
+            )
+
+            outputs = plot_probability_map.generate_plots(source, root / "plots")
+
+            self.assertEqual(len(outputs), 2)
+            for output in outputs:
+                self.assertTrue(output.is_file())
+                with Image.open(output) as image:
+                    self.assertEqual(image.format, "PNG")
+                    self.assertGreater(image.width, 300)
+                    self.assertGreater(image.height, 200)
+                    colors = image.convert("RGB").getcolors(maxcolors=1_000_000)
+                    self.assertIsNotNone(colors)
+                    self.assertGreater(len(colors or []), 2)
+
+
+if __name__ == "__main__":
+    unittest.main()

hash_testing/new_testing/tests/test_probability_map.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Behavior tests for probability_map.py."""
+
+from __future__ import annotations
+
+import csv
+import io
+import os
+import subprocess
+import tempfile
+import unittest
+from contextlib import redirect_stderr
+from pathlib import Path
+
+import generate_input
+import probability_map
+
+
+class PrepareHashTests(unittest.TestCase):
+    def test_compiles_c_source_when_binary_is_missing(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            root = Path(temporary_directory)
+            hash_directory = root / "constant"
+            hash_directory.mkdir()
+            (hash_directory / "bin_hash.c").write_text(
+                "#include <stdio.h>\n"
+                "int main(void) { puts(\"00000001\"); return 0; }\n",
+                encoding="utf-8",
+            )
+
+            executable = probability_map.prepare_hash("constant", root)
+
+            self.assertEqual(executable, hash_directory / "bin_hash")
+            self.assertEqual(
+                subprocess.check_output([executable, "word"], text=True).strip(),
+                "00000001",
+            )
+
+    def test_adds_python_shebang_and_execute_permission(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            root = Path(temporary_directory)
+            hash_directory = root / "python_hash"
+            hash_directory.mkdir()
+            source = hash_directory / "bin_hash.py"
+            source.write_text("print('00000002')\n", encoding="utf-8")
+
+            executable = probability_map.prepare_hash("python_hash", root)
+
+            self.assertEqual(executable, source)
+            self.assertTrue(os.access(source, os.X_OK))
+            self.assertTrue(
+                source.read_text(encoding="utf-8").startswith(
+                    "#!/usr/bin/env python3\n"
+                )
+            )
+            self.assertEqual(
+                subprocess.check_output([executable, "word"], text=True).strip(),
+                "00000002",
+            )
+
+
+class ProbabilityTests(unittest.TestCase):
+    def test_counts_changed_hash_bits_relative_to_source(self) -> None:
+        probabilities = probability_map.bit_probabilities(
+            source_hash=0b0000,
+            changed_hashes=[0b0001, 0b0011, 0b0010, 0b0000],
+            bits=4,
+        )
+
+        self.assertEqual(probabilities, [0.5, 0.5, 0.0, 0.0])
+
+    def test_csv_table_has_operation_rows_and_bit_columns(self) -> None:
+        stream = io.StringIO()
+
+        probability_map.write_csv_table(
+            stream,
+            {
+                "replace": (4, [0.25, 0.75]),
+                "delete": (2, [0.5, 0.0]),
+            },
+        )
+
+        rows = list(csv.reader(io.StringIO(stream.getvalue())))
+        self.assertEqual(rows[0], ["operation", "pairs", "bit_0", "bit_1"])
+        self.assertEqual(rows[1], ["replace", "4", "0.250000", "0.750000"])
+        self.assertEqual(rows[2], ["delete", "2", "0.500000", "0.000000"])
+
+    def test_parser_accepts_multiple_source_words(self) -> None:
+        arguments = probability_map.build_parser().parse_args(["first", "second"])
+
+        self.assertEqual(arguments.words, ["first", "second"])
+
+    def test_aggregates_samples_from_multiple_words_and_warns_on_shortfall(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            executable = Path(temporary_directory) / "hash.py"
+            executable.write_text(
+                "#!/usr/bin/env python3\n"
+                "import sys\n"
+                "print(f'{sum(sys.argv[1].encode()):08x}')\n",
+                encoding="utf-8",
+            )
+            executable.chmod(0o755)
+            warnings = io.StringIO()
+
+            with redirect_stderr(warnings):
+                table = probability_map.build_probability_table(
+                    executable=executable,
+                    sources=["A", "B"],
+                    operations=["delete"],
+                    count=10,
+                    operation_count=1,
+                    alphabet=generate_input.DEFAULT_ALPHABET,
+                    seed=1,
+                    max_attempts=None,
+                )
+
+            pair_count, probabilities = table["delete"]
+            self.assertEqual(pair_count, 2)
+            self.assertEqual(
+                probabilities[:8],
+                [0.5, 0.5, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0],
+            )
+            self.assertEqual(warnings.getvalue().count("delete"), 2)
+
+
+class GenerateWordsTests(unittest.TestCase):
+    def test_count_is_an_upper_bound_when_unique_results_are_exhausted(self) -> None:
+        words = generate_input.generate_words(
+            source="abc",
+            count=100,
+            operation="delete",
+            operation_count=1,
+            seed=42,
+        )
+
+        self.assertEqual(set(words), {"ab", "ac", "bc"})
+
+
+if __name__ == "__main__":
+    unittest.main()

hash_testing/new_testing/tests/test_t1ha2_bin_hash.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""Behavior tests for the standalone t1ha2_atonce hash CLI."""
+
+from __future__ import annotations
+
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SOURCE = ROOT / "hash_funcs" / "t1ha2" / "bin_hash.c"
+
+
+class T1ha2BinHashTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.temporary_directory = tempfile.TemporaryDirectory()
+        cls.binary = Path(cls.temporary_directory.name) / "bin_hash"
+        subprocess.run(
+            [
+                "cc",
+                "-std=c11",
+                "-O2",
+                "-Wall",
+                "-Wextra",
+                "-Wpedantic",
+                "-Werror",
+                str(SOURCE),
+                "-o",
+                str(cls.binary),
+            ],
+            check=True,
+            text=True,
+            capture_output=True,
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.temporary_directory.cleanup()
+
+    def run_hash(
+        self, word: str | None = None, stdin: bytes | None = None
+    ) -> subprocess.CompletedProcess[bytes]:
+        command = [str(self.binary)]
+        if word is not None:
+            command.append(word)
+        return subprocess.run(command, input=stdin, capture_output=True, check=False)
+
+    def test_matches_upstream_t1ha2_atonce_seed_zero_vectors(self) -> None:
+        vectors = {
+            "": b"0000000000000000\n",
+            "hello": b"2a5f2abd74df73b4\n",
+            "HashWord": b"3885d16135ce64f0\n",
+            "abc": b"16bae0f716c45f2e\n",
+            "12345678901234567890123456789012": b"75ed8a8aa66a4602\n",
+        }
+
+        for word, expected in vectors.items():
+            with self.subTest(word=word):
+                result = self.run_hash(word)
+                self.assertEqual(result.returncode, 0)
+                self.assertEqual(result.stdout, expected)
+                self.assertRegex(result.stdout.decode(), r"^[0-9a-f]{16}\n$")
+
+    def test_argv_and_stdin_are_equivalent_and_strip_trailing_ascii_space(self) -> None:
+        argv = self.run_hash("hello")
+        stdin = self.run_hash(stdin=b"hello \t\r\n")
+
+        self.assertEqual(stdin.returncode, 0)
+        self.assertEqual(stdin.stdout, argv.stdout)
+
+    def test_handles_long_ascii_input(self) -> None:
+        payload = b"a" * 100_000
+        result = self.run_hash(stdin=payload)
+
+        self.assertEqual(result.returncode, 0)
+        self.assertEqual(result.stdout, b"bdf3f8539f0504ea\n")
+
+    def test_rejects_non_ascii_input(self) -> None:
+        result = self.run_hash(stdin="ёж".encode())
+
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn(b"ASCII", result.stderr)
+        self.assertEqual(result.stdout, b"")
+
+    def test_unaligned_argv_is_clean_under_undefined_behavior_sanitizer(self) -> None:
+        sanitized = Path(self.temporary_directory.name) / "bin_hash_ubsan"
+        subprocess.run(
+            [
+                "cc",
+                "-std=c11",
+                "-O1",
+                "-g",
+                "-fsanitize=undefined",
+                "-fno-sanitize-recover=undefined",
+                str(SOURCE),
+                "-o",
+                str(sanitized),
+            ],
+            check=True,
+            capture_output=True,
+        )
+
+        for word in ("hello", "12345678", "a" * 33):
+            with self.subTest(word=word):
+                result = subprocess.run(
+                    [str(sanitized), word], capture_output=True, check=False
+                )
+                self.assertEqual(result.returncode, 0, result.stderr.decode())
+
+
+if __name__ == "__main__":
+    unittest.main()

hash_testing/new_testing/tests/test_xxh64_bin_hash.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""Behavior tests for the standalone XXH64 hash CLI."""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SOURCE = ROOT / "hash_funcs" / "xxh64" / "bin_hash.c"
+
+
+class Xxh64BinHashTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.temporary_directory = tempfile.TemporaryDirectory()
+        cls.binary = Path(cls.temporary_directory.name) / "bin_hash"
+        subprocess.run(
+            [
+                "cc",
+                "-std=c11",
+                "-O2",
+                "-Wall",
+                "-Wextra",
+                "-Wpedantic",
+                "-Werror",
+                str(SOURCE),
+                "-o",
+                str(cls.binary),
+            ],
+            check=True,
+            text=True,
+            capture_output=True,
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.temporary_directory.cleanup()
+
+    def run_hash(self, word: str | None = None, stdin: bytes | None = None) -> subprocess.CompletedProcess[bytes]:
+        command = [str(self.binary)]
+        if word is not None:
+            command.append(word)
+        return subprocess.run(command, input=stdin, capture_output=True, check=False)
+
+    def test_matches_official_xxh64_seed_zero_vectors(self) -> None:
+        vectors = {
+            "": b"ef46db3751d8e999\n",
+            "hello": b"26c7827d889f6da3\n",
+            "HashWord": b"3e26fc2935163fbe\n",
+        }
+
+        for word, expected in vectors.items():
+            with self.subTest(word=word):
+                result = self.run_hash(word)
+                self.assertEqual(result.returncode, 0)
+                self.assertEqual(result.stdout, expected)
+                self.assertRegex(result.stdout.decode(), r"^[0-9a-f]{16}\n$")
+
+    def test_argv_and_stdin_are_equivalent_and_strip_trailing_ascii_space(self) -> None:
+        argv = self.run_hash("hello")
+        stdin = self.run_hash(stdin=b"hello \t\r\n")
+
+        self.assertEqual(stdin.returncode, 0)
+        self.assertEqual(stdin.stdout, argv.stdout)
+
+    def test_handles_long_ascii_input(self) -> None:
+        payload = b"a" * 100_000
+        result = self.run_hash(stdin=payload)
+        reference = subprocess.run(
+            ["xxhsum", "-H64"], input=payload, capture_output=True, check=True
+        ).stdout.split()[0]
+
+        self.assertEqual(result.returncode, 0)
+        self.assertEqual(result.stdout.strip(), reference)
+
+    def test_rejects_non_ascii_input(self) -> None:
+        result = self.run_hash(stdin="ёж".encode())
+
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn(b"ASCII", result.stderr)
+        self.assertEqual(result.stdout, b"")
+
+
+if __name__ == "__main__":
+    unittest.main()

new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py

@@ -84,13 +84,16 @@ def sha256_file(path: Path) -> str:
     return digest.hexdigest()
 
 
-def compile_converter(destination: Path) -> None:
+def compile_converter(
+    destination: Path,
+    rpm_include_dir: Path | None = None,
+    rpm_libraries: tuple[Path, ...] = (),
+) -> None:
     include = destination.parent / "compat-include"
     include.mkdir()
     for name in ("rpmlib.h", "system.h", "set.h"):
         (include / name).touch()
-    run(
-        [
+    command = [
             "cc",
             "-O2",
             "-std=gnu11",
@@ -102,16 +105,23 @@ def compile_converter(destination: Path) -> None:
             "-DARSV_WITH_RPM",
             "-I",
             str(include),
+    ]
+    if rpm_include_dir is not None:
+        command.extend(["-I", str(rpm_include_dir)])
+    command.extend(
+        [
             "-include",
             str(ROOT / "scripts/rpmsetcmp/newset_compat.h"),
             str(ROOT / "reimplement/set9.c"),
             str(HERE / "rewrite_sisyphus_pkglist.c"),
-            "-lrpm",
-            "-lrpmio",
-            "-o",
-            str(destination),
         ]
     )
+    if rpm_libraries:
+        command.extend(str(path) for path in rpm_libraries)
+    else:
+        command.extend(["-lrpm", "-lrpmio"])
+    command.extend(["-o", str(destination)])
+    run(command)
 
 
 def find_pkglists(lists_dir: Path) -> dict[str, Path]:
@@ -212,7 +222,12 @@ def convert_one(converter: Path, source: Path, destination: Path) -> dict[str, o
     }
 
 
-def convert_local(output: Path, lists_dir: Path) -> None:
+def convert_local(
+    output: Path,
+    lists_dir: Path,
+    rpm_include_dir: Path | None = None,
+    rpm_libraries: tuple[Path, ...] = (),
+) -> None:
     output.mkdir(parents=True, exist_ok=True)
     if any(output.iterdir()):
         raise RuntimeError(f"output directory is not empty: {output}")
@@ -224,7 +239,7 @@ def convert_local(output: Path, lists_dir: Path) -> None:
     staging = Path(tempfile.mkdtemp(prefix=".d1-staging-", dir=output))
     try:
         converter = staging / "rewrite-sisyphus-pkglist"
-        compile_converter(converter)
+        compile_converter(converter, rpm_include_dir, rpm_libraries)
         architectures: dict[str, object] = {}
         for architecture in ARCHITECTURES:
             destination = staging / f"Sisyphus.{architecture}.pkglist.classic"
@@ -277,6 +292,18 @@ def parse_args() -> argparse.Namespace:
         default=Path("/var/lib/apt/lists"),
         help="APT lists snapshot to convert (default: /var/lib/apt/lists)",
     )
+    parser.add_argument(
+        "--rpm-include-dir",
+        type=Path,
+        help="directory containing rpm/header.h (for an unpacked librpm-devel)",
+    )
+    parser.add_argument(
+        "--rpm-library",
+        type=Path,
+        action="append",
+        default=[],
+        help="versioned RPM library to link; repeat for librpm and librpmio",
+    )
     return parser.parse_args()
 
 
@@ -284,7 +311,20 @@ def main() -> int:
     args = parse_args()
     try:
         output = validate_output(args.output)
-        convert_local(output, args.lists_dir)
+        rpm_include_dir = (
+            args.rpm_include_dir.expanduser().resolve()
+            if args.rpm_include_dir is not None
+            else None
+        )
+        rpm_libraries = tuple(path.expanduser().resolve() for path in args.rpm_library)
+        if rpm_include_dir is not None and not (rpm_include_dir / "rpm/header.h").is_file():
+            raise ValueError(f"rpm/header.h not found below: {rpm_include_dir}")
+        if rpm_libraries and len(rpm_libraries) != 2:
+            raise ValueError("pass exactly two --rpm-library values: librpm and librpmio")
+        for library in rpm_libraries:
+            if not library.is_file():
+                raise ValueError(f"RPM library not found: {library}")
+        convert_local(output, args.lists_dir, rpm_include_dir, rpm_libraries)
     except (OSError, RuntimeError, ValueError) as error:
         print(f"error: {error}", file=sys.stderr)
         return 1

scripts/run-sisyphus-set9-d1-bench.sh

@@ -15,6 +15,9 @@ REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd)
 SET9_C=${SET9_C:-$REPO_ROOT/reimplement/set9.c}
 D1_C=${D1_C:-$REPO_ROOT/new_version/direct_hash/hash_set.c}
 PKGLIST_CONVERTER=${PKGLIST_CONVERTER:-$REPO_ROOT/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py}
+SET_REWRITER_C=$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c
+SET_COMPAT_H=$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h
+ORCHESTRATOR=$(realpath -e -- "${BASH_SOURCE[0]}")
 
 WORK_ROOT=${WORK_ROOT:-$HOME/sisyphus-set9-d1-bench}
 RESULT_DIR=${RESULT_DIR:-$WORK_ROOT/results}
@@ -99,17 +102,38 @@ write_snapshot_fingerprint()
 {
     {
         printf 'sisyphus_mirror=%s\n' "$SISYPHUS_MIRROR"
+        printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')"
         printf 'converter=%s\n' "$(sha256sum "$PKGLIST_CONVERTER" | awk '{print $1}')"
         printf 'set9=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
+        printf 'd1=%s\n' "$(sha256sum "$D1_C" | awk '{print $1}')"
         printf 'rewrite=%s\n' "$(sha256sum "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" | awk '{print $1}')"
         printf 'compat=%s\n' "$(sha256sum "$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h" | awk '{print $1}')"
+        [[ ! -f $SNAPSHOT/devel-input-fingerprint.txt ]] ||
+            cat "$SNAPSHOT/devel-input-fingerprint.txt"
+        write_runtime_fingerprint
     }
 }
 
+write_runtime_fingerprint()
+{
+    local rpm_libdir library resolved owner
+    rpm_libdir=$(rpm --eval '%{_libdir}')
+    for library in librpm.so.7 librpmio.so.7; do
+        resolved=$(realpath -e "$rpm_libdir/$library") ||
+            fail "installed $library not found"
+        owner=$(rpm -qf --qf '%{NAME}|%{SOURCERPM}|%{DISTTAG}' "$resolved") ||
+            fail "cannot identify package owning $resolved"
+        printf 'runtime_%s_owner=%s\n' "$library" "$owner"
+        printf 'runtime_%s_sha256=%s\n' "$library" \
+            "$(sha256sum "$resolved" | awk '{print $1}')"
+    done
+}
+
 validate_snapshot_reuse()
 {
     local current
-    [[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt ]] || return 1
+    [[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt &&
+       -f $SNAPSHOT/devel-input-fingerprint.txt ]] || return 1
     current=$(mktemp)
     write_snapshot_fingerprint >"$current"
     if ! cmp -s "$current" "$SNAPSHOT/input-fingerprint.txt"; then
@@ -163,10 +187,52 @@ prepare_spec()
 " "$spec"
 }
 
+prepare_d1_build_corpus()
+{
+    local source=$1 spec helper_dir
+    spec=$source/alt/rpm.spec
+    helper_dir=$source/alt/arsv-d1-build
+    mkdir -p "$helper_dir"
+    cp "$SET9_C" "$helper_dir/set9.c"
+    cp "$SET_REWRITER_C" "$helper_dir/rewrite_sisyphus_pkglist.c"
+    cp "$SET_COMPAT_H" "$helper_dir/newset_compat.h"
+
+    python3 - "$spec" <<'PY'
+import sys
+from pathlib import Path
+
+spec = Path(sys.argv[1])
+text = spec.read_text()
+needle = "join -o 1.3,2.3 P R |shuf >setcmp-data\n"
+replacement = needle + r'''# The buildroot RPM database contains legacy set9 values.  This D1 build
+# preserves the same relation corpus, but converts both operands before the
+# format-specific setcmp/profile checks instead of feeding incompatible input.
+mkdir -p arsv-d1-compat
+touch arsv-d1-compat/rpmlib.h arsv-d1-compat/system.h arsv-d1-compat/set.h
+%__cc %optflags -std=gnu11 -Wall -Wextra -Werror -D_GNU_SOURCE -DARSV_SET9_EXPORT \
+    -I arsv-d1-compat -include alt/arsv-d1-build/newset_compat.h \
+    alt/arsv-d1-build/set9.c alt/arsv-d1-build/rewrite_sisyphus_pkglist.c \
+    -o arsv-convert-set
+while read -r set1 set2; do
+    d1_set1=$(./arsv-convert-set --convert-set "$set1")
+    d1_set2=$(./arsv-convert-set --convert-set "$set2")
+    printf '%%s %%s\n' "$d1_set1" "$d1_set2"
+done <setcmp-data >setcmp-data.d1
+test "$(wc -l <setcmp-data.d1)" -eq "$(wc -l <setcmp-data)"
+mv setcmp-data.d1 setcmp-data
+'''
+if text.count(needle) != 1:
+    raise SystemExit(f"expected exactly one setcmp corpus creation in {spec}")
+spec.write_text(text.replace(needle, replacement))
+PY
+    git -C "$source" add alt/arsv-d1-build
+}
+
 apt_options()
 {
     local variant=$1
     APT_OPTIONS=(
+        -o 'Debug::NoLocking=true'
         -o 'Dir::Etc::main=-'
         -o 'Dir::Etc::parts=-'
         -o "Dir::Etc::sourcelist=$SNAPSHOT/etc-apt/sources.list"
@@ -287,8 +353,10 @@ run_once()
 
 prepare_snapshot()
 {
-    local converter_output
+    local converter_output devel_cache devel_root rpm_libdir rpm_library rpmio_library
+    local devel_source runtime_source rpmfile
     local -a update_options
+    local -a rpm_devel_rpms popt_devel_rpms
     validate_snapshot_reuse && return
 
     printf '\n===== Preparing one immutable Sisyphus metadata snapshot =====\n'
@@ -319,14 +387,69 @@ prepare_snapshot()
     APT_CONFIG="$COMMON/apt.conf" "$APT_GET" -qq \
         "${update_options[@]}" update
 
+    # The converter needs C headers, but the benchmark host intentionally need
+    # not have development packages installed. Download the p11 packages that
+    # match the host's installed librpm into a private cache and extract only
+    # their headers; no system package is installed or changed.
+    devel_cache="$SNAPSHOT/devel-cache"
+    devel_root="$SNAPSHOT/devel-root"
+    mkdir -p "$devel_cache/archives/partial" "$devel_root"
+    env -u APT_CONFIG "$APT_GET" -qq -y -d \
+        -o "Dir::Cache=$devel_cache" \
+        -o "Dir::Cache::archives=$devel_cache/archives" \
+        -o "Dir::Cache::pkgcache=$devel_cache/pkgcache.bin" \
+        -o "Dir::Cache::srcpkgcache=$devel_cache/srcpkgcache.bin" \
+        install librpm-devel
+    shopt -s nullglob
+    rpm_devel_rpms=("$devel_cache"/archives/librpm-devel_*.rpm)
+    popt_devel_rpms=("$devel_cache"/archives/libpopt-devel_*.rpm)
+    shopt -u nullglob
+    ((${#rpm_devel_rpms[@]} == 1)) ||
+        fail "expected one downloaded librpm-devel RPM, got ${#rpm_devel_rpms[@]}"
+    ((${#popt_devel_rpms[@]} == 1)) ||
+        fail "expected one downloaded libpopt-devel RPM, got ${#popt_devel_rpms[@]}"
+    devel_source=$(rpm -qp --qf '%{SOURCERPM}|%{DISTTAG}' "${rpm_devel_rpms[0]}") ||
+        fail 'cannot identify downloaded librpm-devel source package'
+    for rpmfile in "${rpm_devel_rpms[@]}" "${popt_devel_rpms[@]}"; do
+        (cd "$devel_root" && rpm2cpio "$rpmfile" | cpio -idm --quiet './usr/include/*')
+    done
+    [[ -f $devel_root/usr/include/rpm/header.h && -f $devel_root/usr/include/popt.h ]] ||
+        fail 'failed to extract RPM development headers'
+    rpm_libdir=$(rpm --eval '%{_libdir}')
+    rpm_library=$(realpath -e "$rpm_libdir/librpm.so.7") ||
+        fail 'installed librpm.so.7 not found'
+    rpmio_library=$(realpath -e "$rpm_libdir/librpmio.so.7") ||
+        fail 'installed librpmio.so.7 not found'
+    for rpmfile in "$rpm_library" "$rpmio_library"; do
+        runtime_source=$(rpm -qf --qf '%{SOURCERPM}|%{DISTTAG}' "$rpmfile") ||
+            fail "cannot identify runtime package owning $rpmfile"
+        [[ $runtime_source == "$devel_source" ]] ||
+            fail "downloaded librpm-devel ($devel_source) does not match $rpmfile ($runtime_source)"
+    done
+    {
+        printf 'devel_librpm_identity=%s|%s\n' \
+            "$(rpm -qp --qf '%{NAME}|%{EVR}|%{DISTTAG}|%{ARCH}' "${rpm_devel_rpms[0]}")" \
+            "$devel_source"
+        printf 'devel_librpm_sha256=%s\n' \
+            "$(sha256sum "${rpm_devel_rpms[0]}" | awk '{print $1}')"
+        printf 'devel_popt_identity=%s\n' \
+            "$(rpm -qp --qf '%{NAME}|%{EVR}|%{DISTTAG}|%{ARCH}' "${popt_devel_rpms[0]}")"
+        printf 'devel_popt_sha256=%s\n' \
+            "$(sha256sum "${popt_devel_rpms[0]}" | awk '{print $1}')"
+    } >"$SNAPSHOT/devel-input-fingerprint.txt"
+
     converter_output="$SNAPSHOT/conversion"
     python3 "$PKGLIST_CONVERTER" "$converter_output" \
-        --lists-dir "$SNAPSHOT/lists"
+        --lists-dir "$SNAPSHOT/lists" \
+        --rpm-include-dir "$devel_root/usr/include" \
+        --rpm-library "$rpm_library" \
+        --rpm-library "$rpmio_library"
     mv "$converter_output/d1-pkglists/manifest.json" "$SNAPSHOT/manifest.json"
     mkdir -p "$SNAPSHOT/d1-pkglists"
     mv "$converter_output/d1-pkglists/"*.classic "$SNAPSHOT/d1-pkglists/"
     rmdir "$converter_output/d1-pkglists" "$converter_output"
     rm -rf "$SNAPSHOT/download-cache"
+    rm -rf "$devel_cache" "$devel_root"
     rm -f "$SNAPSHOT/lists/lock"
     rm -rf "$SNAPSHOT/lists/partial"
     mkdir -p "$SNAPSHOT/lists/partial"
@@ -387,9 +510,15 @@ build_variant()
     expected_fingerprint=$(mktemp)
     {
         printf 'base_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
+        printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')"
         printf 'set_source=%s\n' "$(sha256sum "$source_c" | awk '{print $1}')"
         printf 'suffix=%s\n' "$suffix"
         printf 'packager=%s\n' "$PACKAGER"
+        if [[ $name == d1 ]]; then
+            printf 'set9_decoder=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
+            printf 'set_rewriter=%s\n' "$(sha256sum "$SET_REWRITER_C" | awk '{print $1}')"
+            printf 'set_compat=%s\n' "$(sha256sum "$SET_COMPAT_H" | awk '{print $1}')"
+        fi
     } >"$expected_fingerprint"
     if [[ -f $variant/input-fingerprint.txt ]]; then
         if ! cmp -s "$expected_fingerprint" "$variant/input-fingerprint.txt"; then
@@ -407,10 +536,24 @@ build_variant()
         git clone --local "$SOURCE_BASE" "$source"
         cp "$source_c" "$source/lib/set.c"
         prepare_spec "$spec" "$suffix"
+        if [[ $name == d1 ]]; then
+            prepare_d1_build_corpus "$source"
+        fi
+        sha256sum "$spec" >"$variant/prepared-spec.sha256"
         cp "$expected_fingerprint" "$variant/input-fingerprint.txt"
     else
+        [[ -f $variant/prepared-spec.sha256 ]] &&
+            sha256sum -c "$variant/prepared-spec.sha256" >/dev/null ||
+            fail "$name prepared RPM spec changed; use RESET_WORK=1"
         cmp -s "$source_c" "$source/lib/set.c" ||
             fail "$source_c changed; use RESET_WORK=1"
+        if [[ $name == d1 ]]; then
+            cmp -s "$SET9_C" "$source/alt/arsv-d1-build/set9.c" &&
+                cmp -s "$SET_REWRITER_C" \
+                    "$source/alt/arsv-d1-build/rewrite_sisyphus_pkglist.c" &&
+                cmp -s "$SET_COMPAT_H" "$source/alt/arsv-d1-build/newset_compat.h" ||
+                fail 'D1 build corpus converter changed; use RESET_WORK=1'
+        fi
     fi
     rm -f "$expected_fingerprint"
 
@@ -529,9 +672,11 @@ write_provenance()
         printf 'cpu=%s\n' "$CPU"
         printf 'rounds=%s\n' "$ROUNDS"
         printf 'operations=%s\n' "$OPERATIONS"
-        sha256sum "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \
+        sha256sum "$ORCHESTRATOR" "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \
             "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" \
             "$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h"
+        cat "$SNAPSHOT/devel-input-fingerprint.txt"
+        write_runtime_fingerprint
         printf 'set9_librpm='; cat "$SET9_VARIANT/librpm-artifact.sha256"
         printf 'd1_librpm='; cat "$D1_VARIANT/librpm-artifact.sha256"
     } >"$RESULT_DIR/provenance.txt"