Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 krosh
|
452190b8c4 | ||
|
krosh
|
f72d345f0d | ||
|
krosh
|
558450b573 | ||
|
krosh
|
01208f4143 | ||
|
krosh
|
7188ac6b4a | ||
|
krosh
|
2b30d26a56 | ||
|
krosh
|
da79041e88 | ||
|
krosh
|
f472bf6e36 | ||
|
krosh
|
9f17383626 | ||
|
krosh
|
3027f36307 | ||
|
krosh
|
373c5d71dc | ||
|
krosh
|
60ffed1f6f | ||
|
krosh
|
6fe66fa1ba | ||
|
krosh
|
d33caacfef | ||
|
krosh
|
0b7f6ea13a | ||
|
krosh
|
f4c170058e | ||
|
krosh
|
ac894a86b4 | ||
|
krosh
|
53332b0f62 | ||
|
krosh
|
a5e583f600 | ||
|
krosh
|
8b65eb5e37 | ||
|
krosh
|
39b4673421 | ||
|
krosh
|
4ed05f2061 |
+25
-12
@@ -1,17 +1,11 @@
|
||||
## original set.c
|
||||
|
||||
| command | average_seconds | run1_seconds | run2_seconds | run3_seconds | exit_status |
|
||||
| :-------------------------- | :-------------: | :----------: | :----------: | :----------: | ----------: |
|
||||
| -s check | 1.570747 | 1.621496 | 1.545001 | 1.545744 | 0 |
|
||||
| -s autoremove | 2.294502 | 2.312574 | 2.287133 | 2.283798 | 0 |
|
||||
| -s install rpm-build | 2.108121 | 2.142784 | 2.093474 | 2.088105 | 0 |
|
||||
| -s install openuds-server | 8.028098 | 8.147214 | 8.001140 | 7.935940 | 0 |
|
||||
| -s install password-store | 3.373377 | 3.396943 | 3.361290 | 3.361899 | 0 |
|
||||
| --enable-upgrade -s upgrade | 33.915979 | 33.895262 | 33.770192 | 34.082484 | 100 |
|
||||
| -s dist-upgrade | 88.722136 | 95.484250 | 85.808085 | 84.874073 | 100 |
|
||||
|
||||
## original set.c w/out optimizations
|
||||
|
||||
функция `decode_set` идёт по пути функций:
|
||||
|
||||
1. `decode_base62`
|
||||
2. `decode_golomb`
|
||||
3. `decode_delta`
|
||||
|
||||
| command | average_seconds | run1_seconds | run2_seconds | run3_seconds | exit_status |
|
||||
| :-------------------------- | :-------------: | :----------: | :----------: | :----------: | ----------: |
|
||||
| -s check | 4.217343 | 4.310882 | 4.179343 | 4.161804 | 0 |
|
||||
@@ -22,8 +16,27 @@
|
||||
| --enable-upgrade -s upgrade | 36.271838 | 35.511443 | 35.908641 | 37.395430 | 100 |
|
||||
| -s dist-upgrade | 90.550086 | 94.702198 | 92.174160 | 84.773901 | 100 |
|
||||
|
||||
## original set.c
|
||||
|
||||
функция `decode_set` идёт по пути функций:
|
||||
|
||||
1. `decode_base62_golomb`
|
||||
2. `decode_delta`
|
||||
|
||||
| command | average_seconds | run1_seconds | run2_seconds | run3_seconds | exit_status |
|
||||
| :-------------------------- | :-------------: | :----------: | :----------: | :----------: | ----------: |
|
||||
| -s check | 1.570747 | 1.621496 | 1.545001 | 1.545744 | 0 |
|
||||
| -s autoremove | 2.294502 | 2.312574 | 2.287133 | 2.283798 | 0 |
|
||||
| -s install rpm-build | 2.108121 | 2.142784 | 2.093474 | 2.088105 | 0 |
|
||||
| -s install openuds-server | 8.028098 | 8.147214 | 8.001140 | 7.935940 | 0 |
|
||||
| -s install password-store | 3.373377 | 3.396943 | 3.361290 | 3.361899 | 0 |
|
||||
| --enable-upgrade -s upgrade | 33.915979 | 33.895262 | 33.770192 | 34.082484 | 100 |
|
||||
| -s dist-upgrade | 88.722136 | 95.484250 | 85.808085 | 84.874073 | 100 |
|
||||
|
||||
## set9.c
|
||||
|
||||
Переписанная реализация `set.c`
|
||||
|
||||
| command | average_seconds | run1_seconds | run2_seconds | run3_seconds | exit_status |
|
||||
| :-------------------------- | :-------------: | :----------: | :----------: | :----------: | ----------: |
|
||||
| -s check | 1.550342 | 1.596307 | 1.532768 | 1.521950 | 0 |
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
# Работа над `set.c`: краткий план статьи
|
||||
|
||||
## 1. Зачем ALT RPM нужны set-строки
|
||||
|
||||
Обычная зависимость от версии библиотеки не гарантирует, что в ней остались все нужные программе символы: символ можно удалить, не сменив SONAME, а одинаковые SONAME могут скрывать разные наборы экспортов. Поэтому ALT RPM использует версии зависимостей вида `set:<encoded-set>`. В `Provides` такая строка описывает символы, предоставляемые библиотекой, а в `Requires` - символы, которые конкретный потребитель требует именно от неё. Сравниваются не номера версий, а включение множеств: все хэши из `Requires` должны присутствовать в `Provides`. Гарантия вероятностная, поскольку вместо имён хранятся усечённые хэши, но ответ "символ отсутствует", когда он есть мы не получим (ложноположительное срабатывание)
|
||||
|
||||
## 2. Как строится set-строка
|
||||
|
||||
Список символов формируется автодепами `rpm-build`. Для `Provides` из ELF извлекаются отфильтрованные экспортируемые базовые имена символов. Для `Requires` `ldd --bindings` связывает каждый сильный неопределённый символ с конкретной библиотекой, после чего для каждого провайдера строится отдельное множество.
|
||||
|
||||
Далее `mkset` использует API `set_new()` → `set_add()` → `set_fini()`:
|
||||
|
||||
1. для каждого имени вычисляется Jenkins OAAT и оставляются младшие `bpp` бит;
|
||||
2. хэши сортируются, повторы удаляются, о коллизиях выдаётся предупреждение;
|
||||
3. абсолютные значения заменяются дельтами;
|
||||
4. дельты сжимаются кодом Голомба-Райса с параметром `Mshift`;
|
||||
5. битовый поток переводится в RPM-безопасную Base62-строку.
|
||||
|
||||
При сравнении `rpmsetcmp()` выполняет обратное декодирование, приводит строки с разным `bpp` к общей точности и проверяет включение отсортированных наборов. В рабочем вызове первым операндом выгодно передавать `Provides`: исходный код кэширует прежде всего его декодирование. (переформулировать в сторону кэш есть только для)
|
||||
(сказать про подсёт bpp и Mshift на стороне)
|
||||
|
||||
## 3. Разбор исходного кода
|
||||
|
||||
Работа началась с изучения примерно десятилетнего `lib/set.c`. Код оказался большим по объёму и очень плотным: объединённый Base62/Golomb-декодер, таблица всех пар входных байтов, макросы состояний, прыжки по массиву через defined инструкции и подобие LRU-кэш. Отдельно были прослежены реальные точки вызова в `rpm` и `rpm-build`: во время сборки бинарник `mkset` создаёт строки, а `rpmsetcmp()` через `rpmRangesOverlap()` участвует в проверке зависимостей RPM и APT.
|
||||
|
||||
Результатом этого этапа стала документация: описание главных пяти функций API, формата строки, кодирования и декодирования, нормализации `bpp`, кэша, макросов сравнения и встроенного `SELF_TEST`. Таким образом, прежде чем менять алгоритм, для него была построена читаемая модель и зафиксированы его неочевидные инварианты.
|
||||
|
||||
## 4. Переписывание и последовательные оптимизации
|
||||
|
||||
Сначала алгоритм был перенесён в Python как понятная проверочная реализация. Затем появился читаемый C-вариант, а после него - оптимизированный `set9.c`, сохраняющий старый wire-format и публичный API. В нём были опробованы:
|
||||
(добавить про все промежуточные тоже)
|
||||
|
||||
- строковая арена вместо отдельного выделения памяти под каждое имя;
|
||||
- `qsort()` для малых наборов и radix sort для больших;
|
||||
- потоковое кодирование и декодирование без промежуточных массивов битов и дельт; (было и в оригинале, но не было в читаемом)
|
||||
- два раздельных bucketed LRU-кэша для `Provides` и `Requires`; (теперь даже LRU, нет постоянных realloc)
|
||||
- кэширование результата понижения `bpp`; (надо лучше вспомнить разницу)
|
||||
- более простой адаптивный поиск включения в отсортированных массивах;
|
||||
- явная проверка метаданных, размеров и кодов внутренних ошибок. (в оригинале по несколько раз)
|
||||
|
||||
Не все «читаемые» замены оказались быстрыми. Удаление специальных оптимизаций исходного файла замедляло обычные APT-сценарии примерно в 2–3 раза. `set9.c` вернулся к уровню оригинала и в проведённых симуляциях оказался быстрее/медленне него примерно на 1–10%, однако отдельный анализ показал, что старый слитый декодер и прыгающий проход по массивам всё ещё сильны. Поэтому был собран гибридный `set_frank.c`: новые структуры, encoder и кэш соединены с декодером и сравнением из исходного `set.c`.
|
||||
(сказать, что прироста не сильно, читаемость хуже, проверить реальные числа)
|
||||
|
||||
## 5. Как проверялись изменения
|
||||
|
||||
Тесты строились вокруг инвариантов формата: (логично, его и надо сохранить)
|
||||
|
||||
- исходный `SELF_TEST` и расширенные проверки публичного API;
|
||||
- побайтовое совпадение результата `set_fini()` старой и новой реализации для `bpp=10…32`; (уточнить, что на ранд данных)
|
||||
- дифференциальные случайные тесты равенства, включения, несравнимости и обоих направлений сравнения;
|
||||
- повреждённые строки, граничные `bpp`/`Mshift`, разные точности, заполнение и вытеснение кэша;
|
||||
- проверки освобождения памяти; (надо добавить про алгоритм, что этого не было, хе)
|
||||
- проверка реальных `Provides`/`Requires` из Sisyphus;
|
||||
- холодные и прогретые микробенчмарки, затем симуляции и настоящие вызовы ALT APT с проверкой одинакового результата.
|
||||
|
||||
(более того, потом всё равно предполагается запуск в сборочнице. вроде.)
|
||||
|
||||
(куда-то эти мысли про хэш надо впихнуть)
|
||||
Отдельно сравнивались Jenkins OAAT, CityHash и xxHash по скорости и коллизиям на случайных данных и ASCII-строках. Этот эксперимент помог отделить свойства хэш-функции от стоимости остального формата, но простая замена хэша не стала готовым решением: она меняет совместимость строк и не устраняет основную цену декодирования и поиска.
|
||||
|
||||
## 6. Альтернативные форматы и результаты
|
||||
|
||||
Параллельно были проверены варианты, не совместимые со старым форматом.
|
||||
|
||||
**Roaring Bitmap.** Создание bitmap могло быть быстрым, но равномерно распределённые хэши плохо подходят для такого представления. Для 1000 символов при `bpp=32` строка выросла с 3994 до 19924 символов; холодное сравнение было примерно в 4,5 раза, прогретое - в 76 раз медленнее `set9`. Zstd уменьшал строку до 14126 символов, но добавлял ещё больше работы при сравнении. Вариант оставлен как отрицательный эксперимент.
|
||||
|
||||
**Прямой массив хэшей в Base64.** Отказ от delta/Golomb ускорял отдельные микротесты: в одном прогоне холодное сравнение занимало `0,87×`, а прогретое `0,35×` времени `set9`, при росте строки с 3994 до 5340 символов. Но полный поток реальных пар Sisyphus показал обратное (эту стороны надо добить) - текущая реализация была примерно в 1,4 раза медленнее. На отдельном синтетическом APT-графе resolver, напротив, ускорялся на 7–18%, зато `gencaches` замедлялся примерно на 12%. Кроме того, стандартный Base64 использует padding `=` (да, я забыл это пофиксить), который RPM запрещает внутри версии зависимости. Поэтому прямой формат интересен как компромисс между размером и стоимостью декодирования, но в текущем виде не готов заменить set-строки.
|
||||
|
||||
**Другие направления.** Рассматривались отрицательный Bloom-подобный prefilter с обязательной точной проверкой, адаптивный индекс для больших `Provides`, Elias–Fano и глобальные идентификаторы символов. Общий вывод: (в это возможно стоит углубиться и сделать его)
|
||||
|
||||
## 7. Итого
|
||||
|
||||
Суть работы была в понимании "магии" кода. Сначала были восстановлены назначение, формат и реальные пути использования; создана документация и читаемая реализация; после этого каждая оптимизация проверялась на совместимость, безопасность и скорость.
|
||||
Эксперименты показали, что код содержит оправданные низкоуровневые решения, а локально более простой или быстрый формат не обязательно выигрывает на полном потоке RPM/APT (still check). Наиболее практичное продолжение - (что-то)
|
||||
@@ -0,0 +1,241 @@
|
||||
# Исследование алгоритма разрешения зависимостей в ALT RPM
|
||||
|
||||
## Замысел и структура статьи
|
||||
|
||||
Статья будет как решение одной формальной задачи. В ALT RPM пакет-потребитель задаёт множество
|
||||
требуемых символов $R$ (`Requires`), а пакет-поставщик - множество предоставляемых
|
||||
символов $P$ (`Provides`). Зависимость выполнима тогда и только тогда, когда
|
||||
|
||||
$$
|
||||
R \subseteq P.
|
||||
$$
|
||||
|
||||
Явное перечисление тысяч ELF-, Python- и файловых символов раздувает RPM-метаданные,
|
||||
поэтому множества представлены компактными set-строками.
|
||||
|
||||
## 1. Постановка задачи и данные ALT Linux
|
||||
|
||||
Ввести обозначения для bpp(b), множеств, хэш функций(H), длин строк и т.д.
|
||||
|
||||
Проверяемое программой условие имеет вид
|
||||
|
||||
$$
|
||||
H_b(R)\subseteq H_b(P).
|
||||
$$
|
||||
|
||||
Отмечаем вероятностную природу: из $R\subseteq P$ следует
|
||||
$H_b(R)\subseteq H_b(P)$, поэтому коллизия не создаёт ложного отказа. Обратное
|
||||
неверно: отсутствующий символ может совпасть по хешу с символом из $P$ и дать
|
||||
ложное принятие зависимости.
|
||||
|
||||
(Представить данные ниже табличкой)
|
||||
Привести срез Sisyphus `x86_64 + noarch`: 47 367 пакетов, 14 761 `Provides`-отношение и 68 857
|
||||
`Requires`-отношений. Для 68 198 реально сопоставимых пар медианы равны
|
||||
$p=480$, $r=13$, $p/r=28.1$; p75 - 1886, 37 и 80; p90 - 6219, 104 и 257.
|
||||
В 92.4% пар выполняется $p/r\ge 4$. Это к теме, почему рассматриваем разреженные множества.
|
||||
|
||||
## 2. Устройство существующей `set:`-строки
|
||||
|
||||
Даём схему формата:
|
||||
|
||||
```
|
||||
массив строк
|
||||
| (Jenkins OAAT)
|
||||
v
|
||||
массив хэшей
|
||||
| (qsort)
|
||||
v
|
||||
отсортированный массив хэшей
|
||||
| (вычисление разницы между элементами)
|
||||
v
|
||||
массив delta
|
||||
| (Rice-Golomb преобразование)
|
||||
v
|
||||
битовый массив
|
||||
| (base62 преобразование)
|
||||
v
|
||||
set-строка
|
||||
```
|
||||
|
||||
Нужно отдельно разобрать заголовок, диапазоны `bpp` и `Mshift`, внешний префикс
|
||||
`set:` и роль Base62: строка должна оставаться допустимым токеном версии RPM.
|
||||
Полезна небольшая иллюстрация на 5-8 символах: исходные имена, хеши,
|
||||
отсортированный массив, дельты, биты кода и итоговая строка.
|
||||
Но. Это много альт-специфики, может настолько глубоко не стоит (хотя и не очень глубоко)
|
||||
|
||||
### 2.1. Хеширование и вероятность коллизий
|
||||
|
||||
! Всё ещё нет нормального анализа коллизий
|
||||
|
||||
Текущий код использует 32-битный Jenkins OAAT и оставляет $b$ бит. Для модели
|
||||
равномерного независимого хеширования ожидаемое число столкнувшихся пар среди
|
||||
$n$ различных символов равно
|
||||
|
||||
(будет мудрёная формула, теория)
|
||||
|
||||
В статье формулы дополняем разными табличками по реальным данным из Sisyphus: распределения $p,r,b$, ожидаемые коллизии, реальные.
|
||||
|
||||
(Другие хэши идут далее)
|
||||
|
||||
### 2.2. Кодирование Golomb-Rice
|
||||
|
||||
Для отсортированных значений $0\le x_1<\dots<x_n<N$ определить
|
||||
$\delta_1=x_1$, $\delta_i=x_i-x_{i-1}$.
|
||||
|
||||
Занимаеи столько и столько бит. В реализации параметр выбирается из
|
||||
$b$ и $n$ приблизительно как
|
||||
|
||||
$$
|
||||
m=b-\lfloor\log_2 n\rfloor-1
|
||||
$$
|
||||
|
||||
с ограничением допустимым диапазоном формата. При средней дельте
|
||||
$\mathbb E\delta\approx 2^b/n$ это даёт ожидаемую длину порядка
|
||||
|
||||
...
|
||||
|
||||
### 2.3. Base62
|
||||
|
||||
Пояснить экзотичность Base62, его алфавит, принцип кодирования, Z-escape,
|
||||
|
||||
## 3. Построение `set:`-строк в `set.c`
|
||||
|
||||
(фактически его можно в 2 упихать)
|
||||
|
||||
Рассказать путь `set_new()` - `set_add()` - `set_fini()` и отдельно объяснить
|
||||
сложные участки:
|
||||
|
||||
(а тут и нет сложного, всё в cmp запихали)
|
||||
Можно рассказать о слитом энкодере, но это есть в переписанной части
|
||||
|
||||
В оригинальном варианте используется `qsort()`
|
||||
|
||||
Записать итоговую сложность построения:
|
||||
|
||||
...
|
||||
|
||||
И доп памяти:
|
||||
|
||||
...
|
||||
|
||||
## 4. Сравнение `set:`-строк
|
||||
|
||||
### 4.1. Обратное декодирование
|
||||
|
||||
Кратко пройти обратную цепочку:
|
||||
|
||||
```
|
||||
set-строка
|
||||
| (обратное base62 преобразование)
|
||||
v
|
||||
битовый массив
|
||||
| (обратное Rice-Golomb преобразование)
|
||||
v
|
||||
массив delta
|
||||
| (вычисление изначальных значений)
|
||||
v
|
||||
массив хэшей
|
||||
```
|
||||
|
||||
Если `bpp` различается, более точное множество проецируется на меньшее число бит,
|
||||
после чего снова удаляются дубликаты. (btw, тут есть сортировка слиянием, интересный момент)
|
||||
|
||||
### 4.2. Кэш декодированных множеств
|
||||
|
||||
В исходном `set.c` кэш содержит 256 записей (`CACHE_SIZE=256`), при вытеснении
|
||||
использует позицию 243 (`PIVOT_SIZE=243`), `free()` и два `memmove()`. Ключ включает
|
||||
короткий fingerprint, а окончательная проверка требует сравнения строки.
|
||||
|
||||
- что именно кэшируется;
|
||||
- стоимость линейного поиска, `strcmp`, перемещений и аллокаций;
|
||||
- слабое повторное использование при потоке десятков тысяч различных пар;
|
||||
|
||||
(всё надо сильно короче расписывать)
|
||||
|
||||
### 4.3. Проверка включения и «прыжок» по массивам
|
||||
|
||||
После декодирования задача сводится к двум возрастающим массивам. Базовый вариант
|
||||
|
||||
рассказ про IFLT4 и IFLT8
|
||||
|
||||
Худшая асимптотика остаётся
|
||||
|
||||
$$
|
||||
T_{\subseteq}(p,r)=O(p+r),
|
||||
$$
|
||||
|
||||
Полная сложность алгоритма холодного сравнения:
|
||||
|
||||
...
|
||||
|
||||
При попадании в кэш:
|
||||
|
||||
...
|
||||
|
||||
## 5. Оптимизации с сохранением текущего формата
|
||||
|
||||
### 5.1. Новый кэш
|
||||
|
||||
Заменить небольшой кэш на два раздельных bucketed LRU для `Provides` и `Requires`, увеличить вместимость. Хранить длину и более сильный fingerprint, избегать лишних
|
||||
`strlen`/`strcmp`, `free` и полных `memmove`. Отдельно убрать повторные `realloc` при построении входных наборов.
|
||||
Проверить cold, warm, поток меняющихся `Requires` к одному `Provides` и поток
|
||||
полностью уникальных пар.
|
||||
|
||||
### 5.2. Radix sort
|
||||
|
||||
Для больших наборов заменить $O(n\log n)$ `qsort()` стабильной LSD radix sort по
|
||||
байтам хеша. Число проходов
|
||||
|
||||
$$
|
||||
k=\left\lceil\frac b8\right\rceil,
|
||||
\qquad T_{\mathrm{radix}}=O(kn),
|
||||
$$
|
||||
|
||||
дополнительная память $O(n+256)$. Для малых $n$ оставить `qsort()`; в `set9.c`
|
||||
порог равен 128.
|
||||
|
||||
### 5.3. Оптимальный проход по массивам
|
||||
|
||||
Сделать адаптивный алгоритм:
|
||||
|
||||
- равные мощности - `memcmp` и быстрый вывод «равны/несравнимы»;
|
||||
- близкие мощности - обычное линейное слияние;
|
||||
- $p/r$ велико - galloping/jump search с монотонной нижней границей;
|
||||
- немедленный выход при первом отсутствующем хеше.
|
||||
|
||||
## 6. Варианты со сменой формата или API
|
||||
|
||||
### 6.1. Roaring Bitmap - отрицательный результат
|
||||
|
||||
Равномерно распределённые хеши не образуют плотных диапазонов, ради которых создан
|
||||
Roaring. Для 1000 символов при `bpp=32` получено:
|
||||
|
||||
| Представление | Длина строки | Cold compare | Warm compare |
|
||||
| -------------------- | -----------: | -----------: | -----------: |
|
||||
| Golomb/Base62 `set9` | 3994 | 293.87 мкс | 5.24 мкс |
|
||||
| raw Roaring/hex | 19 924 | 1321.17 мкс | 399.20 мкс |
|
||||
|
||||
То есть строка длиннее в 4.99 раза, cold-сравнение медленнее в 4.50 раза, warm -
|
||||
в 76.14 раза. Zstd уменьшал строку до 14 126 символов, но оставлял её в 3.54 раза
|
||||
длиннее `set9`; в соответствующем прогоне cold- и warm-сравнение проиграли в 4.96
|
||||
и 101.86 раза. Этот вариант нужен в статье как важный отрицательный эксперимент.
|
||||
|
||||
### 6.2. Прямые хеши и Base64
|
||||
|
||||
В формате D1 отсортированные усечённые хеши хранятся фиксированной шириной без
|
||||
дельт и Golomb-кода, затем упаковываются в Base64. Теоретическая длина payload:
|
||||
|
||||
...
|
||||
|
||||
Для 1000 символов и `bpp=32` строка выросла с 3994 до 5340
|
||||
символов (+33.7%). Приложить разные результаты тестов. (тесты бы прогнать снова)
|
||||
|
||||
Отдельно про увеличение размера и влияние на apt команды от этого
|
||||
|
||||
выводы насчёт формата
|
||||
|
||||
### 6.3. Альтернативные хэш функции
|
||||
|
||||
тут про разные хэши, их быстрдействие и тесты коллизий
|
||||
|
||||
## 8. Вывод
|
||||
@@ -0,0 +1,385 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure P(hash output bit = 1) over a line-oriented ASCII corpus."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from statistics import fmean
|
||||
from typing import TextIO
|
||||
|
||||
from plot_probability_map import ProbabilityRow, render_bit_probabilities
|
||||
from probability_map import HASHES, HASH_FUNCS_DIR
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
DEFAULT_OUTPUT_DIR = ROOT / "corpus_bit_distribution"
|
||||
|
||||
|
||||
class DistributionError(RuntimeError):
|
||||
"""Raised when a corpus or hash adapter cannot be processed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HashSpec:
|
||||
bits: int
|
||||
expression: str
|
||||
|
||||
|
||||
HASH_SPECS = {
|
||||
"jenkinsOAAT": HashSpec(32, "jenkins_oaat(word, length)"),
|
||||
"xxh64": HashSpec(64, "xxh64(word, length, XXH64_SEED)"),
|
||||
"t1ha2": HashSpec(64, "t1ha2_atonce(word, length, T1HA2_SEED)"),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Distribution:
|
||||
hash_name: str
|
||||
samples: int
|
||||
bits: int
|
||||
ones: tuple[int, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.samples < 1:
|
||||
raise ValueError("число образцов должно быть положительным")
|
||||
if self.bits < 1 or len(self.ones) != self.bits:
|
||||
raise ValueError("число счётчиков должно совпадать с шириной хэша")
|
||||
if any(count < 0 or count > self.samples for count in self.ones):
|
||||
raise ValueError("счётчик единиц должен находиться между 0 и samples")
|
||||
|
||||
@property
|
||||
def probabilities(self) -> tuple[float, ...]:
|
||||
return tuple(count / self.samples for count in self.ones)
|
||||
|
||||
|
||||
def c_include_path(path: Path) -> str:
|
||||
"""Return a C string-safe absolute include path."""
|
||||
return str(path.resolve()).replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
|
||||
def make_adapter_source(source: Path, spec: HashSpec) -> str:
|
||||
"""Build a C program that aggregates bit counts using an existing hash source."""
|
||||
return f'''#define _POSIX_C_SOURCE 200809L
|
||||
#define main arsv_original_hash_cli_main
|
||||
#include "{c_include_path(source)}"
|
||||
#undef main
|
||||
#include <inttypes.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
int main(void)
|
||||
{{
|
||||
char *buffer = NULL;
|
||||
size_t capacity = 0;
|
||||
uint64_t samples = UINT64_C(0);
|
||||
uint64_t ones[{spec.bits}] = {{UINT64_C(0)}};
|
||||
ssize_t bytes_read;
|
||||
|
||||
while ((bytes_read = getline(&buffer, &capacity, stdin)) >= 0) {{
|
||||
size_t length = (size_t)bytes_read;
|
||||
const unsigned char *word = (const unsigned char *)buffer;
|
||||
|
||||
while (length > 0U && is_ascii_trailing_space(word[length - 1U])) {{
|
||||
--length;
|
||||
}}
|
||||
if (length == 0U) {{
|
||||
fprintf(stderr, "corpus contains an empty line\\n");
|
||||
free(buffer);
|
||||
return EXIT_FAILURE;
|
||||
}}
|
||||
for (size_t index = 0; index < length; ++index) {{
|
||||
if (word[index] > 0x7fU) {{
|
||||
fprintf(stderr, "corpus must contain ASCII characters only\\n");
|
||||
free(buffer);
|
||||
return EXIT_FAILURE;
|
||||
}}
|
||||
}}
|
||||
if (samples == UINT64_MAX) {{
|
||||
fprintf(stderr, "sample counter overflow\\n");
|
||||
free(buffer);
|
||||
return EXIT_FAILURE;
|
||||
}}
|
||||
|
||||
uint64_t hash = (uint64_t)({spec.expression});
|
||||
for (unsigned int bit = 0U; bit < {spec.bits}U; ++bit) {{
|
||||
ones[bit] += (hash >> bit) & UINT64_C(1);
|
||||
}}
|
||||
++samples;
|
||||
}}
|
||||
if (ferror(stdin)) {{
|
||||
fprintf(stderr, "failed to read corpus\\n");
|
||||
free(buffer);
|
||||
return EXIT_FAILURE;
|
||||
}}
|
||||
free(buffer);
|
||||
|
||||
printf("%" PRIu64 ",{spec.bits}", samples);
|
||||
for (unsigned int bit = 0U; bit < {spec.bits}U; ++bit) {{
|
||||
printf(",%" PRIu64, ones[bit]);
|
||||
}}
|
||||
putchar('\\n');
|
||||
return EXIT_SUCCESS;
|
||||
}}
|
||||
'''
|
||||
|
||||
|
||||
def measure_hash(
|
||||
hash_name: str,
|
||||
corpus: Path,
|
||||
hash_funcs_dir: Path = HASH_FUNCS_DIR,
|
||||
) -> Distribution:
|
||||
"""Compile a batch adapter and count set output bits for every corpus line."""
|
||||
try:
|
||||
spec = HASH_SPECS[hash_name]
|
||||
except KeyError as error:
|
||||
supported = ", ".join(HASH_SPECS)
|
||||
raise DistributionError(
|
||||
f"нет batch-адаптера для {hash_name!r}; доступны: {supported}"
|
||||
) from error
|
||||
|
||||
source = hash_funcs_dir / hash_name / "bin_hash.c"
|
||||
if not source.is_file():
|
||||
raise DistributionError(f"не найден исходник хэша: {source}")
|
||||
if not corpus.is_file():
|
||||
raise DistributionError(f"не найден corpus: {corpus}")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix=f"arsv-{hash_name}-") as temporary:
|
||||
root = Path(temporary)
|
||||
adapter_source = root / "batch_counts.c"
|
||||
adapter = root / "batch_counts"
|
||||
adapter_source.write_text(make_adapter_source(source, spec), encoding="utf-8")
|
||||
command = [
|
||||
"cc",
|
||||
"-std=c11",
|
||||
"-O3",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Wpedantic",
|
||||
"-Werror",
|
||||
str(adapter_source),
|
||||
"-o",
|
||||
str(adapter),
|
||||
]
|
||||
compiled = subprocess.run(command, text=True, capture_output=True, check=False)
|
||||
if compiled.returncode != 0:
|
||||
details = compiled.stderr.strip() or compiled.stdout.strip()
|
||||
raise DistributionError(f"не удалось собрать адаптер {hash_name}: {details}")
|
||||
|
||||
try:
|
||||
with corpus.open("rb") as stream:
|
||||
measured = subprocess.run(
|
||||
[str(adapter)],
|
||||
stdin=stream,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError as error:
|
||||
raise DistributionError(f"не удалось прочитать {corpus}: {error}") from error
|
||||
if measured.returncode != 0:
|
||||
details = measured.stderr.strip() or measured.stdout.strip()
|
||||
raise DistributionError(f"batch-прогон {hash_name} завершился ошибкой: {details}")
|
||||
|
||||
fields = measured.stdout.strip().split(",")
|
||||
if len(fields) != spec.bits + 2:
|
||||
raise DistributionError(
|
||||
f"batch-прогон {hash_name} вернул {len(fields)} полей вместо {spec.bits + 2}"
|
||||
)
|
||||
try:
|
||||
samples = int(fields[0])
|
||||
bits = int(fields[1])
|
||||
ones = tuple(int(value) for value in fields[2:])
|
||||
except ValueError as error:
|
||||
raise DistributionError(
|
||||
f"batch-прогон {hash_name} вернул некорректные счётчики"
|
||||
) from error
|
||||
if bits != spec.bits:
|
||||
raise DistributionError(
|
||||
f"batch-прогон {hash_name} сообщил ширину {bits} вместо {spec.bits}"
|
||||
)
|
||||
try:
|
||||
return Distribution(hash_name, samples, bits, ones)
|
||||
except ValueError as error:
|
||||
raise DistributionError(f"некорректный результат {hash_name}: {error}") from error
|
||||
|
||||
|
||||
def write_distribution_csv(stream: TextIO, distribution: Distribution) -> None:
|
||||
"""Write bit counts and probabilities from LSB to MSB."""
|
||||
writer = csv.writer(stream, lineterminator="\n")
|
||||
writer.writerow(["bit", "ones", "zeros", "probability", "deviation_from_0.5"])
|
||||
for bit, (ones, probability) in enumerate(
|
||||
zip(distribution.ones, distribution.probabilities, strict=True)
|
||||
):
|
||||
writer.writerow(
|
||||
[
|
||||
bit,
|
||||
ones,
|
||||
distribution.samples - ones,
|
||||
f"{probability:.9f}",
|
||||
f"{abs(probability - 0.5):.9f}",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def write_summary_csv(stream: TextIO, distributions: Sequence[Distribution]) -> None:
|
||||
writer = csv.writer(stream, lineterminator="\n")
|
||||
writer.writerow(
|
||||
[
|
||||
"hash",
|
||||
"samples",
|
||||
"bits",
|
||||
"mean_probability",
|
||||
"mean_abs_deviation",
|
||||
"max_abs_deviation",
|
||||
"min_probability",
|
||||
"max_probability",
|
||||
]
|
||||
)
|
||||
for distribution in distributions:
|
||||
probabilities = distribution.probabilities
|
||||
deviations = [abs(value - 0.5) for value in probabilities]
|
||||
writer.writerow(
|
||||
[
|
||||
distribution.hash_name,
|
||||
distribution.samples,
|
||||
distribution.bits,
|
||||
f"{fmean(probabilities):.9f}",
|
||||
f"{fmean(deviations):.9f}",
|
||||
f"{max(deviations):.9f}",
|
||||
f"{min(probabilities):.9f}",
|
||||
f"{max(probabilities):.9f}",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_manifest(
|
||||
path: Path,
|
||||
corpus: Path,
|
||||
distributions: Sequence[Distribution],
|
||||
) -> None:
|
||||
samples = {distribution.samples for distribution in distributions}
|
||||
if len(samples) != 1:
|
||||
raise DistributionError("хэши обработали разное число строк corpus")
|
||||
content = "\n".join(
|
||||
[
|
||||
f"corpus={corpus}",
|
||||
f"corpus_bytes={corpus.stat().st_size}",
|
||||
f"corpus_sha256={sha256_file(corpus)}",
|
||||
f"samples={samples.pop()}",
|
||||
f"hashes={','.join(item.hash_name for item in distributions)}",
|
||||
"metric=P(hash_output_bit=1)",
|
||||
"bit_order=bit_0_is_LSB",
|
||||
"input_format=one_ASCII_symbol_per_line",
|
||||
"",
|
||||
]
|
||||
)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def run(
|
||||
corpus: Path,
|
||||
output_directory: Path,
|
||||
hash_names: Sequence[str],
|
||||
) -> list[Distribution]:
|
||||
"""Measure all requested hashes and create CSV, manifest, and one PNG."""
|
||||
names = list(dict.fromkeys(hash_names))
|
||||
if not names:
|
||||
raise DistributionError("нужно указать хотя бы один хэш")
|
||||
|
||||
distributions = [measure_hash(name, corpus) for name in names]
|
||||
sample_counts = {item.samples for item in distributions}
|
||||
if len(sample_counts) != 1:
|
||||
raise DistributionError("хэши обработали разное число строк corpus")
|
||||
|
||||
output_directory.mkdir(parents=True, exist_ok=True)
|
||||
for distribution in distributions:
|
||||
path = output_directory / f"{distribution.hash_name}.csv"
|
||||
with path.open("w", encoding="utf-8", newline="") as stream:
|
||||
write_distribution_csv(stream, distribution)
|
||||
|
||||
with (output_directory / "summary.csv").open(
|
||||
"w", encoding="utf-8", newline=""
|
||||
) as stream:
|
||||
write_summary_csv(stream, distributions)
|
||||
write_manifest(output_directory / "manifest.txt", corpus, distributions)
|
||||
|
||||
rows = [
|
||||
ProbabilityRow(
|
||||
operation=item.hash_name,
|
||||
pairs=item.samples,
|
||||
probabilities=list(item.probabilities),
|
||||
)
|
||||
for item in distributions
|
||||
]
|
||||
render_bit_probabilities(
|
||||
rows,
|
||||
output_directory / "bit_distribution.png",
|
||||
"C++ Provides corpus: P(hash output bit = 1)",
|
||||
)
|
||||
return distributions
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Считает прямое распределение единиц по выходным битам хэша "
|
||||
"для ASCII corpus (одна строка — одно значение). bit_0 — младший бит."
|
||||
)
|
||||
)
|
||||
parser.add_argument("corpus", type=Path, help="текстовый corpus")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=DEFAULT_OUTPUT_DIR,
|
||||
help=f"отдельный каталог результатов (по умолчанию: {DEFAULT_OUTPUT_DIR})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hash",
|
||||
dest="hashes",
|
||||
action="append",
|
||||
help="проверить указанный хэш; можно повторять (по умолчанию все)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
arguments = parser.parse_args(argv)
|
||||
try:
|
||||
distributions = run(
|
||||
corpus=arguments.corpus,
|
||||
output_directory=arguments.output,
|
||||
hash_names=arguments.hashes or HASHES,
|
||||
)
|
||||
except (DistributionError, OSError, ValueError) as error:
|
||||
parser.error(str(error))
|
||||
|
||||
for distribution in distributions:
|
||||
probabilities = distribution.probabilities
|
||||
mean_deviation = fmean(abs(value - 0.5) for value in probabilities)
|
||||
print(
|
||||
f"hash={distribution.hash_name} samples={distribution.samples} "
|
||||
f"bits={distribution.bits} mean_abs_deviation={mean_deviation:.9f}"
|
||||
)
|
||||
print(f"wrote {arguments.output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+426
@@ -0,0 +1,426 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract ALT RPM Provides symbol names into a hash-testing corpus.
|
||||
|
||||
Local RPM paths are processed directly. Other positional arguments are resolved
|
||||
as binary package names through the ALT Repository Database (RDB).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO, Sequence, cast
|
||||
|
||||
|
||||
RDB_BASE = "https://rdb.altlinux.org/api"
|
||||
NEWC_MAGICS = {b"070701", b"070702"}
|
||||
NEWC_HEADER_SIZE = 110
|
||||
COPY_CHUNK_SIZE = 1024 * 1024
|
||||
MAX_CPIO_NAME_SIZE = 1024 * 1024
|
||||
|
||||
|
||||
class CorpusError(RuntimeError):
|
||||
"""A user-facing extraction error."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackageInput:
|
||||
label: str
|
||||
path: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackageSymbols:
|
||||
label: str
|
||||
elf_files: int
|
||||
symbols: frozenset[str]
|
||||
|
||||
|
||||
def parse_arguments(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Extract canonical ALT rpm-build Provided ELF symbols from multiple "
|
||||
"RPM packages, one symbol per output line."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"packages",
|
||||
nargs="+",
|
||||
help="local .rpm path or ALT binary package name",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
default="-",
|
||||
help="output file (default: stdout)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cpp-only",
|
||||
action="store_true",
|
||||
help="keep only Itanium C++ ABI mangled names beginning with _Z",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--branch",
|
||||
default="p11",
|
||||
help="ALT branch for package-name resolution (default: p11)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--arch",
|
||||
default="x86_64",
|
||||
help="binary package architecture (default: x86_64)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rpm2cpio",
|
||||
default=shutil.which("rpm2cpio") or "rpm2cpio",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--provided-symbols",
|
||||
default=(
|
||||
"/usr/lib/rpm/provided_symbols"
|
||||
if Path("/usr/lib/rpm/provided_symbols").is_file()
|
||||
else shutil.which("provided_symbols") or "provided_symbols"
|
||||
),
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--file-command",
|
||||
default=shutil.which("file") or "file",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def read_json(url: str) -> dict[str, object]:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=60) as response:
|
||||
return json.load(response)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||
raise CorpusError(f"failed to read ALT RDB response from {url}: {error}") from error
|
||||
|
||||
|
||||
def download_package(name: str, branch: str, arch: str, directory: Path) -> PackageInput:
|
||||
query = urllib.parse.urlencode({"branch": branch, "name": name, "arch": arch})
|
||||
metadata_url = f"{RDB_BASE}/site/pkghash_by_binary_name?{query}"
|
||||
metadata = read_json(metadata_url)
|
||||
package_hash = metadata.get("pkghash")
|
||||
if not isinstance(package_hash, str) or not package_hash:
|
||||
raise CorpusError(f"ALT RDB did not return a package hash for {name!r}")
|
||||
|
||||
download_url = (
|
||||
f"{RDB_BASE}/site/package_downloads_bin/{package_hash}?"
|
||||
f"{urllib.parse.urlencode({'branch': branch, 'arch': arch})}"
|
||||
)
|
||||
download_data = read_json(download_url)
|
||||
downloads = download_data.get("downloads")
|
||||
if not isinstance(downloads, list):
|
||||
raise CorpusError(f"ALT RDB did not return downloads for {name!r}")
|
||||
|
||||
package_record: dict[str, object] | None = None
|
||||
for architecture_record in downloads:
|
||||
if not isinstance(architecture_record, dict):
|
||||
continue
|
||||
if architecture_record.get("arch") != arch:
|
||||
continue
|
||||
packages = architecture_record.get("packages")
|
||||
if isinstance(packages, list):
|
||||
for candidate in packages:
|
||||
if isinstance(candidate, dict):
|
||||
package_record = candidate
|
||||
break
|
||||
if package_record is not None:
|
||||
break
|
||||
|
||||
if package_record is None:
|
||||
raise CorpusError(f"ALT RDB has no {arch} RPM download for {name!r}")
|
||||
|
||||
filename = package_record.get("name")
|
||||
url = package_record.get("url")
|
||||
expected_md5 = package_record.get("md5")
|
||||
if not isinstance(filename, str) or not filename.endswith(".rpm"):
|
||||
raise CorpusError(f"ALT RDB returned an invalid RPM filename for {name!r}")
|
||||
if not isinstance(url, str) or not url.startswith(("https://", "http://")):
|
||||
raise CorpusError(f"ALT RDB returned an invalid RPM URL for {name!r}")
|
||||
|
||||
destination = directory / filename
|
||||
temporary = destination.with_suffix(destination.suffix + ".part")
|
||||
digest = hashlib.md5(usedforsecurity=False)
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=180) as response, temporary.open("wb") as output:
|
||||
while chunk := response.read(COPY_CHUNK_SIZE):
|
||||
output.write(chunk)
|
||||
digest.update(chunk)
|
||||
except OSError as error:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise CorpusError(f"failed to download {name!r} from {url}: {error}") from error
|
||||
|
||||
if isinstance(expected_md5, str) and digest.hexdigest().lower() != expected_md5.lower():
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise CorpusError(f"MD5 mismatch for downloaded package {name!r}")
|
||||
temporary.replace(destination)
|
||||
return PackageInput(f"{name} ({filename})", destination)
|
||||
|
||||
|
||||
def resolve_packages(
|
||||
specifications: Sequence[str], branch: str, arch: str, directory: Path
|
||||
) -> list[PackageInput]:
|
||||
resolved: list[PackageInput] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for specification in specifications:
|
||||
candidate = Path(specification).expanduser()
|
||||
if candidate.is_file():
|
||||
package = PackageInput(candidate.name, candidate.resolve())
|
||||
elif candidate.exists():
|
||||
raise CorpusError(f"package input is not a regular file: {candidate}")
|
||||
elif "/" in specification or specification.endswith(".rpm"):
|
||||
raise CorpusError(f"RPM file does not exist: {candidate}")
|
||||
else:
|
||||
package = download_package(specification, branch, arch, directory)
|
||||
|
||||
identity = (package.label, str(package.path))
|
||||
if identity not in seen:
|
||||
resolved.append(package)
|
||||
seen.add(identity)
|
||||
return resolved
|
||||
|
||||
|
||||
def read_exact(stream: BinaryIO, size: int, context: str) -> bytes:
|
||||
chunks: list[bytes] = []
|
||||
remaining = size
|
||||
while remaining:
|
||||
chunk = stream.read(remaining)
|
||||
if not chunk:
|
||||
raise CorpusError(f"truncated newc stream while reading {context}")
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def discard_exact(stream: BinaryIO, size: int, context: str) -> None:
|
||||
remaining = size
|
||||
while remaining:
|
||||
chunk = stream.read(min(remaining, COPY_CHUNK_SIZE))
|
||||
if not chunk:
|
||||
raise CorpusError(f"truncated newc stream while reading {context}")
|
||||
remaining -= len(chunk)
|
||||
|
||||
|
||||
def copy_exact(stream: BinaryIO, output: BinaryIO, size: int, context: str) -> None:
|
||||
remaining = size
|
||||
while remaining:
|
||||
chunk = stream.read(min(remaining, COPY_CHUNK_SIZE))
|
||||
if not chunk:
|
||||
raise CorpusError(f"truncated newc stream while reading {context}")
|
||||
output.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
|
||||
|
||||
def parse_newc_header(header: bytes) -> tuple[int, int, int]:
|
||||
if len(header) != NEWC_HEADER_SIZE or header[:6] not in NEWC_MAGICS:
|
||||
raise CorpusError("rpm2cpio output is not a valid newc archive")
|
||||
try:
|
||||
fields = [int(header[6 + index * 8 : 14 + index * 8], 16) for index in range(13)]
|
||||
except ValueError as error:
|
||||
raise CorpusError("newc header contains a non-hexadecimal field") from error
|
||||
mode = fields[1]
|
||||
file_size = fields[6]
|
||||
name_size = fields[11]
|
||||
if name_size < 1 or name_size > MAX_CPIO_NAME_SIZE:
|
||||
raise CorpusError(f"invalid newc pathname size: {name_size}")
|
||||
return mode, file_size, name_size
|
||||
|
||||
|
||||
def extract_elf_members(stream: BinaryIO, directory: Path) -> list[Path]:
|
||||
elf_files: list[Path] = []
|
||||
member_index = 0
|
||||
while True:
|
||||
header = read_exact(stream, NEWC_HEADER_SIZE, "header")
|
||||
mode, file_size, name_size = parse_newc_header(header)
|
||||
raw_name = read_exact(stream, name_size, "pathname")
|
||||
if raw_name[-1:] != b"\0":
|
||||
raise CorpusError("newc pathname is not NUL-terminated")
|
||||
discard_exact(stream, -(NEWC_HEADER_SIZE + name_size) % 4, "pathname padding")
|
||||
|
||||
if raw_name[:-1] == b"TRAILER!!!":
|
||||
discard_exact(stream, file_size, "trailer data")
|
||||
discard_exact(stream, -file_size % 4, "trailer padding")
|
||||
break
|
||||
|
||||
member_index += 1
|
||||
prefix_size = min(file_size, 4)
|
||||
prefix = read_exact(stream, prefix_size, "member data")
|
||||
remaining = file_size - prefix_size
|
||||
if stat.S_ISREG(mode) and prefix == b"\x7fELF":
|
||||
destination = directory / f"elf-{member_index:06d}"
|
||||
with destination.open("wb") as output:
|
||||
output.write(prefix)
|
||||
copy_exact(stream, output, remaining, "ELF member data")
|
||||
elf_files.append(destination)
|
||||
else:
|
||||
discard_exact(stream, remaining, "member data")
|
||||
discard_exact(stream, -file_size % 4, "member padding")
|
||||
return elf_files
|
||||
|
||||
|
||||
def extract_package_symbols(
|
||||
package: PackageInput,
|
||||
rpm2cpio: str,
|
||||
provided_symbols: str,
|
||||
file_command: str,
|
||||
directory: Path,
|
||||
) -> PackageSymbols:
|
||||
package_directory = directory / f"package-{len(list(directory.iterdir())):04d}"
|
||||
package_directory.mkdir()
|
||||
process = subprocess.Popen(
|
||||
[rpm2cpio, str(package.path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
assert process.stdout is not None
|
||||
assert process.stderr is not None
|
||||
try:
|
||||
extracted_elf_files = extract_elf_members(
|
||||
cast(BinaryIO, process.stdout), package_directory
|
||||
)
|
||||
except Exception:
|
||||
process.kill()
|
||||
process.communicate()
|
||||
raise
|
||||
finally:
|
||||
process.stdout.close()
|
||||
|
||||
stderr = process.stderr.read().decode(errors="replace")
|
||||
return_code = process.wait()
|
||||
if return_code != 0:
|
||||
raise CorpusError(
|
||||
f"rpm2cpio failed for {package.label} with status {return_code}: {stderr.strip()}"
|
||||
)
|
||||
|
||||
elf_files: list[Path] = []
|
||||
environment = os.environ.copy()
|
||||
environment["LC_ALL"] = "C"
|
||||
for path in extracted_elf_files:
|
||||
result = subprocess.run(
|
||||
[file_command, "--brief", "--", str(path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise CorpusError(
|
||||
f"file failed for {package.label} with status "
|
||||
f"{result.returncode}: {result.stderr.strip()}"
|
||||
)
|
||||
description = f" {result.stdout.strip()} "
|
||||
if (
|
||||
" ELF " in description
|
||||
and " shared object, " in description
|
||||
and " shared object, no machine, " not in description
|
||||
):
|
||||
elf_files.append(path)
|
||||
|
||||
if not elf_files:
|
||||
return PackageSymbols(package.label, 0, frozenset())
|
||||
|
||||
result = subprocess.run(
|
||||
[provided_symbols, *(str(path) for path in elf_files)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise CorpusError(
|
||||
f"provided_symbols failed for {package.label} with status "
|
||||
f"{result.returncode}: {result.stderr.strip()}"
|
||||
)
|
||||
symbols = frozenset(line for line in result.stdout.splitlines() if line)
|
||||
return PackageSymbols(package.label, len(elf_files), symbols)
|
||||
|
||||
|
||||
def render_symbols(symbols: set[str]) -> str:
|
||||
if not symbols:
|
||||
return ""
|
||||
return "\n".join(sorted(symbols)) + "\n"
|
||||
|
||||
|
||||
def write_output(path: str, content: str) -> None:
|
||||
if path == "-":
|
||||
sys.stdout.write(content)
|
||||
return
|
||||
destination = Path(path).expanduser()
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w", encoding="utf-8", dir=destination.parent, delete=False
|
||||
) as temporary:
|
||||
temporary.write(content)
|
||||
temporary_path = Path(temporary.name)
|
||||
temporary_path.replace(destination)
|
||||
|
||||
|
||||
def run(arguments: argparse.Namespace) -> int:
|
||||
with tempfile.TemporaryDirectory(prefix="provided-symbols-") as temporary_name:
|
||||
temporary = Path(temporary_name)
|
||||
download_directory = temporary / "downloads"
|
||||
extraction_directory = temporary / "extract"
|
||||
download_directory.mkdir()
|
||||
extraction_directory.mkdir()
|
||||
packages = resolve_packages(
|
||||
arguments.packages, arguments.branch, arguments.arch, download_directory
|
||||
)
|
||||
|
||||
combined: set[str] = set()
|
||||
total_elf_files = 0
|
||||
for package in packages:
|
||||
package_symbols = extract_package_symbols(
|
||||
package,
|
||||
arguments.rpm2cpio,
|
||||
arguments.provided_symbols,
|
||||
arguments.file_command,
|
||||
extraction_directory,
|
||||
)
|
||||
selected = {
|
||||
symbol
|
||||
for symbol in package_symbols.symbols
|
||||
if not arguments.cpp_only or symbol.startswith("_Z")
|
||||
}
|
||||
combined.update(selected)
|
||||
total_elf_files += package_symbols.elf_files
|
||||
print(
|
||||
f"package={package_symbols.label} "
|
||||
f"elf_files={package_symbols.elf_files} "
|
||||
f"symbols={len(package_symbols.symbols)} selected={len(selected)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
write_output(arguments.output, render_symbols(combined))
|
||||
print(
|
||||
f"packages={len(packages)} elf_files={total_elf_files} "
|
||||
f"unique_symbols={len(combined)} output={arguments.output}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
try:
|
||||
return run(parse_arguments(argv))
|
||||
except (CorpusError, OSError, subprocess.SubprocessError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+264
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate unique words similar to a given word by applying random mutations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import random
|
||||
import string
|
||||
import sys
|
||||
from collections.abc import Callable, Sequence
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_ALPHABET = string.ascii_letters + string.digits + "_"
|
||||
|
||||
OPERATION_ALIASES = {
|
||||
"1": "replace",
|
||||
"replace": "replace",
|
||||
"2": "delete",
|
||||
"delete": "delete",
|
||||
"3": "add",
|
||||
"add": "add",
|
||||
"4": "swap",
|
||||
"swap": "swap",
|
||||
"5": "case",
|
||||
"case": "case",
|
||||
"6": "first",
|
||||
"first": "first",
|
||||
"7": "last",
|
||||
"last": "last",
|
||||
}
|
||||
|
||||
OPERATION_HELP = """операция изменения:
|
||||
1, replace заменить случайный символ
|
||||
2, delete удалить случайный символ
|
||||
3, add добавить символ в случайную позицию
|
||||
4, swap переставить два соседних символа
|
||||
5, case сменить регистр случайного символа
|
||||
6, first изменить первый символ
|
||||
7, last изменить последний символ"""
|
||||
|
||||
|
||||
class MutationError(ValueError):
|
||||
"""Raised when the selected mutation cannot be applied."""
|
||||
|
||||
|
||||
def different_character(current: str, alphabet: str, rng: random.Random) -> str:
|
||||
choices = [character for character in alphabet if character != current]
|
||||
if not choices:
|
||||
raise MutationError("алфавит не содержит символа, отличного от заменяемого")
|
||||
return rng.choice(choices)
|
||||
|
||||
|
||||
def replace_character(word: str, alphabet: str, rng: random.Random) -> str:
|
||||
if not word:
|
||||
raise MutationError("нельзя заменить символ в пустом слове")
|
||||
index = rng.randrange(len(word))
|
||||
replacement = different_character(word[index], alphabet, rng)
|
||||
return word[:index] + replacement + word[index + 1 :]
|
||||
|
||||
|
||||
def delete_character(word: str, _alphabet: str, rng: random.Random) -> str:
|
||||
if not word:
|
||||
raise MutationError("нельзя удалить символ из пустого слова")
|
||||
index = rng.randrange(len(word))
|
||||
return word[:index] + word[index + 1 :]
|
||||
|
||||
|
||||
def add_character(word: str, alphabet: str, rng: random.Random) -> str:
|
||||
index = rng.randrange(len(word) + 1)
|
||||
return word[:index] + rng.choice(alphabet) + word[index:]
|
||||
|
||||
|
||||
def swap_adjacent(word: str, _alphabet: str, rng: random.Random) -> str:
|
||||
indexes = [
|
||||
index for index in range(len(word) - 1) if word[index] != word[index + 1]
|
||||
]
|
||||
if not indexes:
|
||||
raise MutationError(
|
||||
"для перестановки нужны хотя бы два соседних различных символа"
|
||||
)
|
||||
index = rng.choice(indexes)
|
||||
return word[:index] + word[index + 1] + word[index] + word[index + 2 :]
|
||||
|
||||
|
||||
def change_case(word: str, _alphabet: str, rng: random.Random) -> str:
|
||||
indexes = []
|
||||
replacements: dict[int, str] = {}
|
||||
for index, character in enumerate(word):
|
||||
swapped = character.swapcase()
|
||||
if swapped != character and len(swapped) == 1:
|
||||
indexes.append(index)
|
||||
replacements[index] = swapped
|
||||
if not indexes:
|
||||
raise MutationError("в слове нет символов, у которых можно сменить регистр")
|
||||
index = rng.choice(indexes)
|
||||
return word[:index] + replacements[index] + word[index + 1 :]
|
||||
|
||||
|
||||
def change_first(word: str, alphabet: str, rng: random.Random) -> str:
|
||||
if not word:
|
||||
raise MutationError("нельзя изменить первый символ пустого слова")
|
||||
replacement = different_character(word[0], alphabet, rng)
|
||||
return replacement + word[1:]
|
||||
|
||||
|
||||
def change_last(word: str, alphabet: str, rng: random.Random) -> str:
|
||||
if not word:
|
||||
raise MutationError("нельзя изменить последний символ пустого слова")
|
||||
replacement = different_character(word[-1], alphabet, rng)
|
||||
return word[:-1] + replacement
|
||||
|
||||
|
||||
Mutation = Callable[[str, str, random.Random], str]
|
||||
MUTATIONS: dict[str, Mutation] = {
|
||||
"replace": replace_character,
|
||||
"delete": delete_character,
|
||||
"add": add_character,
|
||||
"swap": swap_adjacent,
|
||||
"case": change_case,
|
||||
"first": change_first,
|
||||
"last": change_last,
|
||||
}
|
||||
|
||||
|
||||
def parse_operation(value: str) -> str:
|
||||
try:
|
||||
return OPERATION_ALIASES[value.lower()]
|
||||
except KeyError as error:
|
||||
valid = ", ".join(OPERATION_ALIASES)
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"неизвестная операция {value!r}; допустимы: {valid}"
|
||||
) from error
|
||||
|
||||
|
||||
def generate_words(
|
||||
source: str,
|
||||
count: int,
|
||||
operation: str,
|
||||
operation_count: int,
|
||||
alphabet: str = DEFAULT_ALPHABET,
|
||||
seed: int | None = None,
|
||||
max_attempts: int | None = None,
|
||||
) -> list[str]:
|
||||
"""Generate up to ``count`` unique mutations made by exactly N operations."""
|
||||
if count < 1:
|
||||
raise ValueError("количество выходных слов должно быть положительным")
|
||||
if operation_count < 1:
|
||||
raise ValueError("количество операций должно быть положительным")
|
||||
if not alphabet:
|
||||
raise ValueError("алфавит не должен быть пустым")
|
||||
|
||||
mutation = MUTATIONS[operation]
|
||||
rng = random.Random(seed)
|
||||
attempt_limit = max_attempts or max(10_000, count * 1_000)
|
||||
words: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for _attempt in range(attempt_limit):
|
||||
candidate = source
|
||||
try:
|
||||
for _ in range(operation_count):
|
||||
candidate = mutation(candidate, alphabet, rng)
|
||||
except MutationError:
|
||||
continue
|
||||
|
||||
if candidate != source and candidate not in seen:
|
||||
seen.add(candidate)
|
||||
words.append(candidate)
|
||||
if len(words) == count:
|
||||
return words
|
||||
|
||||
return words
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Создаёт список уникальных слов, похожих на исходное. "
|
||||
"Каждое слово получается независимо от исходного ровно заданным "
|
||||
"числом случайных операций."
|
||||
),
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=(
|
||||
f"{OPERATION_HELP}\n\n"
|
||||
"Пример:\n"
|
||||
" python3 nexus.py example -o swap -n 20 -k 2 --seed 42"
|
||||
),
|
||||
)
|
||||
parser.add_argument("word", help="исходное слово")
|
||||
parser.add_argument(
|
||||
"-o", "--operation", required=True, type=parse_operation, help=OPERATION_HELP
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n",
|
||||
"--count",
|
||||
type=int,
|
||||
default=10,
|
||||
help="верхняя граница числа уникальных выходных слов (по умолчанию: 10)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-k",
|
||||
"--operations",
|
||||
type=int,
|
||||
default=1,
|
||||
help="число операций над каждым словом (по умолчанию: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--alphabet",
|
||||
default=DEFAULT_ALPHABET,
|
||||
help=("символы для добавления и замены " f"(по умолчанию: {DEFAULT_ALPHABET})"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed", type=int, help="seed генератора для воспроизводимого результата"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-attempts",
|
||||
type=int,
|
||||
help="предельное число попыток собрать уникальные слова",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
help="записать слова в файл вместо стандартного вывода",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
if args.max_attempts is not None and args.max_attempts < 1:
|
||||
parser.error("--max-attempts должен быть положительным")
|
||||
|
||||
try:
|
||||
words = generate_words(
|
||||
source=args.word,
|
||||
count=args.count,
|
||||
operation=args.operation,
|
||||
operation_count=args.operations,
|
||||
alphabet=args.alphabet,
|
||||
seed=args.seed,
|
||||
max_attempts=args.max_attempts,
|
||||
)
|
||||
except (MutationError, ValueError) as error:
|
||||
parser.error(str(error))
|
||||
|
||||
if len(words) < args.count:
|
||||
print(
|
||||
f"warning: operation={args.operation}: generated {len(words)} "
|
||||
f"of at most {args.count} unique words",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
output = "".join(f"{word}\n" for word in words)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output, encoding="utf-8")
|
||||
else:
|
||||
sys.stdout.write(output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Binary file not shown.
@@ -0,0 +1,109 @@
|
||||
#include <inttypes.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define JOAAT_SEED UINT32_C(0x9e3779b9)
|
||||
|
||||
static uint32_t jenkins_oaat(const unsigned char *data, size_t length)
|
||||
{
|
||||
uint32_t hash = JOAAT_SEED;
|
||||
|
||||
for (size_t index = 0; index < length; ++index) {
|
||||
hash += data[index];
|
||||
hash += hash << 10;
|
||||
hash ^= hash >> 6;
|
||||
}
|
||||
|
||||
hash += hash << 3;
|
||||
hash ^= hash >> 11;
|
||||
hash += hash << 15;
|
||||
return hash;
|
||||
}
|
||||
|
||||
static int is_ascii_trailing_space(unsigned char character)
|
||||
{
|
||||
return character == ' ' || character == '\t' || character == '\n' ||
|
||||
character == '\r' || character == '\v' || character == '\f';
|
||||
}
|
||||
|
||||
static int read_stdin(unsigned char **data, size_t *length)
|
||||
{
|
||||
size_t capacity = 256;
|
||||
unsigned char *buffer = malloc(capacity);
|
||||
|
||||
if (buffer == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
*length = 0;
|
||||
for (;;) {
|
||||
size_t available = capacity - *length;
|
||||
size_t bytes_read = fread(buffer + *length, 1, available, stdin);
|
||||
*length += bytes_read;
|
||||
|
||||
if (bytes_read < available) {
|
||||
if (ferror(stdin)) {
|
||||
free(buffer);
|
||||
return -1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (capacity > SIZE_MAX / 2) {
|
||||
free(buffer);
|
||||
return -1;
|
||||
}
|
||||
capacity *= 2;
|
||||
|
||||
unsigned char *larger_buffer = realloc(buffer, capacity);
|
||||
if (larger_buffer == NULL) {
|
||||
free(buffer);
|
||||
return -1;
|
||||
}
|
||||
buffer = larger_buffer;
|
||||
}
|
||||
|
||||
*data = buffer;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
const unsigned char *word;
|
||||
unsigned char *stdin_buffer = NULL;
|
||||
size_t length;
|
||||
|
||||
if (argc > 2) {
|
||||
fprintf(stderr, "usage: %s [ASCII_WORD]\n", argv[0]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
if (argc == 2) {
|
||||
word = (const unsigned char *)argv[1];
|
||||
length = strlen(argv[1]);
|
||||
} else {
|
||||
if (read_stdin(&stdin_buffer, &length) != 0) {
|
||||
fprintf(stderr, "failed to read input\n");
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
word = stdin_buffer;
|
||||
}
|
||||
|
||||
while (length > 0 && is_ascii_trailing_space(word[length - 1])) {
|
||||
--length;
|
||||
}
|
||||
|
||||
for (size_t index = 0; index < length; ++index) {
|
||||
if (word[index] > 0x7f) {
|
||||
fprintf(stderr, "input must contain ASCII characters only\n");
|
||||
free(stdin_buffer);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
printf("%08" PRIx32 "\n", jenkins_oaat(word, length));
|
||||
free(stdin_buffer);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
unsigned int hash(const char* str) {
|
||||
unsigned int hash = 0x9e3779b9;
|
||||
const unsigned char* p = (const unsigned char*)str;
|
||||
while (*p) {
|
||||
hash += *p++;
|
||||
hash += (hash << 10);
|
||||
hash ^= (hash >> 6);
|
||||
}
|
||||
hash += (hash << 3);
|
||||
hash ^= (hash >> 11);
|
||||
hash += (hash << 15);
|
||||
return hash;
|
||||
}
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,113 @@
|
||||
/*
|
||||
* Standalone t1ha2_atonce command-line wrapper for avalanche testing.
|
||||
*
|
||||
* Upstream: https://gitflic.ru/project/erthink/t1ha
|
||||
* Commit: 00eb779b6c042ccd831ec2f1ae757409c73f39f6
|
||||
* Algorithm: t1ha2_atonce(data, length, seed=0), stable portable 64-bit mode.
|
||||
*
|
||||
* The vendored upstream implementation is licensed under the zlib License;
|
||||
* see upstream/LICENSE. This wrapper is an altered integration file and is not
|
||||
* represented as an original upstream source file.
|
||||
*/
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define T1HA0_DISABLED
|
||||
#define T1HA1_DISABLED
|
||||
#define T1HA_SYS_UNALIGNED_ACCESS 0
|
||||
#define T1HA_USE_FAST_ONESHOT_READ 0
|
||||
#include "upstream/src/t1ha2.c"
|
||||
|
||||
#define T1HA2_SEED UINT64_C(0)
|
||||
|
||||
static int is_ascii_trailing_space(unsigned char character)
|
||||
{
|
||||
return character == ' ' || character == '\t' || character == '\n' ||
|
||||
character == '\r' || character == '\v' || character == '\f';
|
||||
}
|
||||
|
||||
static int read_stdin(unsigned char **data, size_t *length)
|
||||
{
|
||||
size_t capacity = 256;
|
||||
unsigned char *buffer = malloc(capacity);
|
||||
|
||||
if (buffer == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
*length = 0;
|
||||
for (;;) {
|
||||
size_t available = capacity - *length;
|
||||
size_t bytes_read = fread(buffer + *length, 1, available, stdin);
|
||||
*length += bytes_read;
|
||||
|
||||
if (bytes_read < available) {
|
||||
if (ferror(stdin)) {
|
||||
free(buffer);
|
||||
return -1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (capacity > SIZE_MAX / 2U) {
|
||||
free(buffer);
|
||||
return -1;
|
||||
}
|
||||
capacity *= 2U;
|
||||
|
||||
{
|
||||
unsigned char *larger_buffer = realloc(buffer, capacity);
|
||||
if (larger_buffer == NULL) {
|
||||
free(buffer);
|
||||
return -1;
|
||||
}
|
||||
buffer = larger_buffer;
|
||||
}
|
||||
}
|
||||
|
||||
*data = buffer;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
const unsigned char *word;
|
||||
unsigned char *stdin_buffer = NULL;
|
||||
size_t length;
|
||||
|
||||
if (argc > 2) {
|
||||
fprintf(stderr, "usage: %s [ASCII_WORD]\n", argv[0]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
if (argc == 2) {
|
||||
word = (const unsigned char *)argv[1];
|
||||
length = strlen(argv[1]);
|
||||
} else {
|
||||
if (read_stdin(&stdin_buffer, &length) != 0) {
|
||||
fprintf(stderr, "failed to read input\n");
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
word = stdin_buffer;
|
||||
}
|
||||
|
||||
while (length > 0U && is_ascii_trailing_space(word[length - 1U])) {
|
||||
--length;
|
||||
}
|
||||
|
||||
for (size_t index = 0; index < length; ++index) {
|
||||
if (word[index] > 0x7fU) {
|
||||
fprintf(stderr, "input must contain ASCII characters only\n");
|
||||
free(stdin_buffer);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
printf("%016" PRIx64 "\n", t1ha2_atonce(word, length, T1HA2_SEED));
|
||||
free(stdin_buffer);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
zlib License, see https://en.wikipedia.org/wiki/Zlib_License
|
||||
|
||||
Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com,
|
||||
Fast Positive Hash.
|
||||
|
||||
Portions Copyright (c) 2010-2013 Leonid Yuriev <leo@yuriev.ru>,
|
||||
The 1Hippeus project (t1h).
|
||||
|
||||
This software is provided 'as-is', without any express or implied
|
||||
warranty. In no event will the authors be held liable for any damages
|
||||
arising from the use of this software.
|
||||
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it
|
||||
freely, subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not
|
||||
claim that you wrote the original software. If you use this software
|
||||
in a product, an acknowledgement in the product documentation would be
|
||||
appreciated but is not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be
|
||||
misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
@@ -0,0 +1,383 @@
|
||||
/*
|
||||
* Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com,
|
||||
* Fast Positive Hash.
|
||||
*
|
||||
* Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>,
|
||||
* The 1Hippeus project (t1h).
|
||||
*
|
||||
* This software is provided 'as-is', without any express or implied
|
||||
* warranty. In no event will the authors be held liable for any damages
|
||||
* arising from the use of this software.
|
||||
*
|
||||
* Permission is granted to anyone to use this software for any purpose,
|
||||
* including commercial applications, and to alter it and redistribute it
|
||||
* freely, subject to the following restrictions:
|
||||
*
|
||||
* 1. The origin of this software must not be misrepresented; you must not
|
||||
* claim that you wrote the original software. If you use this software
|
||||
* in a product, an acknowledgement in the product documentation would be
|
||||
* appreciated but is not required.
|
||||
* 2. Altered source versions must be plainly marked as such, and must not be
|
||||
* misrepresented as being the original software.
|
||||
* 3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
|
||||
/*
|
||||
* t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
|
||||
* by [Positive Technologies](https://www.ptsecurity.ru)
|
||||
*
|
||||
* Briefly, it is a 64-bit Hash Function:
|
||||
* 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
|
||||
* but portable and without penalties it can run on any 64-bit CPU.
|
||||
* 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
|
||||
* and all others portable hash-functions (which do not use specific
|
||||
* hardware tricks).
|
||||
* 3. Not suitable for cryptography.
|
||||
*
|
||||
* The Future will (be) Positive. Всё будет хорошо.
|
||||
*
|
||||
* ACKNOWLEDGEMENT:
|
||||
* The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
|
||||
* for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
|
||||
*/
|
||||
|
||||
#ifndef T1HA2_DISABLED
|
||||
#include "t1ha_bits.h"
|
||||
#include "t1ha_selfcheck.h"
|
||||
|
||||
static __always_inline void init_ab(t1ha_state256_t *s, uint64_t x,
|
||||
uint64_t y) {
|
||||
s->n.a = x;
|
||||
s->n.b = y;
|
||||
}
|
||||
|
||||
static __always_inline void init_cd(t1ha_state256_t *s, uint64_t x,
|
||||
uint64_t y) {
|
||||
s->n.c = rot64(y, 23) + ~x;
|
||||
s->n.d = ~y + rot64(x, 19);
|
||||
}
|
||||
|
||||
/* TODO: C++ template in the next version */
|
||||
#define T1HA2_UPDATE(ENDIANNES, ALIGNESS, state, v) \
|
||||
do { \
|
||||
t1ha_state256_t *const s = state; \
|
||||
const uint64_t w0 = fetch64_##ENDIANNES##_##ALIGNESS(v + 0); \
|
||||
const uint64_t w1 = fetch64_##ENDIANNES##_##ALIGNESS(v + 1); \
|
||||
const uint64_t w2 = fetch64_##ENDIANNES##_##ALIGNESS(v + 2); \
|
||||
const uint64_t w3 = fetch64_##ENDIANNES##_##ALIGNESS(v + 3); \
|
||||
\
|
||||
const uint64_t d02 = w0 + rot64(w2 + s->n.d, 56); \
|
||||
const uint64_t c13 = w1 + rot64(w3 + s->n.c, 19); \
|
||||
s->n.d ^= s->n.b + rot64(w1, 38); \
|
||||
s->n.c ^= s->n.a + rot64(w0, 57); \
|
||||
s->n.b ^= prime_6 * (c13 + w2); \
|
||||
s->n.a ^= prime_5 * (d02 + w3); \
|
||||
} while (0)
|
||||
|
||||
static __always_inline void squash(t1ha_state256_t *s) {
|
||||
s->n.a ^= prime_6 * (s->n.c + rot64(s->n.d, 23));
|
||||
s->n.b ^= prime_5 * (rot64(s->n.c, 19) + s->n.d);
|
||||
}
|
||||
|
||||
/* TODO: C++ template in the next version */
|
||||
#define T1HA2_LOOP(ENDIANNES, ALIGNESS, state, data, len) \
|
||||
do { \
|
||||
const void *detent = (const uint8_t *)data + len - 31; \
|
||||
do { \
|
||||
const uint64_t *v = (const uint64_t *)data; \
|
||||
data = (const uint64_t *)data + 4; \
|
||||
prefetch(data); \
|
||||
T1HA2_UPDATE(le, ALIGNESS, state, v); \
|
||||
} while (likely(data < detent)); \
|
||||
} while (0)
|
||||
|
||||
/* TODO: C++ template in the next version */
|
||||
#define T1HA2_TAIL_AB(ENDIANNES, ALIGNESS, state, data, len) \
|
||||
do { \
|
||||
t1ha_state256_t *const s = state; \
|
||||
const uint64_t *v = (const uint64_t *)data; \
|
||||
switch (len) { \
|
||||
default: \
|
||||
mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
|
||||
prime_4); \
|
||||
/* fall through */ \
|
||||
case 24: \
|
||||
case 23: \
|
||||
case 22: \
|
||||
case 21: \
|
||||
case 20: \
|
||||
case 19: \
|
||||
case 18: \
|
||||
case 17: \
|
||||
mixup64(&s->n.b, &s->n.a, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
|
||||
prime_3); \
|
||||
/* fall through */ \
|
||||
case 16: \
|
||||
case 15: \
|
||||
case 14: \
|
||||
case 13: \
|
||||
case 12: \
|
||||
case 11: \
|
||||
case 10: \
|
||||
case 9: \
|
||||
mixup64(&s->n.a, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
|
||||
prime_2); \
|
||||
/* fall through */ \
|
||||
case 8: \
|
||||
case 7: \
|
||||
case 6: \
|
||||
case 5: \
|
||||
case 4: \
|
||||
case 3: \
|
||||
case 2: \
|
||||
case 1: \
|
||||
mixup64(&s->n.b, &s->n.a, tail64_##ENDIANNES##_##ALIGNESS(v, len), \
|
||||
prime_1); \
|
||||
/* fall through */ \
|
||||
case 0: \
|
||||
return final64(s->n.a, s->n.b); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* TODO: C++ template in the next version */
|
||||
#define T1HA2_TAIL_ABCD(ENDIANNES, ALIGNESS, state, data, len) \
|
||||
do { \
|
||||
t1ha_state256_t *const s = state; \
|
||||
const uint64_t *v = (const uint64_t *)data; \
|
||||
switch (len) { \
|
||||
default: \
|
||||
mixup64(&s->n.a, &s->n.d, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
|
||||
prime_4); \
|
||||
/* fall through */ \
|
||||
case 24: \
|
||||
case 23: \
|
||||
case 22: \
|
||||
case 21: \
|
||||
case 20: \
|
||||
case 19: \
|
||||
case 18: \
|
||||
case 17: \
|
||||
mixup64(&s->n.b, &s->n.a, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
|
||||
prime_3); \
|
||||
/* fall through */ \
|
||||
case 16: \
|
||||
case 15: \
|
||||
case 14: \
|
||||
case 13: \
|
||||
case 12: \
|
||||
case 11: \
|
||||
case 10: \
|
||||
case 9: \
|
||||
mixup64(&s->n.c, &s->n.b, fetch64_##ENDIANNES##_##ALIGNESS(v++), \
|
||||
prime_2); \
|
||||
/* fall through */ \
|
||||
case 8: \
|
||||
case 7: \
|
||||
case 6: \
|
||||
case 5: \
|
||||
case 4: \
|
||||
case 3: \
|
||||
case 2: \
|
||||
case 1: \
|
||||
mixup64(&s->n.d, &s->n.c, tail64_##ENDIANNES##_##ALIGNESS(v, len), \
|
||||
prime_1); \
|
||||
/* fall through */ \
|
||||
case 0: \
|
||||
return final128(s->n.a, s->n.b, s->n.c, s->n.d, extra_result); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
static __always_inline uint64_t final128(uint64_t a, uint64_t b, uint64_t c,
|
||||
uint64_t d, uint64_t *h) {
|
||||
mixup64(&a, &b, rot64(c, 41) ^ d, prime_0);
|
||||
mixup64(&b, &c, rot64(d, 23) ^ a, prime_6);
|
||||
mixup64(&c, &d, rot64(a, 19) ^ b, prime_5);
|
||||
mixup64(&d, &a, rot64(b, 31) ^ c, prime_4);
|
||||
*h = c + d;
|
||||
return a ^ b;
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed) {
|
||||
t1ha_state256_t state;
|
||||
init_ab(&state, seed, length);
|
||||
|
||||
#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT
|
||||
if (unlikely(length > 32)) {
|
||||
init_cd(&state, seed, length);
|
||||
#if defined(__LCC__) && __LCC__ > 123
|
||||
/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
|
||||
* в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
|
||||
* говорят, что это нецелесообразно */
|
||||
#pragma comb_oper
|
||||
#endif /* E2K LCC > 1.23 */
|
||||
T1HA2_LOOP(le, unaligned, &state, data, length);
|
||||
squash(&state);
|
||||
length &= 31;
|
||||
}
|
||||
T1HA2_TAIL_AB(le, unaligned, &state, data, length);
|
||||
#else
|
||||
const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
|
||||
if (misaligned) {
|
||||
if (unlikely(length > 32)) {
|
||||
init_cd(&state, seed, length);
|
||||
#if defined(__LCC__) && __LCC__ > 123
|
||||
/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
|
||||
* в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
|
||||
* говорят, что это нецелесообразно */
|
||||
#pragma comb_oper
|
||||
#endif /* E2K LCC > 1.23 */
|
||||
T1HA2_LOOP(le, unaligned, &state, data, length);
|
||||
squash(&state);
|
||||
length &= 31;
|
||||
}
|
||||
T1HA2_TAIL_AB(le, unaligned, &state, data, length);
|
||||
} else {
|
||||
if (unlikely(length > 32)) {
|
||||
init_cd(&state, seed, length);
|
||||
#if defined(__LCC__) && __LCC__ > 123
|
||||
/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
|
||||
* в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
|
||||
* говорят, что это нецелесообразно */
|
||||
#pragma comb_oper
|
||||
#endif /* E2K LCC > 1.23 */
|
||||
T1HA2_LOOP(le, aligned, &state, data, length);
|
||||
squash(&state);
|
||||
length &= 31;
|
||||
}
|
||||
T1HA2_TAIL_AB(le, aligned, &state, data, length);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
|
||||
const void *__restrict data, size_t length,
|
||||
uint64_t seed) {
|
||||
t1ha_state256_t state;
|
||||
init_ab(&state, seed, length);
|
||||
init_cd(&state, seed, length);
|
||||
|
||||
#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT
|
||||
if (unlikely(length > 32)) {
|
||||
#if defined(__LCC__) && __LCC__ > 123
|
||||
/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
|
||||
* в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
|
||||
* говорят, что это нецелесообразно */
|
||||
#pragma comb_oper
|
||||
#endif /* E2K LCC > 1.23 */
|
||||
T1HA2_LOOP(le, unaligned, &state, data, length);
|
||||
length &= 31;
|
||||
}
|
||||
T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
|
||||
#else
|
||||
const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
|
||||
if (misaligned) {
|
||||
if (unlikely(length > 32)) {
|
||||
#if defined(__LCC__) && __LCC__ > 123
|
||||
/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
|
||||
* в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
|
||||
* говорят, что это нецелесообразно */
|
||||
#pragma comb_oper
|
||||
#endif /* E2K LCC > 1.23 */
|
||||
T1HA2_LOOP(le, unaligned, &state, data, length);
|
||||
length &= 31;
|
||||
}
|
||||
T1HA2_TAIL_ABCD(le, unaligned, &state, data, length);
|
||||
} else {
|
||||
if (unlikely(length > 32)) {
|
||||
#if defined(__LCC__) && __LCC__ > 123
|
||||
/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
|
||||
* в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
|
||||
* говорят, что это нецелесообразно */
|
||||
#pragma comb_oper
|
||||
#endif /* E2K LCC > 1.23 */
|
||||
T1HA2_LOOP(le, aligned, &state, data, length);
|
||||
length &= 31;
|
||||
}
|
||||
T1HA2_TAIL_ABCD(le, aligned, &state, data, length);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y) {
|
||||
init_ab(&ctx->state, seed_x, seed_y);
|
||||
init_cd(&ctx->state, seed_x, seed_y);
|
||||
ctx->partial = 0;
|
||||
ctx->total = 0;
|
||||
}
|
||||
|
||||
void t1ha2_update(t1ha_context_t *__restrict ctx, const void *__restrict data,
|
||||
size_t length) {
|
||||
ctx->total += length;
|
||||
|
||||
if (ctx->partial) {
|
||||
const size_t left = 32 - ctx->partial;
|
||||
const size_t chunk = (length >= left) ? left : length;
|
||||
memcpy(ctx->buffer.bytes + ctx->partial, data, chunk);
|
||||
ctx->partial += chunk;
|
||||
if (ctx->partial < 32) {
|
||||
assert(left >= length);
|
||||
return;
|
||||
}
|
||||
ctx->partial = 0;
|
||||
data = (const uint8_t *)data + chunk;
|
||||
length -= chunk;
|
||||
T1HA2_UPDATE(le, aligned, &ctx->state, ctx->buffer.u64);
|
||||
}
|
||||
|
||||
if (length >= 32) {
|
||||
#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT
|
||||
#if defined(__LCC__) && __LCC__ > 123
|
||||
/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
|
||||
* в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
|
||||
* говорят, что это нецелесообразно */
|
||||
#pragma comb_oper
|
||||
#endif /* E2K LCC > 1.23 */
|
||||
T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
|
||||
#else
|
||||
const bool misaligned = (((uintptr_t)data) & (ALIGNMENT_64 - 1)) != 0;
|
||||
if (misaligned) {
|
||||
#if defined(__LCC__) && __LCC__ > 123
|
||||
/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
|
||||
* в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
|
||||
* говорят, что это нецелесообразно */
|
||||
#pragma comb_oper
|
||||
#endif /* E2K LCC > 1.23 */
|
||||
T1HA2_LOOP(le, unaligned, &ctx->state, data, length);
|
||||
} else {
|
||||
#if defined(__LCC__) && __LCC__ > 123
|
||||
/* Форсирует комбинирование пар арифметических операций в двухэтажные операции
|
||||
* в ближайшем после объявления директивы цикле, даже если эвристики оптимизации
|
||||
* говорят, что это нецелесообразно */
|
||||
#pragma comb_oper
|
||||
#endif /* E2K LCC > 1.23 */
|
||||
T1HA2_LOOP(le, aligned, &ctx->state, data, length);
|
||||
}
|
||||
#endif
|
||||
length &= 31;
|
||||
}
|
||||
|
||||
if (length)
|
||||
memcpy(ctx->buffer.bytes, data, ctx->partial = length);
|
||||
}
|
||||
|
||||
uint64_t t1ha2_final(t1ha_context_t *__restrict ctx,
|
||||
uint64_t *__restrict extra_result) {
|
||||
uint64_t bits = (ctx->total << 3) ^ (UINT64_C(1) << 63);
|
||||
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
|
||||
bits = bswap64(bits);
|
||||
#endif
|
||||
t1ha2_update(ctx, &bits, 8);
|
||||
|
||||
if (likely(!extra_result)) {
|
||||
squash(&ctx->state);
|
||||
T1HA2_TAIL_AB(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
|
||||
}
|
||||
|
||||
T1HA2_TAIL_ABCD(le, aligned, &ctx->state, ctx->buffer.u64, ctx->partial);
|
||||
}
|
||||
|
||||
#endif /* T1HA2_DISABLED */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com,
|
||||
* Fast Positive Hash.
|
||||
*
|
||||
* Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>,
|
||||
* The 1Hippeus project (t1h).
|
||||
*
|
||||
* This software is provided 'as-is', without any express or implied
|
||||
* warranty. In no event will the authors be held liable for any damages
|
||||
* arising from the use of this software.
|
||||
*
|
||||
* Permission is granted to anyone to use this software for any purpose,
|
||||
* including commercial applications, and to alter it and redistribute it
|
||||
* freely, subject to the following restrictions:
|
||||
*
|
||||
* 1. The origin of this software must not be misrepresented; you must not
|
||||
* claim that you wrote the original software. If you use this software
|
||||
* in a product, an acknowledgement in the product documentation would be
|
||||
* appreciated but is not required.
|
||||
* 2. Altered source versions must be plainly marked as such, and must not be
|
||||
* misrepresented as being the original software.
|
||||
* 3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
|
||||
/*
|
||||
* t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
|
||||
* by [Positive Technologies](https://www.ptsecurity.ru)
|
||||
*
|
||||
* Briefly, it is a 64-bit Hash Function:
|
||||
* 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
|
||||
* but portable and without penalties it can run on any 64-bit CPU.
|
||||
* 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
|
||||
* and all others portable hash-functions (which do not use specific
|
||||
* hardware tricks).
|
||||
* 3. Not suitable for cryptography.
|
||||
*
|
||||
* The Future will (be) Positive. Всё будет хорошо.
|
||||
*
|
||||
* ACKNOWLEDGEMENT:
|
||||
* The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
|
||||
* for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#if defined(_MSC_VER) && _MSC_VER > 1800
|
||||
#pragma warning(disable : 4464) /* relative include path contains '..' */
|
||||
#endif /* MSVC */
|
||||
#include "../t1ha.h"
|
||||
|
||||
/***************************************************************************/
|
||||
/* Self-checking */
|
||||
|
||||
extern const uint8_t t1ha_test_pattern[64];
|
||||
int t1ha_selfcheck(uint64_t (*hash)(const void *, size_t, uint64_t),
|
||||
const uint64_t *reference_values);
|
||||
|
||||
#ifndef T1HA2_DISABLED
|
||||
extern const uint64_t t1ha_refval_2atonce[81];
|
||||
extern const uint64_t t1ha_refval_2atonce128[81];
|
||||
extern const uint64_t t1ha_refval_2stream[81];
|
||||
extern const uint64_t t1ha_refval_2stream128[81];
|
||||
#endif /* T1HA2_DISABLED */
|
||||
|
||||
#ifndef T1HA1_DISABLED
|
||||
extern const uint64_t t1ha_refval_64le[81];
|
||||
extern const uint64_t t1ha_refval_64be[81];
|
||||
#endif /* T1HA1_DISABLED */
|
||||
|
||||
#ifndef T1HA0_DISABLED
|
||||
extern const uint64_t t1ha_refval_32le[81];
|
||||
extern const uint64_t t1ha_refval_32be[81];
|
||||
#if T1HA0_AESNI_AVAILABLE
|
||||
extern const uint64_t t1ha_refval_ia32aes_a[81];
|
||||
extern const uint64_t t1ha_refval_ia32aes_b[81];
|
||||
#endif /* T1HA0_AESNI_AVAILABLE */
|
||||
#endif /* T1HA0_DISABLED */
|
||||
@@ -0,0 +1,719 @@
|
||||
/*
|
||||
* Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com,
|
||||
* Fast Positive Hash.
|
||||
*
|
||||
* Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>,
|
||||
* The 1Hippeus project (t1h).
|
||||
*
|
||||
* This software is provided 'as-is', without any express or implied
|
||||
* warranty. In no event will the authors be held liable for any damages
|
||||
* arising from the use of this software.
|
||||
*
|
||||
* Permission is granted to anyone to use this software for any purpose,
|
||||
* including commercial applications, and to alter it and redistribute it
|
||||
* freely, subject to the following restrictions:
|
||||
*
|
||||
* 1. The origin of this software must not be misrepresented; you must not
|
||||
* claim that you wrote the original software. If you use this software
|
||||
* in a product, an acknowledgement in the product documentation would be
|
||||
* appreciated but is not required.
|
||||
* 2. Altered source versions must be plainly marked as such, and must not be
|
||||
* misrepresented as being the original software.
|
||||
* 3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
|
||||
/*
|
||||
* t1ha = { Fast Positive Hash, aka "Позитивный Хэш" }
|
||||
* by [Positive Technologies](https://www.ptsecurity.ru)
|
||||
*
|
||||
* Briefly, it is a 64-bit Hash Function:
|
||||
* 1. Created for 64-bit little-endian platforms, in predominantly for x86_64,
|
||||
* but portable and without penalties it can run on any 64-bit CPU.
|
||||
* 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash
|
||||
* and all others portable hash-functions (which do not use specific
|
||||
* hardware tricks).
|
||||
* 3. Not suitable for cryptography.
|
||||
*
|
||||
* The Future will (be) Positive. Всё будет хорошо.
|
||||
*
|
||||
* ACKNOWLEDGEMENT:
|
||||
* The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев)
|
||||
* for The 1Hippeus project - zerocopy messaging in the spirit of Sparta!
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
/*****************************************************************************
|
||||
*
|
||||
* PLEASE PAY ATTENTION TO THE FOLLOWING NOTES
|
||||
* about macros definitions which controls t1ha behaviour and/or performance.
|
||||
*
|
||||
*
|
||||
* 1) T1HA_SYS_UNALIGNED_ACCESS = Defines the system/platform/CPU/architecture
|
||||
* abilities for unaligned data access.
|
||||
*
|
||||
* By default, when the T1HA_SYS_UNALIGNED_ACCESS not defined,
|
||||
* it will defined on the basis hardcoded knowledge about of capabilities
|
||||
* of most common CPU architectures. But you could override this
|
||||
* default behavior when build t1ha library itself:
|
||||
*
|
||||
* // To disable unaligned access at all.
|
||||
* #define T1HA_SYS_UNALIGNED_ACCESS 0
|
||||
*
|
||||
* // To enable unaligned access, but indicate that it significantly slow.
|
||||
* #define T1HA_SYS_UNALIGNED_ACCESS 1
|
||||
*
|
||||
* // To enable unaligned access, and indicate that it effecient.
|
||||
* #define T1HA_SYS_UNALIGNED_ACCESS 2
|
||||
*
|
||||
*
|
||||
* 2) T1HA_USE_FAST_ONESHOT_READ = Controls the data reads at the end of buffer.
|
||||
*
|
||||
* When defined to non-zero, t1ha will use 'one shot' method for reading
|
||||
* up to 8 bytes at the end of data. In this case just the one 64-bit read
|
||||
* will be performed even when the available less than 8 bytes.
|
||||
*
|
||||
* This is little bit faster that switching by length of data tail.
|
||||
* Unfortunately this will triggering a false-positive alarms from Valgrind,
|
||||
* AddressSanitizer and other similar tool.
|
||||
*
|
||||
* By default, t1ha defines it to 1, but you could override this
|
||||
* default behavior when build t1ha library itself:
|
||||
*
|
||||
* // For little bit faster and small code.
|
||||
* #define T1HA_USE_FAST_ONESHOT_READ 1
|
||||
*
|
||||
* // For calmness if doubt.
|
||||
* #define T1HA_USE_FAST_ONESHOT_READ 0
|
||||
*
|
||||
*
|
||||
* 3) T1HA0_RUNTIME_SELECT = Controls choice fastest function in runtime.
|
||||
*
|
||||
* t1ha library offers the t1ha0() function as the fastest for current CPU.
|
||||
* But actual CPU's features/capabilities and may be significantly different,
|
||||
* especially on x86 platform. Therefore, internally, t1ha0() may require
|
||||
* dynamic dispatching for choice best implementation.
|
||||
*
|
||||
* By default, t1ha enables such runtime choice and (may be) corresponding
|
||||
* indirect calls if it reasonable, but you could override this default
|
||||
* behavior when build t1ha library itself:
|
||||
*
|
||||
* // To enable runtime choice of fastest implementation.
|
||||
* #define T1HA0_RUNTIME_SELECT 1
|
||||
*
|
||||
* // To disable runtime choice of fastest implementation.
|
||||
* #define T1HA0_RUNTIME_SELECT 0
|
||||
*
|
||||
* When T1HA0_RUNTIME_SELECT is nonzero the t1ha0_resolve() function could
|
||||
* be used to get actual t1ha0() implementation address at runtime. This is
|
||||
* useful for two cases:
|
||||
* - calling by local pointer-to-function usually is little
|
||||
* bit faster (less overhead) than via a PLT thru the DSO boundary.
|
||||
* - GNU Indirect functions (see below) don't supported by environment
|
||||
* and calling by t1ha0_funcptr is not available and/or expensive.
|
||||
*
|
||||
* 4) T1HA_USE_INDIRECT_FUNCTIONS = Controls usage of GNU Indirect functions.
|
||||
*
|
||||
* In continue of T1HA0_RUNTIME_SELECT the T1HA_USE_INDIRECT_FUNCTIONS
|
||||
* controls usage of ELF indirect functions feature. In general, when
|
||||
* available, this reduces overhead of indirect function's calls though
|
||||
* a DSO-bundary (https://sourceware.org/glibc/wiki/GNU_IFUNC).
|
||||
*
|
||||
* By default, t1ha engage GNU Indirect functions when it available
|
||||
* and useful, but you could override this default behavior when build
|
||||
* t1ha library itself:
|
||||
*
|
||||
* // To enable use of GNU ELF Indirect functions.
|
||||
* #define T1HA_USE_INDIRECT_FUNCTIONS 1
|
||||
*
|
||||
* // To disable use of GNU ELF Indirect functions. This may be useful
|
||||
* // if the actual toolchain or the system's loader don't support ones.
|
||||
* #define T1HA_USE_INDIRECT_FUNCTIONS 0
|
||||
*
|
||||
* 5) T1HA0_AESNI_AVAILABLE = Controls AES-NI detection and dispatching on x86.
|
||||
*
|
||||
* In continue of T1HA0_RUNTIME_SELECT the T1HA0_AESNI_AVAILABLE controls
|
||||
* detection and usage of AES-NI CPU's feature. On the other hand, this
|
||||
* requires compiling parts of t1ha library with certain properly options,
|
||||
* and could be difficult or inconvenient in some cases.
|
||||
*
|
||||
* By default, t1ha engade AES-NI for t1ha0() on the x86 platform, but
|
||||
* you could override this default behavior when build t1ha library itself:
|
||||
*
|
||||
* // To disable detection and usage of AES-NI instructions for t1ha0().
|
||||
* // This may be useful when you unable to build t1ha library properly
|
||||
* // or known that AES-NI will be unavailable at the deploy.
|
||||
* #define T1HA0_AESNI_AVAILABLE 0
|
||||
*
|
||||
* // To force detection and usage of AES-NI instructions for t1ha0(),
|
||||
* // but I don't known reasons to anybody would need this.
|
||||
* #define T1HA0_AESNI_AVAILABLE 1
|
||||
*
|
||||
* 6) T1HA0_DISABLED, T1HA1_DISABLED, T1HA2_DISABLED = Controls availability of
|
||||
* t1ha functions.
|
||||
*
|
||||
* In some cases could be useful to import/use only few of t1ha functions
|
||||
* or just the one. So, this definitions allows disable corresponding parts
|
||||
* of t1ha library.
|
||||
*
|
||||
* // To disable t1ha0(), t1ha0_32le(), t1ha0_32be() and all AES-NI.
|
||||
* #define T1HA0_DISABLED
|
||||
*
|
||||
* // To disable t1ha1_le() and t1ha1_be().
|
||||
* #define T1HA1_DISABLED
|
||||
*
|
||||
* // To disable t1ha2_atonce(), t1ha2_atonce128() and so on.
|
||||
* #define T1HA2_DISABLED
|
||||
*
|
||||
*****************************************************************************/
|
||||
|
||||
#define T1HA_VERSION_MAJOR 2
|
||||
#define T1HA_VERSION_MINOR 1
|
||||
#define T1HA_VERSION_RELEASE 1
|
||||
|
||||
#ifndef __has_attribute
|
||||
#define __has_attribute(x) (0)
|
||||
#endif
|
||||
|
||||
#ifndef __has_include
|
||||
#define __has_include(x) (0)
|
||||
#endif
|
||||
|
||||
#ifndef __GNUC_PREREQ
|
||||
#if defined(__GNUC__) && defined(__GNUC_MINOR__)
|
||||
#define __GNUC_PREREQ(maj, min) \
|
||||
((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
|
||||
#else
|
||||
#define __GNUC_PREREQ(maj, min) 0
|
||||
#endif
|
||||
#endif /* __GNUC_PREREQ */
|
||||
|
||||
#ifndef __CLANG_PREREQ
|
||||
#ifdef __clang__
|
||||
#define __CLANG_PREREQ(maj, min) \
|
||||
((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
|
||||
#else
|
||||
#define __CLANG_PREREQ(maj, min) (0)
|
||||
#endif
|
||||
#endif /* __CLANG_PREREQ */
|
||||
|
||||
#ifndef __LCC_PREREQ
|
||||
#ifdef __LCC__
|
||||
#define __LCC_PREREQ(maj, min) \
|
||||
((__LCC__ << 16) + __LCC_MINOR__ >= ((maj) << 16) + (min))
|
||||
#else
|
||||
#define __LCC_PREREQ(maj, min) (0)
|
||||
#endif
|
||||
#endif /* __LCC_PREREQ */
|
||||
|
||||
/*****************************************************************************/
|
||||
|
||||
#ifdef _MSC_VER
|
||||
/* Avoid '16' bytes padding added after data member 't1ha_context::total'
|
||||
* and other warnings from std-headers if warning-level > 3. */
|
||||
#pragma warning(push, 3)
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus) && __cplusplus >= 201103L
|
||||
#include <climits>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#else
|
||||
#include <limits.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#endif
|
||||
|
||||
/*****************************************************************************/
|
||||
|
||||
#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
|
||||
defined(i486) || defined(__i486) || defined(__i486__) || \
|
||||
defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
|
||||
defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
|
||||
defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
|
||||
defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
|
||||
defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
|
||||
defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
|
||||
#ifndef __ia32__
|
||||
/* LY: define neutral __ia32__ for x86 and x86-64 archs */
|
||||
#define __ia32__ 1
|
||||
#endif /* __ia32__ */
|
||||
#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
|
||||
defined(__amd64) || defined(_M_X64))
|
||||
/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
|
||||
#define __amd64__ 1
|
||||
#endif /* __amd64__ */
|
||||
#endif /* all x86 */
|
||||
|
||||
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
|
||||
!defined(__ORDER_BIG_ENDIAN__)
|
||||
|
||||
/* *INDENT-OFF* */
|
||||
/* clang-format off */
|
||||
|
||||
#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \
|
||||
defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
|
||||
#include <endian.h>
|
||||
#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
|
||||
defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
|
||||
#include <machine/endian.h>
|
||||
#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
|
||||
#include <sys/isa_defs.h>
|
||||
#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
|
||||
(__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
|
||||
#include <sys/endian.h>
|
||||
#include <sys/types.h>
|
||||
#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
|
||||
defined(__NETBSD__) || defined(__NetBSD__) || \
|
||||
defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
|
||||
#include <sys/param.h>
|
||||
#endif /* OS */
|
||||
|
||||
/* *INDENT-ON* */
|
||||
/* clang-format on */
|
||||
|
||||
#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
|
||||
#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
|
||||
#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
|
||||
#define __BYTE_ORDER__ __BYTE_ORDER
|
||||
#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
|
||||
#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
|
||||
#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
|
||||
#define __BYTE_ORDER__ _BYTE_ORDER
|
||||
#else
|
||||
#define __ORDER_LITTLE_ENDIAN__ 1234
|
||||
#define __ORDER_BIG_ENDIAN__ 4321
|
||||
|
||||
#if defined(__LITTLE_ENDIAN__) || \
|
||||
(defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
|
||||
defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
|
||||
defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
|
||||
defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
|
||||
defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
|
||||
defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
|
||||
defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
|
||||
defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
|
||||
defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
|
||||
defined(__WINDOWS__)
|
||||
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
|
||||
|
||||
#elif defined(__BIG_ENDIAN__) || \
|
||||
(defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
|
||||
defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
|
||||
defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
|
||||
defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
|
||||
defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
|
||||
defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
|
||||
defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
|
||||
#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
|
||||
|
||||
#else
|
||||
#error __BYTE_ORDER__ should be defined.
|
||||
#endif /* Arch */
|
||||
|
||||
#endif
|
||||
#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
|
||||
|
||||
/*****************************************************************************/
|
||||
|
||||
#ifndef __dll_export
|
||||
#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__)
|
||||
#if defined(__GNUC__) || __has_attribute(dllexport)
|
||||
#define __dll_export __attribute__((dllexport))
|
||||
#else
|
||||
#define __dll_export __declspec(dllexport)
|
||||
#endif
|
||||
#elif defined(__GNUC__) || __has_attribute(__visibility__)
|
||||
#define __dll_export __attribute__((__visibility__("default")))
|
||||
#else
|
||||
#define __dll_export
|
||||
#endif
|
||||
#endif /* __dll_export */
|
||||
|
||||
#ifndef __dll_import
|
||||
#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__)
|
||||
#if defined(__GNUC__) || __has_attribute(dllimport)
|
||||
#define __dll_import __attribute__((dllimport))
|
||||
#else
|
||||
#define __dll_import __declspec(dllimport)
|
||||
#endif
|
||||
#elif defined(__GNUC__) || __has_attribute(__visibility__)
|
||||
#define __dll_import __attribute__((__visibility__("default")))
|
||||
#else
|
||||
#define __dll_import
|
||||
#endif
|
||||
#endif /* __dll_import */
|
||||
|
||||
#ifndef __force_inline
|
||||
#ifdef _MSC_VER
|
||||
#define __force_inline __forceinline
|
||||
#elif __GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__)
|
||||
#define __force_inline __inline __attribute__((__always_inline__))
|
||||
#else
|
||||
#define __force_inline __inline
|
||||
#endif
|
||||
#endif /* __force_inline */
|
||||
|
||||
#ifndef T1HA_API
|
||||
#if defined(t1ha_EXPORTS)
|
||||
#define T1HA_API __dll_export
|
||||
#elif defined(t1ha_IMPORTS)
|
||||
#define T1HA_API __dll_import
|
||||
#else
|
||||
#define T1HA_API
|
||||
#endif
|
||||
#endif /* T1HA_API */
|
||||
|
||||
#if defined(_MSC_VER) && defined(__ia32__)
|
||||
#define T1HA_ALIGN_PREFIX __declspec(align(32)) /* required only for SIMD */
|
||||
#else
|
||||
#define T1HA_ALIGN_PREFIX
|
||||
#endif /* _MSC_VER */
|
||||
|
||||
#if defined(__GNUC__) && defined(__ia32__)
|
||||
#define T1HA_ALIGN_SUFFIX \
|
||||
__attribute__((__aligned__(32))) /* required only for SIMD */
|
||||
#else
|
||||
#define T1HA_ALIGN_SUFFIX
|
||||
#endif /* GCC x86 */
|
||||
|
||||
#ifndef T1HA_USE_INDIRECT_FUNCTIONS
|
||||
/* GNU ELF indirect functions usage control. For more info please see
|
||||
* https://en.wikipedia.org/wiki/Executable_and_Linkable_Format
|
||||
* and https://sourceware.org/glibc/wiki/GNU_IFUNC */
|
||||
#if defined(__ELF__) && defined(__amd64__) && \
|
||||
(__has_attribute(__ifunc__) || \
|
||||
(!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && \
|
||||
!defined(__SANITIZE_ADDRESS__) && !defined(__SSP_ALL__)))
|
||||
/* Enable gnu_indirect_function by default if :
|
||||
* - ELF AND x86_64
|
||||
* - attribute(__ifunc__) is available OR
|
||||
* GCC >= 4 WITHOUT -fsanitize=address NOR -fstack-protector-all */
|
||||
#define T1HA_USE_INDIRECT_FUNCTIONS 1
|
||||
#else
|
||||
#define T1HA_USE_INDIRECT_FUNCTIONS 0
|
||||
#endif
|
||||
#endif /* T1HA_USE_INDIRECT_FUNCTIONS */
|
||||
|
||||
#if __GNUC_PREREQ(4, 0)
|
||||
#pragma GCC visibility push(hidden)
|
||||
#endif /* __GNUC_PREREQ(4,0) */
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef union T1HA_ALIGN_PREFIX t1ha_state256 {
|
||||
uint8_t bytes[32];
|
||||
uint32_t u32[8];
|
||||
uint64_t u64[4];
|
||||
struct {
|
||||
uint64_t a, b, c, d;
|
||||
} n;
|
||||
} t1ha_state256_t T1HA_ALIGN_SUFFIX;
|
||||
|
||||
typedef struct t1ha_context {
|
||||
t1ha_state256_t state;
|
||||
t1ha_state256_t buffer;
|
||||
size_t partial;
|
||||
uint64_t total;
|
||||
} t1ha_context_t;
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
|
||||
/******************************************************************************
|
||||
*
|
||||
* Self-testing API.
|
||||
*
|
||||
* Unfortunately, some compilers (exactly only Microsoft Visual C/C++) has
|
||||
* a bugs which leads t1ha-functions to produce wrong results. This API allows
|
||||
* check the correctness of the actual code in runtime.
|
||||
*
|
||||
* All check-functions returns 0 on success, or -1 in case the corresponding
|
||||
* hash-function failed verification. PLEASE, always perform such checking at
|
||||
* initialization of your code, if you using MSVC or other troubleful compilers.
|
||||
*/
|
||||
|
||||
T1HA_API int t1ha_selfcheck__all_enabled(void);
|
||||
|
||||
#ifndef T1HA2_DISABLED
|
||||
T1HA_API int t1ha_selfcheck__t1ha2_atonce(void);
|
||||
T1HA_API int t1ha_selfcheck__t1ha2_atonce128(void);
|
||||
T1HA_API int t1ha_selfcheck__t1ha2_stream(void);
|
||||
T1HA_API int t1ha_selfcheck__t1ha2(void);
|
||||
#endif /* T1HA2_DISABLED */
|
||||
|
||||
#ifndef T1HA1_DISABLED
|
||||
T1HA_API int t1ha_selfcheck__t1ha1_le(void);
|
||||
T1HA_API int t1ha_selfcheck__t1ha1_be(void);
|
||||
T1HA_API int t1ha_selfcheck__t1ha1(void);
|
||||
#endif /* T1HA1_DISABLED */
|
||||
|
||||
#ifndef T1HA0_DISABLED
|
||||
T1HA_API int t1ha_selfcheck__t1ha0_32le(void);
|
||||
T1HA_API int t1ha_selfcheck__t1ha0_32be(void);
|
||||
T1HA_API int t1ha_selfcheck__t1ha0(void);
|
||||
|
||||
/* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */
|
||||
#ifndef T1HA0_AESNI_AVAILABLE
|
||||
#if defined(__e2k__) || \
|
||||
(defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800))
|
||||
#define T1HA0_AESNI_AVAILABLE 1
|
||||
#else
|
||||
#define T1HA0_AESNI_AVAILABLE 0
|
||||
#endif
|
||||
#endif /* ifndef T1HA0_AESNI_AVAILABLE */
|
||||
|
||||
#if T1HA0_AESNI_AVAILABLE
|
||||
T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_noavx(void);
|
||||
T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx(void);
|
||||
#ifndef __e2k__
|
||||
T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx2(void);
|
||||
#endif
|
||||
#endif /* if T1HA0_AESNI_AVAILABLE */
|
||||
#endif /* T1HA0_DISABLED */
|
||||
|
||||
/******************************************************************************
|
||||
*
|
||||
* t1ha2 = 64 and 128-bit, SLIGHTLY MORE ATTENTION FOR QUALITY AND STRENGTH.
|
||||
*
|
||||
* - The recommended version of "Fast Positive Hash" with good quality
|
||||
* for checksum, hash tables and fingerprinting.
|
||||
* - Portable and extremely efficiency on modern 64-bit CPUs.
|
||||
* Designed for 64-bit little-endian platforms,
|
||||
* in other cases will runs slowly.
|
||||
* - Great quality of hashing and still faster than other non-t1ha hashes.
|
||||
* Provides streaming mode and 128-bit result.
|
||||
*
|
||||
* Note: Due performance reason 64- and 128-bit results are completely
|
||||
* different each other, i.e. 64-bit result is NOT any part of 128-bit.
|
||||
*/
|
||||
#ifndef T1HA2_DISABLED
|
||||
|
||||
/* The at-once variant with 64-bit result */
|
||||
T1HA_API uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed);
|
||||
|
||||
/* The at-once variant with 128-bit result.
|
||||
* Argument `extra_result` is NOT optional and MUST be valid.
|
||||
* The high 64-bit part of 128-bit hash will be always unconditionally
|
||||
* stored to the address given by `extra_result` argument. */
|
||||
T1HA_API uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result,
|
||||
const void *__restrict data, size_t length,
|
||||
uint64_t seed);
|
||||
|
||||
/* The init/update/final trinity for streaming.
|
||||
* Return 64 or 128-bit result depentently from `extra_result` argument. */
|
||||
T1HA_API void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y);
|
||||
T1HA_API void t1ha2_update(t1ha_context_t *__restrict ctx,
|
||||
const void *__restrict data, size_t length);
|
||||
|
||||
/* Argument `extra_result` is optional and MAY be NULL.
|
||||
* - If `extra_result` is NOT NULL then the 128-bit hash will be calculated,
|
||||
* and high 64-bit part of it will be stored to the address given
|
||||
* by `extra_result` argument.
|
||||
* - Otherwise the 64-bit hash will be calculated
|
||||
* and returned from function directly.
|
||||
*
|
||||
* Note: Due performance reason 64- and 128-bit results are completely
|
||||
* different each other, i.e. 64-bit result is NOT any part of 128-bit. */
|
||||
T1HA_API uint64_t t1ha2_final(t1ha_context_t *__restrict ctx,
|
||||
uint64_t *__restrict extra_result /* optional */);
|
||||
|
||||
#endif /* T1HA2_DISABLED */
|
||||
|
||||
/******************************************************************************
|
||||
*
|
||||
* t1ha1 = 64-bit, BASELINE FAST PORTABLE HASH:
|
||||
*
|
||||
* - Runs faster on 64-bit platforms in other cases may runs slowly.
|
||||
* - Portable and stable, returns same 64-bit result
|
||||
* on all architectures and CPUs.
|
||||
* - Unfortunately it fails the "strict avalanche criteria",
|
||||
* see test results at https://github.com/demerphq/smhasher.
|
||||
*
|
||||
* This flaw is insignificant for the t1ha1() purposes and imperceptible
|
||||
* from a practical point of view.
|
||||
* However, nowadays this issue has resolved in the next t1ha2(),
|
||||
* that was initially planned to providing a bit more quality.
|
||||
*/
|
||||
#ifndef T1HA1_DISABLED
|
||||
|
||||
/* The little-endian variant. */
|
||||
T1HA_API uint64_t t1ha1_le(const void *data, size_t length, uint64_t seed);
|
||||
|
||||
/* The big-endian variant. */
|
||||
T1HA_API uint64_t t1ha1_be(const void *data, size_t length, uint64_t seed);
|
||||
|
||||
#endif /* T1HA1_DISABLED */
|
||||
|
||||
/******************************************************************************
|
||||
*
|
||||
* t1ha0 = 64-bit, JUST ONLY FASTER:
|
||||
*
|
||||
* - Provides fast-as-possible hashing for current CPU, including
|
||||
* 32-bit systems and engaging the available hardware acceleration.
|
||||
* - It is a facade that selects most quick-and-dirty hash
|
||||
* for the current processor. For instance, on IA32 (x86) actual function
|
||||
* will be selected in runtime, depending on current CPU capabilities
|
||||
*
|
||||
* BE CAREFUL!!! THIS IS MEANS:
|
||||
*
|
||||
* 1. The quality of hash is a subject for tradeoffs with performance.
|
||||
* So, the quality and strength of t1ha0() may be lower than t1ha1(),
|
||||
* especially on 32-bit targets, but then much faster.
|
||||
* However, guaranteed that it passes all SMHasher tests.
|
||||
*
|
||||
* 2. No warranty that the hash result will be same for particular
|
||||
* key on another machine or another version of libt1ha.
|
||||
*
|
||||
* Briefly, such hash-results and their derivatives, should be
|
||||
* used only in runtime, but should not be persist or transferred
|
||||
* over a network.
|
||||
*
|
||||
*
|
||||
* When T1HA0_RUNTIME_SELECT is nonzero the t1ha0_resolve() function could
|
||||
* be used to get actual t1ha0() implementation address at runtime. This is
|
||||
* useful for two cases:
|
||||
* - calling by local pointer-to-function usually is little
|
||||
* bit faster (less overhead) than via a PLT thru the DSO boundary.
|
||||
* - GNU Indirect functions (see below) don't supported by environment
|
||||
* and calling by t1ha0_funcptr is not available and/or expensive.
|
||||
*/
|
||||
|
||||
#ifndef T1HA0_DISABLED
|
||||
|
||||
/* The little-endian variant for 32-bit CPU. */
|
||||
uint64_t t1ha0_32le(const void *data, size_t length, uint64_t seed);
|
||||
/* The big-endian variant for 32-bit CPU. */
|
||||
uint64_t t1ha0_32be(const void *data, size_t length, uint64_t seed);
|
||||
|
||||
/* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */
|
||||
#ifndef T1HA0_AESNI_AVAILABLE
|
||||
#if defined(__e2k__) || \
|
||||
(defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800))
|
||||
#define T1HA0_AESNI_AVAILABLE 1
|
||||
#else
|
||||
#define T1HA0_AESNI_AVAILABLE 0
|
||||
#endif
|
||||
#endif /* T1HA0_AESNI_AVAILABLE */
|
||||
|
||||
/* Define T1HA0_RUNTIME_SELECT to 0 for disable dispatching t1ha0 at runtime. */
|
||||
#ifndef T1HA0_RUNTIME_SELECT
|
||||
#if T1HA0_AESNI_AVAILABLE && !defined(__e2k__)
|
||||
#define T1HA0_RUNTIME_SELECT 1
|
||||
#else
|
||||
#define T1HA0_RUNTIME_SELECT 0
|
||||
#endif
|
||||
#endif /* T1HA0_RUNTIME_SELECT */
|
||||
|
||||
#if !T1HA0_RUNTIME_SELECT && !defined(T1HA0_USE_DEFINE)
|
||||
#if defined(__LCC__)
|
||||
#define T1HA0_USE_DEFINE 1
|
||||
#else
|
||||
#define T1HA0_USE_DEFINE 0
|
||||
#endif
|
||||
#endif /* T1HA0_USE_DEFINE */
|
||||
|
||||
#if T1HA0_AESNI_AVAILABLE
|
||||
uint64_t t1ha0_ia32aes_noavx(const void *data, size_t length, uint64_t seed);
|
||||
uint64_t t1ha0_ia32aes_avx(const void *data, size_t length, uint64_t seed);
|
||||
#ifndef __e2k__
|
||||
uint64_t t1ha0_ia32aes_avx2(const void *data, size_t length, uint64_t seed);
|
||||
#endif
|
||||
#endif /* T1HA0_AESNI_AVAILABLE */
|
||||
|
||||
#if T1HA0_RUNTIME_SELECT
|
||||
typedef uint64_t (*t1ha0_function_t)(const void *, size_t, uint64_t);
|
||||
T1HA_API t1ha0_function_t t1ha0_resolve(void);
|
||||
#if T1HA_USE_INDIRECT_FUNCTIONS
|
||||
T1HA_API uint64_t t1ha0(const void *data, size_t length, uint64_t seed);
|
||||
#else
|
||||
/* Otherwise function pointer will be used.
|
||||
* Unfortunately this may cause some overhead calling. */
|
||||
T1HA_API extern uint64_t (*t1ha0_funcptr)(const void *data, size_t length,
|
||||
uint64_t seed);
|
||||
static __force_inline uint64_t t1ha0(const void *data, size_t length,
|
||||
uint64_t seed) {
|
||||
return t1ha0_funcptr(data, length, seed);
|
||||
}
|
||||
#endif /* T1HA_USE_INDIRECT_FUNCTIONS */
|
||||
|
||||
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
|
||||
#if T1HA0_USE_DEFINE
|
||||
|
||||
#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \
|
||||
(!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED))
|
||||
#if defined(T1HA1_DISABLED)
|
||||
#define t1ha0 t1ha2_atonce
|
||||
#else
|
||||
#define t1ha0 t1ha1_be
|
||||
#endif /* T1HA1_DISABLED */
|
||||
#else /* 32/64 */
|
||||
#define t1ha0 t1ha0_32be
|
||||
#endif /* 32/64 */
|
||||
|
||||
#else /* T1HA0_USE_DEFINE */
|
||||
|
||||
static __force_inline uint64_t t1ha0(const void *data, size_t length,
|
||||
uint64_t seed) {
|
||||
#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \
|
||||
(!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED))
|
||||
#if defined(T1HA1_DISABLED)
|
||||
return t1ha2_atonce(data, length, seed);
|
||||
#else
|
||||
return t1ha1_be(data, length, seed);
|
||||
#endif /* T1HA1_DISABLED */
|
||||
#else /* 32/64 */
|
||||
return t1ha0_32be(data, length, seed);
|
||||
#endif /* 32/64 */
|
||||
}
|
||||
|
||||
#endif /* !T1HA0_USE_DEFINE */
|
||||
|
||||
#else /* !T1HA0_RUNTIME_SELECT && __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */
|
||||
|
||||
#if T1HA0_USE_DEFINE
|
||||
|
||||
#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \
|
||||
(!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED))
|
||||
#if defined(T1HA1_DISABLED)
|
||||
#define t1ha0 t1ha2_atonce
|
||||
#else
|
||||
#define t1ha0 t1ha1_le
|
||||
#endif /* T1HA1_DISABLED */
|
||||
#else /* 32/64 */
|
||||
#define t1ha0 t1ha0_32le
|
||||
#endif /* 32/64 */
|
||||
|
||||
#else
|
||||
|
||||
static __force_inline uint64_t t1ha0(const void *data, size_t length,
|
||||
uint64_t seed) {
|
||||
#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \
|
||||
(!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED))
|
||||
#if defined(T1HA1_DISABLED)
|
||||
return t1ha2_atonce(data, length, seed);
|
||||
#else
|
||||
return t1ha1_le(data, length, seed);
|
||||
#endif /* T1HA1_DISABLED */
|
||||
#else /* 32/64 */
|
||||
return t1ha0_32le(data, length, seed);
|
||||
#endif /* 32/64 */
|
||||
}
|
||||
|
||||
#endif /* !T1HA0_USE_DEFINE */
|
||||
|
||||
#endif /* !T1HA0_RUNTIME_SELECT */
|
||||
|
||||
#endif /* T1HA0_DISABLED */
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#if __GNUC_PREREQ(4, 0)
|
||||
#pragma GCC visibility pop
|
||||
#endif /* __GNUC_PREREQ(4,0) */
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,224 @@
|
||||
/*
|
||||
* Standalone XXH64 command-line wrapper for avalanche testing.
|
||||
*
|
||||
* XXH64 algorithm derived from xxHash by Yann Collet:
|
||||
* https://github.com/Cyan4973/xxHash
|
||||
*
|
||||
* Copyright (C) 2012-2023 Yann Collet
|
||||
*
|
||||
* BSD 2-Clause License
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DAMAGES ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
|
||||
* DAMAGE.
|
||||
*/
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define XXH64_SEED UINT64_C(0)
|
||||
#define XXH_PRIME64_1 UINT64_C(11400714785074694791)
|
||||
#define XXH_PRIME64_2 UINT64_C(14029467366897019727)
|
||||
#define XXH_PRIME64_3 UINT64_C(1609587929392839161)
|
||||
#define XXH_PRIME64_4 UINT64_C(9650029242287828579)
|
||||
#define XXH_PRIME64_5 UINT64_C(2870177450012600261)
|
||||
|
||||
static uint64_t rotate_left64(uint64_t value, unsigned int count)
|
||||
{
|
||||
return (value << count) | (value >> (64U - count));
|
||||
}
|
||||
|
||||
static uint32_t read_little_endian32(const unsigned char *data)
|
||||
{
|
||||
return (uint32_t)data[0] | ((uint32_t)data[1] << 8U) |
|
||||
((uint32_t)data[2] << 16U) | ((uint32_t)data[3] << 24U);
|
||||
}
|
||||
|
||||
static uint64_t read_little_endian64(const unsigned char *data)
|
||||
{
|
||||
return (uint64_t)read_little_endian32(data) |
|
||||
((uint64_t)read_little_endian32(data + 4) << 32U);
|
||||
}
|
||||
|
||||
static uint64_t xxh64_round(uint64_t accumulator, uint64_t input)
|
||||
{
|
||||
accumulator += input * XXH_PRIME64_2;
|
||||
accumulator = rotate_left64(accumulator, 31U);
|
||||
accumulator *= XXH_PRIME64_1;
|
||||
return accumulator;
|
||||
}
|
||||
|
||||
static uint64_t xxh64_merge_round(uint64_t accumulator, uint64_t value)
|
||||
{
|
||||
value = xxh64_round(UINT64_C(0), value);
|
||||
accumulator ^= value;
|
||||
accumulator = accumulator * XXH_PRIME64_1 + XXH_PRIME64_4;
|
||||
return accumulator;
|
||||
}
|
||||
|
||||
static uint64_t xxh64(const unsigned char *data, size_t length, uint64_t seed)
|
||||
{
|
||||
const unsigned char *position = data;
|
||||
const unsigned char *const end = data + length;
|
||||
uint64_t hash;
|
||||
|
||||
if (length >= 32U) {
|
||||
const unsigned char *const block_end = end - 32U;
|
||||
uint64_t accumulator1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
|
||||
uint64_t accumulator2 = seed + XXH_PRIME64_2;
|
||||
uint64_t accumulator3 = seed;
|
||||
uint64_t accumulator4 = seed - XXH_PRIME64_1;
|
||||
|
||||
do {
|
||||
accumulator1 = xxh64_round(accumulator1, read_little_endian64(position));
|
||||
position += 8;
|
||||
accumulator2 = xxh64_round(accumulator2, read_little_endian64(position));
|
||||
position += 8;
|
||||
accumulator3 = xxh64_round(accumulator3, read_little_endian64(position));
|
||||
position += 8;
|
||||
accumulator4 = xxh64_round(accumulator4, read_little_endian64(position));
|
||||
position += 8;
|
||||
} while (position <= block_end);
|
||||
|
||||
hash = rotate_left64(accumulator1, 1U) +
|
||||
rotate_left64(accumulator2, 7U) +
|
||||
rotate_left64(accumulator3, 12U) +
|
||||
rotate_left64(accumulator4, 18U);
|
||||
hash = xxh64_merge_round(hash, accumulator1);
|
||||
hash = xxh64_merge_round(hash, accumulator2);
|
||||
hash = xxh64_merge_round(hash, accumulator3);
|
||||
hash = xxh64_merge_round(hash, accumulator4);
|
||||
} else {
|
||||
hash = seed + XXH_PRIME64_5;
|
||||
}
|
||||
|
||||
hash += (uint64_t)length;
|
||||
|
||||
while ((size_t)(end - position) >= 8U) {
|
||||
uint64_t value = xxh64_round(UINT64_C(0), read_little_endian64(position));
|
||||
hash ^= value;
|
||||
hash = rotate_left64(hash, 27U) * XXH_PRIME64_1 + XXH_PRIME64_4;
|
||||
position += 8;
|
||||
}
|
||||
|
||||
if ((size_t)(end - position) >= 4U) {
|
||||
hash ^= (uint64_t)read_little_endian32(position) * XXH_PRIME64_1;
|
||||
hash = rotate_left64(hash, 23U) * XXH_PRIME64_2 + XXH_PRIME64_3;
|
||||
position += 4;
|
||||
}
|
||||
|
||||
while (position < end) {
|
||||
hash ^= (uint64_t)(*position) * XXH_PRIME64_5;
|
||||
hash = rotate_left64(hash, 11U) * XXH_PRIME64_1;
|
||||
++position;
|
||||
}
|
||||
|
||||
hash ^= hash >> 33U;
|
||||
hash *= XXH_PRIME64_2;
|
||||
hash ^= hash >> 29U;
|
||||
hash *= XXH_PRIME64_3;
|
||||
hash ^= hash >> 32U;
|
||||
return hash;
|
||||
}
|
||||
|
||||
static int is_ascii_trailing_space(unsigned char character)
|
||||
{
|
||||
return character == ' ' || character == '\t' || character == '\n' ||
|
||||
character == '\r' || character == '\v' || character == '\f';
|
||||
}
|
||||
|
||||
static int read_stdin(unsigned char **data, size_t *length)
|
||||
{
|
||||
size_t capacity = 256;
|
||||
unsigned char *buffer = malloc(capacity);
|
||||
|
||||
if (buffer == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
*length = 0;
|
||||
for (;;) {
|
||||
size_t available = capacity - *length;
|
||||
size_t bytes_read = fread(buffer + *length, 1, available, stdin);
|
||||
*length += bytes_read;
|
||||
|
||||
if (bytes_read < available) {
|
||||
if (ferror(stdin)) {
|
||||
free(buffer);
|
||||
return -1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (capacity > SIZE_MAX / 2U) {
|
||||
free(buffer);
|
||||
return -1;
|
||||
}
|
||||
capacity *= 2U;
|
||||
|
||||
{
|
||||
unsigned char *larger_buffer = realloc(buffer, capacity);
|
||||
if (larger_buffer == NULL) {
|
||||
free(buffer);
|
||||
return -1;
|
||||
}
|
||||
buffer = larger_buffer;
|
||||
}
|
||||
}
|
||||
|
||||
*data = buffer;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
const unsigned char *word;
|
||||
unsigned char *stdin_buffer = NULL;
|
||||
size_t length;
|
||||
|
||||
if (argc > 2) {
|
||||
fprintf(stderr, "usage: %s [ASCII_WORD]\n", argv[0]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
if (argc == 2) {
|
||||
word = (const unsigned char *)argv[1];
|
||||
length = strlen(argv[1]);
|
||||
} else {
|
||||
if (read_stdin(&stdin_buffer, &length) != 0) {
|
||||
fprintf(stderr, "failed to read input\n");
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
word = stdin_buffer;
|
||||
}
|
||||
|
||||
while (length > 0U && is_ascii_trailing_space(word[length - 1U])) {
|
||||
--length;
|
||||
}
|
||||
|
||||
for (size_t index = 0; index < length; ++index) {
|
||||
if (word[index] > 0x7fU) {
|
||||
fprintf(stderr, "input must contain ASCII characters only\n");
|
||||
free(stdin_buffer);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
printf("%016" PRIx64 "\n", xxh64(word, length, XXH64_SEED));
|
||||
free(stdin_buffer);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
branch=p11
|
||||
arch=x86_64
|
||||
filter=Itanium C++ ABI mangled names beginning with _Z
|
||||
extractor=/usr/lib/rpm/provided_symbols
|
||||
selection=largest x86_64 C++ candidates found by Provides set-string length in apt-cache dumpavail
|
||||
candidate=slicer set_length=509743
|
||||
candidate=libmlir22.1 set_length=303303
|
||||
candidate=libmlir21.1 set_length=284616
|
||||
candidate=libmlir18.1 set_length=204305
|
||||
candidate=libmlir20.1 set_length=182470
|
||||
|
||||
package=slicer (slicer-5.10.0-alt1.x86_64.rpm) elf_files=227 symbols=380667 selected=380530
|
||||
package=libmlir22.1 (libmlir22.1-22.1.1-alt0.1.x86_64.rpm) elf_files=9 symbols=146036 selected=145767
|
||||
package=libmlir21.1 (libmlir21.1-21.1.5-alt0.1.x86_64.rpm) elf_files=8 symbols=136102 selected=135854
|
||||
package=libmlir20.1 (libmlir20.1-20.1.8-alt0.4.x86_64.rpm) elf_files=8 symbols=90904 selected=90656
|
||||
package=libmlir18.1 (libmlir18.1-18.1.8-alt0.2.x86_64.rpm) elf_files=7 symbols=103099 selected=102835
|
||||
packages=5 elf_files=259 unique_symbols=577509 output=cpp_provides_p11.txt
|
||||
similar_lcp12=575206 share=0.996012
|
||||
similar_lcp24=548917 share=0.950491
|
||||
median_max_lcp=61
|
||||
max_lcp=1018
|
||||
sha256=bf8e760bdf4d52b42040cff4625de73988d8dd6b517566efd186c71e3382753d
|
||||
File diff suppressed because it is too large
Load Diff
+445
@@ -0,0 +1,445 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render avalanche bit-probability CSV files as two PNG bar charts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from statistics import fmean
|
||||
from typing import Sequence
|
||||
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
BACKGROUND = "#f7f8fa"
|
||||
PANEL = "#ffffff"
|
||||
GRID = "#d9dee7"
|
||||
TEXT = "#172033"
|
||||
MUTED = "#637083"
|
||||
REFERENCE = "#d24b4b"
|
||||
COLORS = (
|
||||
"#377eb8",
|
||||
"#4daf4a",
|
||||
"#984ea3",
|
||||
"#ff7f00",
|
||||
"#e41a1c",
|
||||
"#00a6a6",
|
||||
"#a65628",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProbabilityRow:
|
||||
operation: str
|
||||
pairs: int
|
||||
probabilities: list[float]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChartScale:
|
||||
minimum: float
|
||||
maximum: float
|
||||
ticks: tuple[float, ...]
|
||||
|
||||
|
||||
def nice_step(value: float) -> float:
|
||||
exponent = math.floor(math.log10(value))
|
||||
fraction = value / 10**exponent
|
||||
nice_fraction = min((1.0, 2.0, 2.5, 5.0, 10.0), key=lambda item: abs(item - fraction))
|
||||
return nice_fraction * 10**exponent
|
||||
|
||||
|
||||
def make_scale(
|
||||
values: Sequence[float],
|
||||
*,
|
||||
hard_limits: tuple[float, float],
|
||||
reference: float | None = None,
|
||||
) -> ChartScale:
|
||||
"""Build a padded shared scale constrained to hard limits."""
|
||||
hard_minimum, hard_maximum = hard_limits
|
||||
finite_values = [value for value in values if math.isfinite(value)]
|
||||
if reference is not None:
|
||||
finite_values.append(reference)
|
||||
if not finite_values:
|
||||
finite_values = [hard_minimum, hard_maximum]
|
||||
|
||||
minimum = min(finite_values)
|
||||
maximum = max(finite_values)
|
||||
if minimum == maximum:
|
||||
expansion = (hard_maximum - hard_minimum) * 0.1
|
||||
minimum -= expansion / 2
|
||||
maximum += expansion / 2
|
||||
|
||||
span = maximum - minimum
|
||||
padded_minimum = max(hard_minimum, minimum - span * 0.1)
|
||||
padded_maximum = min(hard_maximum, maximum + span * 0.1)
|
||||
step = nice_step(max((padded_maximum - padded_minimum) / 5, 1e-12))
|
||||
scaled_minimum = max(hard_minimum, math.floor(padded_minimum / step) * step)
|
||||
scaled_maximum = min(hard_maximum, math.ceil(padded_maximum / step) * step)
|
||||
if scaled_minimum == scaled_maximum:
|
||||
scaled_minimum, scaled_maximum = hard_minimum, hard_maximum
|
||||
|
||||
tick_count = round((scaled_maximum - scaled_minimum) / step)
|
||||
ticks = [scaled_minimum + index * step for index in range(tick_count + 1)]
|
||||
if reference is not None and scaled_minimum <= reference <= scaled_maximum:
|
||||
ticks.append(reference)
|
||||
normalized_ticks = tuple(
|
||||
sorted({round(value, 12) for value in ticks if scaled_minimum <= value <= scaled_maximum})
|
||||
)
|
||||
return ChartScale(scaled_minimum, scaled_maximum, normalized_ticks)
|
||||
|
||||
|
||||
def load_font(size: int, bold: bool = False) -> ImageFont.FreeTypeFont | ImageFont.ImageFont:
|
||||
names = (
|
||||
"DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf",
|
||||
"/usr/share/fonts/TTF/DejaVuSans-Bold.ttf"
|
||||
if bold
|
||||
else "/usr/share/fonts/TTF/DejaVuSans.ttf",
|
||||
"/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf"
|
||||
if bold
|
||||
else "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
|
||||
)
|
||||
for name in names:
|
||||
try:
|
||||
return ImageFont.truetype(name, size)
|
||||
except OSError:
|
||||
continue
|
||||
return ImageFont.load_default()
|
||||
|
||||
|
||||
def read_probability_map(path: Path) -> list[ProbabilityRow]:
|
||||
"""Read operation rows and numerically ordered bit columns from a CSV file."""
|
||||
try:
|
||||
stream = path.open(encoding="utf-8", newline="")
|
||||
except OSError as error:
|
||||
raise ValueError(f"не удалось открыть {path}: {error}") from error
|
||||
|
||||
with stream:
|
||||
reader = csv.DictReader(stream)
|
||||
fields = reader.fieldnames
|
||||
if not fields or "operation" not in fields or "pairs" not in fields:
|
||||
raise ValueError("CSV должен содержать колонки operation и pairs")
|
||||
|
||||
bit_fields: list[tuple[int, str]] = []
|
||||
for field in fields:
|
||||
if not field.startswith("bit_"):
|
||||
continue
|
||||
try:
|
||||
bit_fields.append((int(field.removeprefix("bit_")), field))
|
||||
except ValueError as error:
|
||||
raise ValueError(f"некорректная битовая колонка: {field}") from error
|
||||
bit_fields.sort()
|
||||
if not bit_fields:
|
||||
raise ValueError("CSV не содержит колонок bit_N")
|
||||
expected_bits = list(range(len(bit_fields)))
|
||||
actual_bits = [bit for bit, _field in bit_fields]
|
||||
if actual_bits != expected_bits:
|
||||
raise ValueError("битовые колонки должны непрерывно идти от bit_0")
|
||||
|
||||
rows: list[ProbabilityRow] = []
|
||||
for line_number, row in enumerate(reader, start=2):
|
||||
operation = (row.get("operation") or "").strip()
|
||||
if not operation:
|
||||
raise ValueError(f"строка {line_number}: пустая операция")
|
||||
try:
|
||||
pairs = int(row["pairs"] or "")
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError(
|
||||
f"строка {line_number}: некорректное число пар"
|
||||
) from error
|
||||
if pairs < 0:
|
||||
raise ValueError(f"строка {line_number}: число пар меньше нуля")
|
||||
|
||||
probabilities: list[float] = []
|
||||
for _bit, field in bit_fields:
|
||||
try:
|
||||
value = float(row[field] or "")
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError(
|
||||
f"строка {line_number}: некорректное значение {field}"
|
||||
) from error
|
||||
if not math.isnan(value) and not 0.0 <= value <= 1.0:
|
||||
raise ValueError(
|
||||
f"строка {line_number}: {field} должен быть от 0 до 1"
|
||||
)
|
||||
probabilities.append(value)
|
||||
rows.append(ProbabilityRow(operation, pairs, probabilities))
|
||||
|
||||
if not rows:
|
||||
raise ValueError("CSV не содержит строк с операциями")
|
||||
return rows
|
||||
|
||||
|
||||
def mean_absolute_deviation(probabilities: Sequence[float]) -> float:
|
||||
"""Return the mean |p - 0.5| over finite bit probabilities."""
|
||||
deviations = [
|
||||
abs(probability - 0.5)
|
||||
for probability in probabilities
|
||||
if math.isfinite(probability)
|
||||
]
|
||||
return fmean(deviations) if deviations else float("nan")
|
||||
|
||||
|
||||
def text_width(
|
||||
draw: ImageDraw.ImageDraw,
|
||||
text: str,
|
||||
font: ImageFont.FreeTypeFont | ImageFont.ImageFont,
|
||||
) -> int:
|
||||
box = draw.textbbox((0, 0), text, font=font)
|
||||
return round(box[2] - box[0])
|
||||
|
||||
|
||||
def draw_centered_text(
|
||||
draw: ImageDraw.ImageDraw,
|
||||
center_x: float,
|
||||
y: float,
|
||||
text: str,
|
||||
font: ImageFont.FreeTypeFont | ImageFont.ImageFont,
|
||||
fill: str = TEXT,
|
||||
) -> None:
|
||||
draw.text(
|
||||
(center_x - text_width(draw, text, font) / 2, y),
|
||||
text,
|
||||
font=font,
|
||||
fill=fill,
|
||||
)
|
||||
|
||||
|
||||
def draw_bit_panel(
|
||||
draw: ImageDraw.ImageDraw,
|
||||
bounds: tuple[int, int, int, int],
|
||||
row: ProbabilityRow,
|
||||
color: str,
|
||||
scale: ChartScale,
|
||||
) -> None:
|
||||
left, top, right, bottom = bounds
|
||||
title_font = load_font(22, bold=True)
|
||||
label_font = load_font(14)
|
||||
tick_font = load_font(12)
|
||||
|
||||
draw.rounded_rectangle(bounds, radius=12, fill=PANEL, outline=GRID, width=1)
|
||||
draw.text((left + 18, top + 13), row.operation, font=title_font, fill=TEXT)
|
||||
pairs_text = f"pairs: {row.pairs}"
|
||||
draw.text(
|
||||
(right - 18 - text_width(draw, pairs_text, label_font), top + 17),
|
||||
pairs_text,
|
||||
font=label_font,
|
||||
fill=MUTED,
|
||||
)
|
||||
|
||||
plot_left = left + 54
|
||||
plot_right = right - 18
|
||||
plot_top = top + 55
|
||||
plot_bottom = bottom - 42
|
||||
plot_height = plot_bottom - plot_top
|
||||
|
||||
scale_span = scale.maximum - scale.minimum
|
||||
for probability in scale.ticks:
|
||||
y = round(plot_bottom - (probability - scale.minimum) / scale_span * plot_height)
|
||||
line_color = REFERENCE if probability == 0.5 else GRID
|
||||
line_width = 2 if probability == 0.5 else 1
|
||||
draw.line((plot_left, y, plot_right, y), fill=line_color, width=line_width)
|
||||
label = f"{probability:.3g}"
|
||||
draw.text(
|
||||
(plot_left - 8 - text_width(draw, label, tick_font), y - 7),
|
||||
label,
|
||||
font=tick_font,
|
||||
fill=MUTED,
|
||||
)
|
||||
|
||||
bit_count = len(row.probabilities)
|
||||
slot_width = (plot_right - plot_left) / bit_count
|
||||
bar_width = max(1, int(slot_width * 0.72))
|
||||
for bit, probability in enumerate(row.probabilities):
|
||||
if not math.isfinite(probability):
|
||||
continue
|
||||
center = plot_left + (bit + 0.5) * slot_width
|
||||
x0 = round(center - bar_width / 2)
|
||||
x1 = round(center + bar_width / 2)
|
||||
y = round(
|
||||
plot_bottom
|
||||
- (probability - scale.minimum) / scale_span * plot_height
|
||||
)
|
||||
draw.rectangle((x0, y, x1, plot_bottom), fill=color)
|
||||
|
||||
tick_step = max(1, math.ceil(bit_count / 16))
|
||||
for bit in range(0, bit_count, tick_step):
|
||||
center = plot_left + (bit + 0.5) * slot_width
|
||||
label = str(bit)
|
||||
draw.text(
|
||||
(center - text_width(draw, label, tick_font) / 2, plot_bottom + 7),
|
||||
label,
|
||||
font=tick_font,
|
||||
fill=MUTED,
|
||||
)
|
||||
draw_centered_text(
|
||||
draw,
|
||||
(plot_left + plot_right) / 2,
|
||||
bottom - 21,
|
||||
"output bit (0 = LSB)",
|
||||
tick_font,
|
||||
MUTED,
|
||||
)
|
||||
|
||||
|
||||
def render_bit_probabilities(
|
||||
rows: Sequence[ProbabilityRow], output: Path, title: str
|
||||
) -> None:
|
||||
columns = 2 if len(rows) > 1 else 1
|
||||
panel_width = 760
|
||||
panel_height = 330
|
||||
gap = 18
|
||||
margin = 24
|
||||
title_height = 70
|
||||
row_count = math.ceil(len(rows) / columns)
|
||||
width = margin * 2 + columns * panel_width + (columns - 1) * gap
|
||||
height = title_height + margin + row_count * panel_height + (row_count - 1) * gap
|
||||
|
||||
image = Image.new("RGB", (width, height), BACKGROUND)
|
||||
draw = ImageDraw.Draw(image)
|
||||
scale = make_scale(
|
||||
[probability for row in rows for probability in row.probabilities],
|
||||
reference=0.5,
|
||||
hard_limits=(0.0, 1.0),
|
||||
)
|
||||
draw_centered_text(draw, width / 2, 18, title, load_font(30, bold=True))
|
||||
draw_centered_text(
|
||||
draw,
|
||||
width / 2,
|
||||
52,
|
||||
f"Shared scale {scale.minimum:.3g}–{scale.maximum:.3g}; red line = ideal p=0.5",
|
||||
load_font(14),
|
||||
MUTED,
|
||||
)
|
||||
|
||||
for index, row in enumerate(rows):
|
||||
column = index % columns
|
||||
grid_row = index // columns
|
||||
left = margin + column * (panel_width + gap)
|
||||
top = title_height + grid_row * (panel_height + gap)
|
||||
draw_bit_panel(
|
||||
draw,
|
||||
(left, top, left + panel_width, top + panel_height),
|
||||
row,
|
||||
COLORS[index % len(COLORS)],
|
||||
scale,
|
||||
)
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.save(output, "PNG", optimize=True)
|
||||
|
||||
|
||||
def render_mean_deviations(
|
||||
rows: Sequence[ProbabilityRow], output: Path, title: str
|
||||
) -> None:
|
||||
width, height = 1200, 720
|
||||
image = Image.new("RGB", (width, height), BACKGROUND)
|
||||
draw = ImageDraw.Draw(image)
|
||||
draw_centered_text(draw, width / 2, 22, title, load_font(30, bold=True))
|
||||
draw_centered_text(
|
||||
draw,
|
||||
width / 2,
|
||||
58,
|
||||
"Mean absolute deviation from ideal avalanche probability: mean(|p - 0.5|)",
|
||||
load_font(15),
|
||||
MUTED,
|
||||
)
|
||||
|
||||
plot_left, plot_right = 90, width - 40
|
||||
plot_top, plot_bottom = 110, height - 120
|
||||
plot_height = plot_bottom - plot_top
|
||||
deviations = [mean_absolute_deviation(row.probabilities) for row in rows]
|
||||
scale = make_scale(deviations, hard_limits=(0.0, 0.5))
|
||||
scale_span = scale.maximum - scale.minimum
|
||||
tick_font = load_font(13)
|
||||
label_font = load_font(15)
|
||||
value_font = load_font(14, bold=True)
|
||||
|
||||
for value in scale.ticks:
|
||||
y = round(plot_bottom - (value - scale.minimum) / scale_span * plot_height)
|
||||
draw.line((plot_left, y, plot_right, y), fill=GRID, width=1)
|
||||
label = f"{value:.3g}"
|
||||
draw.text(
|
||||
(plot_left - 10 - text_width(draw, label, tick_font), y - 7),
|
||||
label,
|
||||
font=tick_font,
|
||||
fill=MUTED,
|
||||
)
|
||||
|
||||
slot_width = (plot_right - plot_left) / len(rows)
|
||||
bar_width = min(105, max(20, int(slot_width * 0.62)))
|
||||
for index, (row, deviation) in enumerate(zip(rows, deviations, strict=True)):
|
||||
center = plot_left + (index + 0.5) * slot_width
|
||||
x0 = round(center - bar_width / 2)
|
||||
x1 = round(center + bar_width / 2)
|
||||
if math.isfinite(deviation):
|
||||
y = round(
|
||||
plot_bottom
|
||||
- (deviation - scale.minimum) / scale_span * plot_height
|
||||
)
|
||||
draw.rectangle((x0, y, x1, plot_bottom), fill=COLORS[index % len(COLORS)])
|
||||
value_label = f"{deviation:.4f}"
|
||||
else:
|
||||
y = plot_bottom
|
||||
value_label = "n/a"
|
||||
draw_centered_text(draw, center, max(plot_top, y - 22), value_label, value_font)
|
||||
draw_centered_text(draw, center, plot_bottom + 12, row.operation, label_font)
|
||||
draw_centered_text(
|
||||
draw, center, plot_bottom + 36, f"pairs: {row.pairs}", tick_font, MUTED
|
||||
)
|
||||
|
||||
draw.line((plot_left, plot_top, plot_left, plot_bottom), fill=TEXT, width=2)
|
||||
draw.line((plot_left, plot_bottom, plot_right, plot_bottom), fill=TEXT, width=2)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.save(output, "PNG", optimize=True)
|
||||
|
||||
|
||||
def generate_plots(source: Path, output_directory: Path | None = None) -> tuple[Path, Path]:
|
||||
rows = read_probability_map(source)
|
||||
destination = output_directory or source.parent
|
||||
bit_output = destination / f"{source.stem}_bits.png"
|
||||
deviation_output = destination / f"{source.stem}_deviation.png"
|
||||
display_name = source.stem
|
||||
|
||||
render_bit_probabilities(rows, bit_output, f"{display_name}: per-bit avalanche map")
|
||||
render_mean_deviations(
|
||||
rows,
|
||||
deviation_output,
|
||||
f"{display_name}: deviation from p=0.5",
|
||||
)
|
||||
return bit_output, deviation_output
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Создаёт две PNG-гистограммы из CSV вероятностной карты: "
|
||||
"вероятности по битам для каждой операции и среднее |p-0.5|."
|
||||
)
|
||||
)
|
||||
parser.add_argument("csv_file", type=Path, help="CSV из probability_map.py")
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
help="каталог PNG (по умолчанию каталог исходного CSV)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
outputs = generate_plots(args.csv_file, args.output_dir)
|
||||
except ValueError as error:
|
||||
raise SystemExit(f"error: {error}") from error
|
||||
|
||||
for output in outputs:
|
||||
print(f"wrote {output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+354
@@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build bit-probability maps for hash functions and word mutation types.
|
||||
|
||||
For every hash listed in HASHES, the script compares each source word hash with
|
||||
hashes of generated similar words. A table cell contains the probability that
|
||||
the corresponding output bit changed (XOR with its source word hash).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import TextIO
|
||||
|
||||
from generate_input import (
|
||||
DEFAULT_ALPHABET,
|
||||
MUTATIONS,
|
||||
MutationError,
|
||||
generate_words,
|
||||
parse_operation,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
HASH_FUNCS_DIR = ROOT / "hash_funcs"
|
||||
DEFAULT_OUTPUT_DIR = ROOT / "probability_maps"
|
||||
|
||||
# Add directory names from hash_funcs here to include more implementations.
|
||||
HASHES = [
|
||||
"jenkinsOAAT",
|
||||
"xxh64",
|
||||
"t1ha2",
|
||||
]
|
||||
|
||||
HEX_HASH = re.compile(r"(?:0[xX])?([0-9a-fA-F]+)")
|
||||
ProbabilityRow = tuple[int, list[float]]
|
||||
|
||||
|
||||
class HashToolError(RuntimeError):
|
||||
"""Raised when a hash executable cannot be prepared or invoked."""
|
||||
|
||||
|
||||
def prepare_hash(hash_name: str, hash_funcs_dir: Path = HASH_FUNCS_DIR) -> Path:
|
||||
"""Return an executable hash tool, building or preparing it if necessary."""
|
||||
if not hash_name or Path(hash_name).name != hash_name:
|
||||
raise HashToolError(f"некорректное имя хэша: {hash_name!r}")
|
||||
|
||||
hash_directory = hash_funcs_dir / hash_name
|
||||
if not hash_directory.is_dir():
|
||||
raise HashToolError(f"не найдена папка хэша: {hash_directory}")
|
||||
|
||||
binary = hash_directory / "bin_hash"
|
||||
if binary.is_file():
|
||||
binary.chmod(binary.stat().st_mode | stat.S_IXUSR)
|
||||
return binary
|
||||
|
||||
sources = (
|
||||
(hash_directory / "bin_hash.c", "cc"),
|
||||
(hash_directory / "bin_hash.cpp", "c++"),
|
||||
(hash_directory / "bin_hash.cc", "c++"),
|
||||
(hash_directory / "bin_hash.cxx", "c++"),
|
||||
)
|
||||
for source, compiler in sources:
|
||||
if not source.is_file():
|
||||
continue
|
||||
command = [
|
||||
compiler,
|
||||
"-O2",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Wpedantic",
|
||||
"-Werror",
|
||||
str(source),
|
||||
"-o",
|
||||
str(binary),
|
||||
]
|
||||
if compiler == "cc":
|
||||
command[1:1] = ["-std=c11"]
|
||||
else:
|
||||
command[1:1] = ["-std=c++17"]
|
||||
result = subprocess.run(command, text=True, capture_output=True)
|
||||
if result.returncode != 0:
|
||||
details = result.stderr.strip() or result.stdout.strip()
|
||||
raise HashToolError(f"не удалось скомпилировать {source}: {details}")
|
||||
return binary
|
||||
|
||||
python_source = hash_directory / "bin_hash.py"
|
||||
if python_source.is_file():
|
||||
content = python_source.read_text(encoding="utf-8")
|
||||
if not content.startswith("#!"):
|
||||
python_source.write_text(
|
||||
"#!/usr/bin/env python3\n" + content,
|
||||
encoding="utf-8",
|
||||
)
|
||||
python_source.chmod(
|
||||
python_source.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
|
||||
)
|
||||
return python_source
|
||||
|
||||
raise HashToolError(
|
||||
f"для {hash_name} не найден bin_hash, bin_hash.c/cpp/cc/cxx или bin_hash.py"
|
||||
)
|
||||
|
||||
|
||||
def run_hash(executable: Path, word: str) -> tuple[int, int]:
|
||||
"""Run a hash tool and return its integer value and explicit output width."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(executable), word],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError as error:
|
||||
raise HashToolError(f"не удалось запустить {executable}: {error}") from error
|
||||
|
||||
if result.returncode != 0:
|
||||
details = result.stderr.strip() or result.stdout.strip()
|
||||
raise HashToolError(
|
||||
f"{executable} завершился с кодом {result.returncode}: {details}"
|
||||
)
|
||||
|
||||
output = result.stdout.strip()
|
||||
match = HEX_HASH.fullmatch(output)
|
||||
if match is None:
|
||||
raise HashToolError(f"{executable} вернул не шестнадцатеричный хэш: {output!r}")
|
||||
|
||||
digits = match.group(1)
|
||||
return int(digits, 16), len(digits) * 4
|
||||
|
||||
|
||||
def bit_probabilities(
|
||||
source_hash: int, changed_hashes: Sequence[int], bits: int
|
||||
) -> list[float]:
|
||||
"""Calculate per-bit change probabilities, ordered from LSB to MSB."""
|
||||
if bits < 1:
|
||||
raise ValueError("число бит должно быть положительным")
|
||||
if not changed_hashes:
|
||||
raise ValueError("список изменённых хэшей не должен быть пустым")
|
||||
|
||||
changed_counts = [0] * bits
|
||||
for changed_hash in changed_hashes:
|
||||
difference = source_hash ^ changed_hash
|
||||
for bit in range(bits):
|
||||
changed_counts[bit] += (difference >> bit) & 1
|
||||
|
||||
sample_count = len(changed_hashes)
|
||||
return [count / sample_count for count in changed_counts]
|
||||
|
||||
|
||||
def write_csv_table(stream: TextIO, rows: Mapping[str, ProbabilityRow]) -> None:
|
||||
"""Write one operation-by-bit probability table as CSV."""
|
||||
if not rows:
|
||||
raise ValueError("таблица вероятностей не должна быть пустой")
|
||||
|
||||
widths = {len(probabilities) for _pair_count, probabilities in rows.values()}
|
||||
if len(widths) != 1:
|
||||
raise ValueError("все строки таблицы должны иметь одинаковое число бит")
|
||||
bits = widths.pop()
|
||||
|
||||
writer = csv.writer(stream, lineterminator="\n")
|
||||
writer.writerow(["operation", "pairs", *(f"bit_{bit}" for bit in range(bits))])
|
||||
for operation, (pair_count, probabilities) in rows.items():
|
||||
writer.writerow(
|
||||
[
|
||||
operation,
|
||||
pair_count,
|
||||
*(f"{probability:.6f}" for probability in probabilities),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def build_probability_table(
|
||||
executable: Path,
|
||||
sources: Sequence[str],
|
||||
operations: Sequence[str],
|
||||
count: int,
|
||||
operation_count: int,
|
||||
alphabet: str,
|
||||
seed: int | None,
|
||||
max_attempts: int | None,
|
||||
) -> dict[str, ProbabilityRow]:
|
||||
"""Aggregate avalanche probabilities across all source words."""
|
||||
if not sources:
|
||||
raise ValueError("нужно указать хотя бы одно исходное слово")
|
||||
|
||||
source_hashes: list[int] = []
|
||||
bits: int | None = None
|
||||
for source in sources:
|
||||
source_hash, source_bits = run_hash(executable, source)
|
||||
if bits is None:
|
||||
bits = source_bits
|
||||
elif source_bits != bits:
|
||||
raise HashToolError(
|
||||
f"{executable} вернул хэши разной ширины: "
|
||||
f"{bits} и {source_bits} бит"
|
||||
)
|
||||
source_hashes.append(source_hash)
|
||||
|
||||
assert bits is not None
|
||||
table: dict[str, ProbabilityRow] = {}
|
||||
|
||||
for operation in operations:
|
||||
differences: list[int] = []
|
||||
for source, source_hash in zip(sources, source_hashes, strict=True):
|
||||
words = generate_words(
|
||||
source=source,
|
||||
count=count,
|
||||
operation=operation,
|
||||
operation_count=operation_count,
|
||||
alphabet=alphabet,
|
||||
seed=seed,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
if len(words) < count:
|
||||
print(
|
||||
f"warning: operation={operation} word={source!r}: "
|
||||
f"generated {len(words)} of at most {count} unique words",
|
||||
file=sys.stderr,
|
||||
)
|
||||
for word in words:
|
||||
changed_hash, changed_bits = run_hash(executable, word)
|
||||
if changed_bits != bits:
|
||||
raise HashToolError(
|
||||
f"{executable} вернул хэши разной ширины: "
|
||||
f"{bits} и {changed_bits} бит"
|
||||
)
|
||||
differences.append(source_hash ^ changed_hash)
|
||||
|
||||
if differences:
|
||||
table[operation] = (
|
||||
len(differences),
|
||||
bit_probabilities(0, differences, bits),
|
||||
)
|
||||
else:
|
||||
table[operation] = (0, [float("nan")] * bits)
|
||||
|
||||
return table
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Строит для каждого хэша CSV-таблицу вероятностей изменения "
|
||||
"выходных битов. bit_0 — младший бит."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"words",
|
||||
nargs="+",
|
||||
help="одно или несколько исходных ASCII-слов",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--operation",
|
||||
action="append",
|
||||
type=parse_operation,
|
||||
help=(
|
||||
"тип изменения (номер или имя как в generate_input.py); "
|
||||
"можно повторять, по умолчанию используются все типы"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n",
|
||||
"--count",
|
||||
type=int,
|
||||
default=10,
|
||||
help=(
|
||||
"верхняя граница числа уникальных изменённых слов для каждого "
|
||||
"исходного слова и типа (по умолчанию: 10)"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-k",
|
||||
"--operations",
|
||||
type=int,
|
||||
default=1,
|
||||
help="число операций над каждым словом (по умолчанию: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--alphabet",
|
||||
default=DEFAULT_ALPHABET,
|
||||
help="алфавит для добавления и замены",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed", type=int, help="seed генератора для воспроизводимого результата"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-attempts",
|
||||
type=int,
|
||||
help="предельное число попыток собрать уникальные слова",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=DEFAULT_OUTPUT_DIR,
|
||||
help=("каталог для CSV-таблиц " f"(по умолчанию: {DEFAULT_OUTPUT_DIR})"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hash",
|
||||
dest="hashes",
|
||||
action="append",
|
||||
help="проверить только указанный хэш; можно повторять (по умолчанию HASHES)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.count < 1:
|
||||
parser.error("--count должен быть положительным")
|
||||
if args.operations < 1:
|
||||
parser.error("--operations должен быть положительным")
|
||||
if args.max_attempts is not None and args.max_attempts < 1:
|
||||
parser.error("--max-attempts должен быть положительным")
|
||||
|
||||
operations = list(dict.fromkeys(args.operation or MUTATIONS.keys()))
|
||||
hashes = list(dict.fromkeys(args.hashes or HASHES))
|
||||
if not hashes:
|
||||
parser.error("массив HASHES не должен быть пустым")
|
||||
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
for hash_name in hashes:
|
||||
executable = prepare_hash(hash_name)
|
||||
table = build_probability_table(
|
||||
executable=executable,
|
||||
sources=args.words,
|
||||
operations=operations,
|
||||
count=args.count,
|
||||
operation_count=args.operations,
|
||||
alphabet=args.alphabet,
|
||||
seed=args.seed,
|
||||
max_attempts=args.max_attempts,
|
||||
)
|
||||
output_path = args.output / f"{hash_name}.csv"
|
||||
with output_path.open("w", encoding="utf-8", newline="") as stream:
|
||||
write_csv_table(stream, table)
|
||||
print(f"wrote {output_path}")
|
||||
except (HashToolError, MutationError, ValueError) as error:
|
||||
parser.error(str(error))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,8 @@
|
||||
operation,pairs,bit_0,bit_1,bit_2,bit_3,bit_4,bit_5,bit_6,bit_7,bit_8,bit_9,bit_10,bit_11,bit_12,bit_13,bit_14,bit_15,bit_16,bit_17,bit_18,bit_19,bit_20,bit_21,bit_22,bit_23,bit_24,bit_25,bit_26,bit_27,bit_28,bit_29,bit_30,bit_31
|
||||
replace,1984,0.484375,0.499496,0.494960,0.497984,0.506048,0.490423,0.501512,0.511593,0.496472,0.499496,0.515121,0.504536,0.485383,0.500504,0.496976,0.477319,0.510081,0.490423,0.492440,0.501512,0.502016,0.484879,0.490927,0.490927,0.511593,0.494960,0.512097,0.523185,0.498992,0.505544,0.496472,0.500504
|
||||
delete,31,0.612903,0.451613,0.612903,0.516129,0.419355,0.548387,0.451613,0.612903,0.419355,0.548387,0.677419,0.419355,0.580645,0.516129,0.451613,0.516129,0.516129,0.645161,0.548387,0.516129,0.548387,0.419355,0.516129,0.419355,0.645161,0.612903,0.516129,0.580645,0.516129,0.419355,0.580645,0.354839
|
||||
add,2173,0.501611,0.492867,0.500690,0.505292,0.515877,0.505292,0.505292,0.505752,0.497929,0.502991,0.485044,0.503451,0.496088,0.524160,0.517257,0.507593,0.489646,0.476760,0.496549,0.521859,0.514956,0.514036,0.513116,0.508053,0.492867,0.503451,0.518638,0.506673,0.495628,0.506213,0.518178,0.489185
|
||||
swap,28,0.607143,0.500000,0.428571,0.535714,0.392857,0.535714,0.500000,0.357143,0.571429,0.428571,0.678571,0.750000,0.428571,0.464286,0.392857,0.464286,0.500000,0.428571,0.428571,0.428571,0.464286,0.428571,0.500000,0.678571,0.571429,0.464286,0.607143,0.571429,0.571429,0.535714,0.428571,0.607143
|
||||
case,29,0.344828,0.551724,0.517241,0.448276,0.448276,0.310345,0.482759,0.344828,0.379310,0.413793,0.413793,0.448276,0.620690,0.620690,0.448276,0.586207,0.517241,0.551724,0.482759,0.448276,0.413793,0.517241,0.275862,0.655172,0.551724,0.586207,0.551724,0.482759,0.448276,0.551724,0.517241,0.655172
|
||||
first,186,0.500000,0.500000,0.494624,0.467742,0.521505,0.451613,0.489247,0.478495,0.521505,0.543011,0.516129,0.510753,0.478495,0.510753,0.510753,0.510753,0.456989,0.473118,0.456989,0.543011,0.489247,0.478495,0.456989,0.467742,0.500000,0.483871,0.483871,0.489247,0.559140,0.559140,0.462366,0.483871
|
||||
last,186,0.510753,0.516129,0.510753,0.467742,0.510753,0.505376,0.473118,0.516129,0.500000,0.516129,0.516129,0.521505,0.483871,0.526882,0.510753,0.451613,0.548387,0.489247,0.505376,0.430108,0.526882,0.456989,0.478495,0.505376,0.516129,0.537634,0.526882,0.478495,0.467742,0.494624,0.446237,0.548387
|
||||
|
Binary file not shown.
|
After Width: | Height: | Size: 38 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
@@ -0,0 +1,8 @@
|
||||
operation,pairs,bit_0,bit_1,bit_2,bit_3,bit_4,bit_5,bit_6,bit_7,bit_8,bit_9,bit_10,bit_11,bit_12,bit_13,bit_14,bit_15,bit_16,bit_17,bit_18,bit_19,bit_20,bit_21,bit_22,bit_23,bit_24,bit_25,bit_26,bit_27,bit_28,bit_29,bit_30,bit_31,bit_32,bit_33,bit_34,bit_35,bit_36,bit_37,bit_38,bit_39,bit_40,bit_41,bit_42,bit_43,bit_44,bit_45,bit_46,bit_47,bit_48,bit_49,bit_50,bit_51,bit_52,bit_53,bit_54,bit_55,bit_56,bit_57,bit_58,bit_59,bit_60,bit_61,bit_62,bit_63
|
||||
replace,1984,0.511593,0.515121,0.510081,0.525706,0.499496,0.504032,0.495464,0.494960,0.522681,0.521169,0.507056,0.484879,0.502016,0.482359,0.513609,0.513105,0.500504,0.503024,0.496472,0.509073,0.491431,0.488911,0.495464,0.491431,0.494960,0.491431,0.496976,0.496976,0.502016,0.511593,0.502016,0.496976,0.504536,0.499496,0.494456,0.496976,0.496976,0.490927,0.523690,0.489919,0.495968,0.478327,0.492944,0.529234,0.495968,0.496472,0.500000,0.493448,0.519657,0.510585,0.493448,0.496976,0.495464,0.501512,0.512097,0.500504,0.494960,0.494456,0.512097,0.494456,0.494960,0.495464,0.494456,0.515121
|
||||
delete,31,0.741935,0.387097,0.548387,0.354839,0.483871,0.451613,0.483871,0.612903,0.451613,0.645161,0.516129,0.419355,0.419355,0.483871,0.516129,0.483871,0.548387,0.741935,0.354839,0.483871,0.645161,0.516129,0.516129,0.548387,0.516129,0.354839,0.387097,0.387097,0.354839,0.548387,0.419355,0.516129,0.548387,0.516129,0.322581,0.419355,0.387097,0.483871,0.483871,0.419355,0.451613,0.612903,0.451613,0.419355,0.516129,0.580645,0.548387,0.483871,0.387097,0.612903,0.419355,0.516129,0.483871,0.548387,0.516129,0.516129,0.677419,0.451613,0.483871,0.451613,0.516129,0.612903,0.516129,0.483871
|
||||
add,2173,0.496088,0.508514,0.489646,0.488725,0.500230,0.514496,0.491486,0.479521,0.499770,0.484123,0.488725,0.495168,0.499310,0.484584,0.498850,0.530143,0.488265,0.484123,0.519098,0.494248,0.497469,0.505752,0.501611,0.520018,0.497929,0.512195,0.498389,0.474919,0.498389,0.504372,0.491026,0.514496,0.496549,0.479521,0.492867,0.499310,0.499770,0.498389,0.480902,0.493787,0.492407,0.505292,0.492867,0.499310,0.494708,0.511735,0.524620,0.506213,0.518638,0.518178,0.492867,0.498850,0.498389,0.502531,0.509434,0.498850,0.492867,0.499310,0.501611,0.506213,0.482743,0.503451,0.511275,0.493787
|
||||
swap,28,0.571429,0.571429,0.607143,0.535714,0.428571,0.392857,0.428571,0.535714,0.571429,0.535714,0.571429,0.464286,0.285714,0.714286,0.607143,0.500000,0.500000,0.357143,0.392857,0.571429,0.678571,0.464286,0.428571,0.392857,0.357143,0.392857,0.607143,0.571429,0.392857,0.642857,0.464286,0.678571,0.750000,0.428571,0.607143,0.428571,0.535714,0.500000,0.571429,0.642857,0.428571,0.357143,0.535714,0.571429,0.535714,0.571429,0.535714,0.464286,0.607143,0.642857,0.642857,0.678571,0.285714,0.500000,0.642857,0.571429,0.500000,0.357143,0.464286,0.428571,0.571429,0.571429,0.678571,0.535714
|
||||
case,29,0.413793,0.413793,0.620690,0.586207,0.379310,0.586207,0.482759,0.448276,0.551724,0.448276,0.655172,0.586207,0.655172,0.586207,0.379310,0.551724,0.517241,0.310345,0.517241,0.517241,0.310345,0.620690,0.482759,0.655172,0.551724,0.379310,0.413793,0.482759,0.517241,0.517241,0.482759,0.448276,0.586207,0.448276,0.620690,0.448276,0.448276,0.448276,0.551724,0.586207,0.551724,0.413793,0.413793,0.586207,0.517241,0.482759,0.551724,0.482759,0.586207,0.586207,0.379310,0.482759,0.586207,0.379310,0.482759,0.344828,0.620690,0.482759,0.448276,0.551724,0.482759,0.551724,0.344828,0.482759
|
||||
first,186,0.500000,0.500000,0.526882,0.526882,0.489247,0.483871,0.456989,0.532258,0.510753,0.483871,0.537634,0.456989,0.500000,0.424731,0.500000,0.435484,0.532258,0.456989,0.478495,0.532258,0.430108,0.489247,0.521505,0.478495,0.478495,0.532258,0.478495,0.532258,0.500000,0.473118,0.473118,0.537634,0.575269,0.526882,0.505376,0.526882,0.424731,0.462366,0.548387,0.478495,0.462366,0.532258,0.500000,0.473118,0.473118,0.559140,0.510753,0.526882,0.564516,0.521505,0.532258,0.521505,0.564516,0.537634,0.467742,0.473118,0.500000,0.424731,0.424731,0.462366,0.494624,0.489247,0.537634,0.559140
|
||||
last,186,0.510753,0.543011,0.580645,0.516129,0.516129,0.564516,0.500000,0.478495,0.489247,0.494624,0.494624,0.478495,0.521505,0.462366,0.526882,0.521505,0.564516,0.451613,0.489247,0.543011,0.537634,0.467742,0.467742,0.526882,0.500000,0.526882,0.478495,0.532258,0.462366,0.473118,0.569892,0.505376,0.516129,0.494624,0.456989,0.478495,0.537634,0.473118,0.516129,0.494624,0.440860,0.505376,0.537634,0.548387,0.408602,0.569892,0.537634,0.559140,0.494624,0.559140,0.526882,0.559140,0.489247,0.569892,0.537634,0.543011,0.543011,0.494624,0.537634,0.505376,0.473118,0.494624,0.532258,0.483871
|
||||
|
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
@@ -0,0 +1,8 @@
|
||||
operation,pairs,bit_0,bit_1,bit_2,bit_3,bit_4,bit_5,bit_6,bit_7,bit_8,bit_9,bit_10,bit_11,bit_12,bit_13,bit_14,bit_15,bit_16,bit_17,bit_18,bit_19,bit_20,bit_21,bit_22,bit_23,bit_24,bit_25,bit_26,bit_27,bit_28,bit_29,bit_30,bit_31,bit_32,bit_33,bit_34,bit_35,bit_36,bit_37,bit_38,bit_39,bit_40,bit_41,bit_42,bit_43,bit_44,bit_45,bit_46,bit_47,bit_48,bit_49,bit_50,bit_51,bit_52,bit_53,bit_54,bit_55,bit_56,bit_57,bit_58,bit_59,bit_60,bit_61,bit_62,bit_63
|
||||
replace,1984,0.491431,0.497984,0.532258,0.504536,0.494456,0.519657,0.516129,0.493448,0.501008,0.519657,0.490423,0.495968,0.516129,0.498488,0.491935,0.509073,0.520665,0.487903,0.497984,0.487903,0.502016,0.504536,0.502520,0.510081,0.512097,0.492440,0.505544,0.492944,0.491431,0.500000,0.509577,0.511593,0.491431,0.485383,0.481351,0.493448,0.498992,0.498992,0.499496,0.504032,0.495968,0.483367,0.495464,0.498488,0.512097,0.489415,0.502520,0.496472,0.501008,0.495464,0.503528,0.507056,0.510081,0.504032,0.487399,0.523690,0.511089,0.472278,0.496976,0.501008,0.503528,0.504032,0.494456,0.491935
|
||||
delete,31,0.548387,0.516129,0.451613,0.580645,0.548387,0.483871,0.548387,0.387097,0.451613,0.451613,0.451613,0.387097,0.677419,0.419355,0.354839,0.612903,0.387097,0.419355,0.548387,0.645161,0.451613,0.548387,0.451613,0.322581,0.516129,0.483871,0.516129,0.387097,0.483871,0.483871,0.483871,0.451613,0.483871,0.516129,0.645161,0.483871,0.580645,0.612903,0.677419,0.516129,0.516129,0.516129,0.516129,0.516129,0.677419,0.419355,0.516129,0.612903,0.612903,0.451613,0.483871,0.419355,0.516129,0.419355,0.580645,0.483871,0.645161,0.419355,0.419355,0.677419,0.451613,0.483871,0.419355,0.516129
|
||||
add,2173,0.506213,0.502531,0.484584,0.506213,0.505292,0.469397,0.497929,0.499310,0.487345,0.516797,0.502071,0.504832,0.515416,0.498389,0.486884,0.492867,0.505752,0.484123,0.513116,0.503451,0.508974,0.494248,0.501150,0.509894,0.485964,0.482743,0.490106,0.501150,0.487345,0.491486,0.515416,0.502071,0.497009,0.480902,0.499310,0.510815,0.509894,0.499310,0.511735,0.510354,0.498389,0.485504,0.497009,0.511275,0.504832,0.490566,0.495628,0.495628,0.498850,0.497009,0.485504,0.498850,0.506673,0.488265,0.492407,0.509434,0.495168,0.489185,0.500230,0.489646,0.498850,0.493327,0.493327,0.504832
|
||||
swap,28,0.535714,0.607143,0.428571,0.321429,0.535714,0.464286,0.500000,0.357143,0.464286,0.500000,0.464286,0.607143,0.535714,0.357143,0.607143,0.321429,0.428571,0.571429,0.464286,0.357143,0.357143,0.464286,0.464286,0.607143,0.500000,0.642857,0.642857,0.535714,0.428571,0.464286,0.535714,0.535714,0.285714,0.607143,0.500000,0.571429,0.428571,0.642857,0.500000,0.607143,0.571429,0.392857,0.500000,0.535714,0.535714,0.392857,0.571429,0.285714,0.642857,0.500000,0.535714,0.321429,0.428571,0.428571,0.464286,0.678571,0.642857,0.571429,0.607143,0.535714,0.535714,0.571429,0.285714,0.357143
|
||||
case,29,0.448276,0.551724,0.586207,0.620690,0.482759,0.655172,0.655172,0.586207,0.517241,0.379310,0.551724,0.448276,0.482759,0.482759,0.586207,0.448276,0.517241,0.448276,0.448276,0.517241,0.551724,0.620690,0.655172,0.655172,0.551724,0.586207,0.655172,0.586207,0.482759,0.379310,0.586207,0.517241,0.517241,0.551724,0.517241,0.655172,0.413793,0.551724,0.551724,0.448276,0.517241,0.379310,0.517241,0.517241,0.586207,0.310345,0.620690,0.379310,0.413793,0.448276,0.586207,0.689655,0.482759,0.310345,0.551724,0.413793,0.448276,0.517241,0.586207,0.551724,0.517241,0.448276,0.448276,0.448276
|
||||
first,186,0.521505,0.456989,0.543011,0.516129,0.505376,0.494624,0.569892,0.516129,0.564516,0.500000,0.500000,0.500000,0.516129,0.446237,0.580645,0.532258,0.521505,0.521505,0.473118,0.462366,0.462366,0.473118,0.526882,0.500000,0.500000,0.516129,0.532258,0.532258,0.489247,0.559140,0.462366,0.564516,0.521505,0.467742,0.456989,0.516129,0.537634,0.483871,0.548387,0.500000,0.532258,0.516129,0.478495,0.478495,0.462366,0.483871,0.494624,0.526882,0.537634,0.478495,0.446237,0.473118,0.478495,0.564516,0.435484,0.467742,0.526882,0.494624,0.494624,0.494624,0.543011,0.478495,0.430108,0.494624
|
||||
last,186,0.526882,0.446237,0.526882,0.521505,0.494624,0.521505,0.521505,0.483871,0.516129,0.500000,0.478495,0.500000,0.564516,0.500000,0.381720,0.500000,0.537634,0.473118,0.532258,0.543011,0.505376,0.510753,0.483871,0.456989,0.564516,0.489247,0.596774,0.526882,0.510753,0.462366,0.494624,0.494624,0.408602,0.548387,0.505376,0.543011,0.494624,0.467742,0.462366,0.440860,0.548387,0.505376,0.516129,0.510753,0.543011,0.462366,0.478495,0.505376,0.478495,0.532258,0.510753,0.586022,0.526882,0.478495,0.467742,0.516129,0.543011,0.478495,0.510753,0.478495,0.494624,0.505376,0.548387,0.478495
|
||||
|
Binary file not shown.
|
After Width: | Height: | Size: 35 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 39 KiB |
@@ -0,0 +1,33 @@
|
||||
bit,ones,zeros,probability,deviation_from_0.5
|
||||
0,287790,289719,0.498329896,0.001670104
|
||||
1,288092,289417,0.498852832,0.001147168
|
||||
2,288750,288759,0.499992208,0.000007792
|
||||
3,288546,288963,0.499638967,0.000361033
|
||||
4,288778,288731,0.500040692,0.000040692
|
||||
5,289060,288449,0.500528996,0.000528996
|
||||
6,288814,288695,0.500103029,0.000103029
|
||||
7,288788,288721,0.500058008,0.000058008
|
||||
8,288712,288797,0.499926408,0.000073592
|
||||
9,288209,289300,0.499055426,0.000944574
|
||||
10,289080,288429,0.500563628,0.000563628
|
||||
11,288964,288545,0.500362765,0.000362765
|
||||
12,289040,288469,0.500494365,0.000494365
|
||||
13,289175,288334,0.500728127,0.000728127
|
||||
14,288883,288626,0.500222507,0.000222507
|
||||
15,288908,288601,0.500265797,0.000265797
|
||||
16,288259,289250,0.499142005,0.000857995
|
||||
17,288493,289016,0.499547193,0.000452807
|
||||
18,288101,289408,0.498868416,0.001131584
|
||||
19,289164,288345,0.500709080,0.000709080
|
||||
20,288817,288692,0.500108223,0.000108223
|
||||
21,288723,288786,0.499945455,0.000054545
|
||||
22,288577,288932,0.499692645,0.000307355
|
||||
23,288523,288986,0.499599140,0.000400860
|
||||
24,288625,288884,0.499775761,0.000224239
|
||||
25,288128,289381,0.498915168,0.001084832
|
||||
26,288821,288688,0.500115150,0.000115150
|
||||
27,288762,288747,0.500012987,0.000012987
|
||||
28,289539,287970,0.501358420,0.001358420
|
||||
29,288493,289016,0.499547193,0.000452807
|
||||
30,288614,288895,0.499756714,0.000243286
|
||||
31,288681,288828,0.499872729,0.000127271
|
||||
|
@@ -0,0 +1,8 @@
|
||||
corpus=input_symbols/cpp_provides_p11.txt
|
||||
corpus_bytes=54941449
|
||||
corpus_sha256=bf8e760bdf4d52b42040cff4625de73988d8dd6b517566efd186c71e3382753d
|
||||
samples=577509
|
||||
hashes=jenkinsOAAT,xxh64,t1ha2
|
||||
metric=P(hash_output_bit=1)
|
||||
bit_order=bit_0_is_LSB
|
||||
input_format=one_ASCII_symbol_per_line
|
||||
@@ -0,0 +1,4 @@
|
||||
hash,samples,bits,mean_probability,mean_abs_deviation,max_abs_deviation,min_probability,max_probability
|
||||
jenkinsOAAT,577509,32,0.499879060,0.000475425,0.001670104,0.498329896,0.501358420
|
||||
xxh64,577509,64,0.499871404,0.000497665,0.001704735,0.498295265,0.501221626
|
||||
t1ha2,577509,64,0.500008550,0.000532513,0.001876161,0.498123839,0.501256257
|
||||
|
@@ -0,0 +1,65 @@
|
||||
bit,ones,zeros,probability,deviation_from_0.5
|
||||
0,288524,288985,0.499600872,0.000399128
|
||||
1,289273,288236,0.500897822,0.000897822
|
||||
2,289241,288268,0.500842411,0.000842411
|
||||
3,289050,288459,0.500511680,0.000511680
|
||||
4,288827,288682,0.500125539,0.000125539
|
||||
5,289341,288168,0.501015569,0.001015569
|
||||
6,288570,288939,0.499680524,0.000319476
|
||||
7,289480,288029,0.501256257,0.001256257
|
||||
8,289469,288040,0.501237210,0.001237210
|
||||
9,289359,288150,0.501046737,0.001046737
|
||||
10,288340,289169,0.499282262,0.000717738
|
||||
11,288850,288659,0.500165365,0.000165365
|
||||
12,288401,289108,0.499387888,0.000612112
|
||||
13,288828,288681,0.500127271,0.000127271
|
||||
14,288809,288700,0.500094371,0.000094371
|
||||
15,288437,289072,0.499450225,0.000549775
|
||||
16,287671,289838,0.498123839,0.001876161
|
||||
17,288776,288733,0.500037229,0.000037229
|
||||
18,288058,289451,0.498793958,0.001206042
|
||||
19,289300,288209,0.500944574,0.000944574
|
||||
20,288721,288788,0.499941992,0.000058008
|
||||
21,288403,289106,0.499391351,0.000608649
|
||||
22,288749,288760,0.499990476,0.000009524
|
||||
23,289382,288127,0.501086563,0.001086563
|
||||
24,288503,289006,0.499564509,0.000435491
|
||||
25,289237,288272,0.500835485,0.000835485
|
||||
26,288918,288591,0.500283112,0.000283112
|
||||
27,288889,288620,0.500232897,0.000232897
|
||||
28,288205,289304,0.499048500,0.000951500
|
||||
29,288637,288872,0.499796540,0.000203460
|
||||
30,289047,288462,0.500506486,0.000506486
|
||||
31,288479,289030,0.499522951,0.000477049
|
||||
32,289147,288362,0.500679643,0.000679643
|
||||
33,288429,289080,0.499436372,0.000563628
|
||||
34,288675,288834,0.499862340,0.000137660
|
||||
35,288817,288692,0.500108223,0.000108223
|
||||
36,288302,289207,0.499216462,0.000783538
|
||||
37,288197,289312,0.499034647,0.000965353
|
||||
38,289472,288037,0.501242405,0.001242405
|
||||
39,289181,288328,0.500738517,0.000738517
|
||||
40,289046,288463,0.500504754,0.000504754
|
||||
41,288910,288599,0.500269260,0.000269260
|
||||
42,288608,288901,0.499746324,0.000253676
|
||||
43,288745,288764,0.499983550,0.000016450
|
||||
44,288429,289080,0.499436372,0.000563628
|
||||
45,288738,288771,0.499971429,0.000028571
|
||||
46,288425,289084,0.499429446,0.000570554
|
||||
47,288857,288652,0.500177486,0.000177486
|
||||
48,288795,288714,0.500070129,0.000070129
|
||||
49,288155,289354,0.498961921,0.001038079
|
||||
50,288639,288870,0.499800003,0.000199997
|
||||
51,289013,288496,0.500447612,0.000447612
|
||||
52,288717,288792,0.499935066,0.000064934
|
||||
53,289130,288379,0.500650206,0.000650206
|
||||
54,288936,288573,0.500314281,0.000314281
|
||||
55,288904,288605,0.500258870,0.000258870
|
||||
56,288616,288893,0.499760177,0.000239823
|
||||
57,288873,288636,0.500205192,0.000205192
|
||||
58,288340,289169,0.499282262,0.000717738
|
||||
59,288507,289002,0.499571435,0.000428565
|
||||
60,288986,288523,0.500400860,0.000400860
|
||||
61,288162,289347,0.498974042,0.001025958
|
||||
62,288354,289155,0.499306504,0.000693496
|
||||
63,288725,288784,0.499948919,0.000051081
|
||||
|
@@ -0,0 +1,65 @@
|
||||
bit,ones,zeros,probability,deviation_from_0.5
|
||||
0,288778,288731,0.500040692,0.000040692
|
||||
1,288503,289006,0.499564509,0.000435491
|
||||
2,288868,288641,0.500196534,0.000196534
|
||||
3,288638,288871,0.499798272,0.000201728
|
||||
4,288821,288688,0.500115150,0.000115150
|
||||
5,288588,288921,0.499711693,0.000288307
|
||||
6,288882,288627,0.500220776,0.000220776
|
||||
7,289460,288049,0.501221626,0.001221626
|
||||
8,288866,288643,0.500193071,0.000193071
|
||||
9,288162,289347,0.498974042,0.001025958
|
||||
10,289120,288389,0.500632891,0.000632891
|
||||
11,288842,288667,0.500151513,0.000151513
|
||||
12,288456,289053,0.499483125,0.000516875
|
||||
13,288410,289099,0.499403472,0.000596528
|
||||
14,289202,288307,0.500774880,0.000774880
|
||||
15,288850,288659,0.500165365,0.000165365
|
||||
16,288499,289010,0.499557583,0.000442417
|
||||
17,288163,289346,0.498975774,0.001024226
|
||||
18,288220,289289,0.499074473,0.000925527
|
||||
19,288892,288617,0.500238092,0.000238092
|
||||
20,288695,288814,0.499896971,0.000103029
|
||||
21,288797,288712,0.500073592,0.000073592
|
||||
22,288795,288714,0.500070129,0.000070129
|
||||
23,288188,289321,0.499019063,0.000980937
|
||||
24,288864,288645,0.500189607,0.000189607
|
||||
25,288653,288856,0.499824245,0.000175755
|
||||
26,289025,288484,0.500468391,0.000468391
|
||||
27,288750,288759,0.499992208,0.000007792
|
||||
28,289267,288242,0.500887432,0.000887432
|
||||
29,288448,289061,0.499469272,0.000530728
|
||||
30,288742,288767,0.499978355,0.000021645
|
||||
31,288247,289262,0.499121226,0.000878774
|
||||
32,288629,288880,0.499782687,0.000217313
|
||||
33,289449,288060,0.501202579,0.001202579
|
||||
34,288761,288748,0.500011255,0.000011255
|
||||
35,288208,289301,0.499053694,0.000946306
|
||||
36,288308,289201,0.499226852,0.000773148
|
||||
37,289073,288436,0.500551507,0.000551507
|
||||
38,288394,289115,0.499375767,0.000624233
|
||||
39,289142,288367,0.500670985,0.000670985
|
||||
40,288574,288935,0.499687451,0.000312549
|
||||
41,288209,289300,0.499055426,0.000944574
|
||||
42,288292,289217,0.499199147,0.000800853
|
||||
43,288781,288728,0.500045887,0.000045887
|
||||
44,287770,289739,0.498295265,0.001704735
|
||||
45,289014,288495,0.500449344,0.000449344
|
||||
46,289209,288300,0.500787001,0.000787001
|
||||
47,289366,288143,0.501058858,0.001058858
|
||||
48,288445,289064,0.499464078,0.000535922
|
||||
49,288384,289125,0.499358452,0.000641548
|
||||
50,288509,289000,0.499574898,0.000425102
|
||||
51,288599,288910,0.499730740,0.000269260
|
||||
52,288771,288738,0.500028571,0.000028571
|
||||
53,287887,289622,0.498497859,0.001502141
|
||||
54,288871,288638,0.500201728,0.000201728
|
||||
55,288294,289215,0.499202610,0.000797390
|
||||
56,288557,288952,0.499658014,0.000341986
|
||||
57,288416,289093,0.499413862,0.000586138
|
||||
58,288674,288835,0.499860608,0.000139392
|
||||
59,288644,288865,0.499808661,0.000191339
|
||||
60,288679,288830,0.499869266,0.000130734
|
||||
61,288828,288681,0.500127271,0.000127271
|
||||
62,288905,288604,0.500260602,0.000260602
|
||||
63,289202,288307,0.500774880,0.000774880
|
||||
|
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Behavior tests for corpus_bit_distribution.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
import corpus_bit_distribution
|
||||
import probability_map
|
||||
|
||||
|
||||
class CorpusBitDistributionTests(unittest.TestCase):
|
||||
def test_batch_counts_match_scalar_cli_for_every_hash(self) -> None:
|
||||
words = ["a", "abc", "HashWord", "_ZN4llvm3fooEv"]
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
corpus = Path(temporary_directory) / "corpus.txt"
|
||||
corpus.write_text("".join(f"{word}\n" for word in words), encoding="ascii")
|
||||
|
||||
for hash_name in probability_map.HASHES:
|
||||
with self.subTest(hash_name=hash_name):
|
||||
distribution = corpus_bit_distribution.measure_hash(
|
||||
hash_name,
|
||||
corpus,
|
||||
)
|
||||
executable = probability_map.prepare_hash(hash_name)
|
||||
scalar = [
|
||||
probability_map.run_hash(executable, word)[0] for word in words
|
||||
]
|
||||
expected_ones = tuple(
|
||||
sum((value >> bit) & 1 for value in scalar)
|
||||
for bit in range(distribution.bits)
|
||||
)
|
||||
|
||||
self.assertEqual(distribution.samples, len(words))
|
||||
self.assertEqual(distribution.ones, expected_ones)
|
||||
|
||||
def test_writes_one_csv_row_per_lsb_first_bit(self) -> None:
|
||||
distribution = corpus_bit_distribution.Distribution(
|
||||
hash_name="tiny",
|
||||
samples=4,
|
||||
bits=3,
|
||||
ones=(1, 2, 4),
|
||||
)
|
||||
stream = io.StringIO()
|
||||
|
||||
corpus_bit_distribution.write_distribution_csv(stream, distribution)
|
||||
|
||||
rows = list(csv.reader(io.StringIO(stream.getvalue())))
|
||||
self.assertEqual(
|
||||
rows[0],
|
||||
["bit", "ones", "zeros", "probability", "deviation_from_0.5"],
|
||||
)
|
||||
self.assertEqual(rows[1], ["0", "1", "3", "0.250000000", "0.250000000"])
|
||||
self.assertEqual(rows[2], ["1", "2", "2", "0.500000000", "0.000000000"])
|
||||
self.assertEqual(rows[3], ["2", "4", "0", "1.000000000", "0.500000000"])
|
||||
|
||||
def test_run_creates_separate_csv_summary_manifest_and_png(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
corpus = root / "corpus.txt"
|
||||
corpus.write_text("a\nabc\nHashWord\n", encoding="ascii")
|
||||
output = root / "results"
|
||||
|
||||
result = corpus_bit_distribution.run(
|
||||
corpus=corpus,
|
||||
output_directory=output,
|
||||
hash_names=["jenkinsOAAT"],
|
||||
)
|
||||
|
||||
self.assertEqual(result[0].samples, 3)
|
||||
self.assertTrue((output / "jenkinsOAAT.csv").is_file())
|
||||
self.assertTrue((output / "summary.csv").is_file())
|
||||
self.assertTrue((output / "manifest.txt").is_file())
|
||||
image_path = output / "bit_distribution.png"
|
||||
self.assertTrue(image_path.is_file())
|
||||
with Image.open(image_path) as image:
|
||||
self.assertEqual(image.format, "PNG")
|
||||
self.assertGreater(image.width, 300)
|
||||
self.assertGreater(image.height, 200)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Behavior tests for extract_provided_symbols.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "extract_provided_symbols.py"
|
||||
|
||||
|
||||
def newc_entry(name: str, data: bytes, mode: int = stat.S_IFREG | 0o644) -> bytes:
|
||||
encoded_name = name.encode("utf-8") + b"\0"
|
||||
fields = (
|
||||
1,
|
||||
mode,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
0,
|
||||
len(data),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
len(encoded_name),
|
||||
0,
|
||||
)
|
||||
header = b"070701" + b"".join(f"{value:08x}".encode() for value in fields)
|
||||
record = header + encoded_name
|
||||
record += b"\0" * (-len(record) % 4)
|
||||
record += data
|
||||
record += b"\0" * (-len(data) % 4)
|
||||
return record
|
||||
|
||||
|
||||
def newc_archive(entries: list[tuple[str, bytes, int]]) -> bytes:
|
||||
payload = b"".join(newc_entry(*entry) for entry in entries)
|
||||
return payload + newc_entry("TRAILER!!!", b"", 0)
|
||||
|
||||
|
||||
class ExtractProvidedSymbolsTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.work = Path(self.temporary_directory.name)
|
||||
self.rpm2cpio = self.work / "rpm2cpio"
|
||||
self.provided_symbols = self.work / "provided_symbols"
|
||||
self.file_command = self.work / "file"
|
||||
|
||||
self.rpm2cpio.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import pathlib, sys\n"
|
||||
"sys.stdout.buffer.write(pathlib.Path(sys.argv[1]).read_bytes())\n"
|
||||
)
|
||||
self.provided_symbols.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import pathlib, sys\n"
|
||||
"symbols = set()\n"
|
||||
"for name in sys.argv[1:]:\n"
|
||||
" symbols.update(pathlib.Path(name).read_bytes()[4:].decode().splitlines())\n"
|
||||
"print(*sorted(symbols), sep='\\n')\n"
|
||||
)
|
||||
self.file_command.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import pathlib, sys\n"
|
||||
"data = pathlib.Path(sys.argv[-1]).read_bytes()\n"
|
||||
"kind = 'pie executable' if b'EXECUTABLE' in data else 'shared object'\n"
|
||||
"print(f'ELF 64-bit LSB {kind}, x86-64, version 1 (SYSV)')\n"
|
||||
)
|
||||
self.rpm2cpio.chmod(0o755)
|
||||
self.provided_symbols.chmod(0o755)
|
||||
self.file_command.chmod(0o755)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.temporary_directory.cleanup()
|
||||
|
||||
def run_script(self, *arguments: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--rpm2cpio",
|
||||
str(self.rpm2cpio),
|
||||
"--provided-symbols",
|
||||
str(self.provided_symbols),
|
||||
"--file-command",
|
||||
str(self.file_command),
|
||||
*arguments,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_combines_unique_cpp_symbols_from_multiple_packages(self) -> None:
|
||||
first = self.work / "first.rpm"
|
||||
second = self.work / "second.rpm"
|
||||
output = self.work / "symbols.txt"
|
||||
first.write_bytes(
|
||||
newc_archive(
|
||||
[
|
||||
("./usr/lib64/libfirst.so", b"\x7fELF_ZN3Foo3barEv\nplain_c_symbol\n", stat.S_IFREG | 0o755),
|
||||
("./usr/share/doc/readme", b"not ELF", stat.S_IFREG | 0o644),
|
||||
("./usr/lib64/libalias.so", b"libfirst.so", stat.S_IFLNK | 0o777),
|
||||
]
|
||||
)
|
||||
)
|
||||
second.write_bytes(
|
||||
newc_archive(
|
||||
[
|
||||
(
|
||||
"./usr/lib64/libsecond.so",
|
||||
b"\x7fELF_ZN3Foo3barEv\n_ZN3Foo3bazEv\nanother_c_symbol\n",
|
||||
stat.S_IFREG | 0o755,
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
result = self.run_script(
|
||||
"--cpp-only", "--output", str(output), str(first), str(second)
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(
|
||||
output.read_text().splitlines(),
|
||||
["_ZN3Foo3barEv", "_ZN3Foo3bazEv"],
|
||||
)
|
||||
self.assertIn("package=first.rpm elf_files=1 symbols=2 selected=1", result.stderr)
|
||||
self.assertIn("package=second.rpm elf_files=1 symbols=3 selected=2", result.stderr)
|
||||
self.assertIn("unique_symbols=2", result.stderr)
|
||||
self.assertEqual(result.stdout, "")
|
||||
|
||||
def test_without_cpp_filter_preserves_all_provided_symbols(self) -> None:
|
||||
package = self.work / "all.rpm"
|
||||
package.write_bytes(
|
||||
newc_archive(
|
||||
[
|
||||
(
|
||||
"./usr/lib64/liball.so",
|
||||
b"\x7fELF_ZN3Foo3barEv\nplain_c_symbol\n",
|
||||
stat.S_IFREG | 0o755,
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
result = self.run_script(str(package))
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.splitlines(), ["_ZN3Foo3barEv", "plain_c_symbol"])
|
||||
|
||||
def test_excludes_elf_executables_that_do_not_generate_library_provides(self) -> None:
|
||||
package = self.work / "mixed.rpm"
|
||||
package.write_bytes(
|
||||
newc_archive(
|
||||
[
|
||||
(
|
||||
"./usr/lib64/libmixed.so",
|
||||
b"\x7fELF_ZN3Lib3runEv\n",
|
||||
stat.S_IFREG | 0o755,
|
||||
),
|
||||
(
|
||||
"./usr/bin/mixed",
|
||||
b"\x7fELFEXECUTABLE\n_ZN4Exec3runEv\n",
|
||||
stat.S_IFREG | 0o755,
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
result = self.run_script("--cpp-only", str(package))
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.splitlines(), ["_ZN3Lib3runEv"])
|
||||
self.assertIn("elf_files=1", result.stderr)
|
||||
|
||||
def test_rejects_malformed_cpio_stream(self) -> None:
|
||||
package = self.work / "broken.rpm"
|
||||
package.write_bytes(b"not a newc archive")
|
||||
|
||||
result = self.run_script(str(package))
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("newc", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Behavior tests for plot_probability_map.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
import plot_probability_map
|
||||
|
||||
|
||||
class ProbabilityMapPlotTests(unittest.TestCase):
|
||||
def test_reads_bit_columns_and_ignores_operation_and_pairs(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
path = Path(temporary_directory) / "hash.csv"
|
||||
path.write_text(
|
||||
"operation,pairs,bit_0,bit_1\n"
|
||||
"replace,4,0.25,0.75\n"
|
||||
"delete,2,0.5,nan\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
rows = plot_probability_map.read_probability_map(path)
|
||||
|
||||
self.assertEqual(rows[0].operation, "replace")
|
||||
self.assertEqual(rows[0].pairs, 4)
|
||||
self.assertEqual(rows[0].probabilities, [0.25, 0.75])
|
||||
self.assertEqual(rows[1].operation, "delete")
|
||||
self.assertEqual(rows[1].pairs, 2)
|
||||
self.assertEqual(len(rows[1].probabilities), 2)
|
||||
|
||||
def test_mean_absolute_deviation_from_half_ignores_nan(self) -> None:
|
||||
result = plot_probability_map.mean_absolute_deviation(
|
||||
[0.25, 0.5, 0.75, float("nan")]
|
||||
)
|
||||
|
||||
self.assertEqual(result, 1 / 6)
|
||||
|
||||
def test_shared_scale_zooms_to_all_values_and_reference(self) -> None:
|
||||
scale = plot_probability_map.make_scale(
|
||||
[0.45, 0.48, 0.52, 0.55],
|
||||
reference=0.5,
|
||||
hard_limits=(0.0, 1.0),
|
||||
)
|
||||
|
||||
self.assertGreater(scale.minimum, 0.0)
|
||||
self.assertLess(scale.maximum, 1.0)
|
||||
self.assertLessEqual(scale.minimum, 0.45)
|
||||
self.assertGreaterEqual(scale.maximum, 0.55)
|
||||
self.assertIn(0.5, scale.ticks)
|
||||
|
||||
def test_deviation_scale_uses_data_range_instead_of_fixed_half(self) -> None:
|
||||
scale = plot_probability_map.make_scale(
|
||||
[0.05, 0.08], hard_limits=(0.0, 0.5)
|
||||
)
|
||||
|
||||
self.assertGreater(scale.minimum, 0.0)
|
||||
self.assertLess(scale.maximum, 0.5)
|
||||
self.assertLessEqual(scale.minimum, 0.05)
|
||||
self.assertGreaterEqual(scale.maximum, 0.08)
|
||||
|
||||
def test_generates_two_nonempty_png_files(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
source = root / "sample.csv"
|
||||
source.write_text(
|
||||
"operation,pairs,bit_0,bit_1,bit_2,bit_3\n"
|
||||
"replace,4,0.25,0.50,0.75,1.0\n"
|
||||
"delete,2,0.10,0.20,0.30,0.40\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
outputs = plot_probability_map.generate_plots(source, root / "plots")
|
||||
|
||||
self.assertEqual(len(outputs), 2)
|
||||
for output in outputs:
|
||||
self.assertTrue(output.is_file())
|
||||
with Image.open(output) as image:
|
||||
self.assertEqual(image.format, "PNG")
|
||||
self.assertGreater(image.width, 300)
|
||||
self.assertGreater(image.height, 200)
|
||||
colors = image.convert("RGB").getcolors(maxcolors=1_000_000)
|
||||
self.assertIsNotNone(colors)
|
||||
self.assertGreater(len(colors or []), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Behavior tests for probability_map.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stderr
|
||||
from pathlib import Path
|
||||
|
||||
import generate_input
|
||||
import probability_map
|
||||
|
||||
|
||||
class PrepareHashTests(unittest.TestCase):
|
||||
def test_compiles_c_source_when_binary_is_missing(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
hash_directory = root / "constant"
|
||||
hash_directory.mkdir()
|
||||
(hash_directory / "bin_hash.c").write_text(
|
||||
"#include <stdio.h>\n"
|
||||
"int main(void) { puts(\"00000001\"); return 0; }\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
executable = probability_map.prepare_hash("constant", root)
|
||||
|
||||
self.assertEqual(executable, hash_directory / "bin_hash")
|
||||
self.assertEqual(
|
||||
subprocess.check_output([executable, "word"], text=True).strip(),
|
||||
"00000001",
|
||||
)
|
||||
|
||||
def test_adds_python_shebang_and_execute_permission(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
hash_directory = root / "python_hash"
|
||||
hash_directory.mkdir()
|
||||
source = hash_directory / "bin_hash.py"
|
||||
source.write_text("print('00000002')\n", encoding="utf-8")
|
||||
|
||||
executable = probability_map.prepare_hash("python_hash", root)
|
||||
|
||||
self.assertEqual(executable, source)
|
||||
self.assertTrue(os.access(source, os.X_OK))
|
||||
self.assertTrue(
|
||||
source.read_text(encoding="utf-8").startswith(
|
||||
"#!/usr/bin/env python3\n"
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
subprocess.check_output([executable, "word"], text=True).strip(),
|
||||
"00000002",
|
||||
)
|
||||
|
||||
|
||||
class ProbabilityTests(unittest.TestCase):
|
||||
def test_counts_changed_hash_bits_relative_to_source(self) -> None:
|
||||
probabilities = probability_map.bit_probabilities(
|
||||
source_hash=0b0000,
|
||||
changed_hashes=[0b0001, 0b0011, 0b0010, 0b0000],
|
||||
bits=4,
|
||||
)
|
||||
|
||||
self.assertEqual(probabilities, [0.5, 0.5, 0.0, 0.0])
|
||||
|
||||
def test_csv_table_has_operation_rows_and_bit_columns(self) -> None:
|
||||
stream = io.StringIO()
|
||||
|
||||
probability_map.write_csv_table(
|
||||
stream,
|
||||
{
|
||||
"replace": (4, [0.25, 0.75]),
|
||||
"delete": (2, [0.5, 0.0]),
|
||||
},
|
||||
)
|
||||
|
||||
rows = list(csv.reader(io.StringIO(stream.getvalue())))
|
||||
self.assertEqual(rows[0], ["operation", "pairs", "bit_0", "bit_1"])
|
||||
self.assertEqual(rows[1], ["replace", "4", "0.250000", "0.750000"])
|
||||
self.assertEqual(rows[2], ["delete", "2", "0.500000", "0.000000"])
|
||||
|
||||
def test_parser_accepts_multiple_source_words(self) -> None:
|
||||
arguments = probability_map.build_parser().parse_args(["first", "second"])
|
||||
|
||||
self.assertEqual(arguments.words, ["first", "second"])
|
||||
|
||||
def test_aggregates_samples_from_multiple_words_and_warns_on_shortfall(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
executable = Path(temporary_directory) / "hash.py"
|
||||
executable.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import sys\n"
|
||||
"print(f'{sum(sys.argv[1].encode()):08x}')\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
executable.chmod(0o755)
|
||||
warnings = io.StringIO()
|
||||
|
||||
with redirect_stderr(warnings):
|
||||
table = probability_map.build_probability_table(
|
||||
executable=executable,
|
||||
sources=["A", "B"],
|
||||
operations=["delete"],
|
||||
count=10,
|
||||
operation_count=1,
|
||||
alphabet=generate_input.DEFAULT_ALPHABET,
|
||||
seed=1,
|
||||
max_attempts=None,
|
||||
)
|
||||
|
||||
pair_count, probabilities = table["delete"]
|
||||
self.assertEqual(pair_count, 2)
|
||||
self.assertEqual(
|
||||
probabilities[:8],
|
||||
[0.5, 0.5, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0],
|
||||
)
|
||||
self.assertEqual(warnings.getvalue().count("delete"), 2)
|
||||
|
||||
|
||||
class GenerateWordsTests(unittest.TestCase):
|
||||
def test_count_is_an_upper_bound_when_unique_results_are_exhausted(self) -> None:
|
||||
words = generate_input.generate_words(
|
||||
source="abc",
|
||||
count=100,
|
||||
operation="delete",
|
||||
operation_count=1,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
self.assertEqual(set(words), {"ab", "ac", "bc"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Behavior tests for the standalone t1ha2_atonce hash CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCE = ROOT / "hash_funcs" / "t1ha2" / "bin_hash.c"
|
||||
|
||||
|
||||
class T1ha2BinHashTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.temporary_directory = tempfile.TemporaryDirectory()
|
||||
cls.binary = Path(cls.temporary_directory.name) / "bin_hash"
|
||||
subprocess.run(
|
||||
[
|
||||
"cc",
|
||||
"-std=c11",
|
||||
"-O2",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Wpedantic",
|
||||
"-Werror",
|
||||
str(SOURCE),
|
||||
"-o",
|
||||
str(cls.binary),
|
||||
],
|
||||
check=True,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cls.temporary_directory.cleanup()
|
||||
|
||||
def run_hash(
|
||||
self, word: str | None = None, stdin: bytes | None = None
|
||||
) -> subprocess.CompletedProcess[bytes]:
|
||||
command = [str(self.binary)]
|
||||
if word is not None:
|
||||
command.append(word)
|
||||
return subprocess.run(command, input=stdin, capture_output=True, check=False)
|
||||
|
||||
def test_matches_upstream_t1ha2_atonce_seed_zero_vectors(self) -> None:
|
||||
vectors = {
|
||||
"": b"0000000000000000\n",
|
||||
"hello": b"2a5f2abd74df73b4\n",
|
||||
"HashWord": b"3885d16135ce64f0\n",
|
||||
"abc": b"16bae0f716c45f2e\n",
|
||||
"12345678901234567890123456789012": b"75ed8a8aa66a4602\n",
|
||||
}
|
||||
|
||||
for word, expected in vectors.items():
|
||||
with self.subTest(word=word):
|
||||
result = self.run_hash(word)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, expected)
|
||||
self.assertRegex(result.stdout.decode(), r"^[0-9a-f]{16}\n$")
|
||||
|
||||
def test_argv_and_stdin_are_equivalent_and_strip_trailing_ascii_space(self) -> None:
|
||||
argv = self.run_hash("hello")
|
||||
stdin = self.run_hash(stdin=b"hello \t\r\n")
|
||||
|
||||
self.assertEqual(stdin.returncode, 0)
|
||||
self.assertEqual(stdin.stdout, argv.stdout)
|
||||
|
||||
def test_handles_long_ascii_input(self) -> None:
|
||||
payload = b"a" * 100_000
|
||||
result = self.run_hash(stdin=payload)
|
||||
|
||||
self.assertEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, b"bdf3f8539f0504ea\n")
|
||||
|
||||
def test_rejects_non_ascii_input(self) -> None:
|
||||
result = self.run_hash(stdin="ёж".encode())
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(b"ASCII", result.stderr)
|
||||
self.assertEqual(result.stdout, b"")
|
||||
|
||||
def test_unaligned_argv_is_clean_under_undefined_behavior_sanitizer(self) -> None:
|
||||
sanitized = Path(self.temporary_directory.name) / "bin_hash_ubsan"
|
||||
subprocess.run(
|
||||
[
|
||||
"cc",
|
||||
"-std=c11",
|
||||
"-O1",
|
||||
"-g",
|
||||
"-fsanitize=undefined",
|
||||
"-fno-sanitize-recover=undefined",
|
||||
str(SOURCE),
|
||||
"-o",
|
||||
str(sanitized),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
for word in ("hello", "12345678", "a" * 33):
|
||||
with self.subTest(word=word):
|
||||
result = subprocess.run(
|
||||
[str(sanitized), word], capture_output=True, check=False
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Behavior tests for the standalone XXH64 hash CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCE = ROOT / "hash_funcs" / "xxh64" / "bin_hash.c"
|
||||
|
||||
|
||||
class Xxh64BinHashTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.temporary_directory = tempfile.TemporaryDirectory()
|
||||
cls.binary = Path(cls.temporary_directory.name) / "bin_hash"
|
||||
subprocess.run(
|
||||
[
|
||||
"cc",
|
||||
"-std=c11",
|
||||
"-O2",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Wpedantic",
|
||||
"-Werror",
|
||||
str(SOURCE),
|
||||
"-o",
|
||||
str(cls.binary),
|
||||
],
|
||||
check=True,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cls.temporary_directory.cleanup()
|
||||
|
||||
def run_hash(self, word: str | None = None, stdin: bytes | None = None) -> subprocess.CompletedProcess[bytes]:
|
||||
command = [str(self.binary)]
|
||||
if word is not None:
|
||||
command.append(word)
|
||||
return subprocess.run(command, input=stdin, capture_output=True, check=False)
|
||||
|
||||
def test_matches_official_xxh64_seed_zero_vectors(self) -> None:
|
||||
vectors = {
|
||||
"": b"ef46db3751d8e999\n",
|
||||
"hello": b"26c7827d889f6da3\n",
|
||||
"HashWord": b"3e26fc2935163fbe\n",
|
||||
}
|
||||
|
||||
for word, expected in vectors.items():
|
||||
with self.subTest(word=word):
|
||||
result = self.run_hash(word)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, expected)
|
||||
self.assertRegex(result.stdout.decode(), r"^[0-9a-f]{16}\n$")
|
||||
|
||||
def test_argv_and_stdin_are_equivalent_and_strip_trailing_ascii_space(self) -> None:
|
||||
argv = self.run_hash("hello")
|
||||
stdin = self.run_hash(stdin=b"hello \t\r\n")
|
||||
|
||||
self.assertEqual(stdin.returncode, 0)
|
||||
self.assertEqual(stdin.stdout, argv.stdout)
|
||||
|
||||
def test_handles_long_ascii_input(self) -> None:
|
||||
payload = b"a" * 100_000
|
||||
result = self.run_hash(stdin=payload)
|
||||
reference = subprocess.run(
|
||||
["xxhsum", "-H64"], input=payload, capture_output=True, check=True
|
||||
).stdout.split()[0]
|
||||
|
||||
self.assertEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout.strip(), reference)
|
||||
|
||||
def test_rejects_non_ascii_input(self) -> None:
|
||||
result = self.run_hash(stdin="ёж".encode())
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(b"ASCII", result.stderr)
|
||||
self.assertEqual(result.stdout, b"")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
Before Width: | Height: | Size: 37 KiB After Width: | Height: | Size: 37 KiB |
|
Before Width: | Height: | Size: 38 KiB After Width: | Height: | Size: 38 KiB |
|
Before Width: | Height: | Size: 9.7 KiB After Width: | Height: | Size: 9.7 KiB |
@@ -0,0 +1,296 @@
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifdef ARSV_WITH_RPM
|
||||
#include <rpm/header.h>
|
||||
#include <rpm/rpmio.h>
|
||||
#include <rpm/rpmlib.h>
|
||||
#include <rpm/rpmtag.h>
|
||||
#include <rpm/rpmtd.h>
|
||||
#endif
|
||||
|
||||
int arsv_set9_decode(const char* source, unsigned** hashes, size_t* count, unsigned* bpp);
|
||||
|
||||
#define D1_PREFIX "set:D1"
|
||||
#define D1_HEADER_LEN 8
|
||||
|
||||
static const char base64_alphabet[] =
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
static size_t base64_encoded_size(size_t byte_count) {
|
||||
if (byte_count > SIZE_MAX - 2) return SIZE_MAX;
|
||||
size_t groups = (byte_count + 2) / 3;
|
||||
if (groups > SIZE_MAX / 4) return SIZE_MAX;
|
||||
size_t size = groups * 4;
|
||||
size_t remainder = byte_count % 3;
|
||||
if (remainder) size -= 3 - remainder;
|
||||
return size;
|
||||
}
|
||||
|
||||
static void base64_encode(const unsigned char* input, size_t input_len, char* output) {
|
||||
while (input_len >= 3) {
|
||||
uint32_t value = ((uint32_t)input[0] << 16) | ((uint32_t)input[1] << 8) | input[2];
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
output[2] = base64_alphabet[(value >> 6) & 0x3f];
|
||||
output[3] = base64_alphabet[value & 0x3f];
|
||||
input += 3;
|
||||
input_len -= 3;
|
||||
output += 4;
|
||||
}
|
||||
if (input_len == 1) {
|
||||
uint32_t value = (uint32_t)input[0] << 16;
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
} else if (input_len == 2) {
|
||||
uint32_t value = ((uint32_t)input[0] << 16) | ((uint32_t)input[1] << 8);
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
output[2] = base64_alphabet[(value >> 6) & 0x3f];
|
||||
}
|
||||
}
|
||||
|
||||
static int encode_d1(const unsigned* hashes, size_t count, unsigned bpp, char** result) {
|
||||
if (!hashes || !count || !result || bpp < 10 || bpp > 32) return -EINVAL;
|
||||
if (count > (SIZE_MAX - 7) / bpp) return -EOVERFLOW;
|
||||
|
||||
size_t bit_count = count * bpp;
|
||||
size_t byte_count = (bit_count + 7) / 8;
|
||||
unsigned char* bytes = calloc(byte_count, 1);
|
||||
if (!bytes) return -ENOMEM;
|
||||
|
||||
unsigned char* output = bytes;
|
||||
uint64_t bits = 0;
|
||||
unsigned filled = 0;
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
bits |= (uint64_t)hashes[i] << filled;
|
||||
filled += bpp;
|
||||
while (filled >= 8) {
|
||||
*output++ = (unsigned char)bits;
|
||||
bits >>= 8;
|
||||
filled -= 8;
|
||||
}
|
||||
}
|
||||
if (filled) *output++ = (unsigned char)bits;
|
||||
if ((size_t)(output - bytes) != byte_count) {
|
||||
free(bytes);
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
size_t payload_len = base64_encoded_size(byte_count);
|
||||
if (payload_len == SIZE_MAX || payload_len > SIZE_MAX - D1_HEADER_LEN - 1) {
|
||||
free(bytes);
|
||||
return -EOVERFLOW;
|
||||
}
|
||||
char* encoded = malloc(D1_HEADER_LEN + payload_len + 1);
|
||||
if (!encoded) {
|
||||
free(bytes);
|
||||
return -ENOMEM;
|
||||
}
|
||||
memcpy(encoded, D1_PREFIX, sizeof(D1_PREFIX) - 1);
|
||||
encoded[6] = (char)('0' + bpp / 10);
|
||||
encoded[7] = (char)('0' + bpp % 10);
|
||||
base64_encode(bytes, byte_count, encoded + D1_HEADER_LEN);
|
||||
encoded[D1_HEADER_LEN + payload_len] = '\0';
|
||||
free(bytes);
|
||||
*result = encoded;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int convert_set(const char* source, char** result, size_t* hash_count, unsigned* bpp) {
|
||||
if (!source || strncmp(source, "set:", 4) != 0 || strncmp(source, D1_PREFIX, 6) == 0)
|
||||
return -EINVAL;
|
||||
|
||||
unsigned* hashes = NULL;
|
||||
size_t count = 0;
|
||||
unsigned precision = 0;
|
||||
int rc = arsv_set9_decode(source, &hashes, &count, &precision);
|
||||
if (rc == 0) rc = encode_d1(hashes, count, precision, result);
|
||||
free(hashes);
|
||||
if (rc < 0) return rc;
|
||||
if (hash_count) *hash_count = count;
|
||||
if (bpp) *bpp = precision;
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef ARSV_WITH_RPM
|
||||
struct statistics {
|
||||
unsigned long headers;
|
||||
unsigned long set_occurrences;
|
||||
unsigned long set_bytes_old;
|
||||
unsigned long set_bytes_new;
|
||||
unsigned long hashes;
|
||||
};
|
||||
|
||||
static int rewrite_version_tag(Header header, rpmTagVal tag, const char* package,
|
||||
struct statistics* stats) {
|
||||
struct rpmtd_s values;
|
||||
memset(&values, 0, sizeof(values));
|
||||
if (headerGet(header, tag, &values, HEADERGET_MINMEM) != 1) return 0;
|
||||
if (rpmtdType(&values) != RPM_STRING_ARRAY_TYPE) {
|
||||
rpmtdFreeData(&values);
|
||||
fprintf(stderr, "%s: tag %d is not a string array\n", package, (int)tag);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rpm_count_t count = rpmtdCount(&values);
|
||||
const char** rewritten = calloc((size_t)count, sizeof(*rewritten));
|
||||
if (!rewritten) {
|
||||
rpmtdFreeData(&values);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
int rc = 0;
|
||||
int changed = 0;
|
||||
rpmtdInit(&values);
|
||||
for (rpm_count_t i = 0; i < count; ++i) {
|
||||
const char* value = rpmtdNextString(&values);
|
||||
if (!value) {
|
||||
rc = -EINVAL;
|
||||
break;
|
||||
}
|
||||
if (strncmp(value, "set:", 4) == 0) {
|
||||
char* converted = NULL;
|
||||
size_t hashes = 0;
|
||||
unsigned bpp = 0;
|
||||
rc = convert_set(value, &converted, &hashes, &bpp);
|
||||
if (rc < 0) {
|
||||
fprintf(stderr, "%s: cannot convert tag %d index %u (rc=%d)\n", package,
|
||||
(int)tag, (unsigned)i, rc);
|
||||
break;
|
||||
}
|
||||
rewritten[i] = converted;
|
||||
changed = 1;
|
||||
++stats->set_occurrences;
|
||||
stats->set_bytes_old += strlen(value);
|
||||
stats->set_bytes_new += strlen(converted);
|
||||
stats->hashes += hashes;
|
||||
} else {
|
||||
rewritten[i] = strdup(value);
|
||||
if (!rewritten[i]) {
|
||||
rc = -ENOMEM;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rc == 0 && changed) {
|
||||
headerDel(header, tag);
|
||||
if (!headerPutStringArray(header, tag, rewritten, count)) rc = -EIO;
|
||||
}
|
||||
for (rpm_count_t i = 0; i < count; ++i) free((void*)rewritten[i]);
|
||||
free(rewritten);
|
||||
rpmtdFreeData(&values);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int rewrite_pkglist(const char* input_path, const char* output_path, unsigned long limit) {
|
||||
FD_t input = Fopen(input_path, "r.ufdio");
|
||||
if (!input || Ferror(input)) {
|
||||
fprintf(stderr, "%s: %s\n", input_path, input ? Fstrerror(input) : "cannot open");
|
||||
return 1;
|
||||
}
|
||||
FD_t output = Fopen(output_path, "w.ufdio");
|
||||
if (!output || Ferror(output)) {
|
||||
fprintf(stderr, "%s: %s\n", output_path, output ? Fstrerror(output) : "cannot open");
|
||||
Fclose(input);
|
||||
return 1;
|
||||
}
|
||||
|
||||
struct statistics stats = {0};
|
||||
Header header;
|
||||
int failed = 0;
|
||||
const rpmTagVal tags[] = {
|
||||
RPMTAG_REQUIREVERSION,
|
||||
RPMTAG_PROVIDEVERSION,
|
||||
RPMTAG_CONFLICTVERSION,
|
||||
RPMTAG_OBSOLETEVERSION,
|
||||
RPMTAG_RECOMMENDVERSION,
|
||||
RPMTAG_SUGGESTVERSION,
|
||||
RPMTAG_SUPPLEMENTVERSION,
|
||||
RPMTAG_ENHANCEVERSION,
|
||||
};
|
||||
while ((!limit || stats.headers < limit) &&
|
||||
(header = headerRead(input, HEADER_MAGIC_YES)) != NULL) {
|
||||
const char* package = headerGetString(header, RPMTAG_NAME);
|
||||
if (!package) package = "<unknown>";
|
||||
for (size_t i = 0; i < sizeof(tags) / sizeof(tags[0]); ++i) {
|
||||
int rc = rewrite_version_tag(header, tags[i], package, &stats);
|
||||
if (rc < 0) {
|
||||
failed = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!failed && headerWrite(output, header, HEADER_MAGIC_YES) != 0) {
|
||||
fprintf(stderr, "%s: failed to write header for %s\n", output_path, package);
|
||||
failed = 1;
|
||||
}
|
||||
headerFree(header);
|
||||
if (failed) break;
|
||||
++stats.headers;
|
||||
}
|
||||
if (!failed && Ferror(input)) {
|
||||
fprintf(stderr, "%s: read error: %s\n", input_path, Fstrerror(input));
|
||||
failed = 1;
|
||||
}
|
||||
if (Fclose(output) != 0) failed = 1;
|
||||
Fclose(input);
|
||||
|
||||
if (failed) {
|
||||
remove(output_path);
|
||||
return 1;
|
||||
}
|
||||
fprintf(stderr,
|
||||
"headers=%lu set_occurrences=%lu hashes=%lu old_set_bytes=%lu "
|
||||
"new_set_bytes=%lu\n",
|
||||
stats.headers, stats.set_occurrences, stats.hashes, stats.set_bytes_old,
|
||||
stats.set_bytes_new);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
static void usage(const char* program) {
|
||||
fprintf(stderr, "usage: %s --convert-set set:VALUE\n", program);
|
||||
#ifdef ARSV_WITH_RPM
|
||||
fprintf(stderr, " %s --rewrite INPUT OUTPUT [--max-headers N]\n", program);
|
||||
#endif
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc == 3 && strcmp(argv[1], "--convert-set") == 0) {
|
||||
char* converted = NULL;
|
||||
int rc = convert_set(argv[2], &converted, NULL, NULL);
|
||||
if (rc < 0) {
|
||||
fprintf(stderr, "cannot convert set value (rc=%d)\n", rc);
|
||||
return 1;
|
||||
}
|
||||
puts(converted);
|
||||
free(converted);
|
||||
return 0;
|
||||
}
|
||||
#ifdef ARSV_WITH_RPM
|
||||
if ((argc == 4 || argc == 6) && strcmp(argv[1], "--rewrite") == 0) {
|
||||
unsigned long limit = 0;
|
||||
if (argc == 6) {
|
||||
if (strcmp(argv[4], "--max-headers") != 0) {
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
char* end = NULL;
|
||||
errno = 0;
|
||||
limit = strtoul(argv[5], &end, 10);
|
||||
if (errno || !end || *end || !limit) {
|
||||
fprintf(stderr, "invalid --max-headers value: %s\n", argv[5]);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
return rewrite_pkglist(argv[2], argv[3], limit);
|
||||
}
|
||||
#endif
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create D1 copies of the real Sisyphus x86_64 and noarch pkglist files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[2]
|
||||
ARCHITECTURES = ("x86_64", "noarch")
|
||||
SUMMARY_RE = re.compile(
|
||||
r"headers=(?P<headers>\d+) set_occurrences=(?P<set_occurrences>\d+) "
|
||||
r"hashes=(?P<hashes>\d+) old_set_bytes=(?P<old_set_bytes>\d+) "
|
||||
r"new_set_bytes=(?P<new_set_bytes>\d+)"
|
||||
)
|
||||
VERSION_FORMAT = (
|
||||
"[%{REQUIREVERSION}\\n]"
|
||||
"[%{PROVIDEVERSION}\\n]"
|
||||
"[%{CONFLICTVERSION}\\n]"
|
||||
"[%{OBSOLETEVERSION}\\n]"
|
||||
"[%{RECOMMENDVERSION}\\n]"
|
||||
"[%{SUGGESTVERSION}\\n]"
|
||||
"[%{SUPPLEMENTVERSION}\\n]"
|
||||
"[%{ENHANCEVERSION}\\n]"
|
||||
)
|
||||
|
||||
|
||||
def classify_pkglist(path: Path) -> str | None:
|
||||
name = path.name
|
||||
if not name.endswith("_base_pkglist.classic"):
|
||||
return None
|
||||
if "_Sisyphus_x86%5f64_" in name or "_Sisyphus_x86_64_" in name:
|
||||
return "x86_64"
|
||||
if "_Sisyphus_noarch_" in name:
|
||||
return "noarch"
|
||||
return None
|
||||
|
||||
|
||||
def validate_output(path: Path) -> Path:
|
||||
resolved = path.expanduser().resolve()
|
||||
home = Path.home().resolve()
|
||||
source = ROOT.resolve()
|
||||
if resolved == home or resolved == source or source in resolved.parents:
|
||||
raise ValueError(f"refusing protected output path: {resolved}")
|
||||
if resolved == Path("/"):
|
||||
raise ValueError("refusing filesystem root as output")
|
||||
return resolved
|
||||
|
||||
|
||||
def run(
|
||||
command: list[str],
|
||||
*,
|
||||
check: bool = True,
|
||||
cwd: Path | None = None,
|
||||
stdout: int | None = None,
|
||||
) -> subprocess.CompletedProcess[bytes]:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
stdout=stdout if stdout is not None else subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode != 0:
|
||||
stderr = result.stderr.decode(errors="replace")
|
||||
raise RuntimeError(f"command failed ({result.returncode}): {' '.join(command)}\n{stderr}")
|
||||
return result
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def compile_converter(
|
||||
destination: Path,
|
||||
rpm_include_dir: Path | None = None,
|
||||
rpm_libraries: tuple[Path, ...] = (),
|
||||
) -> None:
|
||||
include = destination.parent / "compat-include"
|
||||
include.mkdir()
|
||||
for name in ("rpmlib.h", "system.h", "set.h"):
|
||||
(include / name).touch()
|
||||
command = [
|
||||
"cc",
|
||||
"-O2",
|
||||
"-std=gnu11",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
"-D_GNU_SOURCE",
|
||||
"-DARSV_SET9_EXPORT",
|
||||
"-DARSV_WITH_RPM",
|
||||
"-I",
|
||||
str(include),
|
||||
]
|
||||
if rpm_include_dir is not None:
|
||||
command.extend(["-I", str(rpm_include_dir)])
|
||||
command.extend(
|
||||
[
|
||||
"-include",
|
||||
str(ROOT / "scripts/rpmsetcmp/newset_compat.h"),
|
||||
str(ROOT / "reimplement/set9.c"),
|
||||
str(HERE / "rewrite_sisyphus_pkglist.c"),
|
||||
]
|
||||
)
|
||||
if rpm_libraries:
|
||||
command.extend(str(path) for path in rpm_libraries)
|
||||
else:
|
||||
command.extend(["-lrpm", "-lrpmio"])
|
||||
command.extend(["-o", str(destination)])
|
||||
run(command)
|
||||
|
||||
|
||||
def find_pkglists(lists_dir: Path) -> dict[str, Path]:
|
||||
found: dict[str, Path] = {}
|
||||
for path in lists_dir.glob("*_base_pkglist.classic"):
|
||||
architecture = classify_pkglist(path)
|
||||
if architecture is None:
|
||||
continue
|
||||
if architecture in found:
|
||||
raise RuntimeError(f"multiple Sisyphus {architecture} pkglist files")
|
||||
found[architecture] = path
|
||||
missing = set(ARCHITECTURES) - set(found)
|
||||
if missing:
|
||||
raise RuntimeError(f"missing Sisyphus pkglist for: {', '.join(sorted(missing))}")
|
||||
return found
|
||||
|
||||
|
||||
def query_header_count(path: Path) -> int:
|
||||
process = subprocess.Popen(
|
||||
["pkglist-query", "%{NAME}\\n", str(path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
assert process.stdout is not None
|
||||
count = sum(1 for _ in process.stdout)
|
||||
stderr = process.stderr.read() if process.stderr else b""
|
||||
status = process.wait()
|
||||
if status != 0:
|
||||
raise RuntimeError(f"pkglist-query failed for {path}: {stderr.decode(errors='replace')}")
|
||||
return count
|
||||
|
||||
|
||||
def query_set_counts(path: Path) -> dict[str, int]:
|
||||
process = subprocess.Popen(
|
||||
["pkglist-query", VERSION_FORMAT, str(path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
assert process.stdout is not None
|
||||
old = direct = 0
|
||||
for raw_line in process.stdout:
|
||||
value = raw_line.strip()
|
||||
if value.startswith(b"set:D1"):
|
||||
direct += 1
|
||||
elif value.startswith(b"set:"):
|
||||
old += 1
|
||||
stderr = process.stderr.read() if process.stderr else b""
|
||||
status = process.wait()
|
||||
if status != 0:
|
||||
raise RuntimeError(f"pkglist-query failed for {path}: {stderr.decode(errors='replace')}")
|
||||
return {"set9": old, "d1": direct}
|
||||
|
||||
|
||||
def conversion_summary(stderr: bytes) -> dict[str, int]:
|
||||
text = stderr.decode(errors="replace")
|
||||
match = SUMMARY_RE.search(text)
|
||||
if not match:
|
||||
raise RuntimeError(f"converter did not report summary:\n{text}")
|
||||
return {name: int(value) for name, value in match.groupdict().items()}
|
||||
|
||||
|
||||
def convert_one(converter: Path, source: Path, destination: Path) -> dict[str, object]:
|
||||
result = run([str(converter), "--rewrite", str(source), str(destination)])
|
||||
summary = conversion_summary(result.stderr)
|
||||
source_headers = query_header_count(source)
|
||||
output_headers = query_header_count(destination)
|
||||
source_sets = query_set_counts(source)
|
||||
output_sets = query_set_counts(destination)
|
||||
if source_headers != output_headers or output_headers != summary["headers"]:
|
||||
raise RuntimeError(
|
||||
f"header count mismatch: source={source_headers} output={output_headers} "
|
||||
f"converter={summary['headers']}"
|
||||
)
|
||||
if source_sets["d1"] != 0:
|
||||
raise RuntimeError(f"source already contains {source_sets['d1']} D1 values")
|
||||
if output_sets["set9"] != 0:
|
||||
raise RuntimeError(f"output still contains {output_sets['set9']} set9 values")
|
||||
if source_sets["set9"] != output_sets["d1"] or output_sets["d1"] != summary["set_occurrences"]:
|
||||
raise RuntimeError(
|
||||
"set occurrence mismatch: "
|
||||
f"source={source_sets['set9']} output={output_sets['d1']} "
|
||||
f"converter={summary['set_occurrences']}"
|
||||
)
|
||||
if summary["set_occurrences"] == 0:
|
||||
raise RuntimeError("source pkglist contains no set versions")
|
||||
return {
|
||||
**summary,
|
||||
"source": {
|
||||
"path": source.name,
|
||||
"size": source.stat().st_size,
|
||||
"sha256": sha256_file(source),
|
||||
},
|
||||
"output": {
|
||||
"path": destination.name,
|
||||
"size": destination.stat().st_size,
|
||||
"sha256": sha256_file(destination),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def convert_local(
|
||||
output: Path,
|
||||
lists_dir: Path,
|
||||
rpm_include_dir: Path | None = None,
|
||||
rpm_libraries: tuple[Path, ...] = (),
|
||||
) -> None:
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
if any(output.iterdir()):
|
||||
raise RuntimeError(f"output directory is not empty: {output}")
|
||||
|
||||
lists_dir = lists_dir.expanduser().resolve()
|
||||
if not lists_dir.is_dir():
|
||||
raise RuntimeError(f"APT lists directory does not exist: {lists_dir}")
|
||||
pkglists = find_pkglists(lists_dir)
|
||||
staging = Path(tempfile.mkdtemp(prefix=".d1-staging-", dir=output))
|
||||
try:
|
||||
converter = staging / "rewrite-sisyphus-pkglist"
|
||||
compile_converter(converter, rpm_include_dir, rpm_libraries)
|
||||
architectures: dict[str, object] = {}
|
||||
for architecture in ARCHITECTURES:
|
||||
destination = staging / f"Sisyphus.{architecture}.pkglist.classic"
|
||||
architectures[architecture] = convert_one(
|
||||
converter, pkglists[architecture], destination
|
||||
)
|
||||
converter.unlink()
|
||||
shutil.rmtree(staging / "compat-include")
|
||||
manifest = {
|
||||
"format": 1,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"input": {"lists_dir": str(lists_dir)},
|
||||
"rpm": run(["rpm", "--version"]).stdout.decode().strip(),
|
||||
"apt": run(["rpmquery", "--qf", "%{VERSION}-%{RELEASE}", "apt"])
|
||||
.stdout.decode()
|
||||
.strip(),
|
||||
"converter_source": {
|
||||
"set9_sha256": sha256_file(ROOT / "reimplement/set9.c"),
|
||||
"rewrite_sha256": sha256_file(HERE / "rewrite_sisyphus_pkglist.c"),
|
||||
},
|
||||
"architectures": architectures,
|
||||
}
|
||||
(staging / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
final = output / "d1-pkglists"
|
||||
staging.replace(final)
|
||||
print(final)
|
||||
for architecture in ARCHITECTURES:
|
||||
data = architectures[architecture]
|
||||
assert isinstance(data, dict)
|
||||
print(
|
||||
f"{architecture}: headers={data['headers']} "
|
||||
f"set_occurrences={data['set_occurrences']} "
|
||||
f"output={final / f'Sisyphus.{architecture}.pkglist.classic'}"
|
||||
)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Create D1 copies of local Sisyphus x86_64/noarch pkglist files"
|
||||
)
|
||||
parser.add_argument("output", type=Path, help="new or empty output directory")
|
||||
parser.add_argument(
|
||||
"--lists-dir",
|
||||
type=Path,
|
||||
default=Path("/var/lib/apt/lists"),
|
||||
help="APT lists snapshot to convert (default: /var/lib/apt/lists)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rpm-include-dir",
|
||||
type=Path,
|
||||
help="directory containing rpm/header.h (for an unpacked librpm-devel)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rpm-library",
|
||||
type=Path,
|
||||
action="append",
|
||||
default=[],
|
||||
help="versioned RPM library to link; repeat for librpm and librpmio",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
output = validate_output(args.output)
|
||||
rpm_include_dir = (
|
||||
args.rpm_include_dir.expanduser().resolve()
|
||||
if args.rpm_include_dir is not None
|
||||
else None
|
||||
)
|
||||
rpm_libraries = tuple(path.expanduser().resolve() for path in args.rpm_library)
|
||||
if rpm_include_dir is not None and not (rpm_include_dir / "rpm/header.h").is_file():
|
||||
raise ValueError(f"rpm/header.h not found below: {rpm_include_dir}")
|
||||
if rpm_libraries and len(rpm_libraries) != 2:
|
||||
raise ValueError("pass exactly two --rpm-library values: librpm and librpmio")
|
||||
for library in rpm_libraries:
|
||||
if not library.is_file():
|
||||
raise ValueError(f"RPM library not found: {library}")
|
||||
convert_local(output, args.lists_dir, rpm_include_dir, rpm_libraries)
|
||||
except (OSError, RuntimeError, ValueError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -15,7 +15,7 @@
|
||||
* This is intentionally a new set-string format. It is not compatible with
|
||||
* the Golomb-Rice/base62 strings produced by the original lib/set.c.
|
||||
*
|
||||
* D1<two decimal bpp digits><RFC 4648 base64 of packed hashes>
|
||||
* D1<two decimal bpp digits><unpadded RFC 4648 base64 of packed hashes>
|
||||
*
|
||||
* Sorted unique hashes are packed least-significant bit first, using exactly
|
||||
* bpp bits per hash. Base64 is only a textual representation of those bytes;
|
||||
@@ -209,7 +209,10 @@ static size_t base64_encoded_size(size_t byte_count) {
|
||||
size_t groups = (byte_count + 2) / 3;
|
||||
if (groups > (SIZE_MAX - FORMAT_HEADER_LEN - 1) / 4) abort();
|
||||
|
||||
return groups * 4;
|
||||
size_t size = groups * 4;
|
||||
size_t remainder = byte_count % 3;
|
||||
if (remainder != 0) size -= 3 - remainder;
|
||||
return size;
|
||||
}
|
||||
|
||||
static void base64_encode(const unsigned char* input, size_t input_len, char* output) {
|
||||
@@ -228,14 +231,11 @@ static void base64_encode(const unsigned char* input, size_t input_len, char* ou
|
||||
uint32_t value = (uint32_t)input[0] << 16;
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
output[2] = '=';
|
||||
output[3] = '=';
|
||||
} else if (input_len == 2) {
|
||||
uint32_t value = ((uint32_t)input[0] << 16) | ((uint32_t)input[1] << 8);
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
output[2] = base64_alphabet[(value >> 6) & 0x3f];
|
||||
output[3] = '=';
|
||||
}
|
||||
|
||||
return;
|
||||
@@ -334,6 +334,20 @@ static const unsigned char base64_values[256] = {
|
||||
|
||||
static inline int base64_value(unsigned char c) { return (int)base64_values[c] - 1; }
|
||||
|
||||
static int base64_decoded_size(size_t input_len, size_t* byte_count) {
|
||||
size_t remainder = input_len % 4;
|
||||
if (input_len == 0 || remainder == 1) return -1;
|
||||
|
||||
size_t groups = input_len / 4;
|
||||
if (groups > SIZE_MAX / 3) return -1;
|
||||
size_t size = groups * 3;
|
||||
size_t tail_size = remainder == 0 ? 0 : remainder - 1;
|
||||
if (size > SIZE_MAX - tail_size) return -1;
|
||||
|
||||
*byte_count = size + tail_size;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int has_set_prefix(const char* str) {
|
||||
return str[0] == 's' && str[1] == 'e' && str[2] == 't' && str[3] == ':';
|
||||
}
|
||||
@@ -343,11 +357,11 @@ static int set_meta_init(const char* source, struct set_meta* meta) {
|
||||
if (has_set_prefix(str)) str += 4;
|
||||
if (has_set_prefix(str)) return -1;
|
||||
|
||||
/* A valid direct set has at least one complete Base64 quartet. Checking
|
||||
* this fixed prefix makes cache hits independent of total key length. */
|
||||
/* With bpp >= 10, a valid direct set has at least three Base64 characters.
|
||||
* Checking this fixed prefix makes cache hits independent of total key length. */
|
||||
if (str[0] != FORMAT_PREFIX[0] || str[1] != FORMAT_PREFIX[1]) return -1;
|
||||
if (str[2] < '0' || str[2] > '9' || str[3] < '0' || str[3] > '9') return -1;
|
||||
if (str[4] == '\0' || str[5] == '\0' || str[6] == '\0' || str[7] == '\0') return -1;
|
||||
if (str[4] == '\0' || str[5] == '\0' || str[6] == '\0') return -1;
|
||||
|
||||
unsigned bpp = (unsigned)(str[2] - '0') * 10 + (unsigned)(str[3] - '0');
|
||||
if (bpp < 10 || bpp > 32) return -1;
|
||||
@@ -396,36 +410,34 @@ static int decode_base64_bytes(const char* input, size_t input_len, unsigned cha
|
||||
size_t output_len) {
|
||||
unsigned char* const output_end = output + output_len;
|
||||
size_t offset = 0;
|
||||
for (; offset + 4 < input_len; offset += 4) {
|
||||
while (input_len - offset >= 4) {
|
||||
int v0 = base64_value((unsigned char)input[offset]);
|
||||
int v1 = base64_value((unsigned char)input[offset + 1]);
|
||||
int v2 = base64_value((unsigned char)input[offset + 2]);
|
||||
int v3 = base64_value((unsigned char)input[offset + 3]);
|
||||
if ((v0 | v1 | v2 | v3) < 0) return -1;
|
||||
if ((v0 | v1 | v2 | v3) < 0 || output_end - output < 3) return -1;
|
||||
output[0] = (unsigned char)((v0 << 2) | (v1 >> 4));
|
||||
output[1] = (unsigned char)(((v1 & 0x0f) << 4) | (v2 >> 2));
|
||||
output[2] = (unsigned char)(((v2 & 0x03) << 6) | v3);
|
||||
output += 3;
|
||||
offset += 4;
|
||||
}
|
||||
|
||||
size_t remainder = input_len - offset;
|
||||
if (remainder == 0) return output == output_end ? 0 : -1;
|
||||
if (remainder == 1) return -1;
|
||||
|
||||
int v0 = base64_value((unsigned char)input[offset]);
|
||||
int v1 = base64_value((unsigned char)input[offset + 1]);
|
||||
if ((v0 | v1) < 0 || output == output_end) return -1;
|
||||
*output++ = (unsigned char)((v0 << 2) | (v1 >> 4));
|
||||
|
||||
if (input[offset + 2] == '=') {
|
||||
return input[offset + 3] == '=' && (v1 & 0x0f) == 0 && output == output_end ? 0 : -1;
|
||||
}
|
||||
if (remainder == 2) return (v1 & 0x0f) == 0 && output == output_end ? 0 : -1;
|
||||
|
||||
int v2 = base64_value((unsigned char)input[offset + 2]);
|
||||
if (v2 < 0 || output == output_end) return -1;
|
||||
*output++ = (unsigned char)(((v1 & 0x0f) << 4) | (v2 >> 2));
|
||||
if (input[offset + 3] == '=') return (v2 & 0x03) == 0 && output == output_end ? 0 : -1;
|
||||
|
||||
int v3 = base64_value((unsigned char)input[offset + 3]);
|
||||
if (v3 < 0 || output == output_end) return -1;
|
||||
*output++ = (unsigned char)(((v2 & 0x03) << 6) | v3);
|
||||
return output == output_end ? 0 : -1;
|
||||
return (v2 & 0x03) == 0 && output == output_end ? 0 : -1;
|
||||
}
|
||||
|
||||
static inline int decode_base64_triplet(const char* input, uint32_t* triplet) {
|
||||
@@ -502,11 +514,8 @@ static int decode_set_sized(const char* str, size_t str_len, struct decoded_set*
|
||||
|
||||
const char* input = str + FORMAT_HEADER_LEN;
|
||||
size_t input_len = str_len - FORMAT_HEADER_LEN;
|
||||
if (input_len == 0 || input_len % 4 != 0 || input_len / 4 > SIZE_MAX / 3) return -1;
|
||||
|
||||
size_t byte_count = input_len / 4 * 3;
|
||||
if (input[input_len - 1] == '=') --byte_count;
|
||||
if (input[input_len - 2] == '=') --byte_count;
|
||||
size_t byte_count;
|
||||
if (base64_decoded_size(input_len, &byte_count) < 0) return -1;
|
||||
if (byte_count > SIZE_MAX / 8) return -1;
|
||||
|
||||
size_t count = byte_count * 8 / bpp;
|
||||
@@ -563,32 +572,35 @@ static int decode_set_sized(const char* str, size_t str_len, struct decoded_set*
|
||||
.bpp = bpp,
|
||||
};
|
||||
|
||||
for (size_t offset = 0; offset < input_len; offset += 4) {
|
||||
size_t full_len = input_len - input_len % 4;
|
||||
for (size_t offset = 0; offset < full_len; offset += 4) {
|
||||
int v0 = base64_value((unsigned char)input[offset]);
|
||||
int v1 = base64_value((unsigned char)input[offset + 1]);
|
||||
int last = offset + 4 == input_len;
|
||||
if (v0 < 0 || v1 < 0) goto invalid;
|
||||
int v2 = base64_value((unsigned char)input[offset + 2]);
|
||||
int v3 = base64_value((unsigned char)input[offset + 3]);
|
||||
if ((v0 | v1 | v2 | v3) < 0) goto invalid;
|
||||
|
||||
uint32_t bytes = (uint32_t)((v0 << 2) | (v1 >> 4)) |
|
||||
(uint32_t)(((v1 & 0x0f) << 4) | (v2 >> 2)) << 8 |
|
||||
(uint32_t)(((v2 & 0x03) << 6) | v3) << 16;
|
||||
if (decode_writer_put(&writer, bytes, 3) < 0) goto invalid;
|
||||
}
|
||||
|
||||
size_t remainder = input_len - full_len;
|
||||
if (remainder != 0) {
|
||||
int v0 = base64_value((unsigned char)input[full_len]);
|
||||
int v1 = base64_value((unsigned char)input[full_len + 1]);
|
||||
if ((v0 | v1) < 0) goto invalid;
|
||||
|
||||
uint32_t bytes = (uint32_t)((v0 << 2) | (v1 >> 4));
|
||||
if (input[offset + 2] == '=') {
|
||||
if (!last || input[offset + 3] != '=' || (v1 & 0x0f) != 0 ||
|
||||
decode_writer_put(&writer, bytes, 1) < 0)
|
||||
goto invalid;
|
||||
continue;
|
||||
if (remainder == 2) {
|
||||
if ((v1 & 0x0f) != 0 || decode_writer_put(&writer, bytes, 1) < 0) goto invalid;
|
||||
} else {
|
||||
int v2 = base64_value((unsigned char)input[full_len + 2]);
|
||||
if (v2 < 0 || (v2 & 0x03) != 0) goto invalid;
|
||||
bytes |= (uint32_t)(((v1 & 0x0f) << 4) | (v2 >> 2)) << 8;
|
||||
if (decode_writer_put(&writer, bytes, 2) < 0) goto invalid;
|
||||
}
|
||||
|
||||
int v2 = base64_value((unsigned char)input[offset + 2]);
|
||||
if (v2 < 0) goto invalid;
|
||||
bytes |= (uint32_t)(((v1 & 0x0f) << 4) | (v2 >> 2)) << 8;
|
||||
if (input[offset + 3] == '=') {
|
||||
if (!last || (v2 & 0x03) != 0 || decode_writer_put(&writer, bytes, 2) < 0) goto invalid;
|
||||
continue;
|
||||
}
|
||||
|
||||
int v3 = base64_value((unsigned char)input[offset + 3]);
|
||||
if (v3 < 0) goto invalid;
|
||||
bytes |= (uint32_t)(((v2 & 0x03) << 6) | v3) << 16;
|
||||
if (decode_writer_put(&writer, bytes, 3) < 0) goto invalid;
|
||||
}
|
||||
|
||||
if (writer.written != count || writer.bits != 0) goto invalid;
|
||||
@@ -803,8 +815,6 @@ static int cache_decode_set(const struct set_meta* meta, unsigned target_bpp, un
|
||||
}
|
||||
|
||||
size_t len = strlen(meta->str);
|
||||
size_t input_len = len - FORMAT_HEADER_LEN;
|
||||
if (input_len == 0 || input_len % 4 != 0 || input_len / 4 > SIZE_MAX / 3) return -1;
|
||||
|
||||
struct decoded_set decoded;
|
||||
if (decode_set_sized(meta->str, len, &decoded) < 0) return -1;
|
||||
|
||||
+154
-122
@@ -2,6 +2,7 @@
|
||||
#undef NDEBUG
|
||||
#endif
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
@@ -15,26 +16,25 @@
|
||||
#include "set.h"
|
||||
#include "system.h"
|
||||
|
||||
#define CACHE_SIZE 512
|
||||
#define SET_HASH_SEED UINT32_C(0x9e3779b9)
|
||||
|
||||
#define CACHE_FINGERPRINT_MULTIPLIER UINT32_C(0x9e3779b1)
|
||||
#define CACHE_TARGET_BPP_MULTIPLIER UINT32_C(0x85ebca6b)
|
||||
enum {
|
||||
CACHE_SIZE = 512,
|
||||
CACHE_BUCKETS = 1024,
|
||||
};
|
||||
|
||||
enum {
|
||||
CACHE_BUCKETS = 1024,
|
||||
|
||||
SET_HEADER_SIZE = 2,
|
||||
SET_PARAM_CHAR_OFFSET = 7,
|
||||
SET_BPP_MIN = 10,
|
||||
SET_BPP_MAX = 32,
|
||||
SET_MSHIFT_MIN = 7,
|
||||
SET_MSHIFT_MAX = 31,
|
||||
};
|
||||
|
||||
enum {
|
||||
BASE62_ESCAPE_BITS = 4,
|
||||
BASE62_VALUE_BITS = 6,
|
||||
BASE62_ESCAPE_CHUNK_BITS = BASE62_ESCAPE_BITS + BASE62_VALUE_BITS,
|
||||
BASE62_MIN_BITS_PER_CHAR = BASE62_VALUE_BITS - 1,
|
||||
BASE62_MIN_BITS_PER_CHAR = BASE62_ESCAPE_CHUNK_BITS / 2,
|
||||
BASE62_MAX_PADDING_BITS = BASE62_VALUE_BITS - 1,
|
||||
BASE62_LOWERCASE_OFFSET = 10,
|
||||
BASE62_UPPERCASE_OFFSET = 36,
|
||||
@@ -42,12 +42,14 @@ enum {
|
||||
BASE62_VALUE_COUNT = 62,
|
||||
BASE62_ESCAPE_LOW_MASK = (1u << BASE62_ESCAPE_BITS) - 1,
|
||||
BASE62_ESCAPE_HIGH_MASK = 3u << BASE62_ESCAPE_BITS,
|
||||
};
|
||||
|
||||
enum {
|
||||
BASE62_INVALID = 0xee,
|
||||
BASE62_END = UCHAR_MAX,
|
||||
};
|
||||
|
||||
_Static_assert(CHAR_BIT == 8, "set:version requires 8-bit bytes");
|
||||
_Static_assert(CHAR_BIT == 8, "set:version relies on 8-bit");
|
||||
|
||||
struct set {
|
||||
size_t cnt;
|
||||
@@ -82,7 +84,7 @@ void set_add(struct set* set, const char* sym) {
|
||||
size_t length = strlen(sym) + 1;
|
||||
size_t required = set->strings_len + length;
|
||||
if (required > set->strings_cap) {
|
||||
size_t capacity = set->strings_cap ? set->strings_cap : 4096;
|
||||
size_t capacity = set->strings_cap ?: 4096;
|
||||
while (capacity < required) capacity *= 2;
|
||||
|
||||
set->strings = xrealloc(set->strings, capacity);
|
||||
@@ -94,8 +96,6 @@ void set_add(struct set* set, const char* sym) {
|
||||
memcpy(set->strings + set->strings_len, sym, length);
|
||||
set->strings_len = required;
|
||||
set->cnt++;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
struct set* set_free(struct set* set) {
|
||||
@@ -108,10 +108,8 @@ struct set* set_free(struct set* set) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
static unsigned hash(const char* str) {
|
||||
unsigned hash = SET_HASH_SEED;
|
||||
unsigned hash = UINT32_C(0x9e3779b9);
|
||||
const unsigned char* p = (const unsigned char*)str;
|
||||
|
||||
while (*p) {
|
||||
@@ -171,12 +169,8 @@ static void sort_symbols(struct symbols* values, size_t count, int bpp) {
|
||||
}
|
||||
|
||||
if (source != values) memcpy(values, source, count * sizeof(*values));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
static int log2i(int n) {
|
||||
int m = 0;
|
||||
while (n /= 2) m++;
|
||||
@@ -184,13 +178,15 @@ static int log2i(int n) {
|
||||
return m;
|
||||
}
|
||||
|
||||
// Calculate Mshift paramter for encoding.
|
||||
/* Calculate Mshift parameter for encoding. */
|
||||
static int encode_golomb_Mshift(int cnt, int bpp) {
|
||||
// XXX Slightly better Mshift estimations are probably possible.
|
||||
// Recheck "Compression and coding algorithms" by Moffat & Turpin.
|
||||
/*
|
||||
* XXX Slightly better Mshift estimations are probably possible.
|
||||
* Recheck "Compression and coding algorithms" by Moffat & Turpin.
|
||||
*/
|
||||
int Mshift = bpp - log2i(cnt) - 1;
|
||||
|
||||
// Adjust out-of-range values.
|
||||
/* Adjust out-of-range values. */
|
||||
Mshift = (Mshift < SET_MSHIFT_MIN) ? SET_MSHIFT_MIN : Mshift;
|
||||
Mshift = (Mshift > SET_MSHIFT_MAX) ? SET_MSHIFT_MAX : Mshift;
|
||||
assert(Mshift < bpp);
|
||||
@@ -198,31 +194,33 @@ static int encode_golomb_Mshift(int cnt, int bpp) {
|
||||
return Mshift;
|
||||
}
|
||||
|
||||
// Estimate how many bits can be filled up.
|
||||
/* Estimate how many bits can be filled up. */
|
||||
static inline int encode_golomb_size(int cnt, int Mshift) {
|
||||
// XXX No precise estimation. However, we do not expect unary-encoded bits
|
||||
// to take more than binary-encoded Mshift bits.
|
||||
/*
|
||||
* XXX No precise estimation. However, we do not expect unary-encoded bits
|
||||
* to take more than binary-encoded Mshift bits.
|
||||
*/
|
||||
return 2 * Mshift * cnt + 16;
|
||||
}
|
||||
|
||||
// Estimate base62 buffer size required to encode a given number of bits.
|
||||
/* Estimate base62 buffer size required to encode a given number of bits. */
|
||||
static inline int encode_base62_size(int bit_cnt) {
|
||||
// In the worst case, which is ZxZxZx..., five bits can make a character;
|
||||
// the remaining bits can make a character, too. And the string must be
|
||||
// null-terminated.
|
||||
/*
|
||||
* In the worst case, which is ZxZxZx..., five bits can make a character;
|
||||
* the remaining bits can make a character, too. And the string must be
|
||||
* null-terminated.
|
||||
*/
|
||||
return bit_cnt / BASE62_MIN_BITS_PER_CHAR + 2;
|
||||
}
|
||||
|
||||
static int encode_set_size(int cnt, int bpp) {
|
||||
int Mshift = encode_golomb_Mshift(cnt, bpp);
|
||||
int bit_cnt = encode_golomb_size(cnt, Mshift);
|
||||
// The leading characters encode bpp and Mshift.
|
||||
/* The leading characters encode bpp and Mshift. */
|
||||
return SET_HEADER_SIZE + encode_base62_size(bit_cnt);
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
// Main base62 encoding routine: pack bit_arr into base62 string.
|
||||
/* Main base62 encoding routine: pack bit_arr into base62 string. */
|
||||
/*
|
||||
* Base62 routines - encode bits with alnum characters.
|
||||
*
|
||||
@@ -236,8 +234,6 @@ static int encode_set_size(int cnt, int bpp) {
|
||||
* how multiple escapes are avoided.
|
||||
*/
|
||||
|
||||
// ---
|
||||
|
||||
static inline char encode_bpp(int bpp) { return (char)(bpp - SET_PARAM_CHAR_OFFSET + 'a'); }
|
||||
|
||||
struct encode_writer {
|
||||
@@ -283,8 +279,10 @@ static inline void encode_writer_flush(struct encode_writer* writer) {
|
||||
}
|
||||
|
||||
static inline void encode_writer_zeros(struct encode_writer* writer, unsigned count) {
|
||||
// encode_writer_flush() leaves fewer than BASE62_VALUE_BITS bits buffered.
|
||||
// Adding at most 56 bits therefore cannot overflow uint64_t.
|
||||
/*
|
||||
* encode_writer_flush() leaves fewer than BASE62_VALUE_BITS bits buffered.
|
||||
* Adding at most 56 bits therefore cannot overflow uint64_t.
|
||||
*/
|
||||
while (count) {
|
||||
unsigned take = count > 56 ? 56 : count;
|
||||
writer->filled += take;
|
||||
@@ -292,16 +290,12 @@ static inline void encode_writer_zeros(struct encode_writer* writer, unsigned co
|
||||
|
||||
encode_writer_flush(writer);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
static inline void encode_writer_put(struct encode_writer* writer, uint64_t value, unsigned width) {
|
||||
writer->bits |= value << writer->filled;
|
||||
writer->filled += width;
|
||||
encode_writer_flush(writer);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
static int encode_set(int cnt, const unsigned* hash_arr, int bpp, char* base62_str) {
|
||||
@@ -337,7 +331,7 @@ static int encode_set(int cnt, const unsigned* hash_arr, int bpp, char* base62_s
|
||||
}
|
||||
|
||||
const char* set_fini(struct set* set, int bpp) {
|
||||
// Implementation for finalizing the set
|
||||
/* Implementation for finalizing the set */
|
||||
|
||||
assert(set != NULL);
|
||||
assert(set->cnt > 0);
|
||||
@@ -351,20 +345,20 @@ const char* set_fini(struct set* set, int bpp) {
|
||||
|
||||
sort_symbols(set->symbols_v, set->cnt, bpp);
|
||||
|
||||
// warn on hash collizions
|
||||
/* warn on hash collisions */
|
||||
for (size_t i = 0; i < set->cnt - 1; ++i) {
|
||||
if (set->symbols_v[i].hash != set->symbols_v[i + 1].hash) continue;
|
||||
const char* left = set->strings + set->symbols_v[i].offset;
|
||||
const char* right = set->strings + set->symbols_v[i + 1].offset;
|
||||
if (!strcmp(left, right)) continue;
|
||||
|
||||
fprintf(stderr, "warning: hash collision: %s %s\n", left, right);
|
||||
fprintf(stderr, "warning: set-version hash collision: %s %s\n", left, right);
|
||||
}
|
||||
|
||||
unsigned unique_hash[set->cnt];
|
||||
int unique_cnt = 0;
|
||||
|
||||
// delete duplicates
|
||||
/* delete duplicates */
|
||||
for (size_t i = 0; i < set->cnt; ++i) {
|
||||
while (i + 1 < set->cnt && set->symbols_v[i].hash == set->symbols_v[i + 1].hash) {
|
||||
++i;
|
||||
@@ -378,8 +372,6 @@ const char* set_fini(struct set* set, int bpp) {
|
||||
return xstrdup(base62_str);
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
struct set_meta {
|
||||
const char* str;
|
||||
const char* payload;
|
||||
@@ -392,15 +384,15 @@ struct set_meta {
|
||||
};
|
||||
|
||||
static int set_meta_init(const char* str, struct set_meta* meta) {
|
||||
// The header must be followed by at least one payload character.
|
||||
if (!str[0] || !str[1] || !str[SET_HEADER_SIZE]) return -4;
|
||||
/* The header must be followed by at least one payload character. */
|
||||
if (!str[0] || !str[1] || !str[SET_HEADER_SIZE]) return -EPIPE;
|
||||
|
||||
int bpp = str[0] + SET_PARAM_CHAR_OFFSET - 'a';
|
||||
if (bpp < SET_BPP_MIN || bpp > SET_BPP_MAX) return -1;
|
||||
if (bpp < SET_BPP_MIN || bpp > SET_BPP_MAX) return -ERANGE;
|
||||
|
||||
int Mshift = str[1] + SET_PARAM_CHAR_OFFSET - 'a';
|
||||
if (Mshift < SET_MSHIFT_MIN || Mshift > SET_MSHIFT_MAX) return -2;
|
||||
if (Mshift >= bpp) return -3;
|
||||
if (Mshift < SET_MSHIFT_MIN || Mshift > SET_MSHIFT_MAX) return -ERANGE;
|
||||
if (Mshift >= bpp) return -EINVAL;
|
||||
|
||||
*meta = (struct set_meta){
|
||||
.str = str,
|
||||
@@ -420,7 +412,7 @@ static int set_meta_fini(struct set_meta* meta) {
|
||||
int bit_capacity = (int)payload_len * BASE62_VALUE_BITS;
|
||||
int value_capacity = bit_capacity / (meta->Mshift + 1);
|
||||
|
||||
if (value_capacity < 1) return -4;
|
||||
if (value_capacity < 1) return -EINVAL;
|
||||
|
||||
meta->len = len;
|
||||
meta->payload_len = payload_len;
|
||||
@@ -429,9 +421,9 @@ static int set_meta_fini(struct set_meta* meta) {
|
||||
|
||||
return 0;
|
||||
}
|
||||
// clang-format off
|
||||
/* clang-format off */
|
||||
__extension__ static const unsigned char char_to_num[UCHAR_MAX + 1] = {
|
||||
[0] = BASE62_END, // конец строки
|
||||
[0] = BASE62_END, /* end of string */
|
||||
[1 ... ('0' - 1)] = BASE62_INVALID,
|
||||
|
||||
['0'] = 0, ['1'] = 1, ['2'] = 2, ['3'] = 3, ['4'] = 4,
|
||||
@@ -457,10 +449,12 @@ __extension__ static const unsigned char char_to_num[UCHAR_MAX + 1] = {
|
||||
|
||||
[('z' + 1) ... UCHAR_MAX] = BASE62_INVALID,
|
||||
};
|
||||
// clang-format on
|
||||
/* clang-format on */
|
||||
|
||||
// Decode base62 and Golomb-Rice in one pass. Base62 is LSB-first; a Z escape contributes
|
||||
// BASE62_ESCAPE_CHUNK_BITS stream bits.
|
||||
/*
|
||||
* Decode base62 and Golomb-Rice in one pass. Base62 is LSB-first; a Z escape contributes
|
||||
* BASE62_ESCAPE_CHUNK_BITS stream bits.
|
||||
*/
|
||||
static inline int decode_chunk(const unsigned char** input, uint64_t* chunk, unsigned* width) {
|
||||
unsigned value = char_to_num[*(*input)++];
|
||||
|
||||
@@ -470,14 +464,14 @@ static inline int decode_chunk(const unsigned char** input, uint64_t* chunk, uns
|
||||
return 1;
|
||||
}
|
||||
if (value == BASE62_END) return 0;
|
||||
if (value == BASE62_INVALID) return -1;
|
||||
if (value == BASE62_INVALID) return -EINVAL;
|
||||
|
||||
unsigned escaped = char_to_num[*(*input)++];
|
||||
if (escaped == BASE62_END) return -2;
|
||||
if (escaped == BASE62_INVALID) return -3;
|
||||
if (escaped == BASE62_END) return -EPIPE;
|
||||
if (escaped == BASE62_INVALID) return -EINVAL;
|
||||
|
||||
unsigned high = escaped & BASE62_ESCAPE_HIGH_MASK;
|
||||
if (high == BASE62_ESCAPE_HIGH_MASK) return -4;
|
||||
if (high == BASE62_ESCAPE_HIGH_MASK) return -EILSEQ;
|
||||
|
||||
*chunk = (BASE62_ESCAPE_VALUE + (high >> BASE62_ESCAPE_BITS)) |
|
||||
((uint64_t)(escaped & BASE62_ESCAPE_LOW_MASK) << BASE62_VALUE_BITS);
|
||||
@@ -497,7 +491,7 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
|
||||
int count = 0;
|
||||
|
||||
for (;;) {
|
||||
// Unary quotient: zero bits terminated by one.
|
||||
/* Unary quotient: zero bits terminated by one. */
|
||||
for (;;) {
|
||||
if (filled == 0) {
|
||||
uint64_t chunk;
|
||||
@@ -505,7 +499,7 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
|
||||
int rc = decode_chunk(&input, &chunk, &width);
|
||||
|
||||
if (rc < 0) return rc;
|
||||
if (rc == 0) return q <= BASE62_MAX_PADDING_BITS ? count : -10;
|
||||
if (rc == 0) return q <= BASE62_MAX_PADDING_BITS ? count : -EINVAL;
|
||||
|
||||
bits = chunk;
|
||||
filled = width;
|
||||
@@ -531,13 +525,13 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Fixed-width remainder. At most 31+10 bits are held at once.
|
||||
/* Fixed-width remainder. At most 31+10 bits are held at once. */
|
||||
while (filled < Mshift) {
|
||||
uint64_t chunk;
|
||||
unsigned width;
|
||||
int rc = decode_chunk(&input, &chunk, &width);
|
||||
if (rc < 0) return rc;
|
||||
if (rc == 0) return -11;
|
||||
if (rc == 0) return -EINVAL;
|
||||
bits |= chunk << filled;
|
||||
filled += width;
|
||||
}
|
||||
@@ -552,13 +546,45 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
|
||||
}
|
||||
}
|
||||
|
||||
// Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates.
|
||||
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp);
|
||||
#ifdef ARSV_SET9_EXPORT
|
||||
/* Test-only bridge used to translate existing repository metadata without
|
||||
* reconstructing unavailable symbol names. The caller owns *hashes. */
|
||||
int arsv_set9_decode(const char* source, unsigned** hashes, size_t* count, unsigned* bpp) {
|
||||
if (!source || !hashes || !count || !bpp) return -EINVAL;
|
||||
|
||||
const char* str = source;
|
||||
if (strncmp(str, "set:", 4) == 0) str += 4;
|
||||
|
||||
struct set_meta meta;
|
||||
int rc = set_meta_init(str, &meta);
|
||||
if (rc < 0) return rc;
|
||||
rc = set_meta_fini(&meta);
|
||||
if (rc < 0) return rc;
|
||||
|
||||
unsigned* values = malloc((size_t)meta.value_capacity * sizeof(*values));
|
||||
if (!values) return -ENOMEM;
|
||||
|
||||
int decoded = decode_set(&meta, values);
|
||||
if (decoded <= 0) {
|
||||
free(values);
|
||||
return decoded < 0 ? decoded : -EINVAL;
|
||||
}
|
||||
|
||||
*hashes = values;
|
||||
*count = (size_t)decoded;
|
||||
*bpp = (unsigned)meta.bpp;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates. */
|
||||
static int downsample_set(const unsigned* hash_pt, size_t hash_cnt, unsigned* dest_pt,
|
||||
int target_bpp);
|
||||
|
||||
static inline unsigned cache_bucket(uint32_t fingerprint, int target_bpp) {
|
||||
uint32_t mixed = fingerprint ^ ((uint32_t)target_bpp * CACHE_TARGET_BPP_MULTIPLIER);
|
||||
uint32_t mixed = fingerprint ^ ((uint32_t)target_bpp * UINT32_C(0x85ebca6b));
|
||||
mixed ^= mixed >> 11;
|
||||
mixed *= CACHE_FINGERPRINT_MULTIPLIER;
|
||||
mixed *= UINT32_C(0x9e3779b1);
|
||||
mixed ^= mixed >> 16;
|
||||
|
||||
return mixed & (CACHE_BUCKETS - 1);
|
||||
@@ -609,7 +635,8 @@ static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigne
|
||||
return ent->cnt;
|
||||
}
|
||||
|
||||
if (set_meta_fini(meta) < 0) return -4;
|
||||
int meta_status = set_meta_fini(meta);
|
||||
if (meta_status < 0) return meta_status;
|
||||
|
||||
int len = (int)meta->len;
|
||||
int capacity = meta->value_capacity;
|
||||
@@ -630,7 +657,7 @@ static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigne
|
||||
unsigned* destination = temporary;
|
||||
|
||||
for (int bpp = meta->bpp - 1; bpp >= target_bpp; --bpp) {
|
||||
cnt = downsample_set(cnt, current, destination, bpp);
|
||||
cnt = downsample_set(current, (size_t)cnt, destination, bpp);
|
||||
unsigned* swap = current;
|
||||
current = destination;
|
||||
destination = swap;
|
||||
@@ -678,15 +705,16 @@ static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigne
|
||||
return cnt;
|
||||
}
|
||||
|
||||
// Reduce a set of (bpp + 1) values to a set of bpp values.
|
||||
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp) {
|
||||
unsigned mask = (1u << bpp) - 1;
|
||||
/* Reduce a set of (bpp + 1) values to a set of bpp values. */
|
||||
static int downsample_set(const unsigned* hash_pt, size_t hash_cnt, unsigned* dest_pt,
|
||||
int target_bpp) {
|
||||
unsigned mask = (1u << target_bpp) - 1;
|
||||
|
||||
// find the first element with high bit set
|
||||
int l = 0;
|
||||
int u = cnt;
|
||||
/* find the first element with high bit set */
|
||||
size_t l = 0;
|
||||
size_t u = hash_cnt;
|
||||
while (l < u) {
|
||||
int i = (l + u) / 2;
|
||||
size_t i = (l + u) / 2;
|
||||
|
||||
if (hash_pt[i] <= mask) {
|
||||
l = i + 1;
|
||||
@@ -695,33 +723,33 @@ static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int
|
||||
}
|
||||
}
|
||||
|
||||
// initialize parts
|
||||
const unsigned* ds_start = ds_pt;
|
||||
/* initialize parts */
|
||||
const unsigned* ds_start = dest_pt;
|
||||
const unsigned *v1 = hash_pt + 0, *v1_end = hash_pt + u;
|
||||
const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + cnt;
|
||||
const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + hash_cnt;
|
||||
|
||||
// merge v1 and v2 into w
|
||||
/* merge v1 and v2 into w */
|
||||
if (v1 < v1_end && v2 < v2_end) {
|
||||
unsigned v1_val = *v1;
|
||||
unsigned v2_val = *v2 & mask;
|
||||
|
||||
while (1) {
|
||||
if (v1_val < v2_val) {
|
||||
*ds_pt++ = v1_val;
|
||||
*dest_pt++ = v1_val;
|
||||
v1++;
|
||||
|
||||
if (v1 == v1_end) break;
|
||||
|
||||
v1_val = *v1;
|
||||
} else if (v2_val < v1_val) {
|
||||
*ds_pt++ = v2_val;
|
||||
*dest_pt++ = v2_val;
|
||||
v2++;
|
||||
|
||||
if (v2 == v2_end) break;
|
||||
|
||||
v2_val = *v2 & mask;
|
||||
} else {
|
||||
*ds_pt++ = v1_val;
|
||||
*dest_pt++ = v1_val;
|
||||
v1++;
|
||||
v2++;
|
||||
|
||||
@@ -734,11 +762,11 @@ static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int
|
||||
}
|
||||
}
|
||||
|
||||
// append what's left
|
||||
while (v1 < v1_end) *ds_pt++ = *v1++;
|
||||
while (v2 < v2_end) *ds_pt++ = *v2++ & mask;
|
||||
/* append what's left */
|
||||
while (v1 < v1_end) *dest_pt++ = *v1++;
|
||||
while (v2 < v2_end) *dest_pt++ = *v2++ & mask;
|
||||
|
||||
return (int)(ds_pt - ds_start);
|
||||
return (int)(dest_pt - ds_start);
|
||||
}
|
||||
|
||||
static const unsigned* step_lower_bound(const unsigned* first, const unsigned* last, unsigned value,
|
||||
@@ -780,8 +808,10 @@ static int sorted_subset(const unsigned* small, size_t small_count, const unsign
|
||||
const unsigned* const large_end = large + large_count;
|
||||
size_t jump = large_count / small_count;
|
||||
|
||||
// Dense sets favor a conventional merge; sparse sets skip by approximately
|
||||
// the mean distance between required values and then refine the last block.
|
||||
/*
|
||||
* Dense sets favor a conventional merge; sparse sets skip by approximately
|
||||
* the mean distance between required values and then refine the last block.
|
||||
*/
|
||||
if (jump < 4) {
|
||||
while (small < small_end) {
|
||||
unsigned value = *small++;
|
||||
@@ -803,7 +833,7 @@ static int sorted_subset(const unsigned* small, size_t small_count, const unsign
|
||||
return 1;
|
||||
}
|
||||
|
||||
// main API routine
|
||||
/* main API routine */
|
||||
int rpmsetcmp(const char* str1, const char* str2) {
|
||||
if (strncmp(str1, "set:", 4) == 0) str1 += 4;
|
||||
if (strncmp(str2, "set:", 4) == 0) str2 += 4;
|
||||
@@ -816,22 +846,26 @@ int rpmsetcmp(const char* str1, const char* str2) {
|
||||
|
||||
int target_bpp = meta1.bpp < meta2.bpp ? meta1.bpp : meta2.bpp;
|
||||
|
||||
// Decode and cache the first operand at the comparison precision.
|
||||
/* Decode and cache the first operand at the comparison precision. */
|
||||
const unsigned* hash_arr1 = NULL;
|
||||
int cnt1 = cache_decode_set(&meta1, target_bpp, &hash_arr1, 0);
|
||||
if (cnt1 < 0) return -3;
|
||||
|
||||
// Metadata for both operands has already been validated, and set1 has been
|
||||
// decoded, so this preserves set8's malformed-input error precedence.
|
||||
/*
|
||||
* Metadata for both operands has already been validated, and set1 has been
|
||||
* decoded, so this preserves set8's malformed-input error precedence.
|
||||
*/
|
||||
if (str1 == str2 || strcmp(str1, str2) == 0) return 0;
|
||||
|
||||
// Requirement sets are frequently reused by dependency solvers too.
|
||||
/* Requirement sets are frequently reused by dependency solvers too. */
|
||||
const unsigned* hash_arr2 = NULL;
|
||||
int cnt2 = cache_decode_set(&meta2, target_bpp, &hash_arr2, 1);
|
||||
if (cnt2 < 0) return -4;
|
||||
|
||||
// Cardinality determines which strict-inclusion result is even possible.
|
||||
// For equal cardinalities, sorted unique sets are equal iff their bytes match.
|
||||
/*
|
||||
* Cardinality determines which strict-inclusion result is even possible.
|
||||
* For equal cardinalities, sorted unique sets are equal iff their bytes match.
|
||||
*/
|
||||
if (cnt1 == cnt2) {
|
||||
return memcmp(hash_arr1, hash_arr2, (size_t)cnt1 * sizeof(*hash_arr1)) == 0 ? 0 : -2;
|
||||
}
|
||||
@@ -841,8 +875,6 @@ int rpmsetcmp(const char* str1, const char* str2) {
|
||||
return sorted_subset(hash_arr1, (size_t)cnt1, hash_arr2, (size_t)cnt2) ? -1 : -2;
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
#ifdef SELF_TEST
|
||||
static void test_hash(void) {
|
||||
assert(hash("") == UINT32_C(0xecd739e9));
|
||||
@@ -924,7 +956,7 @@ static void test_encode_decode(void) {
|
||||
uint64_t mask = bpp < 32 ? (UINT64_C(1) << bpp) - 1 : UINT32_MAX;
|
||||
unsigned values[VALUE_COUNT];
|
||||
|
||||
// uniform distribution of values
|
||||
/* uniform distribution of values */
|
||||
for (int i = 0; i < VALUE_COUNT; ++i) {
|
||||
values[i] = (unsigned)(((uint64_t)(i + 1) * mask) / (VALUE_COUNT + 1));
|
||||
}
|
||||
@@ -971,10 +1003,10 @@ static void test_metadata_and_chunks(void) {
|
||||
{.input = "Zg", .rc = 1, .chunk = 62, .width = 10},
|
||||
{.input = "Zw", .rc = 1, .chunk = 63, .width = 10},
|
||||
{.input = "", .rc = 0, .chunk = 0, .width = 0},
|
||||
{.input = "!", .rc = -1, .chunk = 0, .width = 0},
|
||||
{.input = "Z", .rc = -2, .chunk = 0, .width = 0},
|
||||
{.input = "Z!", .rc = -3, .chunk = 0, .width = 0},
|
||||
{.input = "ZM", .rc = -4, .chunk = 0, .width = 0},
|
||||
{.input = "!", .rc = -EINVAL, .chunk = 0, .width = 0},
|
||||
{.input = "Z", .rc = -EPIPE, .chunk = 0, .width = 0},
|
||||
{.input = "Z!", .rc = -EINVAL, .chunk = 0, .width = 0},
|
||||
{.input = "ZM", .rc = -EILSEQ, .chunk = 0, .width = 0},
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(*cases); ++i) {
|
||||
@@ -991,17 +1023,17 @@ static void test_metadata_and_chunks(void) {
|
||||
}
|
||||
|
||||
struct set_meta meta;
|
||||
assert(set_meta_init("", &meta) == -4); // too short
|
||||
assert(set_meta_init("da", &meta) == -4);
|
||||
assert(set_meta_init("ca0", &meta) == -1); // incorrect bpp
|
||||
assert(set_meta_init("{a0", &meta) == -1);
|
||||
assert(set_meta_init("d`0", &meta) == -2); // incorrect Mshift
|
||||
assert(set_meta_init("dz0", &meta) == -2);
|
||||
assert(set_meta_init("dd0", &meta) == -3); // Mshift == bpp
|
||||
assert(set_meta_init("", &meta) == -EPIPE); /* too short */
|
||||
assert(set_meta_init("da", &meta) == -EPIPE);
|
||||
assert(set_meta_init("ca0", &meta) == -ERANGE); /* incorrect bpp */
|
||||
assert(set_meta_init("{a0", &meta) == -ERANGE);
|
||||
assert(set_meta_init("d`0", &meta) == -ERANGE); /* incorrect Mshift */
|
||||
assert(set_meta_init("dz0", &meta) == -ERANGE);
|
||||
assert(set_meta_init("dd0", &meta) == -EINVAL); /* Mshift == bpp */
|
||||
assert(set_meta_init("da0", &meta) == 0);
|
||||
assert(set_meta_fini(&meta) == -4); // not enough data
|
||||
assert(set_meta_fini(&meta) == -EINVAL); /* not enough data */
|
||||
assert(set_meta_init("da00", &meta) == 0);
|
||||
assert(set_meta_fini(&meta) == 0); // ok
|
||||
assert(set_meta_fini(&meta) == 0); /* ok */
|
||||
assert(meta.len == 4);
|
||||
assert(meta.payload_len == 2);
|
||||
assert(meta.bit_capacity == 12);
|
||||
@@ -1015,19 +1047,19 @@ static void test_downsample(void) {
|
||||
const unsigned mixed_expected[] = {0, 2, 5, 7};
|
||||
|
||||
unsigned result[sizeof(mixed) / sizeof(*mixed)];
|
||||
int count = downsample_set((int)(sizeof(mixed) / sizeof(*mixed)), mixed, result, 3);
|
||||
int count = downsample_set(mixed, sizeof(mixed) / sizeof(*mixed), result, 3);
|
||||
assert(count == (int)(sizeof(mixed_expected) / sizeof(*mixed_expected)));
|
||||
assert(memcmp(result, mixed_expected, sizeof(mixed_expected)) == 0);
|
||||
|
||||
const unsigned low[] = {1, 2, 3};
|
||||
count = downsample_set((int)(sizeof(low) / sizeof(*low)), low, result,
|
||||
3); // sizeof(result) >= sizeof(low)
|
||||
count = downsample_set(low, sizeof(low) / sizeof(*low), result,
|
||||
3); /* sizeof(result) >= sizeof(low) */
|
||||
assert(count == (int)(sizeof(low) / sizeof(*low)));
|
||||
assert(memcmp(result, low, sizeof(low)) == 0);
|
||||
|
||||
const unsigned high[] = {8, 9};
|
||||
const unsigned high_expected[] = {0, 1};
|
||||
count = downsample_set((int)(sizeof(high) / sizeof(*high)), high, result, 3);
|
||||
count = downsample_set(high, sizeof(high) / sizeof(*high), result, 3);
|
||||
assert(count == (int)(sizeof(high_expected) / sizeof(*high_expected)));
|
||||
assert(memcmp(result, high_expected, sizeof(high_expected)) == 0);
|
||||
|
||||
@@ -1039,7 +1071,7 @@ static void test_subset(void) {
|
||||
const unsigned dense_small[] = {2, 4, 6};
|
||||
const unsigned dense_missing[] = {2, 4, 8};
|
||||
assert(sorted_subset(dense_small, 3, dense_large, 7) == 1);
|
||||
assert(sorted_subset(dense_missing, 3, dense_large, 7) == 0); // 0 - incompatible
|
||||
assert(sorted_subset(dense_missing, 3, dense_large, 7) == 0); /* 0 - incompatible */
|
||||
|
||||
unsigned sparse_large[64];
|
||||
for (size_t i = 0; i < sizeof(sparse_large) / sizeof(*sparse_large); ++i) {
|
||||
@@ -1076,7 +1108,7 @@ static void test_cache(void) {
|
||||
assert(second == first);
|
||||
|
||||
unsigned downsampled[value_count];
|
||||
int downsampled_count = downsample_set(value_count, values, downsampled, 15);
|
||||
int downsampled_count = downsample_set(values, (size_t)value_count, downsampled, 15);
|
||||
assert(set_meta_init(encoded, &meta) == 0);
|
||||
assert(cache_decode_set(&meta, 15, &second, 0) == downsampled_count);
|
||||
assert(memcmp(second, downsampled, (size_t)downsampled_count * sizeof(*downsampled)) == 0);
|
||||
|
||||
Executable
+936
@@ -0,0 +1,936 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
export LC_ALL=C
|
||||
|
||||
# A/B benchmark:
|
||||
# set9: librpm из Sisyphus с reimplement/set9.c и исходными pkglist Sisyphus;
|
||||
# d1: librpm из того же commit с direct_hash/hash_set.c и теми же pkglist,
|
||||
# заранее преобразованными run_sisyphus_pkglist.py в формат set:D1.
|
||||
#
|
||||
# Все измеряемые команды работают без сети. Конвертация, сборка и gencaches
|
||||
# выполняются до таймера. Порядок замеров в каждом раунде: set9/d1/d1/set9.
|
||||
|
||||
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
||||
REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd)
|
||||
SET9_C=${SET9_C:-$REPO_ROOT/reimplement/set9.c}
|
||||
D1_C=${D1_C:-$REPO_ROOT/new_version/direct_hash/hash_set.c}
|
||||
PKGLIST_CONVERTER=${PKGLIST_CONVERTER:-$REPO_ROOT/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py}
|
||||
SET_REWRITER_C=$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c
|
||||
SET_COMPAT_H=$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h
|
||||
ORCHESTRATOR=$(realpath -e -- "${BASH_SOURCE[0]}")
|
||||
|
||||
WORK_ROOT=${WORK_ROOT:-$HOME/sisyphus-set9-d1-bench}
|
||||
RESULT_DIR=${RESULT_DIR:-$WORK_ROOT/results}
|
||||
SISYPHUS_MIRROR=${SISYPHUS_MIRROR:-https://ftp.altlinux.org/pub/distributions/ALTLinux}
|
||||
RPM_GIT=${RPM_GIT:-https://git.altlinux.org/gears/r/rpm.git}
|
||||
RPM_BRANCH=${RPM_BRANCH:-sisyphus}
|
||||
PACKAGER=${PACKAGER:-krosh <gudovdo@my.msu.ru>}
|
||||
CPU=${CPU:-0}
|
||||
ROUNDS=${ROUNDS:-2}
|
||||
RESET_WORK=${RESET_WORK:-1} # 0 — продолжить подготовку/сборки, 1 — начать заново.
|
||||
OPERATIONS=${OPERATIONS:-unmet install-rpm-build install-openuds-server install-password-store}
|
||||
APT_GET=${APT_GET:-/usr/lib/apt/apt-get}
|
||||
APT_CACHE=${APT_CACHE:-$(command -v apt-cache 2>/dev/null || true)}
|
||||
|
||||
usage()
|
||||
{
|
||||
cat <<'EOF'
|
||||
Usage: scripts/run-sisyphus-set9-d1-bench.sh
|
||||
|
||||
The script has no positional arguments. Configuration is passed through env:
|
||||
CPU=2 ROUNDS=3 RESET_WORK=1 ./scripts/run-sisyphus-set9-d1-bench.sh
|
||||
RESET_WORK=0 OPERATIONS='unmet check' ./scripts/run-sisyphus-set9-d1-bench.sh
|
||||
|
||||
Main variables:
|
||||
WORK_ROOT, RESULT_DIR, SISYPHUS_MIRROR, RPM_GIT, RPM_BRANCH, PACKAGER,
|
||||
SET9_C, D1_C, PKGLIST_CONVERTER, CPU, ROUNDS, RESET_WORK, OPERATIONS.
|
||||
|
||||
RESET_WORK=1 removes WORK_ROOT after running hsh --cleanup-only for old hasher
|
||||
workdirs. Timed runs never update repositories and never install packages.
|
||||
Snapshot preparation downloads x86_64 and noarch metadata from SISYPHUS_MIRROR
|
||||
into an isolated APT directory under WORK_ROOT; host APT configuration is untouched.
|
||||
EOF
|
||||
}
|
||||
|
||||
fail()
|
||||
{
|
||||
printf 'error: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
safe_remove_work_root()
|
||||
{
|
||||
local marker="$WORK_ROOT/.arsv-sisyphus-set-bench-root"
|
||||
|
||||
[[ -n $WORK_ROOT && $WORK_ROOT == /* && $WORK_ROOT != / &&
|
||||
$WORK_ROOT != "$HOME_REAL" && $HOME_REAL != "$WORK_ROOT/"* ]] ||
|
||||
fail "unsafe WORK_ROOT for removal: $WORK_ROOT"
|
||||
[[ $WORK_ROOT != "$REPO_ROOT" && $REPO_ROOT != "$WORK_ROOT/"* &&
|
||||
$WORK_ROOT != "$REPO_ROOT/"* && $WORK_ROOT != "$CWD_REAL" &&
|
||||
$CWD_REAL != "$WORK_ROOT/"* && $WORK_ROOT != "$CWD_REAL/"* ]] ||
|
||||
fail "WORK_ROOT overlaps the source repository or current directory: $WORK_ROOT"
|
||||
[[ ! -e $WORK_ROOT || (-f $marker && ! -L $marker) ]] ||
|
||||
fail "refusing to remove unmarked WORK_ROOT: $WORK_ROOT"
|
||||
if [[ -f $marker ]]; then
|
||||
grep -Fx 'ARSV Sisyphus set9/D1 benchmark work root' "$marker" >/dev/null ||
|
||||
fail "invalid WORK_ROOT ownership marker: $marker"
|
||||
fi
|
||||
}
|
||||
|
||||
canonical_work_subdir()
|
||||
{
|
||||
local path=$1 label=$2 resolved
|
||||
[[ ! -L $path ]] || fail "$label must not be a symlink: $path"
|
||||
resolved=$(realpath -e -- "$path") || fail "$label does not resolve: $path"
|
||||
[[ $resolved == "$WORK_ROOT/"* ]] ||
|
||||
fail "$label resolves outside WORK_ROOT: $path -> $resolved"
|
||||
printf '%s' "$resolved"
|
||||
}
|
||||
|
||||
ensure_work_subdir()
|
||||
{
|
||||
local path=$1 label=$2 resolved
|
||||
[[ ! -L $path ]] || fail "$label must not be a symlink: $path"
|
||||
resolved=$(realpath -m -- "$path")
|
||||
[[ $resolved == "$WORK_ROOT/"* ]] ||
|
||||
fail "$label resolves outside WORK_ROOT: $path -> $resolved"
|
||||
mkdir -p -- "$path"
|
||||
canonical_work_subdir "$path" "$label" >/dev/null
|
||||
}
|
||||
|
||||
write_snapshot_fingerprint()
|
||||
{
|
||||
{
|
||||
printf 'sisyphus_mirror=%s\n' "$SISYPHUS_MIRROR"
|
||||
printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')"
|
||||
printf 'converter=%s\n' "$(sha256sum "$PKGLIST_CONVERTER" | awk '{print $1}')"
|
||||
printf 'set9=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
|
||||
printf 'd1=%s\n' "$(sha256sum "$D1_C" | awk '{print $1}')"
|
||||
printf 'rewrite=%s\n' "$(sha256sum "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" | awk '{print $1}')"
|
||||
printf 'compat=%s\n' "$(sha256sum "$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h" | awk '{print $1}')"
|
||||
[[ ! -f $SNAPSHOT/devel-input-fingerprint.txt ]] ||
|
||||
cat "$SNAPSHOT/devel-input-fingerprint.txt"
|
||||
write_runtime_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
write_runtime_fingerprint()
|
||||
{
|
||||
local rpm_libdir library resolved owner
|
||||
rpm_libdir=$(rpm --eval '%{_libdir}')
|
||||
for library in librpm.so.7 librpmio.so.7; do
|
||||
resolved=$(realpath -e "$rpm_libdir/$library") ||
|
||||
fail "installed $library not found"
|
||||
owner=$(rpm -qf --qf '%{NAME}|%{SOURCERPM}|%{DISTTAG}' "$resolved") ||
|
||||
fail "cannot identify package owning $resolved"
|
||||
printf 'runtime_%s_owner=%s\n' "$library" "$owner"
|
||||
printf 'runtime_%s_sha256=%s\n' "$library" \
|
||||
"$(sha256sum "$resolved" | awk '{print $1}')"
|
||||
done
|
||||
}
|
||||
|
||||
validate_snapshot_reuse()
|
||||
{
|
||||
local current
|
||||
[[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt &&
|
||||
-f $SNAPSHOT/devel-input-fingerprint.txt ]] || return 1
|
||||
current=$(mktemp)
|
||||
write_snapshot_fingerprint >"$current"
|
||||
if ! cmp -s "$current" "$SNAPSHOT/input-fingerprint.txt"; then
|
||||
rm -f "$current"
|
||||
fail 'snapshot inputs changed; use RESET_WORK=1'
|
||||
fi
|
||||
rm -f "$current"
|
||||
return 0
|
||||
}
|
||||
|
||||
write_source_fingerprint()
|
||||
{
|
||||
{
|
||||
printf 'rpm_git=%s\n' "$RPM_GIT"
|
||||
printf 'rpm_branch=%s\n' "$RPM_BRANCH"
|
||||
printf 'rpm_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
|
||||
}
|
||||
}
|
||||
|
||||
validate_source_reuse()
|
||||
{
|
||||
local current
|
||||
[[ -f $SOURCE_FINGERPRINT ]] ||
|
||||
fail 'source fingerprint missing; use RESET_WORK=1'
|
||||
[[ -z $(git -C "$SOURCE_BASE" status --porcelain) ]] ||
|
||||
fail 'RPM base source has local changes; use RESET_WORK=1'
|
||||
current=$(mktemp)
|
||||
write_source_fingerprint >"$current"
|
||||
if ! cmp -s "$current" "$SOURCE_FINGERPRINT"; then
|
||||
rm -f "$current"
|
||||
fail 'RPM source URL, branch, or commit changed; use RESET_WORK=1'
|
||||
fi
|
||||
rm -f "$current"
|
||||
}
|
||||
|
||||
prepare_spec()
|
||||
{
|
||||
local spec=$1 suffix=$2 version release new_release date
|
||||
|
||||
version=$(sed -n 's/^Version:[[:space:]]*//p' "$spec" | sed -n '1p')
|
||||
release=$(sed -n 's/^Release:[[:space:]]*//p' "$spec" | sed -n '1p')
|
||||
[[ -n $version && -n $release ]] || fail "cannot read Version/Release from $spec"
|
||||
|
||||
new_release="$release.$suffix"
|
||||
sed -i "0,/^Release:[[:space:]]*$release$/s//Release: $new_release/" "$spec"
|
||||
|
||||
date=$(date '+%a %b %d %Y')
|
||||
sed -i "/^%changelog/a\\
|
||||
* $date $PACKAGER $version-$new_release\\
|
||||
- Local Sisyphus set format benchmark build.\\
|
||||
" "$spec"
|
||||
}
|
||||
|
||||
prepare_d1_build_corpus()
|
||||
{
|
||||
local source=$1 spec helper_dir
|
||||
spec=$source/alt/rpm.spec
|
||||
helper_dir=$source/alt/arsv-d1-build
|
||||
mkdir -p "$helper_dir"
|
||||
cp "$SET9_C" "$helper_dir/set9.c"
|
||||
cp "$SET_REWRITER_C" "$helper_dir/rewrite_sisyphus_pkglist.c"
|
||||
cp "$SET_COMPAT_H" "$helper_dir/newset_compat.h"
|
||||
|
||||
python3 - "$spec" <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
spec = Path(sys.argv[1])
|
||||
text = spec.read_text()
|
||||
needle = "join -o 1.3,2.3 P R |shuf >setcmp-data\n"
|
||||
replacement = needle + r'''# The buildroot RPM database contains legacy set9 values. This D1 build
|
||||
# preserves the same relation corpus, but converts both operands before the
|
||||
# format-specific setcmp/profile checks instead of feeding incompatible input.
|
||||
mkdir -p arsv-d1-compat
|
||||
touch arsv-d1-compat/rpmlib.h arsv-d1-compat/system.h arsv-d1-compat/set.h
|
||||
%__cc %optflags -std=gnu11 -Wall -Wextra -Werror -D_GNU_SOURCE -DARSV_SET9_EXPORT \
|
||||
-I arsv-d1-compat -include alt/arsv-d1-build/newset_compat.h \
|
||||
alt/arsv-d1-build/set9.c alt/arsv-d1-build/rewrite_sisyphus_pkglist.c \
|
||||
-o arsv-convert-set
|
||||
while read -r set1 set2; do
|
||||
d1_set1=$(./arsv-convert-set --convert-set "$set1")
|
||||
d1_set2=$(./arsv-convert-set --convert-set "$set2")
|
||||
printf '%%s %%s\n' "$d1_set1" "$d1_set2"
|
||||
done <setcmp-data >setcmp-data.d1
|
||||
test "$(wc -l <setcmp-data.d1)" -eq "$(wc -l <setcmp-data)"
|
||||
mv setcmp-data.d1 setcmp-data
|
||||
'''
|
||||
if text.count(needle) != 1:
|
||||
raise SystemExit(f"expected exactly one setcmp corpus creation in {spec}")
|
||||
spec.write_text(text.replace(needle, replacement))
|
||||
PY
|
||||
git -C "$source" add alt/arsv-d1-build
|
||||
}
|
||||
|
||||
apt_options()
|
||||
{
|
||||
local variant=$1
|
||||
APT_OPTIONS=(
|
||||
-o 'Debug::NoLocking=true'
|
||||
-o 'Dir::Etc::main=-'
|
||||
-o 'Dir::Etc::parts=-'
|
||||
-o "Dir::Etc::sourcelist=$SNAPSHOT/etc-apt/sources.list"
|
||||
-o "Dir::Etc::sourceparts=$SNAPSHOT/etc-apt/sources.list.d"
|
||||
-o 'Dir::Etc::preferences=-'
|
||||
-o 'Dir::Etc::preferencesparts=-'
|
||||
-o "Dir::State::lists=$variant/apt/lists/"
|
||||
-o "Dir::State::status=$COMMON/status"
|
||||
-o "Dir::Cache=$variant/apt/cache/"
|
||||
-o "Dir::Cache::archives=$variant/apt/cache/archives"
|
||||
-o "Dir::Cache::pkgcache=$variant/apt/cache/pkgcache.bin"
|
||||
-o "Dir::Cache::srcpkgcache=$variant/apt/cache/srcpkgcache.bin"
|
||||
-o "RPM::RootDir=$COMMON/root"
|
||||
)
|
||||
}
|
||||
|
||||
operation_command()
|
||||
{
|
||||
local operation=$1 variant=$2
|
||||
apt_options "$variant"
|
||||
|
||||
case $operation in
|
||||
unmet)
|
||||
COMMAND=("$APT_CACHE" -q "${APT_OPTIONS[@]}" unmet)
|
||||
;;
|
||||
check)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s check)
|
||||
;;
|
||||
autoremove)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s autoremove)
|
||||
;;
|
||||
install-rpm-build)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s install rpm-build)
|
||||
;;
|
||||
install-openuds-server)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s install openuds-server)
|
||||
;;
|
||||
install-password-store)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s install password-store)
|
||||
;;
|
||||
upgrade)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -o APT::Get::EnableUpgrade=true -s upgrade)
|
||||
;;
|
||||
dist-upgrade)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s dist-upgrade)
|
||||
;;
|
||||
*)
|
||||
fail "unknown operation: $operation"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
variant_dir()
|
||||
{
|
||||
case $1 in
|
||||
set9) printf '%s' "$SET9_VARIANT" ;;
|
||||
d1) printf '%s' "$D1_VARIANT" ;;
|
||||
*) fail "unknown variant: $1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
variant_libdir()
|
||||
{
|
||||
local variant
|
||||
variant=$(variant_dir "$1")
|
||||
printf '%s' "$variant/lib/usr/lib64"
|
||||
}
|
||||
|
||||
prepare_command()
|
||||
{
|
||||
local operation=$1 variant_name=$2 variant
|
||||
variant=$(variant_dir "$variant_name")
|
||||
RUN_LIBDIR=$(variant_libdir "$variant_name")
|
||||
operation_command "$operation" "$variant"
|
||||
}
|
||||
|
||||
execute_command()
|
||||
{
|
||||
local stdout=$1 stderr=$2 status
|
||||
|
||||
if env LC_ALL=C APT_CONFIG="$COMMON/apt.conf" LD_LIBRARY_PATH="$RUN_LIBDIR" \
|
||||
taskset -c "$CPU" "${COMMAND[@]}" >"$stdout" 2>"$stderr"; then
|
||||
status=0
|
||||
else
|
||||
status=$?
|
||||
fi
|
||||
printf '%s' "$status"
|
||||
}
|
||||
|
||||
run_once()
|
||||
{
|
||||
local operation=$1 variant_name=$2 sequence=$3 sample=$4
|
||||
local variant raw stdout stderr start end status elapsed
|
||||
local stdout_sha stderr_sha stdout_bytes stderr_bytes
|
||||
variant=$(variant_dir "$variant_name")
|
||||
raw="$RESULT_DIR/raw/$operation/$variant_name"
|
||||
mkdir -p "$raw"
|
||||
stdout="$raw/$sample.stdout"
|
||||
stderr="$raw/$sample.stderr"
|
||||
|
||||
prepare_command "$operation" "$variant_name"
|
||||
start=$(date +%s%N)
|
||||
status=$(execute_command "$stdout" "$stderr")
|
||||
end=$(date +%s%N)
|
||||
elapsed=$(awk -v start="$start" -v end="$end" \
|
||||
'BEGIN { printf "%.6f", (end - start) / 1000000000 }')
|
||||
|
||||
stdout_sha=$(sha256sum "$stdout" | awk '{print $1}')
|
||||
stderr_sha=$(sha256sum "$stderr" | awk '{print $1}')
|
||||
stdout_bytes=$(stat -c %s "$stdout")
|
||||
stderr_bytes=$(stat -c %s "$stderr")
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$operation" "$sequence" "$variant_name" "$sample" "$elapsed" "$status" \
|
||||
"$stdout_sha" "$stdout_bytes" "$stderr_sha" "$stderr_bytes" >>"$RAW_RESULTS"
|
||||
printf '%-28s sequence=%-2s %-4s sample=%-2s %ss status=%s\n' \
|
||||
"$operation" "$sequence" "$variant_name" "$sample" "$elapsed" "$status"
|
||||
}
|
||||
|
||||
prepare_snapshot()
|
||||
{
|
||||
local converter_output devel_cache devel_root rpm_libdir rpm_library rpmio_library
|
||||
local devel_source runtime_source rpmfile
|
||||
local -a update_options
|
||||
local -a rpm_devel_rpms popt_devel_rpms
|
||||
validate_snapshot_reuse && return
|
||||
|
||||
printf '\n===== Preparing one immutable Sisyphus metadata snapshot =====\n'
|
||||
rm -rf "$SNAPSHOT"
|
||||
mkdir -p "$SNAPSHOT/lists/partial" \
|
||||
"$SNAPSHOT/etc-apt/sources.list.d" \
|
||||
"$SNAPSHOT/download-cache/archives/partial"
|
||||
printf '%s\n' \
|
||||
"rpm [alt] $SISYPHUS_MIRROR Sisyphus/x86_64 classic" \
|
||||
"rpm [alt] $SISYPHUS_MIRROR Sisyphus/noarch classic" \
|
||||
>"$SNAPSHOT/etc-apt/sources.list"
|
||||
|
||||
update_options=(
|
||||
-o 'Dir::Etc::main=-'
|
||||
-o 'Dir::Etc::parts=-'
|
||||
-o "Dir::Etc::sourcelist=$SNAPSHOT/etc-apt/sources.list"
|
||||
-o "Dir::Etc::sourceparts=$SNAPSHOT/etc-apt/sources.list.d"
|
||||
-o 'Dir::Etc::preferences=-'
|
||||
-o 'Dir::Etc::preferencesparts=-'
|
||||
-o "Dir::State::lists=$SNAPSHOT/lists"
|
||||
-o "Dir::State::status=$COMMON/status"
|
||||
-o "Dir::Cache=$SNAPSHOT/download-cache"
|
||||
-o "Dir::Cache::archives=$SNAPSHOT/download-cache/archives"
|
||||
-o "Dir::Cache::pkgcache=$SNAPSHOT/download-cache/pkgcache.bin"
|
||||
-o "Dir::Cache::srcpkgcache=$SNAPSHOT/download-cache/srcpkgcache.bin"
|
||||
-o "RPM::RootDir=$COMMON/root"
|
||||
)
|
||||
APT_CONFIG="$COMMON/apt.conf" "$APT_GET" -qq \
|
||||
"${update_options[@]}" update
|
||||
|
||||
# The converter needs C headers, but the benchmark host intentionally need
|
||||
# not have development packages installed. Download the p11 packages that
|
||||
# match the host's installed librpm into a private cache and extract only
|
||||
# their headers; no system package is installed or changed.
|
||||
devel_cache="$SNAPSHOT/devel-cache"
|
||||
devel_root="$SNAPSHOT/devel-root"
|
||||
mkdir -p "$devel_cache/archives/partial" "$devel_root"
|
||||
env -u APT_CONFIG "$APT_GET" -qq -y -d \
|
||||
-o "Dir::Cache=$devel_cache" \
|
||||
-o "Dir::Cache::archives=$devel_cache/archives" \
|
||||
-o "Dir::Cache::pkgcache=$devel_cache/pkgcache.bin" \
|
||||
-o "Dir::Cache::srcpkgcache=$devel_cache/srcpkgcache.bin" \
|
||||
install librpm-devel
|
||||
shopt -s nullglob
|
||||
rpm_devel_rpms=("$devel_cache"/archives/librpm-devel_*.rpm)
|
||||
popt_devel_rpms=("$devel_cache"/archives/libpopt-devel_*.rpm)
|
||||
shopt -u nullglob
|
||||
((${#rpm_devel_rpms[@]} == 1)) ||
|
||||
fail "expected one downloaded librpm-devel RPM, got ${#rpm_devel_rpms[@]}"
|
||||
((${#popt_devel_rpms[@]} == 1)) ||
|
||||
fail "expected one downloaded libpopt-devel RPM, got ${#popt_devel_rpms[@]}"
|
||||
devel_source=$(rpm -qp --qf '%{SOURCERPM}|%{DISTTAG}' "${rpm_devel_rpms[0]}") ||
|
||||
fail 'cannot identify downloaded librpm-devel source package'
|
||||
for rpmfile in "${rpm_devel_rpms[@]}" "${popt_devel_rpms[@]}"; do
|
||||
(cd "$devel_root" && rpm2cpio "$rpmfile" | cpio -idm --quiet './usr/include/*')
|
||||
done
|
||||
[[ -f $devel_root/usr/include/rpm/header.h && -f $devel_root/usr/include/popt.h ]] ||
|
||||
fail 'failed to extract RPM development headers'
|
||||
rpm_libdir=$(rpm --eval '%{_libdir}')
|
||||
rpm_library=$(realpath -e "$rpm_libdir/librpm.so.7") ||
|
||||
fail 'installed librpm.so.7 not found'
|
||||
rpmio_library=$(realpath -e "$rpm_libdir/librpmio.so.7") ||
|
||||
fail 'installed librpmio.so.7 not found'
|
||||
for rpmfile in "$rpm_library" "$rpmio_library"; do
|
||||
runtime_source=$(rpm -qf --qf '%{SOURCERPM}|%{DISTTAG}' "$rpmfile") ||
|
||||
fail "cannot identify runtime package owning $rpmfile"
|
||||
[[ $runtime_source == "$devel_source" ]] ||
|
||||
fail "downloaded librpm-devel ($devel_source) does not match $rpmfile ($runtime_source)"
|
||||
done
|
||||
{
|
||||
printf 'devel_librpm_identity=%s|%s\n' \
|
||||
"$(rpm -qp --qf '%{NAME}|%{EVR}|%{DISTTAG}|%{ARCH}' "${rpm_devel_rpms[0]}")" \
|
||||
"$devel_source"
|
||||
printf 'devel_librpm_sha256=%s\n' \
|
||||
"$(sha256sum "${rpm_devel_rpms[0]}" | awk '{print $1}')"
|
||||
printf 'devel_popt_identity=%s\n' \
|
||||
"$(rpm -qp --qf '%{NAME}|%{EVR}|%{DISTTAG}|%{ARCH}' "${popt_devel_rpms[0]}")"
|
||||
printf 'devel_popt_sha256=%s\n' \
|
||||
"$(sha256sum "${popt_devel_rpms[0]}" | awk '{print $1}')"
|
||||
} >"$SNAPSHOT/devel-input-fingerprint.txt"
|
||||
|
||||
converter_output="$SNAPSHOT/conversion"
|
||||
python3 "$PKGLIST_CONVERTER" "$converter_output" \
|
||||
--lists-dir "$SNAPSHOT/lists" \
|
||||
--rpm-include-dir "$devel_root/usr/include" \
|
||||
--rpm-library "$rpm_library" \
|
||||
--rpm-library "$rpmio_library"
|
||||
mv "$converter_output/d1-pkglists/manifest.json" "$SNAPSHOT/manifest.json"
|
||||
mkdir -p "$SNAPSHOT/d1-pkglists"
|
||||
mv "$converter_output/d1-pkglists/"*.classic "$SNAPSHOT/d1-pkglists/"
|
||||
rmdir "$converter_output/d1-pkglists" "$converter_output"
|
||||
rm -rf "$SNAPSHOT/download-cache"
|
||||
rm -rf "$devel_cache" "$devel_root"
|
||||
rm -f "$SNAPSHOT/lists/lock"
|
||||
rm -rf "$SNAPSHOT/lists/partial"
|
||||
mkdir -p "$SNAPSHOT/lists/partial"
|
||||
write_snapshot_fingerprint >"$SNAPSHOT/input-fingerprint.txt"
|
||||
: >"$SNAPSHOT/.complete"
|
||||
}
|
||||
|
||||
prepare_variant_lists()
|
||||
{
|
||||
local variant=$1 format=$2 variant_apt
|
||||
if [[ -d $variant/apt ]]; then
|
||||
variant_apt=$(canonical_work_subdir "$variant/apt" 'APT variant directory')
|
||||
chmod -R u+w "$variant_apt"
|
||||
fi
|
||||
rm -rf "$variant/apt"
|
||||
mkdir -p "$variant/apt/lists" "$variant/apt/cache/archives/partial"
|
||||
cp -a "$SNAPSHOT/lists/." "$variant/apt/lists/"
|
||||
|
||||
if [[ $format == d1 ]]; then
|
||||
python3 - "$SNAPSHOT/manifest.json" "$SNAPSHOT/lists" \
|
||||
"$SNAPSHOT/d1-pkglists" "$variant/apt/lists" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
manifest_path, original_dir, d1_dir, target_dir = map(Path, sys.argv[1:])
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
for architecture, data in manifest["architectures"].items():
|
||||
source = original_dir / data["source"]["path"]
|
||||
converted = d1_dir / data["output"]["path"]
|
||||
target = target_dir / data["source"]["path"]
|
||||
if hashlib.sha256(source.read_bytes()).hexdigest() != data["source"]["sha256"]:
|
||||
raise SystemExit(f"source checksum mismatch for {architecture}: {source}")
|
||||
if hashlib.sha256(converted.read_bytes()).hexdigest() != data["output"]["sha256"]:
|
||||
raise SystemExit(f"D1 checksum mismatch for {architecture}: {converted}")
|
||||
shutil.copyfile(converted, target)
|
||||
PY
|
||||
fi
|
||||
}
|
||||
|
||||
build_variant()
|
||||
{
|
||||
local name=$1 source_c=$2 suffix=$3 variant source hasher repo spec
|
||||
local expected_fingerprint artifact_fingerprint
|
||||
local -a librpm_rpms
|
||||
variant=$(variant_dir "$name")
|
||||
source="$variant/src/rpm"
|
||||
hasher="$variant/hasher"
|
||||
repo="$hasher/repo/x86_64/RPMS.hasher"
|
||||
spec="$source/alt/rpm.spec"
|
||||
|
||||
printf '\n===== Building %s with %s =====\n' "$name" "$source_c"
|
||||
mkdir -p "$variant/src" "$variant/logs" "$hasher"
|
||||
shopt -s nullglob
|
||||
librpm_rpms=("$repo"/librpm7-[0-9]*".$suffix".x86_64.rpm)
|
||||
expected_fingerprint=$(mktemp)
|
||||
{
|
||||
printf 'base_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
|
||||
printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')"
|
||||
printf 'set_source=%s\n' "$(sha256sum "$source_c" | awk '{print $1}')"
|
||||
printf 'suffix=%s\n' "$suffix"
|
||||
printf 'packager=%s\n' "$PACKAGER"
|
||||
if [[ $name == d1 ]]; then
|
||||
printf 'set9_decoder=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
|
||||
printf 'set_rewriter=%s\n' "$(sha256sum "$SET_REWRITER_C" | awk '{print $1}')"
|
||||
printf 'set_compat=%s\n' "$(sha256sum "$SET_COMPAT_H" | awk '{print $1}')"
|
||||
fi
|
||||
} >"$expected_fingerprint"
|
||||
if [[ -f $variant/input-fingerprint.txt ]]; then
|
||||
if ! cmp -s "$expected_fingerprint" "$variant/input-fingerprint.txt"; then
|
||||
rm -f "$expected_fingerprint"
|
||||
fail "$name build inputs changed; use RESET_WORK=1"
|
||||
fi
|
||||
elif ((${#librpm_rpms[@]} > 0)); then
|
||||
rm -f "$expected_fingerprint"
|
||||
fail "$name RPM exists without its input fingerprint; use RESET_WORK=1"
|
||||
fi
|
||||
if ((${#librpm_rpms[@]} == 0)) && [[ -d $source ]]; then
|
||||
rm -rf "$source"
|
||||
fi
|
||||
if [[ ! -d $source/.git ]]; then
|
||||
git clone --local "$SOURCE_BASE" "$source"
|
||||
cp "$source_c" "$source/lib/set.c"
|
||||
prepare_spec "$spec" "$suffix"
|
||||
if [[ $name == d1 ]]; then
|
||||
prepare_d1_build_corpus "$source"
|
||||
fi
|
||||
sha256sum "$spec" >"$variant/prepared-spec.sha256"
|
||||
cp "$expected_fingerprint" "$variant/input-fingerprint.txt"
|
||||
else
|
||||
[[ -f $variant/prepared-spec.sha256 ]] &&
|
||||
sha256sum -c "$variant/prepared-spec.sha256" >/dev/null ||
|
||||
fail "$name prepared RPM spec changed; use RESET_WORK=1"
|
||||
cmp -s "$source_c" "$source/lib/set.c" ||
|
||||
fail "$source_c changed; use RESET_WORK=1"
|
||||
if [[ $name == d1 ]]; then
|
||||
cmp -s "$SET9_C" "$source/alt/arsv-d1-build/set9.c" &&
|
||||
cmp -s "$SET_REWRITER_C" \
|
||||
"$source/alt/arsv-d1-build/rewrite_sisyphus_pkglist.c" &&
|
||||
cmp -s "$SET_COMPAT_H" "$source/alt/arsv-d1-build/newset_compat.h" ||
|
||||
fail 'D1 build corpus converter changed; use RESET_WORK=1'
|
||||
fi
|
||||
fi
|
||||
rm -f "$expected_fingerprint"
|
||||
|
||||
librpm_rpms=("$repo"/librpm7-[0-9]*".$suffix".x86_64.rpm)
|
||||
if ((${#librpm_rpms[@]} == 0)); then
|
||||
(
|
||||
cd "$source"
|
||||
gear-hsh \
|
||||
--commit \
|
||||
--with-stuff \
|
||||
--packager="$PACKAGER" \
|
||||
--no-sisyphus-check=changelog,packager,gpg \
|
||||
--mountpoints=/proc \
|
||||
--workdir="$hasher" \
|
||||
--target=x86_64
|
||||
) 2>&1 | tee "$variant/logs/rpm.log"
|
||||
else
|
||||
printf 'already built: %s\n' "${librpm_rpms[0]}"
|
||||
fi
|
||||
|
||||
librpm_rpms=("$repo"/librpm7-[0-9]*".$suffix".x86_64.rpm)
|
||||
((${#librpm_rpms[@]} == 1)) ||
|
||||
fail "expected one librpm7 package for $name, got ${#librpm_rpms[@]}"
|
||||
artifact_fingerprint=$(sha256sum "${librpm_rpms[0]}")
|
||||
if [[ -f $variant/librpm-artifact.sha256 ]]; then
|
||||
[[ $artifact_fingerprint == "$(<"$variant/librpm-artifact.sha256")" ]] ||
|
||||
fail "$name librpm artifact changed; use RESET_WORK=1"
|
||||
else
|
||||
printf '%s\n' "$artifact_fingerprint" >"$variant/librpm-artifact.sha256"
|
||||
fi
|
||||
|
||||
rm -rf "$variant/lib"
|
||||
mkdir -p "$variant/lib"
|
||||
(
|
||||
cd "$variant/lib"
|
||||
rpm2cpio "${librpm_rpms[0]}" | cpio -idm --quiet
|
||||
)
|
||||
[[ -e $variant/lib/usr/lib64/librpm.so.7 && -e $variant/lib/usr/lib64/librpmio.so.7 ]] ||
|
||||
fail "librpm libraries were not extracted for $name"
|
||||
for apt_binary in "$APT_GET" "$APT_CACHE"; do
|
||||
LD_LIBRARY_PATH="$variant/lib/usr/lib64" ldd "$apt_binary" |
|
||||
awk -v expected="$variant/lib/usr/lib64/librpm.so.7" '
|
||||
index($0, expected) { found = 1 }
|
||||
END { exit found ? 0 : 1 }
|
||||
' || fail "$apt_binary does not load the built librpm for $name"
|
||||
done
|
||||
}
|
||||
|
||||
build_cache()
|
||||
{
|
||||
local name=$1 variant libdir start end elapsed stdout stderr status
|
||||
variant=$(variant_dir "$name")
|
||||
libdir=$(variant_libdir "$name")
|
||||
apt_options "$variant"
|
||||
stdout="$variant/logs/gencaches.stdout"
|
||||
stderr="$variant/logs/gencaches.stderr"
|
||||
|
||||
chmod -R u+w "$variant/apt/cache"
|
||||
rm -f "$variant/apt/cache/pkgcache.bin" "$variant/apt/cache/srcpkgcache.bin"
|
||||
start=$(date +%s%N)
|
||||
if env LC_ALL=C APT_CONFIG="$COMMON/apt.conf" LD_LIBRARY_PATH="$libdir" \
|
||||
"$APT_CACHE" -q "${APT_OPTIONS[@]}" gencaches >"$stdout" 2>"$stderr"; then
|
||||
status=0
|
||||
else
|
||||
status=$?
|
||||
fi
|
||||
end=$(date +%s%N)
|
||||
elapsed=$(awk -v start="$start" -v end="$end" \
|
||||
'BEGIN { printf "%.6f", (end - start) / 1000000000 }')
|
||||
printf '%s\t%s\t%s\n' "$name" "$elapsed" "$status" >>"$CACHE_RESULTS"
|
||||
[[ $status -eq 0 && -s $variant/apt/cache/pkgcache.bin ]] ||
|
||||
fail "gencaches failed for $name; see $stderr"
|
||||
chmod -R a-w "$variant/apt/cache"
|
||||
}
|
||||
|
||||
record_cache_checksums()
|
||||
{
|
||||
local name variant
|
||||
: >"$RESULT_DIR/cache-files.before.txt"
|
||||
for name in set9 d1; do
|
||||
variant=$(variant_dir "$name")
|
||||
for cache_file in "$variant/apt/cache/pkgcache.bin" \
|
||||
"$variant/apt/cache/srcpkgcache.bin"; do
|
||||
stat -c '%n\t%D\t%i\t%s\t%Y\t%A' "$cache_file"
|
||||
sha256sum "$cache_file"
|
||||
done >>"$RESULT_DIR/cache-files.before.txt"
|
||||
done
|
||||
}
|
||||
|
||||
verify_cache_checksums()
|
||||
{
|
||||
local name variant
|
||||
: >"$RESULT_DIR/cache-files.after.txt"
|
||||
for name in set9 d1; do
|
||||
variant=$(variant_dir "$name")
|
||||
for cache_file in "$variant/apt/cache/pkgcache.bin" \
|
||||
"$variant/apt/cache/srcpkgcache.bin"; do
|
||||
stat -c '%n\t%D\t%i\t%s\t%Y\t%A' "$cache_file"
|
||||
sha256sum "$cache_file"
|
||||
done >>"$RESULT_DIR/cache-files.after.txt"
|
||||
done
|
||||
cmp -s "$RESULT_DIR/cache-files.before.txt" \
|
||||
"$RESULT_DIR/cache-files.after.txt" ||
|
||||
fail "APT cache changed during timed runs"
|
||||
}
|
||||
|
||||
write_provenance()
|
||||
{
|
||||
{
|
||||
printf 'rpm_git=%s\n' "$RPM_GIT"
|
||||
printf 'rpm_branch=%s\n' "$RPM_BRANCH"
|
||||
printf 'rpm_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
|
||||
printf 'sisyphus_mirror=%s\n' "$SISYPHUS_MIRROR"
|
||||
printf 'apt=%s\n' "$(rpmquery --qf '%{VERSION}-%{RELEASE}' apt)"
|
||||
printf 'rpm=%s\n' "$(rpm --version)"
|
||||
printf 'cpu=%s\n' "$CPU"
|
||||
printf 'rounds=%s\n' "$ROUNDS"
|
||||
printf 'operations=%s\n' "$OPERATIONS"
|
||||
sha256sum "$ORCHESTRATOR" "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \
|
||||
"$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" \
|
||||
"$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h"
|
||||
cat "$SNAPSHOT/devel-input-fingerprint.txt"
|
||||
write_runtime_fingerprint
|
||||
printf 'set9_librpm='; cat "$SET9_VARIANT/librpm-artifact.sha256"
|
||||
printf 'd1_librpm='; cat "$D1_VARIANT/librpm-artifact.sha256"
|
||||
} >"$RESULT_DIR/provenance.txt"
|
||||
cp "$SNAPSHOT/manifest.json" "$RESULT_DIR/pkglist-manifest.json"
|
||||
}
|
||||
|
||||
summarize_results()
|
||||
{
|
||||
python3 - "$RAW_RESULTS" "$RESULT_DIR/summary.tsv" "$RESULT_DIR/summary.md" <<'PY'
|
||||
import csv
|
||||
import statistics
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
raw_path, tsv_path, markdown_path = map(Path, sys.argv[1:])
|
||||
with raw_path.open(newline="") as stream:
|
||||
rows = list(csv.DictReader(stream, delimiter="\t"))
|
||||
|
||||
groups = defaultdict(list)
|
||||
for row in rows:
|
||||
groups[row["operation"]].append(row)
|
||||
|
||||
summary = []
|
||||
all_equivalent = True
|
||||
for operation, items in groups.items():
|
||||
by_variant = defaultdict(list)
|
||||
signatures = set()
|
||||
statuses = set()
|
||||
for item in items:
|
||||
by_variant[item["variant"]].append(float(item["seconds"]))
|
||||
statuses.add(item["status"])
|
||||
signatures.add(
|
||||
(
|
||||
item["status"],
|
||||
item["stdout_sha256"],
|
||||
item["stdout_bytes"],
|
||||
item["stderr_sha256"],
|
||||
item["stderr_bytes"],
|
||||
)
|
||||
)
|
||||
if set(by_variant) != {"set9", "d1"}:
|
||||
raise SystemExit(f"missing variant samples for {operation}")
|
||||
if len(by_variant["set9"]) != len(by_variant["d1"]):
|
||||
raise SystemExit(f"unbalanced variant samples for {operation}")
|
||||
set9 = statistics.median(by_variant["set9"])
|
||||
d1 = statistics.median(by_variant["d1"])
|
||||
equivalent = len(signatures) == 1
|
||||
all_equivalent &= equivalent
|
||||
status = next(iter(statuses)) if len(statuses) == 1 else "mixed:" + ",".join(sorted(statuses))
|
||||
path = "normal" if status == "0" else "failure-path"
|
||||
summary.append(
|
||||
(
|
||||
operation,
|
||||
len(by_variant["set9"]),
|
||||
set9,
|
||||
min(by_variant["set9"]),
|
||||
max(by_variant["set9"]),
|
||||
d1,
|
||||
min(by_variant["d1"]),
|
||||
max(by_variant["d1"]),
|
||||
d1 / set9,
|
||||
status,
|
||||
path,
|
||||
equivalent,
|
||||
)
|
||||
)
|
||||
|
||||
with tsv_path.open("w", newline="") as stream:
|
||||
writer = csv.writer(stream, delimiter="\t", lineterminator="\n")
|
||||
writer.writerow(
|
||||
[
|
||||
"operation", "runs_per_variant", "set9_median_seconds", "set9_min_seconds",
|
||||
"set9_max_seconds", "d1_median_seconds", "d1_min_seconds", "d1_max_seconds",
|
||||
"d1/set9", "exit_status", "benchmark_path", "outputs_equal"
|
||||
]
|
||||
)
|
||||
for operation, count, set9, set9_min, set9_max, d1, d1_min, d1_max, ratio, status, path, equivalent in summary:
|
||||
writer.writerow(
|
||||
[
|
||||
operation, count, f"{set9:.6f}", f"{set9_min:.6f}", f"{set9_max:.6f}",
|
||||
f"{d1:.6f}", f"{d1_min:.6f}", f"{d1_max:.6f}", f"{ratio:.4f}",
|
||||
status, path, "yes" if equivalent else "NO"
|
||||
]
|
||||
)
|
||||
|
||||
lines = [
|
||||
"# Sisyphus set9 vs D1 benchmark",
|
||||
"",
|
||||
"Timed scope: resolver commands only; conversion, builds and `gencaches` are excluded.",
|
||||
"Each round uses the balanced order `set9 / d1 / d1 / set9` after one warm-up per variant.",
|
||||
"",
|
||||
"| operation | runs/variant | set9 median [min–max], s | D1 median [min–max], s | D1/set9 | status/path | output equivalence |",
|
||||
"|---|---:|---:|---:|---:|:---:|:---:|",
|
||||
]
|
||||
for operation, count, set9, set9_min, set9_max, d1, d1_min, d1_max, ratio, status, path, equivalent in summary:
|
||||
lines.append(
|
||||
f"| `{operation}` | {count} | {set9:.6f} [{set9_min:.6f}–{set9_max:.6f}] | "
|
||||
f"{d1:.6f} [{d1_min:.6f}–{d1_max:.6f}] | {ratio:.4f} | "
|
||||
f"{status}/{path} | {'yes' if equivalent else '**NO**'} |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"`D1/set9 < 1` means that the D1 variant was faster.",
|
||||
"Rows with non-zero status are explicitly labelled `failure-path`; do not treat them as successful resolver workloads.",
|
||||
"Output equivalence includes exit status plus exact stdout and stderr hashes for every repeat and both variants.",
|
||||
]
|
||||
)
|
||||
markdown_path.write_text("\n".join(lines) + "\n")
|
||||
if not all_equivalent:
|
||||
raise SystemExit(2)
|
||||
PY
|
||||
}
|
||||
|
||||
if [[ ${1:-} == --help || ${1:-} == -h ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
(($# == 0)) || fail "unexpected arguments; use --help"
|
||||
|
||||
for command in git gear-hsh hsh rpm rpmquery rpm2cpio cpio taskset awk sed cc \
|
||||
pkglist-query date sha256sum stat ldd python3 cmp cp mv tee realpath mktemp grep; do
|
||||
command -v "$command" >/dev/null || fail "required command not found: $command"
|
||||
done
|
||||
[[ -n $APT_CACHE && -x $APT_CACHE ]] || fail "apt-cache not found: $APT_CACHE"
|
||||
[[ -x $APT_GET ]] || fail "apt-get executable not found: $APT_GET"
|
||||
[[ -f $SET9_C ]] || fail "set9 source not found: $SET9_C"
|
||||
[[ -f $D1_C ]] || fail "D1 source not found: $D1_C"
|
||||
[[ -f $PKGLIST_CONVERTER ]] || fail "pkglist converter not found: $PKGLIST_CONVERTER"
|
||||
[[ $ROUNDS =~ ^[1-9][0-9]*$ ]] || fail "ROUNDS must be a positive integer"
|
||||
[[ $RESET_WORK =~ ^[01]$ ]] || fail "RESET_WORK must be 0 or 1"
|
||||
taskset -c "$CPU" true >/dev/null 2>&1 || fail "CPU $CPU is unavailable to taskset"
|
||||
read -r -a OPERATION_LIST <<<"$OPERATIONS"
|
||||
((${#OPERATION_LIST[@]} > 0)) || fail "OPERATIONS is empty"
|
||||
for operation in "${OPERATION_LIST[@]}"; do
|
||||
case $operation in
|
||||
unmet|check|autoremove|install-rpm-build|install-openuds-server|install-password-store|upgrade|dist-upgrade) ;;
|
||||
*) fail "unknown operation in OPERATIONS: $operation" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
HOME_REAL=$(realpath -m "$HOME")
|
||||
CWD_REAL=$(realpath -m "$PWD")
|
||||
REPO_ROOT=$(realpath -m "$REPO_ROOT")
|
||||
SET9_C=$(realpath -m "$SET9_C")
|
||||
D1_C=$(realpath -m "$D1_C")
|
||||
PKGLIST_CONVERTER=$(realpath -m "$PKGLIST_CONVERTER")
|
||||
WORK_ROOT=$(realpath -m "$WORK_ROOT")
|
||||
RESULT_DIR=$(realpath -m "$RESULT_DIR")
|
||||
[[ $RESULT_DIR == "$WORK_ROOT"/* ]] ||
|
||||
fail "RESULT_DIR must be inside WORK_ROOT: $RESULT_DIR"
|
||||
|
||||
COMMON="$WORK_ROOT/common"
|
||||
SNAPSHOT="$COMMON/snapshot"
|
||||
SOURCE_BASE="$WORK_ROOT/src/rpm-base"
|
||||
SOURCE_FINGERPRINT="$WORK_ROOT/src/rpm-base.fingerprint.txt"
|
||||
SET9_VARIANT="$WORK_ROOT/variants/set9"
|
||||
D1_VARIANT="$WORK_ROOT/variants/d1"
|
||||
|
||||
WORK_ROOT_EXISTED=0
|
||||
[[ -e $WORK_ROOT ]] && WORK_ROOT_EXISTED=1
|
||||
if ((RESET_WORK)); then
|
||||
safe_remove_work_root
|
||||
for old_hasher in "$SET9_VARIANT/hasher" "$D1_VARIANT/hasher"; do
|
||||
[[ -d $old_hasher ]] || continue
|
||||
old_hasher=$(canonical_work_subdir "$old_hasher" 'hasher workdir')
|
||||
hsh --cleanup-only --workdir="$old_hasher" ||
|
||||
fail "hasher cleanup failed: $old_hasher"
|
||||
done
|
||||
for old_apt in "$SET9_VARIANT/apt" "$D1_VARIANT/apt"; do
|
||||
[[ -d $old_apt ]] || continue
|
||||
old_apt=$(canonical_work_subdir "$old_apt" 'APT workdir')
|
||||
chmod -R u+w "$old_apt"
|
||||
done
|
||||
rm -rf "$WORK_ROOT"
|
||||
WORK_ROOT_EXISTED=0
|
||||
elif ((WORK_ROOT_EXISTED)); then
|
||||
marker="$WORK_ROOT/.arsv-sisyphus-set-bench-root"
|
||||
[[ -f $marker && ! -L $marker ]] ||
|
||||
fail "refusing unmarked existing WORK_ROOT: $WORK_ROOT"
|
||||
grep -Fx 'ARSV Sisyphus set9/D1 benchmark work root' "$marker" >/dev/null ||
|
||||
fail "invalid WORK_ROOT ownership marker: $marker"
|
||||
fi
|
||||
mkdir -p "$WORK_ROOT"
|
||||
MARKER="$WORK_ROOT/.arsv-sisyphus-set-bench-root"
|
||||
if [[ -e $MARKER ]]; then
|
||||
grep -Fx 'ARSV Sisyphus set9/D1 benchmark work root' "$MARKER" >/dev/null ||
|
||||
fail "invalid WORK_ROOT ownership marker: $MARKER"
|
||||
else
|
||||
printf '%s\n' 'ARSV Sisyphus set9/D1 benchmark work root' >"$MARKER"
|
||||
fi
|
||||
ensure_work_subdir "$WORK_ROOT/src" 'source directory'
|
||||
ensure_work_subdir "$WORK_ROOT/variants" 'variants directory'
|
||||
ensure_work_subdir "$COMMON/root/var/lib/rpm" 'isolated RPM database directory'
|
||||
ensure_work_subdir "$RESULT_DIR" 'results directory'
|
||||
: >"$COMMON/apt.conf"
|
||||
: >"$COMMON/status"
|
||||
|
||||
prepare_snapshot
|
||||
if [[ ! -d $SOURCE_BASE/.git ]]; then
|
||||
git clone --branch "$RPM_BRANCH" --single-branch "$RPM_GIT" "$SOURCE_BASE"
|
||||
write_source_fingerprint >"$SOURCE_FINGERPRINT"
|
||||
else
|
||||
validate_source_reuse
|
||||
fi
|
||||
|
||||
build_variant set9 "$SET9_C" arsvset9
|
||||
build_variant d1 "$D1_C" arsvd1
|
||||
prepare_variant_lists "$SET9_VARIANT" set9
|
||||
prepare_variant_lists "$D1_VARIANT" d1
|
||||
|
||||
mkdir -p "$RESULT_DIR/raw"
|
||||
CACHE_RESULTS="$RESULT_DIR/cache-build.tsv"
|
||||
printf 'variant\tseconds\texit_status\n' >"$CACHE_RESULTS"
|
||||
build_cache set9
|
||||
build_cache d1
|
||||
record_cache_checksums
|
||||
write_provenance
|
||||
|
||||
RAW_RESULTS="$RESULT_DIR/raw.tsv"
|
||||
printf 'operation\tsequence\tvariant\tsample\tseconds\tstatus\tstdout_sha256\tstdout_bytes\tstderr_sha256\tstderr_bytes\n' >"$RAW_RESULTS"
|
||||
declare -A SAMPLE_COUNTS=([set9]=0 [d1]=0)
|
||||
sequence=0
|
||||
|
||||
for operation in "${OPERATION_LIST[@]}"; do
|
||||
printf '\n===== Warm-up: %s =====\n' "$operation"
|
||||
mkdir -p "$RESULT_DIR/warmup/$operation"
|
||||
for variant_name in set9 d1; do
|
||||
prepare_command "$operation" "$variant_name"
|
||||
warm_status=$(execute_command \
|
||||
"$RESULT_DIR/warmup/$operation/$variant_name.stdout" \
|
||||
"$RESULT_DIR/warmup/$operation/$variant_name.stderr")
|
||||
printf '%s\n' "$warm_status" >"$RESULT_DIR/warmup/$operation/$variant_name.status"
|
||||
done
|
||||
|
||||
printf '===== Timed ABBA: %s =====\n' "$operation"
|
||||
for ((round = 1; round <= ROUNDS; ++round)); do
|
||||
for variant_name in set9 d1 d1 set9; do
|
||||
sequence=$((sequence + 1))
|
||||
SAMPLE_COUNTS[$variant_name]=$((SAMPLE_COUNTS[$variant_name] + 1))
|
||||
run_once "$operation" "$variant_name" "$sequence" "${SAMPLE_COUNTS[$variant_name]}"
|
||||
done
|
||||
done
|
||||
SAMPLE_COUNTS[set9]=0
|
||||
SAMPLE_COUNTS[d1]=0
|
||||
done
|
||||
|
||||
verify_cache_checksums
|
||||
if ! summarize_results; then
|
||||
fail "variant/repeat outputs differ; inspect $RESULT_DIR/raw and $RESULT_DIR/summary.md"
|
||||
fi
|
||||
|
||||
printf '\nDone. Results:\n'
|
||||
printf ' %s\n' "$RESULT_DIR/summary.md" "$RESULT_DIR/summary.tsv" \
|
||||
"$RESULT_DIR/raw.tsv" "$RESULT_DIR/cache-build.tsv" "$RESULT_DIR/provenance.txt" \
|
||||
"$RESULT_DIR/pkglist-manifest.json"
|
||||
Reference in New Issue
Block a user