From 6584a628ced53a849dbc5146928c8bf9eaf6959c Mon Sep 17 00:00:00 2001 From: Dmitrii Krosh Date: Mon, 3 Aug 2026 03:12:54 +0300 Subject: [PATCH] more plans + testing for roaring --- new_version/main_ideas.md | 75 +++++++++ new_version/roaring_bitmap/about.md | 3 + new_version/roaring_bitmap/benchmark.py | 206 ++++++++++++++++++++++++ new_version/roaring_bitmap/bitmap_set.c | 87 ++-------- new_version/roaring_bitmap/build.sh | 20 +-- 5 files changed, 309 insertions(+), 82 deletions(-) create mode 100644 new_version/main_ideas.md create mode 100644 new_version/roaring_bitmap/about.md create mode 100755 new_version/roaring_bitmap/benchmark.py diff --git a/new_version/main_ideas.md b/new_version/main_ideas.md new file mode 100644 index 0000000..170f6b4 --- /dev/null +++ b/new_version/main_ideas.md @@ -0,0 +1,75 @@ +# Другой дизайн + +## Roaring map + +По крайней мере в примитивном варианте скорость создания остаётся примерно прежней, скорость сравнения становится кратно меньше + +Также кратно выше становится сама длина строки + +```text +symbols=1000 required=500 bpp=32 +implementation set_chars format +set9 3994 golomb +bitmap 19924 R1 +``` + +```text +operation set9 bitmap bitmap/set9 +set_fini only 266.24 us 202.89 us 0.76x +new+add+fini 1696.05 us 1600.44 us 0.94x +rpmsetcmp cold 293.87 us 1321.17 us 4.50x +rpmsetcmp warm 5.24 us 399.20 us 76.14x +``` + +### Использование zstd + +```text +symbols=1000 required=500 bpp=32 +implementation set_chars format +set9 3994 golomb +bitmap 14126 R2 +``` + +без zstd строка становится ~40% длинее + +```text +operation set9 bitmap bitmap/set9 +set_fini only 171.84 us 314.20 us 1.83x +new+add+fini 1185.33 us 1726.32 us 1.46x +rpmsetcmp cold 285.04 us 1412.84 us 4.96x +rpmsetcmp warm 4.82 us 490.94 us 101.86x +``` + +## хранить сразу расшифрованный set + +- огромная часть ресурсов уходит не на просмотр включения множеств, а на декодирование set-строк + - при возможности хранить больше данных за более дешёвое сравнение - прекрасно + - тупо условный формат: + - ` <последовательно упакованные bpp-битные хеши>` +- Shannon–Fano–Elias coding как одна из идей, но надо глубже копать + +## Группировать, а не хэшировать + +- если была бы возможность точно делать соответствия между label и id, то provides стал бы в большинстве последовательным, а required было бы легко искать в P. + +# Доработки на текущий дизайн + +## битовый prefilter + +- Позволяет отбросить заведомо ложные варианты + - но так ли часто это будет срабатывать, но вызывает доп расходы при высчитывании +- фильтр Блума как пример + - В теории хэш можно заменить на него + - необходимо знать, насколько больше будет ложноположительных срабатываний + +## улучшить проход по массивам + +- Можно provides хранить не в виде массива, а сразу как хэш-структурку + - если provides строка кэшируется не так часто, смысла не будет + +# Прочие улучшения + +## улучшение работы с хэшем + +- если условно "отсортировать" массив provides/requires, можно получить лучшую работу с кжшом + - (надеюсь, что под капотом оно уже и так это делает, но проверить стоит) diff --git a/new_version/roaring_bitmap/about.md b/new_version/roaring_bitmap/about.md new file mode 100644 index 0000000..6c466cc --- /dev/null +++ b/new_version/roaring_bitmap/about.md @@ -0,0 +1,3 @@ +попытка воссоздания алгоритма с roaring bitmap + +Даже эффективней в создании, но проигрывает в дешифровке (сравнении) и занимаемому месту diff --git a/new_version/roaring_bitmap/benchmark.py b/new_version/roaring_bitmap/benchmark.py new file mode 100755 index 0000000..a132fe4 --- /dev/null +++ b/new_version/roaring_bitmap/benchmark.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +import argparse +import ctypes +import gc +import os +import statistics +import subprocess +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +BUILD = HERE / "build" +LIBC = ctypes.CDLL(None) +LIBC.free.argtypes = [ctypes.c_void_p] + + +class SetAPI: + def __init__(self, path: Path, result_needs_free: bool): + self.lib = ctypes.CDLL(str(path)) + self.result_needs_free = result_needs_free + self.lib.set_new.restype = ctypes.c_void_p + self.lib.set_add.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + self.lib.set_fini.argtypes = [ctypes.c_void_p, ctypes.c_int] + self.lib.set_fini.restype = ctypes.c_void_p + self.lib.set_free.argtypes = [ctypes.c_void_p] + self.lib.set_free.restype = ctypes.c_void_p + self.lib.rpmsetcmp.argtypes = [ctypes.c_char_p, ctypes.c_char_p] + self.lib.rpmsetcmp.restype = ctypes.c_int + + def new_with_symbols(self, symbols): + value = self.lib.set_new() + if not value: + raise RuntimeError("set_new returned NULL") + for symbol in symbols: + self.lib.set_add(value, symbol) + return value + + def release(self, value, result): + if self.result_needs_free and result: + LIBC.free(result) + self.lib.set_free(value) + + def encode(self, symbols, bpp): + value = self.new_with_symbols(symbols) + result = self.lib.set_fini(value, bpp) + if not result: + raise RuntimeError("set_fini returned NULL") + encoded = ctypes.string_at(result) + self.release(value, result) + return encoded + + +def median_fini(api, symbols, bpp, calls, rounds): + samples = [] + for _ in range(rounds): + sets = [api.new_with_symbols(symbols) for _ in range(calls)] + results = [] + start = time.perf_counter_ns() + for value in sets: + results.append(api.lib.set_fini(value, bpp)) + samples.append((time.perf_counter_ns() - start) / calls) + if not all(results): + raise RuntimeError("set_fini returned NULL") + encoded = [ctypes.string_at(result) for result in results] + if len(set(encoded)) != 1: + raise RuntimeError("set_fini is not deterministic") + for value, result in zip(sets, results): + api.release(value, result) + return statistics.median(samples) + + +def median_build(api, symbols, bpp, calls, rounds): + samples = [] + for _ in range(rounds): + sets = [] + results = [] + start = time.perf_counter_ns() + for _ in range(calls): + value = api.new_with_symbols(symbols) + result = api.lib.set_fini(value, bpp) + sets.append(value) + results.append(result) + samples.append((time.perf_counter_ns() - start) / calls) + if not all(results): + raise RuntimeError("set_fini returned NULL") + for value, result in zip(sets, results): + api.release(value, result) + return statistics.median(samples) + + +def median_cmp(api, provider, requirement, calls, rounds): + expected = api.lib.rpmsetcmp(provider, requirement) + if expected != 1 or api.lib.rpmsetcmp(provider, provider) != 0: + raise RuntimeError(f"unexpected rpmsetcmp result: {expected}") + for _ in range(100): + api.lib.rpmsetcmp(provider, requirement) + + samples = [] + for _ in range(rounds): + checksum = 0 + start = time.perf_counter_ns() + for _ in range(calls): + checksum += api.lib.rpmsetcmp(provider, requirement) + samples.append((time.perf_counter_ns() - start) / calls) + if checksum != calls: + raise RuntimeError("rpmsetcmp result changed during benchmark") + return statistics.median(samples) + + +def cold_cmp_once(api, provider, requirement): + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + os.close(read_fd) + start = time.perf_counter_ns() + result = api.lib.rpmsetcmp(provider, requirement) + elapsed = time.perf_counter_ns() - start + os.write(write_fd, f"{elapsed} {result}".encode()) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + payload = b"" + while chunk := os.read(read_fd, 128): + payload += chunk + os.close(read_fd) + _, status = os.waitpid(pid, 0) + if status != 0: + raise RuntimeError(f"cold rpmsetcmp child failed: status={status}") + elapsed, result = map(int, payload.split()) + if result != 1: + raise RuntimeError(f"unexpected cold rpmsetcmp result: {result}") + return elapsed + + +def median_cmp_cold(api, provider, requirement, calls, rounds): + samples = [] + for _ in range(rounds): + total = sum(cold_cmp_once(api, provider, requirement) for _ in range(calls)) + samples.append(total / calls) + return statistics.median(samples) + + +def format_time(ns): + return f"{ns / 1000:.2f} us" + + +def main(): + parser = argparse.ArgumentParser(description="Compare set9 and CRoaring set APIs") + parser.add_argument("--symbols", type=int, default=1000) + parser.add_argument("--bpp", type=int, default=32) + parser.add_argument("--rounds", type=int, default=7) + parser.add_argument("--fini-calls", type=int, default=5) + parser.add_argument("--cmp-calls", type=int, default=2000) + parser.add_argument("--cold-calls", type=int, default=20) + parser.add_argument("--skip-build", action="store_true") + args = parser.parse_args() + if args.symbols < 2 or not 10 <= args.bpp <= 32: + parser.error("symbols must be >= 2 and bpp must be in 10..32") + if min(args.rounds, args.fini_calls, args.cmp_calls, args.cold_calls) < 1: + parser.error("rounds and call counts must be positive") + + if not args.skip_build: + subprocess.run([str(HERE / "build.sh")], check=True) + + symbols = tuple( + f"symbol_{i:08d}_version_ALT_{i % 97}".encode() for i in range(args.symbols) + ) + required = symbols[::2] + apis = { + "set9": SetAPI(BUILD / "libset9.so", result_needs_free=True), + "bitmap": SetAPI(BUILD / "libbitmap-set.so", result_needs_free=False), + } + + gc.disable() + try: + timings = {} + lengths = {} + for name, api in apis.items(): + provider = api.encode(symbols, args.bpp) + requirement = api.encode(required, args.bpp) + wire_format = provider[:2].decode() if provider.startswith(b"R1") else "golomb" + lengths[name] = (len(provider), wire_format) + timings[name] = ( + median_fini(api, symbols, args.bpp, args.fini_calls, args.rounds), + median_build(api, symbols, args.bpp, args.fini_calls, args.rounds), + median_cmp_cold(api, provider, requirement, args.cold_calls, args.rounds), + median_cmp(api, provider, requirement, args.cmp_calls, args.rounds), + ) + finally: + gc.enable() + + print(f"symbols={args.symbols} required={len(required)} bpp={args.bpp}") + print("implementation set_chars format") + for name in apis: + print(f"{name:<14} {lengths[name][0]:>9} {lengths[name][1]}") + print("\noperation set9 bitmap bitmap/set9") + labels = ("set_fini only", "new+add+fini", "rpmsetcmp cold", "rpmsetcmp warm") + for index, label in enumerate(labels): + old = timings["set9"][index] + new = timings["bitmap"][index] + print(f"{label:<22} {format_time(old):>10} {format_time(new):>10} {new / old:>12.2f}x") + + +if __name__ == "__main__": + main() diff --git a/new_version/roaring_bitmap/bitmap_set.c b/new_version/roaring_bitmap/bitmap_set.c index adf769d..bc7c987 100644 --- a/new_version/roaring_bitmap/bitmap_set.c +++ b/new_version/roaring_bitmap/bitmap_set.c @@ -4,17 +4,14 @@ #include #include #include -#include /* * This is intentionally a new set-string format. It is not compatible with * the Rice-Golomb/base62 strings produced by the original lib/set.c. * * R1 - * R2 */ -#define FORMAT_RAW "R1" -#define FORMAT_ZSTD "R2" +#define FORMAT_PREFIX "R1" #define FORMAT_HEADER_LEN 4 #define MAX_SERIALIZED_SIZE (64u * 1024u * 1024u) @@ -92,14 +89,8 @@ static char hex_digit(unsigned value) { const char* set_fini(struct set* set, int bpp) { roaring_bitmap_t* truncated; size_t portable_size; - size_t compressed_capacity; - size_t compressed_size; size_t written; unsigned char* portable; - unsigned char* compressed; - const unsigned char* payload; - const char* prefix; - size_t payload_size; char* output; if (!set || set->added == 0 || bpp < 10 || bpp > 32) return NULL; @@ -118,35 +109,19 @@ const char* set_fini(struct set* set, int bpp) { portable = xmalloc(portable_size); written = roaring_bitmap_portable_serialize(truncated, (char*)portable); if (written != portable_size) abort(); + if (portable_size > (SIZE_MAX - FORMAT_HEADER_LEN - 1) / 2) abort(); - compressed_capacity = ZSTD_compressBound(portable_size); - compressed = xmalloc(compressed_capacity); - compressed_size = - ZSTD_compress(compressed, compressed_capacity, portable, portable_size, ZSTD_CLEVEL_DEFAULT); - if (ZSTD_isError(compressed_size)) abort(); - if (compressed_size < portable_size) { - prefix = FORMAT_ZSTD; - payload = compressed; - payload_size = compressed_size; - } else { - prefix = FORMAT_RAW; - payload = portable; - payload_size = portable_size; - } - if (payload_size > (SIZE_MAX - FORMAT_HEADER_LEN - 1) / 2) abort(); - - output = xmalloc(FORMAT_HEADER_LEN + payload_size * 2 + 1); - memcpy(output, prefix, 2); + output = xmalloc(FORMAT_HEADER_LEN + portable_size * 2 + 1); + memcpy(output, FORMAT_PREFIX, sizeof(FORMAT_PREFIX) - 1); output[2] = (char)('0' + bpp / 10); output[3] = (char)('0' + bpp % 10); - for (size_t i = 0; i < payload_size; ++i) { - output[FORMAT_HEADER_LEN + i * 2] = hex_digit(payload[i] >> 4); - output[FORMAT_HEADER_LEN + i * 2 + 1] = hex_digit(payload[i] & 0x0f); + for (size_t i = 0; i < portable_size; ++i) { + output[FORMAT_HEADER_LEN + i * 2] = hex_digit(portable[i] >> 4); + output[FORMAT_HEADER_LEN + i * 2 + 1] = hex_digit(portable[i] & 0x0f); } - output[FORMAT_HEADER_LEN + payload_size * 2] = '\0'; + output[FORMAT_HEADER_LEN + portable_size * 2] = '\0'; - free(compressed); free(portable); roaring_bitmap_free(truncated); free(set->encoded); @@ -173,22 +148,18 @@ static int hex_value(char c) { static roaring_bitmap_t* decode_bitmap(const char* str, unsigned* bpp) { roaring_bitmap_t* bitmap; unsigned char* bytes; - unsigned char* portable; const char* hex; const char* reason = NULL; - char version; size_t hex_len; size_t str_len; size_t byte_count; - size_t portable_size; size_t expected; if (!str || !bpp) return NULL; if (strncmp(str, "set:", 4) == 0) str += 4; str_len = strlen(str); if (str_len < FORMAT_HEADER_LEN) return NULL; - if (str[0] != 'R' || (str[1] != '1' && str[1] != '2')) return NULL; - version = str[1]; + if (strncmp(str, FORMAT_PREFIX, sizeof(FORMAT_PREFIX) - 1) != 0) return NULL; if (str[2] < '0' || str[2] > '9' || str[3] < '0' || str[3] > '9') return NULL; *bpp = (unsigned)(str[2] - '0') * 10u + (unsigned)(str[3] - '0'); @@ -211,44 +182,14 @@ static roaring_bitmap_t* decode_bitmap(const char* str, unsigned* bpp) { bytes[i] = (unsigned char)((high << 4) | low); } - if (version == '1') { - portable = bytes; - portable_size = byte_count; - } else { - unsigned long long frame_content_size; - size_t frame_size = ZSTD_findFrameCompressedSize(bytes, byte_count); - - if (ZSTD_isError(frame_size) || frame_size != byte_count) { - free(bytes); - return NULL; - } - - frame_content_size = ZSTD_getFrameContentSize(bytes, byte_count); - if (frame_content_size == ZSTD_CONTENTSIZE_ERROR || - frame_content_size == ZSTD_CONTENTSIZE_UNKNOWN || frame_content_size == 0 || - frame_content_size > MAX_SERIALIZED_SIZE || frame_content_size > SIZE_MAX) { - free(bytes); - return NULL; - } - - portable_size = (size_t)frame_content_size; - portable = xmalloc(portable_size); - expected = ZSTD_decompress(portable, portable_size, bytes, byte_count); + expected = roaring_bitmap_portable_deserialize_size((const char*)bytes, byte_count); + if (expected != byte_count) { free(bytes); - if (ZSTD_isError(expected) || expected != portable_size) { - free(portable); - return NULL; - } - } - - expected = roaring_bitmap_portable_deserialize_size((const char*)portable, portable_size); - if (expected != portable_size) { - free(portable); return NULL; } - bitmap = roaring_bitmap_portable_deserialize_safe((const char*)portable, portable_size); - free(portable); + bitmap = roaring_bitmap_portable_deserialize_safe((const char*)bytes, byte_count); + free(bytes); if (!bitmap) return NULL; if (!roaring_bitmap_internal_validate(bitmap, &reason) || roaring_bitmap_is_empty(bitmap)) { roaring_bitmap_free(bitmap); @@ -334,8 +275,8 @@ int main(void) { assert(rpmsetcmp(small16, large16) == -1); assert(rpmsetcmp(small16, other16) == -2); assert(rpmsetcmp("bad", small16) == -3); + assert(rpmsetcmp("R21600", small16) == -3); assert(rpmsetcmp(small16, "R116xyz") == -4); - assert(rpmsetcmp(small16, "R216xyz") == -4); assert(rpmsetcmp("R", small16) == -3); free(prefixed); diff --git a/new_version/roaring_bitmap/build.sh b/new_version/roaring_bitmap/build.sh index d56bb90..464c04f 100755 --- a/new_version/roaring_bitmap/build.sh +++ b/new_version/roaring_bitmap/build.sh @@ -13,35 +13,37 @@ if [[ -z ${ROARING_CFLAGS+x} || -z ${ROARING_LIBS+x} ]]; then ROARING_LIBS=$(pkg-config --libs roaring) else CROARING_SRC="$BUILD/CRoaring" - CROARING_BUILD="$BUILD/CRoaring-build" + CROARING_BUILD="$BUILD/CRoaring-pic-build" if [[ ! -d $CROARING_SRC/.git ]]; then rm -rf "$CROARING_SRC" git clone --depth 1 https://github.com/RoaringBitmap/CRoaring.git "$CROARING_SRC" fi if [[ ! -f $CROARING_BUILD/src/libroaring.a ]]; then cmake -S "$CROARING_SRC" -B "$CROARING_BUILD" \ - -DROARING_BUILD_STATIC=ON -DENABLE_ROARING_TESTS=OFF -DCMAKE_BUILD_TYPE=Release + -DROARING_BUILD_STATIC=ON -DENABLE_ROARING_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON cmake --build "$CROARING_BUILD" --parallel fi ROARING_CFLAGS="-I$CROARING_SRC/include" ROARING_LIBS="$CROARING_BUILD/src/libroaring.a" fi fi -ZSTD_CFLAGS=${ZSTD_CFLAGS-$(pkg-config --cflags libzstd)} -ZSTD_LIBS=${ZSTD_LIBS-$(pkg-config --libs libzstd)} read -r -a ROARING_CFLAGS_A <<<"$ROARING_CFLAGS" read -r -a ROARING_LIBS_A <<<"$ROARING_LIBS" -read -r -a ZSTD_CFLAGS_A <<<"$ZSTD_CFLAGS" -read -r -a ZSTD_LIBS_A <<<"$ZSTD_LIBS" CFLAGS=(-O2 -std=gnu11 -D_GNU_SOURCE -Wall -Wextra -I"$HERE" -I"$BUILD") for tool in mkset setcmp; do cc "${CFLAGS[@]}" -include "$ROOT/scripts/rpmsetcmp/newset_compat.h" \ "$ROOT/reimplement/set9.c" "$ROOT/scripts/rpmsetcmp/$tool.c" \ -o "$BUILD/$tool-set9" - cc "${CFLAGS[@]}" "${ROARING_CFLAGS_A[@]}" "${ZSTD_CFLAGS_A[@]}" \ + cc "${CFLAGS[@]}" "${ROARING_CFLAGS_A[@]}" \ "$HERE/bitmap_set.c" "$ROOT/scripts/rpmsetcmp/$tool.c" \ - "${ROARING_LIBS_A[@]}" "${ZSTD_LIBS_A[@]}" -o "$BUILD/$tool-bitmap" + "${ROARING_LIBS_A[@]}" -o "$BUILD/$tool-bitmap" done -printf 'Built in %s: mkset-{set9,bitmap} setcmp-{set9,bitmap}\n' "$BUILD" +cc "${CFLAGS[@]}" -fPIC -shared -include "$ROOT/scripts/rpmsetcmp/newset_compat.h" \ + "$ROOT/reimplement/set9.c" -o "$BUILD/libset9.so" +cc "${CFLAGS[@]}" -fPIC -shared "${ROARING_CFLAGS_A[@]}" \ + "$HERE/bitmap_set.c" "${ROARING_LIBS_A[@]}" -o "$BUILD/libbitmap-set.so" + +printf 'Built tools and benchmark libraries in %s\n' "$BUILD"