more plans + testing for roaring
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
попытка воссоздания алгоритма с roaring bitmap
|
||||
|
||||
Даже эффективней в создании, но проигрывает в дешифровке (сравнении) и занимаемому месту
|
||||
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import ctypes
|
||||
import gc
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
BUILD = HERE / "build"
|
||||
LIBC = ctypes.CDLL(None)
|
||||
LIBC.free.argtypes = [ctypes.c_void_p]
|
||||
|
||||
|
||||
class SetAPI:
|
||||
def __init__(self, path: Path, result_needs_free: bool):
|
||||
self.lib = ctypes.CDLL(str(path))
|
||||
self.result_needs_free = result_needs_free
|
||||
self.lib.set_new.restype = ctypes.c_void_p
|
||||
self.lib.set_add.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
|
||||
self.lib.set_fini.argtypes = [ctypes.c_void_p, ctypes.c_int]
|
||||
self.lib.set_fini.restype = ctypes.c_void_p
|
||||
self.lib.set_free.argtypes = [ctypes.c_void_p]
|
||||
self.lib.set_free.restype = ctypes.c_void_p
|
||||
self.lib.rpmsetcmp.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
|
||||
self.lib.rpmsetcmp.restype = ctypes.c_int
|
||||
|
||||
def new_with_symbols(self, symbols):
|
||||
value = self.lib.set_new()
|
||||
if not value:
|
||||
raise RuntimeError("set_new returned NULL")
|
||||
for symbol in symbols:
|
||||
self.lib.set_add(value, symbol)
|
||||
return value
|
||||
|
||||
def release(self, value, result):
|
||||
if self.result_needs_free and result:
|
||||
LIBC.free(result)
|
||||
self.lib.set_free(value)
|
||||
|
||||
def encode(self, symbols, bpp):
|
||||
value = self.new_with_symbols(symbols)
|
||||
result = self.lib.set_fini(value, bpp)
|
||||
if not result:
|
||||
raise RuntimeError("set_fini returned NULL")
|
||||
encoded = ctypes.string_at(result)
|
||||
self.release(value, result)
|
||||
return encoded
|
||||
|
||||
|
||||
def median_fini(api, symbols, bpp, calls, rounds):
|
||||
samples = []
|
||||
for _ in range(rounds):
|
||||
sets = [api.new_with_symbols(symbols) for _ in range(calls)]
|
||||
results = []
|
||||
start = time.perf_counter_ns()
|
||||
for value in sets:
|
||||
results.append(api.lib.set_fini(value, bpp))
|
||||
samples.append((time.perf_counter_ns() - start) / calls)
|
||||
if not all(results):
|
||||
raise RuntimeError("set_fini returned NULL")
|
||||
encoded = [ctypes.string_at(result) for result in results]
|
||||
if len(set(encoded)) != 1:
|
||||
raise RuntimeError("set_fini is not deterministic")
|
||||
for value, result in zip(sets, results):
|
||||
api.release(value, result)
|
||||
return statistics.median(samples)
|
||||
|
||||
|
||||
def median_build(api, symbols, bpp, calls, rounds):
|
||||
samples = []
|
||||
for _ in range(rounds):
|
||||
sets = []
|
||||
results = []
|
||||
start = time.perf_counter_ns()
|
||||
for _ in range(calls):
|
||||
value = api.new_with_symbols(symbols)
|
||||
result = api.lib.set_fini(value, bpp)
|
||||
sets.append(value)
|
||||
results.append(result)
|
||||
samples.append((time.perf_counter_ns() - start) / calls)
|
||||
if not all(results):
|
||||
raise RuntimeError("set_fini returned NULL")
|
||||
for value, result in zip(sets, results):
|
||||
api.release(value, result)
|
||||
return statistics.median(samples)
|
||||
|
||||
|
||||
def median_cmp(api, provider, requirement, calls, rounds):
|
||||
expected = api.lib.rpmsetcmp(provider, requirement)
|
||||
if expected != 1 or api.lib.rpmsetcmp(provider, provider) != 0:
|
||||
raise RuntimeError(f"unexpected rpmsetcmp result: {expected}")
|
||||
for _ in range(100):
|
||||
api.lib.rpmsetcmp(provider, requirement)
|
||||
|
||||
samples = []
|
||||
for _ in range(rounds):
|
||||
checksum = 0
|
||||
start = time.perf_counter_ns()
|
||||
for _ in range(calls):
|
||||
checksum += api.lib.rpmsetcmp(provider, requirement)
|
||||
samples.append((time.perf_counter_ns() - start) / calls)
|
||||
if checksum != calls:
|
||||
raise RuntimeError("rpmsetcmp result changed during benchmark")
|
||||
return statistics.median(samples)
|
||||
|
||||
|
||||
def cold_cmp_once(api, provider, requirement):
|
||||
read_fd, write_fd = os.pipe()
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
os.close(read_fd)
|
||||
start = time.perf_counter_ns()
|
||||
result = api.lib.rpmsetcmp(provider, requirement)
|
||||
elapsed = time.perf_counter_ns() - start
|
||||
os.write(write_fd, f"{elapsed} {result}".encode())
|
||||
os.close(write_fd)
|
||||
os._exit(0)
|
||||
|
||||
os.close(write_fd)
|
||||
payload = b""
|
||||
while chunk := os.read(read_fd, 128):
|
||||
payload += chunk
|
||||
os.close(read_fd)
|
||||
_, status = os.waitpid(pid, 0)
|
||||
if status != 0:
|
||||
raise RuntimeError(f"cold rpmsetcmp child failed: status={status}")
|
||||
elapsed, result = map(int, payload.split())
|
||||
if result != 1:
|
||||
raise RuntimeError(f"unexpected cold rpmsetcmp result: {result}")
|
||||
return elapsed
|
||||
|
||||
|
||||
def median_cmp_cold(api, provider, requirement, calls, rounds):
|
||||
samples = []
|
||||
for _ in range(rounds):
|
||||
total = sum(cold_cmp_once(api, provider, requirement) for _ in range(calls))
|
||||
samples.append(total / calls)
|
||||
return statistics.median(samples)
|
||||
|
||||
|
||||
def format_time(ns):
|
||||
return f"{ns / 1000:.2f} us"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Compare set9 and CRoaring set APIs")
|
||||
parser.add_argument("--symbols", type=int, default=1000)
|
||||
parser.add_argument("--bpp", type=int, default=32)
|
||||
parser.add_argument("--rounds", type=int, default=7)
|
||||
parser.add_argument("--fini-calls", type=int, default=5)
|
||||
parser.add_argument("--cmp-calls", type=int, default=2000)
|
||||
parser.add_argument("--cold-calls", type=int, default=20)
|
||||
parser.add_argument("--skip-build", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.symbols < 2 or not 10 <= args.bpp <= 32:
|
||||
parser.error("symbols must be >= 2 and bpp must be in 10..32")
|
||||
if min(args.rounds, args.fini_calls, args.cmp_calls, args.cold_calls) < 1:
|
||||
parser.error("rounds and call counts must be positive")
|
||||
|
||||
if not args.skip_build:
|
||||
subprocess.run([str(HERE / "build.sh")], check=True)
|
||||
|
||||
symbols = tuple(
|
||||
f"symbol_{i:08d}_version_ALT_{i % 97}".encode() for i in range(args.symbols)
|
||||
)
|
||||
required = symbols[::2]
|
||||
apis = {
|
||||
"set9": SetAPI(BUILD / "libset9.so", result_needs_free=True),
|
||||
"bitmap": SetAPI(BUILD / "libbitmap-set.so", result_needs_free=False),
|
||||
}
|
||||
|
||||
gc.disable()
|
||||
try:
|
||||
timings = {}
|
||||
lengths = {}
|
||||
for name, api in apis.items():
|
||||
provider = api.encode(symbols, args.bpp)
|
||||
requirement = api.encode(required, args.bpp)
|
||||
wire_format = provider[:2].decode() if provider.startswith(b"R1") else "golomb"
|
||||
lengths[name] = (len(provider), wire_format)
|
||||
timings[name] = (
|
||||
median_fini(api, symbols, args.bpp, args.fini_calls, args.rounds),
|
||||
median_build(api, symbols, args.bpp, args.fini_calls, args.rounds),
|
||||
median_cmp_cold(api, provider, requirement, args.cold_calls, args.rounds),
|
||||
median_cmp(api, provider, requirement, args.cmp_calls, args.rounds),
|
||||
)
|
||||
finally:
|
||||
gc.enable()
|
||||
|
||||
print(f"symbols={args.symbols} required={len(required)} bpp={args.bpp}")
|
||||
print("implementation set_chars format")
|
||||
for name in apis:
|
||||
print(f"{name:<14} {lengths[name][0]:>9} {lengths[name][1]}")
|
||||
print("\noperation set9 bitmap bitmap/set9")
|
||||
labels = ("set_fini only", "new+add+fini", "rpmsetcmp cold", "rpmsetcmp warm")
|
||||
for index, label in enumerate(labels):
|
||||
old = timings["set9"][index]
|
||||
new = timings["bitmap"][index]
|
||||
print(f"{label:<22} {format_time(old):>10} {format_time(new):>10} {new / old:>12.2f}x")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,17 +4,14 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <zstd.h>
|
||||
|
||||
/*
|
||||
* This is intentionally a new set-string format. It is not compatible with
|
||||
* the Rice-Golomb/base62 strings produced by the original lib/set.c.
|
||||
*
|
||||
* R1<two decimal bpp digits><hex CRoaring portable serialization>
|
||||
* R2<two decimal bpp digits><hex Zstd-compressed portable serialization>
|
||||
*/
|
||||
#define FORMAT_RAW "R1"
|
||||
#define FORMAT_ZSTD "R2"
|
||||
#define FORMAT_PREFIX "R1"
|
||||
#define FORMAT_HEADER_LEN 4
|
||||
#define MAX_SERIALIZED_SIZE (64u * 1024u * 1024u)
|
||||
|
||||
@@ -92,14 +89,8 @@ static char hex_digit(unsigned value) {
|
||||
const char* set_fini(struct set* set, int bpp) {
|
||||
roaring_bitmap_t* truncated;
|
||||
size_t portable_size;
|
||||
size_t compressed_capacity;
|
||||
size_t compressed_size;
|
||||
size_t written;
|
||||
unsigned char* portable;
|
||||
unsigned char* compressed;
|
||||
const unsigned char* payload;
|
||||
const char* prefix;
|
||||
size_t payload_size;
|
||||
char* output;
|
||||
|
||||
if (!set || set->added == 0 || bpp < 10 || bpp > 32) return NULL;
|
||||
@@ -118,35 +109,19 @@ const char* set_fini(struct set* set, int bpp) {
|
||||
portable = xmalloc(portable_size);
|
||||
written = roaring_bitmap_portable_serialize(truncated, (char*)portable);
|
||||
if (written != portable_size) abort();
|
||||
if (portable_size > (SIZE_MAX - FORMAT_HEADER_LEN - 1) / 2) abort();
|
||||
|
||||
compressed_capacity = ZSTD_compressBound(portable_size);
|
||||
compressed = xmalloc(compressed_capacity);
|
||||
compressed_size =
|
||||
ZSTD_compress(compressed, compressed_capacity, portable, portable_size, ZSTD_CLEVEL_DEFAULT);
|
||||
if (ZSTD_isError(compressed_size)) abort();
|
||||
if (compressed_size < portable_size) {
|
||||
prefix = FORMAT_ZSTD;
|
||||
payload = compressed;
|
||||
payload_size = compressed_size;
|
||||
} else {
|
||||
prefix = FORMAT_RAW;
|
||||
payload = portable;
|
||||
payload_size = portable_size;
|
||||
}
|
||||
if (payload_size > (SIZE_MAX - FORMAT_HEADER_LEN - 1) / 2) abort();
|
||||
|
||||
output = xmalloc(FORMAT_HEADER_LEN + payload_size * 2 + 1);
|
||||
memcpy(output, prefix, 2);
|
||||
output = xmalloc(FORMAT_HEADER_LEN + portable_size * 2 + 1);
|
||||
memcpy(output, FORMAT_PREFIX, sizeof(FORMAT_PREFIX) - 1);
|
||||
output[2] = (char)('0' + bpp / 10);
|
||||
output[3] = (char)('0' + bpp % 10);
|
||||
|
||||
for (size_t i = 0; i < payload_size; ++i) {
|
||||
output[FORMAT_HEADER_LEN + i * 2] = hex_digit(payload[i] >> 4);
|
||||
output[FORMAT_HEADER_LEN + i * 2 + 1] = hex_digit(payload[i] & 0x0f);
|
||||
for (size_t i = 0; i < portable_size; ++i) {
|
||||
output[FORMAT_HEADER_LEN + i * 2] = hex_digit(portable[i] >> 4);
|
||||
output[FORMAT_HEADER_LEN + i * 2 + 1] = hex_digit(portable[i] & 0x0f);
|
||||
}
|
||||
output[FORMAT_HEADER_LEN + payload_size * 2] = '\0';
|
||||
output[FORMAT_HEADER_LEN + portable_size * 2] = '\0';
|
||||
|
||||
free(compressed);
|
||||
free(portable);
|
||||
roaring_bitmap_free(truncated);
|
||||
free(set->encoded);
|
||||
@@ -173,22 +148,18 @@ static int hex_value(char c) {
|
||||
static roaring_bitmap_t* decode_bitmap(const char* str, unsigned* bpp) {
|
||||
roaring_bitmap_t* bitmap;
|
||||
unsigned char* bytes;
|
||||
unsigned char* portable;
|
||||
const char* hex;
|
||||
const char* reason = NULL;
|
||||
char version;
|
||||
size_t hex_len;
|
||||
size_t str_len;
|
||||
size_t byte_count;
|
||||
size_t portable_size;
|
||||
size_t expected;
|
||||
|
||||
if (!str || !bpp) return NULL;
|
||||
if (strncmp(str, "set:", 4) == 0) str += 4;
|
||||
str_len = strlen(str);
|
||||
if (str_len < FORMAT_HEADER_LEN) return NULL;
|
||||
if (str[0] != 'R' || (str[1] != '1' && str[1] != '2')) return NULL;
|
||||
version = str[1];
|
||||
if (strncmp(str, FORMAT_PREFIX, sizeof(FORMAT_PREFIX) - 1) != 0) return NULL;
|
||||
if (str[2] < '0' || str[2] > '9' || str[3] < '0' || str[3] > '9') return NULL;
|
||||
|
||||
*bpp = (unsigned)(str[2] - '0') * 10u + (unsigned)(str[3] - '0');
|
||||
@@ -211,44 +182,14 @@ static roaring_bitmap_t* decode_bitmap(const char* str, unsigned* bpp) {
|
||||
bytes[i] = (unsigned char)((high << 4) | low);
|
||||
}
|
||||
|
||||
if (version == '1') {
|
||||
portable = bytes;
|
||||
portable_size = byte_count;
|
||||
} else {
|
||||
unsigned long long frame_content_size;
|
||||
size_t frame_size = ZSTD_findFrameCompressedSize(bytes, byte_count);
|
||||
|
||||
if (ZSTD_isError(frame_size) || frame_size != byte_count) {
|
||||
free(bytes);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
frame_content_size = ZSTD_getFrameContentSize(bytes, byte_count);
|
||||
if (frame_content_size == ZSTD_CONTENTSIZE_ERROR ||
|
||||
frame_content_size == ZSTD_CONTENTSIZE_UNKNOWN || frame_content_size == 0 ||
|
||||
frame_content_size > MAX_SERIALIZED_SIZE || frame_content_size > SIZE_MAX) {
|
||||
free(bytes);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
portable_size = (size_t)frame_content_size;
|
||||
portable = xmalloc(portable_size);
|
||||
expected = ZSTD_decompress(portable, portable_size, bytes, byte_count);
|
||||
expected = roaring_bitmap_portable_deserialize_size((const char*)bytes, byte_count);
|
||||
if (expected != byte_count) {
|
||||
free(bytes);
|
||||
if (ZSTD_isError(expected) || expected != portable_size) {
|
||||
free(portable);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
expected = roaring_bitmap_portable_deserialize_size((const char*)portable, portable_size);
|
||||
if (expected != portable_size) {
|
||||
free(portable);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bitmap = roaring_bitmap_portable_deserialize_safe((const char*)portable, portable_size);
|
||||
free(portable);
|
||||
bitmap = roaring_bitmap_portable_deserialize_safe((const char*)bytes, byte_count);
|
||||
free(bytes);
|
||||
if (!bitmap) return NULL;
|
||||
if (!roaring_bitmap_internal_validate(bitmap, &reason) || roaring_bitmap_is_empty(bitmap)) {
|
||||
roaring_bitmap_free(bitmap);
|
||||
@@ -334,8 +275,8 @@ int main(void) {
|
||||
assert(rpmsetcmp(small16, large16) == -1);
|
||||
assert(rpmsetcmp(small16, other16) == -2);
|
||||
assert(rpmsetcmp("bad", small16) == -3);
|
||||
assert(rpmsetcmp("R21600", small16) == -3);
|
||||
assert(rpmsetcmp(small16, "R116xyz") == -4);
|
||||
assert(rpmsetcmp(small16, "R216xyz") == -4);
|
||||
assert(rpmsetcmp("R", small16) == -3);
|
||||
|
||||
free(prefixed);
|
||||
|
||||
@@ -13,35 +13,37 @@ if [[ -z ${ROARING_CFLAGS+x} || -z ${ROARING_LIBS+x} ]]; then
|
||||
ROARING_LIBS=$(pkg-config --libs roaring)
|
||||
else
|
||||
CROARING_SRC="$BUILD/CRoaring"
|
||||
CROARING_BUILD="$BUILD/CRoaring-build"
|
||||
CROARING_BUILD="$BUILD/CRoaring-pic-build"
|
||||
if [[ ! -d $CROARING_SRC/.git ]]; then
|
||||
rm -rf "$CROARING_SRC"
|
||||
git clone --depth 1 https://github.com/RoaringBitmap/CRoaring.git "$CROARING_SRC"
|
||||
fi
|
||||
if [[ ! -f $CROARING_BUILD/src/libroaring.a ]]; then
|
||||
cmake -S "$CROARING_SRC" -B "$CROARING_BUILD" \
|
||||
-DROARING_BUILD_STATIC=ON -DENABLE_ROARING_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
|
||||
-DROARING_BUILD_STATIC=ON -DENABLE_ROARING_TESTS=OFF \
|
||||
-DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON
|
||||
cmake --build "$CROARING_BUILD" --parallel
|
||||
fi
|
||||
ROARING_CFLAGS="-I$CROARING_SRC/include"
|
||||
ROARING_LIBS="$CROARING_BUILD/src/libroaring.a"
|
||||
fi
|
||||
fi
|
||||
ZSTD_CFLAGS=${ZSTD_CFLAGS-$(pkg-config --cflags libzstd)}
|
||||
ZSTD_LIBS=${ZSTD_LIBS-$(pkg-config --libs libzstd)}
|
||||
read -r -a ROARING_CFLAGS_A <<<"$ROARING_CFLAGS"
|
||||
read -r -a ROARING_LIBS_A <<<"$ROARING_LIBS"
|
||||
read -r -a ZSTD_CFLAGS_A <<<"$ZSTD_CFLAGS"
|
||||
read -r -a ZSTD_LIBS_A <<<"$ZSTD_LIBS"
|
||||
CFLAGS=(-O2 -std=gnu11 -D_GNU_SOURCE -Wall -Wextra -I"$HERE" -I"$BUILD")
|
||||
|
||||
for tool in mkset setcmp; do
|
||||
cc "${CFLAGS[@]}" -include "$ROOT/scripts/rpmsetcmp/newset_compat.h" \
|
||||
"$ROOT/reimplement/set9.c" "$ROOT/scripts/rpmsetcmp/$tool.c" \
|
||||
-o "$BUILD/$tool-set9"
|
||||
cc "${CFLAGS[@]}" "${ROARING_CFLAGS_A[@]}" "${ZSTD_CFLAGS_A[@]}" \
|
||||
cc "${CFLAGS[@]}" "${ROARING_CFLAGS_A[@]}" \
|
||||
"$HERE/bitmap_set.c" "$ROOT/scripts/rpmsetcmp/$tool.c" \
|
||||
"${ROARING_LIBS_A[@]}" "${ZSTD_LIBS_A[@]}" -o "$BUILD/$tool-bitmap"
|
||||
"${ROARING_LIBS_A[@]}" -o "$BUILD/$tool-bitmap"
|
||||
done
|
||||
|
||||
printf 'Built in %s: mkset-{set9,bitmap} setcmp-{set9,bitmap}\n' "$BUILD"
|
||||
cc "${CFLAGS[@]}" -fPIC -shared -include "$ROOT/scripts/rpmsetcmp/newset_compat.h" \
|
||||
"$ROOT/reimplement/set9.c" -o "$BUILD/libset9.so"
|
||||
cc "${CFLAGS[@]}" -fPIC -shared "${ROARING_CFLAGS_A[@]}" \
|
||||
"$HERE/bitmap_set.c" "${ROARING_LIBS_A[@]}" -o "$BUILD/libbitmap-set.so"
|
||||
|
||||
printf 'Built tools and benchmark libraries in %s\n' "$BUILD"
|
||||
|
||||
Reference in New Issue
Block a user