Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6584a628ce | ||
|
|
1c9a9be086 |
@@ -0,0 +1,75 @@
|
|||||||
|
# Другой дизайн
|
||||||
|
|
||||||
|
## Roaring map
|
||||||
|
|
||||||
|
По крайней мере в примитивном варианте скорость создания остаётся примерно прежней, скорость сравнения становится кратно меньше
|
||||||
|
|
||||||
|
Также кратно выше становится сама длина строки
|
||||||
|
|
||||||
|
```text
|
||||||
|
symbols=1000 required=500 bpp=32
|
||||||
|
implementation set_chars format
|
||||||
|
set9 3994 golomb
|
||||||
|
bitmap 19924 R1
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
operation set9 bitmap bitmap/set9
|
||||||
|
set_fini only 266.24 us 202.89 us 0.76x
|
||||||
|
new+add+fini 1696.05 us 1600.44 us 0.94x
|
||||||
|
rpmsetcmp cold 293.87 us 1321.17 us 4.50x
|
||||||
|
rpmsetcmp warm 5.24 us 399.20 us 76.14x
|
||||||
|
```
|
||||||
|
|
||||||
|
### Использование zstd
|
||||||
|
|
||||||
|
```text
|
||||||
|
symbols=1000 required=500 bpp=32
|
||||||
|
implementation set_chars format
|
||||||
|
set9 3994 golomb
|
||||||
|
bitmap 14126 R2
|
||||||
|
```
|
||||||
|
|
||||||
|
без zstd строка становится ~40% длинее
|
||||||
|
|
||||||
|
```text
|
||||||
|
operation set9 bitmap bitmap/set9
|
||||||
|
set_fini only 171.84 us 314.20 us 1.83x
|
||||||
|
new+add+fini 1185.33 us 1726.32 us 1.46x
|
||||||
|
rpmsetcmp cold 285.04 us 1412.84 us 4.96x
|
||||||
|
rpmsetcmp warm 4.82 us 490.94 us 101.86x
|
||||||
|
```
|
||||||
|
|
||||||
|
## хранить сразу расшифрованный set
|
||||||
|
|
||||||
|
- огромная часть ресурсов уходит не на просмотр включения множеств, а на декодирование set-строк
|
||||||
|
- при возможности хранить больше данных за более дешёвое сравнение - прекрасно
|
||||||
|
- тупо условный формат:
|
||||||
|
- `<bpp> <последовательно упакованные bpp-битные хеши>`
|
||||||
|
- Shannon–Fano–Elias coding как одна из идей, но надо глубже копать
|
||||||
|
|
||||||
|
## Группировать, а не хэшировать
|
||||||
|
|
||||||
|
- если была бы возможность точно делать соответствия между label и id, то provides стал бы в большинстве последовательным, а required было бы легко искать в P.
|
||||||
|
|
||||||
|
# Доработки на текущий дизайн
|
||||||
|
|
||||||
|
## битовый prefilter
|
||||||
|
|
||||||
|
- Позволяет отбросить заведомо ложные варианты
|
||||||
|
- но так ли часто это будет срабатывать, но вызывает доп расходы при высчитывании
|
||||||
|
- фильтр Блума как пример
|
||||||
|
- В теории хэш можно заменить на него
|
||||||
|
- необходимо знать, насколько больше будет ложноположительных срабатываний
|
||||||
|
|
||||||
|
## улучшить проход по массивам
|
||||||
|
|
||||||
|
- Можно provides хранить не в виде массива, а сразу как хэш-структурку
|
||||||
|
- если provides строка кэшируется не так часто, смысла не будет
|
||||||
|
|
||||||
|
# Прочие улучшения
|
||||||
|
|
||||||
|
## улучшение работы с хэшем
|
||||||
|
|
||||||
|
- если условно "отсортировать" массив provides/requires, можно получить лучшую работу с кжшом
|
||||||
|
- (надеюсь, что под капотом оно уже и так это делает, но проверить стоит)
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
попытка воссоздания алгоритма с roaring bitmap
|
||||||
|
|
||||||
|
Даже эффективней в создании, но проигрывает в дешифровке (сравнении) и занимаемому месту
|
||||||
Executable
+206
@@ -0,0 +1,206 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import argparse
|
||||||
|
import ctypes
|
||||||
|
import gc
|
||||||
|
import os
|
||||||
|
import statistics
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
HERE = Path(__file__).resolve().parent
|
||||||
|
BUILD = HERE / "build"
|
||||||
|
LIBC = ctypes.CDLL(None)
|
||||||
|
LIBC.free.argtypes = [ctypes.c_void_p]
|
||||||
|
|
||||||
|
|
||||||
|
class SetAPI:
|
||||||
|
def __init__(self, path: Path, result_needs_free: bool):
|
||||||
|
self.lib = ctypes.CDLL(str(path))
|
||||||
|
self.result_needs_free = result_needs_free
|
||||||
|
self.lib.set_new.restype = ctypes.c_void_p
|
||||||
|
self.lib.set_add.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
|
||||||
|
self.lib.set_fini.argtypes = [ctypes.c_void_p, ctypes.c_int]
|
||||||
|
self.lib.set_fini.restype = ctypes.c_void_p
|
||||||
|
self.lib.set_free.argtypes = [ctypes.c_void_p]
|
||||||
|
self.lib.set_free.restype = ctypes.c_void_p
|
||||||
|
self.lib.rpmsetcmp.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
|
||||||
|
self.lib.rpmsetcmp.restype = ctypes.c_int
|
||||||
|
|
||||||
|
def new_with_symbols(self, symbols):
|
||||||
|
value = self.lib.set_new()
|
||||||
|
if not value:
|
||||||
|
raise RuntimeError("set_new returned NULL")
|
||||||
|
for symbol in symbols:
|
||||||
|
self.lib.set_add(value, symbol)
|
||||||
|
return value
|
||||||
|
|
||||||
|
def release(self, value, result):
|
||||||
|
if self.result_needs_free and result:
|
||||||
|
LIBC.free(result)
|
||||||
|
self.lib.set_free(value)
|
||||||
|
|
||||||
|
def encode(self, symbols, bpp):
|
||||||
|
value = self.new_with_symbols(symbols)
|
||||||
|
result = self.lib.set_fini(value, bpp)
|
||||||
|
if not result:
|
||||||
|
raise RuntimeError("set_fini returned NULL")
|
||||||
|
encoded = ctypes.string_at(result)
|
||||||
|
self.release(value, result)
|
||||||
|
return encoded
|
||||||
|
|
||||||
|
|
||||||
|
def median_fini(api, symbols, bpp, calls, rounds):
|
||||||
|
samples = []
|
||||||
|
for _ in range(rounds):
|
||||||
|
sets = [api.new_with_symbols(symbols) for _ in range(calls)]
|
||||||
|
results = []
|
||||||
|
start = time.perf_counter_ns()
|
||||||
|
for value in sets:
|
||||||
|
results.append(api.lib.set_fini(value, bpp))
|
||||||
|
samples.append((time.perf_counter_ns() - start) / calls)
|
||||||
|
if not all(results):
|
||||||
|
raise RuntimeError("set_fini returned NULL")
|
||||||
|
encoded = [ctypes.string_at(result) for result in results]
|
||||||
|
if len(set(encoded)) != 1:
|
||||||
|
raise RuntimeError("set_fini is not deterministic")
|
||||||
|
for value, result in zip(sets, results):
|
||||||
|
api.release(value, result)
|
||||||
|
return statistics.median(samples)
|
||||||
|
|
||||||
|
|
||||||
|
def median_build(api, symbols, bpp, calls, rounds):
|
||||||
|
samples = []
|
||||||
|
for _ in range(rounds):
|
||||||
|
sets = []
|
||||||
|
results = []
|
||||||
|
start = time.perf_counter_ns()
|
||||||
|
for _ in range(calls):
|
||||||
|
value = api.new_with_symbols(symbols)
|
||||||
|
result = api.lib.set_fini(value, bpp)
|
||||||
|
sets.append(value)
|
||||||
|
results.append(result)
|
||||||
|
samples.append((time.perf_counter_ns() - start) / calls)
|
||||||
|
if not all(results):
|
||||||
|
raise RuntimeError("set_fini returned NULL")
|
||||||
|
for value, result in zip(sets, results):
|
||||||
|
api.release(value, result)
|
||||||
|
return statistics.median(samples)
|
||||||
|
|
||||||
|
|
||||||
|
def median_cmp(api, provider, requirement, calls, rounds):
|
||||||
|
expected = api.lib.rpmsetcmp(provider, requirement)
|
||||||
|
if expected != 1 or api.lib.rpmsetcmp(provider, provider) != 0:
|
||||||
|
raise RuntimeError(f"unexpected rpmsetcmp result: {expected}")
|
||||||
|
for _ in range(100):
|
||||||
|
api.lib.rpmsetcmp(provider, requirement)
|
||||||
|
|
||||||
|
samples = []
|
||||||
|
for _ in range(rounds):
|
||||||
|
checksum = 0
|
||||||
|
start = time.perf_counter_ns()
|
||||||
|
for _ in range(calls):
|
||||||
|
checksum += api.lib.rpmsetcmp(provider, requirement)
|
||||||
|
samples.append((time.perf_counter_ns() - start) / calls)
|
||||||
|
if checksum != calls:
|
||||||
|
raise RuntimeError("rpmsetcmp result changed during benchmark")
|
||||||
|
return statistics.median(samples)
|
||||||
|
|
||||||
|
|
||||||
|
def cold_cmp_once(api, provider, requirement):
|
||||||
|
read_fd, write_fd = os.pipe()
|
||||||
|
pid = os.fork()
|
||||||
|
if pid == 0:
|
||||||
|
os.close(read_fd)
|
||||||
|
start = time.perf_counter_ns()
|
||||||
|
result = api.lib.rpmsetcmp(provider, requirement)
|
||||||
|
elapsed = time.perf_counter_ns() - start
|
||||||
|
os.write(write_fd, f"{elapsed} {result}".encode())
|
||||||
|
os.close(write_fd)
|
||||||
|
os._exit(0)
|
||||||
|
|
||||||
|
os.close(write_fd)
|
||||||
|
payload = b""
|
||||||
|
while chunk := os.read(read_fd, 128):
|
||||||
|
payload += chunk
|
||||||
|
os.close(read_fd)
|
||||||
|
_, status = os.waitpid(pid, 0)
|
||||||
|
if status != 0:
|
||||||
|
raise RuntimeError(f"cold rpmsetcmp child failed: status={status}")
|
||||||
|
elapsed, result = map(int, payload.split())
|
||||||
|
if result != 1:
|
||||||
|
raise RuntimeError(f"unexpected cold rpmsetcmp result: {result}")
|
||||||
|
return elapsed
|
||||||
|
|
||||||
|
|
||||||
|
def median_cmp_cold(api, provider, requirement, calls, rounds):
|
||||||
|
samples = []
|
||||||
|
for _ in range(rounds):
|
||||||
|
total = sum(cold_cmp_once(api, provider, requirement) for _ in range(calls))
|
||||||
|
samples.append(total / calls)
|
||||||
|
return statistics.median(samples)
|
||||||
|
|
||||||
|
|
||||||
|
def format_time(ns):
|
||||||
|
return f"{ns / 1000:.2f} us"
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Compare set9 and CRoaring set APIs")
|
||||||
|
parser.add_argument("--symbols", type=int, default=1000)
|
||||||
|
parser.add_argument("--bpp", type=int, default=32)
|
||||||
|
parser.add_argument("--rounds", type=int, default=7)
|
||||||
|
parser.add_argument("--fini-calls", type=int, default=5)
|
||||||
|
parser.add_argument("--cmp-calls", type=int, default=2000)
|
||||||
|
parser.add_argument("--cold-calls", type=int, default=20)
|
||||||
|
parser.add_argument("--skip-build", action="store_true")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.symbols < 2 or not 10 <= args.bpp <= 32:
|
||||||
|
parser.error("symbols must be >= 2 and bpp must be in 10..32")
|
||||||
|
if min(args.rounds, args.fini_calls, args.cmp_calls, args.cold_calls) < 1:
|
||||||
|
parser.error("rounds and call counts must be positive")
|
||||||
|
|
||||||
|
if not args.skip_build:
|
||||||
|
subprocess.run([str(HERE / "build.sh")], check=True)
|
||||||
|
|
||||||
|
symbols = tuple(
|
||||||
|
f"symbol_{i:08d}_version_ALT_{i % 97}".encode() for i in range(args.symbols)
|
||||||
|
)
|
||||||
|
required = symbols[::2]
|
||||||
|
apis = {
|
||||||
|
"set9": SetAPI(BUILD / "libset9.so", result_needs_free=True),
|
||||||
|
"bitmap": SetAPI(BUILD / "libbitmap-set.so", result_needs_free=False),
|
||||||
|
}
|
||||||
|
|
||||||
|
gc.disable()
|
||||||
|
try:
|
||||||
|
timings = {}
|
||||||
|
lengths = {}
|
||||||
|
for name, api in apis.items():
|
||||||
|
provider = api.encode(symbols, args.bpp)
|
||||||
|
requirement = api.encode(required, args.bpp)
|
||||||
|
wire_format = provider[:2].decode() if provider.startswith(b"R1") else "golomb"
|
||||||
|
lengths[name] = (len(provider), wire_format)
|
||||||
|
timings[name] = (
|
||||||
|
median_fini(api, symbols, args.bpp, args.fini_calls, args.rounds),
|
||||||
|
median_build(api, symbols, args.bpp, args.fini_calls, args.rounds),
|
||||||
|
median_cmp_cold(api, provider, requirement, args.cold_calls, args.rounds),
|
||||||
|
median_cmp(api, provider, requirement, args.cmp_calls, args.rounds),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
gc.enable()
|
||||||
|
|
||||||
|
print(f"symbols={args.symbols} required={len(required)} bpp={args.bpp}")
|
||||||
|
print("implementation set_chars format")
|
||||||
|
for name in apis:
|
||||||
|
print(f"{name:<14} {lengths[name][0]:>9} {lengths[name][1]}")
|
||||||
|
print("\noperation set9 bitmap bitmap/set9")
|
||||||
|
labels = ("set_fini only", "new+add+fini", "rpmsetcmp cold", "rpmsetcmp warm")
|
||||||
|
for index, label in enumerate(labels):
|
||||||
|
old = timings["set9"][index]
|
||||||
|
new = timings["bitmap"][index]
|
||||||
|
print(f"{label:<22} {format_time(old):>10} {format_time(new):>10} {new / old:>12.2f}x")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
#include <roaring/roaring.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This is intentionally a new set-string format. It is not compatible with
|
||||||
|
* the Rice-Golomb/base62 strings produced by the original lib/set.c.
|
||||||
|
*
|
||||||
|
* R1<two decimal bpp digits><hex CRoaring portable serialization>
|
||||||
|
*/
|
||||||
|
#define FORMAT_PREFIX "R1"
|
||||||
|
#define FORMAT_HEADER_LEN 4
|
||||||
|
#define MAX_SERIALIZED_SIZE (64u * 1024u * 1024u)
|
||||||
|
|
||||||
|
struct set {
|
||||||
|
roaring_bitmap_t* hashes;
|
||||||
|
size_t added;
|
||||||
|
char* encoded;
|
||||||
|
};
|
||||||
|
|
||||||
|
static void* xmalloc(size_t size) {
|
||||||
|
void* ptr = malloc(size);
|
||||||
|
if (!ptr) abort();
|
||||||
|
return ptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint32_t hash_symbol(const char* str) {
|
||||||
|
uint32_t hash = UINT32_C(0x9e3779b9);
|
||||||
|
const unsigned char* p = (const unsigned char*)str;
|
||||||
|
|
||||||
|
while (*p) {
|
||||||
|
hash += *p++;
|
||||||
|
hash += hash << 10;
|
||||||
|
hash ^= hash >> 6;
|
||||||
|
}
|
||||||
|
|
||||||
|
hash += hash << 3;
|
||||||
|
hash ^= hash >> 11;
|
||||||
|
hash += hash << 15;
|
||||||
|
return hash;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct set* set_new(void) {
|
||||||
|
struct set* set = xmalloc(sizeof(*set));
|
||||||
|
set->hashes = roaring_bitmap_create();
|
||||||
|
if (!set->hashes) abort();
|
||||||
|
set->added = 0;
|
||||||
|
set->encoded = NULL;
|
||||||
|
return set;
|
||||||
|
}
|
||||||
|
|
||||||
|
void set_add(struct set* set, const char* sym) {
|
||||||
|
if (!set || !sym) return;
|
||||||
|
roaring_bitmap_add(set->hashes, hash_symbol(sym));
|
||||||
|
set->added++;
|
||||||
|
}
|
||||||
|
|
||||||
|
static roaring_bitmap_t* truncate_bitmap(const roaring_bitmap_t* source, unsigned bpp) {
|
||||||
|
roaring_bitmap_t* result;
|
||||||
|
roaring_uint32_iterator_t iterator;
|
||||||
|
uint32_t mask;
|
||||||
|
|
||||||
|
if (bpp == 32) {
|
||||||
|
result = roaring_bitmap_copy(source);
|
||||||
|
if (!result) abort();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
mask = (UINT32_C(1) << bpp) - 1;
|
||||||
|
result = roaring_bitmap_create();
|
||||||
|
if (!result) abort();
|
||||||
|
|
||||||
|
roaring_iterator_init(source, &iterator);
|
||||||
|
while (iterator.has_value) {
|
||||||
|
roaring_bitmap_add(result, iterator.current_value & mask);
|
||||||
|
roaring_uint32_iterator_advance(&iterator);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
static char hex_digit(unsigned value) {
|
||||||
|
return (char)(value < 10 ? '0' + value : 'a' + value - 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* set_fini(struct set* set, int bpp) {
|
||||||
|
roaring_bitmap_t* truncated;
|
||||||
|
size_t portable_size;
|
||||||
|
size_t written;
|
||||||
|
unsigned char* portable;
|
||||||
|
char* output;
|
||||||
|
|
||||||
|
if (!set || set->added == 0 || bpp < 10 || bpp > 32) return NULL;
|
||||||
|
|
||||||
|
truncated = truncate_bitmap(set->hashes, (unsigned)bpp);
|
||||||
|
if (roaring_bitmap_get_cardinality(truncated) < set->added)
|
||||||
|
fprintf(stderr, "warning: hash collision\n");
|
||||||
|
|
||||||
|
roaring_bitmap_run_optimize(truncated);
|
||||||
|
portable_size = roaring_bitmap_portable_size_in_bytes(truncated);
|
||||||
|
if (portable_size > MAX_SERIALIZED_SIZE) {
|
||||||
|
roaring_bitmap_free(truncated);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
portable = xmalloc(portable_size);
|
||||||
|
written = roaring_bitmap_portable_serialize(truncated, (char*)portable);
|
||||||
|
if (written != portable_size) abort();
|
||||||
|
if (portable_size > (SIZE_MAX - FORMAT_HEADER_LEN - 1) / 2) abort();
|
||||||
|
|
||||||
|
output = xmalloc(FORMAT_HEADER_LEN + portable_size * 2 + 1);
|
||||||
|
memcpy(output, FORMAT_PREFIX, sizeof(FORMAT_PREFIX) - 1);
|
||||||
|
output[2] = (char)('0' + bpp / 10);
|
||||||
|
output[3] = (char)('0' + bpp % 10);
|
||||||
|
|
||||||
|
for (size_t i = 0; i < portable_size; ++i) {
|
||||||
|
output[FORMAT_HEADER_LEN + i * 2] = hex_digit(portable[i] >> 4);
|
||||||
|
output[FORMAT_HEADER_LEN + i * 2 + 1] = hex_digit(portable[i] & 0x0f);
|
||||||
|
}
|
||||||
|
output[FORMAT_HEADER_LEN + portable_size * 2] = '\0';
|
||||||
|
|
||||||
|
free(portable);
|
||||||
|
roaring_bitmap_free(truncated);
|
||||||
|
free(set->encoded);
|
||||||
|
set->encoded = output;
|
||||||
|
return set->encoded;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct set* set_free(struct set* set) {
|
||||||
|
if (set) {
|
||||||
|
roaring_bitmap_free(set->hashes);
|
||||||
|
free(set->encoded);
|
||||||
|
free(set);
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int hex_value(char c) {
|
||||||
|
if (c >= '0' && c <= '9') return c - '0';
|
||||||
|
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
|
||||||
|
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static roaring_bitmap_t* decode_bitmap(const char* str, unsigned* bpp) {
|
||||||
|
roaring_bitmap_t* bitmap;
|
||||||
|
unsigned char* bytes;
|
||||||
|
const char* hex;
|
||||||
|
const char* reason = NULL;
|
||||||
|
size_t hex_len;
|
||||||
|
size_t str_len;
|
||||||
|
size_t byte_count;
|
||||||
|
size_t expected;
|
||||||
|
|
||||||
|
if (!str || !bpp) return NULL;
|
||||||
|
if (strncmp(str, "set:", 4) == 0) str += 4;
|
||||||
|
str_len = strlen(str);
|
||||||
|
if (str_len < FORMAT_HEADER_LEN) return NULL;
|
||||||
|
if (strncmp(str, FORMAT_PREFIX, sizeof(FORMAT_PREFIX) - 1) != 0) return NULL;
|
||||||
|
if (str[2] < '0' || str[2] > '9' || str[3] < '0' || str[3] > '9') return NULL;
|
||||||
|
|
||||||
|
*bpp = (unsigned)(str[2] - '0') * 10u + (unsigned)(str[3] - '0');
|
||||||
|
if (*bpp < 10 || *bpp > 32) return NULL;
|
||||||
|
|
||||||
|
hex = str + FORMAT_HEADER_LEN;
|
||||||
|
hex_len = str_len - FORMAT_HEADER_LEN;
|
||||||
|
if (hex_len == 0 || (hex_len & 1u) != 0) return NULL;
|
||||||
|
byte_count = hex_len / 2;
|
||||||
|
if (byte_count > MAX_SERIALIZED_SIZE) return NULL;
|
||||||
|
bytes = xmalloc(byte_count);
|
||||||
|
|
||||||
|
for (size_t i = 0; i < byte_count; ++i) {
|
||||||
|
int high = hex_value(hex[i * 2]);
|
||||||
|
int low = hex_value(hex[i * 2 + 1]);
|
||||||
|
if (high < 0 || low < 0) {
|
||||||
|
free(bytes);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
bytes[i] = (unsigned char)((high << 4) | low);
|
||||||
|
}
|
||||||
|
|
||||||
|
expected = roaring_bitmap_portable_deserialize_size((const char*)bytes, byte_count);
|
||||||
|
if (expected != byte_count) {
|
||||||
|
free(bytes);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
bitmap = roaring_bitmap_portable_deserialize_safe((const char*)bytes, byte_count);
|
||||||
|
free(bytes);
|
||||||
|
if (!bitmap) return NULL;
|
||||||
|
if (!roaring_bitmap_internal_validate(bitmap, &reason) || roaring_bitmap_is_empty(bitmap)) {
|
||||||
|
roaring_bitmap_free(bitmap);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
return bitmap;
|
||||||
|
}
|
||||||
|
|
||||||
|
int rpmsetcmp(const char* set1, const char* set2) {
|
||||||
|
roaring_bitmap_t* bitmap1;
|
||||||
|
roaring_bitmap_t* bitmap2;
|
||||||
|
unsigned bpp1;
|
||||||
|
unsigned bpp2;
|
||||||
|
unsigned common_bpp;
|
||||||
|
bool one_in_two;
|
||||||
|
bool two_in_one;
|
||||||
|
|
||||||
|
bitmap1 = decode_bitmap(set1, &bpp1);
|
||||||
|
if (!bitmap1) return -3;
|
||||||
|
bitmap2 = decode_bitmap(set2, &bpp2);
|
||||||
|
if (!bitmap2) {
|
||||||
|
roaring_bitmap_free(bitmap1);
|
||||||
|
return -4;
|
||||||
|
}
|
||||||
|
|
||||||
|
common_bpp = bpp1 < bpp2 ? bpp1 : bpp2;
|
||||||
|
if (bpp1 != common_bpp) {
|
||||||
|
roaring_bitmap_t* truncated = truncate_bitmap(bitmap1, common_bpp);
|
||||||
|
roaring_bitmap_free(bitmap1);
|
||||||
|
bitmap1 = truncated;
|
||||||
|
}
|
||||||
|
if (bpp2 != common_bpp) {
|
||||||
|
roaring_bitmap_t* truncated = truncate_bitmap(bitmap2, common_bpp);
|
||||||
|
roaring_bitmap_free(bitmap2);
|
||||||
|
bitmap2 = truncated;
|
||||||
|
}
|
||||||
|
|
||||||
|
one_in_two = roaring_bitmap_is_subset(bitmap1, bitmap2);
|
||||||
|
two_in_one = roaring_bitmap_is_subset(bitmap2, bitmap1);
|
||||||
|
roaring_bitmap_free(bitmap1);
|
||||||
|
roaring_bitmap_free(bitmap2);
|
||||||
|
|
||||||
|
if (one_in_two && two_in_one) return 0;
|
||||||
|
if (two_in_one) return 1;
|
||||||
|
if (one_in_two) return -1;
|
||||||
|
return -2;
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef SELF_TEST
|
||||||
|
#include <assert.h>
|
||||||
|
|
||||||
|
static char* make_set(const char* const* symbols, size_t count, int bpp) {
|
||||||
|
struct set* set = set_new();
|
||||||
|
const char* encoded;
|
||||||
|
char* copy;
|
||||||
|
|
||||||
|
for (size_t i = 0; i < count; ++i) set_add(set, symbols[i]);
|
||||||
|
encoded = set_fini(set, bpp);
|
||||||
|
assert(encoded);
|
||||||
|
copy = xmalloc(strlen(encoded) + 1);
|
||||||
|
strcpy(copy, encoded);
|
||||||
|
set = set_free(set);
|
||||||
|
assert(!set);
|
||||||
|
return copy;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(void) {
|
||||||
|
static const char* const small[] = {"malloc", "printf"};
|
||||||
|
static const char* const large[] = {"free", "malloc", "printf"};
|
||||||
|
static const char* const other[] = {"calloc", "malloc"};
|
||||||
|
char* small16 = make_set(small, 2, 16);
|
||||||
|
char* small32 = make_set(small, 2, 32);
|
||||||
|
char* large16 = make_set(large, 3, 16);
|
||||||
|
char* other16 = make_set(other, 2, 16);
|
||||||
|
char* prefixed = xmalloc(strlen(small16) + 5);
|
||||||
|
|
||||||
|
sprintf(prefixed, "set:%s", small16);
|
||||||
|
assert(rpmsetcmp(small16, small16) == 0);
|
||||||
|
assert(rpmsetcmp(prefixed, small16) == 0);
|
||||||
|
assert(rpmsetcmp(small32, small16) == 0);
|
||||||
|
assert(rpmsetcmp(large16, small16) == 1);
|
||||||
|
assert(rpmsetcmp(small16, large16) == -1);
|
||||||
|
assert(rpmsetcmp(small16, other16) == -2);
|
||||||
|
assert(rpmsetcmp("bad", small16) == -3);
|
||||||
|
assert(rpmsetcmp("R21600", small16) == -3);
|
||||||
|
assert(rpmsetcmp(small16, "R116xyz") == -4);
|
||||||
|
assert(rpmsetcmp("R", small16) == -3);
|
||||||
|
|
||||||
|
free(prefixed);
|
||||||
|
free(other16);
|
||||||
|
free(large16);
|
||||||
|
free(small32);
|
||||||
|
free(small16);
|
||||||
|
puts("bitmap_set self-test: OK");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
Executable
+49
@@ -0,0 +1,49 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||||
|
ROOT=$(cd "$HERE/../.." && pwd)
|
||||||
|
BUILD="$HERE/build"
|
||||||
|
mkdir -p "$BUILD"
|
||||||
|
touch "$BUILD/rpmlib.h" "$BUILD/system.h"
|
||||||
|
|
||||||
|
if [[ -z ${ROARING_CFLAGS+x} || -z ${ROARING_LIBS+x} ]]; then
|
||||||
|
if pkg-config --exists roaring; then
|
||||||
|
ROARING_CFLAGS=$(pkg-config --cflags roaring)
|
||||||
|
ROARING_LIBS=$(pkg-config --libs roaring)
|
||||||
|
else
|
||||||
|
CROARING_SRC="$BUILD/CRoaring"
|
||||||
|
CROARING_BUILD="$BUILD/CRoaring-pic-build"
|
||||||
|
if [[ ! -d $CROARING_SRC/.git ]]; then
|
||||||
|
rm -rf "$CROARING_SRC"
|
||||||
|
git clone --depth 1 https://github.com/RoaringBitmap/CRoaring.git "$CROARING_SRC"
|
||||||
|
fi
|
||||||
|
if [[ ! -f $CROARING_BUILD/src/libroaring.a ]]; then
|
||||||
|
cmake -S "$CROARING_SRC" -B "$CROARING_BUILD" \
|
||||||
|
-DROARING_BUILD_STATIC=ON -DENABLE_ROARING_TESTS=OFF \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON
|
||||||
|
cmake --build "$CROARING_BUILD" --parallel
|
||||||
|
fi
|
||||||
|
ROARING_CFLAGS="-I$CROARING_SRC/include"
|
||||||
|
ROARING_LIBS="$CROARING_BUILD/src/libroaring.a"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
read -r -a ROARING_CFLAGS_A <<<"$ROARING_CFLAGS"
|
||||||
|
read -r -a ROARING_LIBS_A <<<"$ROARING_LIBS"
|
||||||
|
CFLAGS=(-O2 -std=gnu11 -D_GNU_SOURCE -Wall -Wextra -I"$HERE" -I"$BUILD")
|
||||||
|
|
||||||
|
for tool in mkset setcmp; do
|
||||||
|
cc "${CFLAGS[@]}" -include "$ROOT/scripts/rpmsetcmp/newset_compat.h" \
|
||||||
|
"$ROOT/reimplement/set9.c" "$ROOT/scripts/rpmsetcmp/$tool.c" \
|
||||||
|
-o "$BUILD/$tool-set9"
|
||||||
|
cc "${CFLAGS[@]}" "${ROARING_CFLAGS_A[@]}" \
|
||||||
|
"$HERE/bitmap_set.c" "$ROOT/scripts/rpmsetcmp/$tool.c" \
|
||||||
|
"${ROARING_LIBS_A[@]}" -o "$BUILD/$tool-bitmap"
|
||||||
|
done
|
||||||
|
|
||||||
|
cc "${CFLAGS[@]}" -fPIC -shared -include "$ROOT/scripts/rpmsetcmp/newset_compat.h" \
|
||||||
|
"$ROOT/reimplement/set9.c" -o "$BUILD/libset9.so"
|
||||||
|
cc "${CFLAGS[@]}" -fPIC -shared "${ROARING_CFLAGS_A[@]}" \
|
||||||
|
"$HERE/bitmap_set.c" "${ROARING_LIBS_A[@]}" -o "$BUILD/libbitmap-set.so"
|
||||||
|
|
||||||
|
printf 'Built tools and benchmark libraries in %s\n' "$BUILD"
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
#ifndef ARSV_SET_H
|
||||||
|
#define ARSV_SET_H
|
||||||
|
|
||||||
|
struct set;
|
||||||
|
|
||||||
|
int rpmsetcmp(const char *set1, const char *set2);
|
||||||
|
struct set *set_new(void);
|
||||||
|
void set_add(struct set *set, const char *sym);
|
||||||
|
const char *set_fini(struct set *set, int bpp);
|
||||||
|
struct set *set_free(struct set *set);
|
||||||
|
|
||||||
|
#endif
|
||||||
Reference in New Issue
Block a user