Compare commits

..
3 Commits
Author SHA1 Message Date
krosh 8b65eb5e37 Use standard errno error codes 2026-08-10 03:17:51 +03:00
krosh 39b4673421 Rename hash and downsample parameters 2026-08-10 03:11:37 +03:00
krosh 4ed05f2061 Refactor set version hashing constants 2026-08-10 03:06:12 +03:00
+46 -51
View File
@@ -2,6 +2,7 @@
#undef NDEBUG #undef NDEBUG
#endif #endif
#include <assert.h> #include <assert.h>
#include <errno.h>
#include <limits.h> #include <limits.h>
#include <stdint.h> #include <stdint.h>
#include <stdlib.h> #include <stdlib.h>
@@ -15,26 +16,25 @@
#include "set.h" #include "set.h"
#include "system.h" #include "system.h"
#define CACHE_SIZE 512 enum {
#define SET_HASH_SEED UINT32_C(0x9e3779b9) CACHE_SIZE = 512,
CACHE_BUCKETS = 1024,
#define CACHE_FINGERPRINT_MULTIPLIER UINT32_C(0x9e3779b1) };
#define CACHE_TARGET_BPP_MULTIPLIER UINT32_C(0x85ebca6b)
enum { enum {
CACHE_BUCKETS = 1024,
SET_HEADER_SIZE = 2, SET_HEADER_SIZE = 2,
SET_PARAM_CHAR_OFFSET = 7, SET_PARAM_CHAR_OFFSET = 7,
SET_BPP_MIN = 10, SET_BPP_MIN = 10,
SET_BPP_MAX = 32, SET_BPP_MAX = 32,
SET_MSHIFT_MIN = 7, SET_MSHIFT_MIN = 7,
SET_MSHIFT_MAX = 31, SET_MSHIFT_MAX = 31,
};
enum {
BASE62_ESCAPE_BITS = 4, BASE62_ESCAPE_BITS = 4,
BASE62_VALUE_BITS = 6, BASE62_VALUE_BITS = 6,
BASE62_ESCAPE_CHUNK_BITS = BASE62_ESCAPE_BITS + BASE62_VALUE_BITS, BASE62_ESCAPE_CHUNK_BITS = BASE62_ESCAPE_BITS + BASE62_VALUE_BITS,
BASE62_MIN_BITS_PER_CHAR = BASE62_VALUE_BITS - 1, BASE62_MIN_BITS_PER_CHAR = BASE62_ESCAPE_CHUNK_BITS / 2,
BASE62_MAX_PADDING_BITS = BASE62_VALUE_BITS - 1, BASE62_MAX_PADDING_BITS = BASE62_VALUE_BITS - 1,
BASE62_LOWERCASE_OFFSET = 10, BASE62_LOWERCASE_OFFSET = 10,
BASE62_UPPERCASE_OFFSET = 36, BASE62_UPPERCASE_OFFSET = 36,
@@ -42,12 +42,14 @@ enum {
BASE62_VALUE_COUNT = 62, BASE62_VALUE_COUNT = 62,
BASE62_ESCAPE_LOW_MASK = (1u << BASE62_ESCAPE_BITS) - 1, BASE62_ESCAPE_LOW_MASK = (1u << BASE62_ESCAPE_BITS) - 1,
BASE62_ESCAPE_HIGH_MASK = 3u << BASE62_ESCAPE_BITS, BASE62_ESCAPE_HIGH_MASK = 3u << BASE62_ESCAPE_BITS,
};
enum {
BASE62_INVALID = 0xee, BASE62_INVALID = 0xee,
BASE62_END = UCHAR_MAX, BASE62_END = UCHAR_MAX,
}; };
_Static_assert(CHAR_BIT == 8, "set:version requires 8-bit bytes"); _Static_assert(CHAR_BIT == 8, "set:version relies on 8-bit");
struct set { struct set {
size_t cnt; size_t cnt;
@@ -82,7 +84,7 @@ void set_add(struct set* set, const char* sym) {
size_t length = strlen(sym) + 1; size_t length = strlen(sym) + 1;
size_t required = set->strings_len + length; size_t required = set->strings_len + length;
if (required > set->strings_cap) { if (required > set->strings_cap) {
size_t capacity = set->strings_cap ? set->strings_cap : 4096; size_t capacity = set->strings_cap ?: 4096;
while (capacity < required) capacity *= 2; while (capacity < required) capacity *= 2;
set->strings = xrealloc(set->strings, capacity); set->strings = xrealloc(set->strings, capacity);
@@ -94,8 +96,6 @@ void set_add(struct set* set, const char* sym) {
memcpy(set->strings + set->strings_len, sym, length); memcpy(set->strings + set->strings_len, sym, length);
set->strings_len = required; set->strings_len = required;
set->cnt++; set->cnt++;
return;
} }
struct set* set_free(struct set* set) { struct set* set_free(struct set* set) {
@@ -111,7 +111,7 @@ struct set* set_free(struct set* set) {
// --- // ---
static unsigned hash(const char* str) { static unsigned hash(const char* str) {
unsigned hash = SET_HASH_SEED; unsigned hash = UINT32_C(0x9e3779b9);
const unsigned char* p = (const unsigned char*)str; const unsigned char* p = (const unsigned char*)str;
while (*p) { while (*p) {
@@ -171,8 +171,6 @@ static void sort_symbols(struct symbols* values, size_t count, int bpp) {
} }
if (source != values) memcpy(values, source, count * sizeof(*values)); if (source != values) memcpy(values, source, count * sizeof(*values));
return;
} }
// --- // ---
@@ -292,16 +290,12 @@ static inline void encode_writer_zeros(struct encode_writer* writer, unsigned co
encode_writer_flush(writer); encode_writer_flush(writer);
} }
return;
} }
static inline void encode_writer_put(struct encode_writer* writer, uint64_t value, unsigned width) { static inline void encode_writer_put(struct encode_writer* writer, uint64_t value, unsigned width) {
writer->bits |= value << writer->filled; writer->bits |= value << writer->filled;
writer->filled += width; writer->filled += width;
encode_writer_flush(writer); encode_writer_flush(writer);
return;
} }
static int encode_set(int cnt, const unsigned* hash_arr, int bpp, char* base62_str) { static int encode_set(int cnt, const unsigned* hash_arr, int bpp, char* base62_str) {
@@ -358,7 +352,7 @@ const char* set_fini(struct set* set, int bpp) {
const char* right = set->strings + set->symbols_v[i + 1].offset; const char* right = set->strings + set->symbols_v[i + 1].offset;
if (!strcmp(left, right)) continue; if (!strcmp(left, right)) continue;
fprintf(stderr, "warning: hash collision: %s %s\n", left, right); fprintf(stderr, "warning: set-version hash collision: %s %s\n", left, right);
} }
unsigned unique_hash[set->cnt]; unsigned unique_hash[set->cnt];
@@ -393,14 +387,14 @@ struct set_meta {
static int set_meta_init(const char* str, struct set_meta* meta) { static int set_meta_init(const char* str, struct set_meta* meta) {
// The header must be followed by at least one payload character. // The header must be followed by at least one payload character.
if (!str[0] || !str[1] || !str[SET_HEADER_SIZE]) return -4; if (!str[0] || !str[1] || !str[SET_HEADER_SIZE]) return -EINVAL;
int bpp = str[0] + SET_PARAM_CHAR_OFFSET - 'a'; int bpp = str[0] + SET_PARAM_CHAR_OFFSET - 'a';
if (bpp < SET_BPP_MIN || bpp > SET_BPP_MAX) return -1; if (bpp < SET_BPP_MIN || bpp > SET_BPP_MAX) return -ERANGE;
int Mshift = str[1] + SET_PARAM_CHAR_OFFSET - 'a'; int Mshift = str[1] + SET_PARAM_CHAR_OFFSET - 'a';
if (Mshift < SET_MSHIFT_MIN || Mshift > SET_MSHIFT_MAX) return -2; if (Mshift < SET_MSHIFT_MIN || Mshift > SET_MSHIFT_MAX) return -ERANGE;
if (Mshift >= bpp) return -3; if (Mshift >= bpp) return -EINVAL;
*meta = (struct set_meta){ *meta = (struct set_meta){
.str = str, .str = str,
@@ -420,7 +414,7 @@ static int set_meta_fini(struct set_meta* meta) {
int bit_capacity = (int)payload_len * BASE62_VALUE_BITS; int bit_capacity = (int)payload_len * BASE62_VALUE_BITS;
int value_capacity = bit_capacity / (meta->Mshift + 1); int value_capacity = bit_capacity / (meta->Mshift + 1);
if (value_capacity < 1) return -4; if (value_capacity < 1) return -EINVAL;
meta->len = len; meta->len = len;
meta->payload_len = payload_len; meta->payload_len = payload_len;
@@ -470,14 +464,14 @@ static inline int decode_chunk(const unsigned char** input, uint64_t* chunk, uns
return 1; return 1;
} }
if (value == BASE62_END) return 0; if (value == BASE62_END) return 0;
if (value == BASE62_INVALID) return -1; if (value == BASE62_INVALID) return -EINVAL;
unsigned escaped = char_to_num[*(*input)++]; unsigned escaped = char_to_num[*(*input)++];
if (escaped == BASE62_END) return -2; if (escaped == BASE62_END) return -EINVAL;
if (escaped == BASE62_INVALID) return -3; if (escaped == BASE62_INVALID) return -EINVAL;
unsigned high = escaped & BASE62_ESCAPE_HIGH_MASK; unsigned high = escaped & BASE62_ESCAPE_HIGH_MASK;
if (high == BASE62_ESCAPE_HIGH_MASK) return -4; if (high == BASE62_ESCAPE_HIGH_MASK) return -EINVAL;
*chunk = (BASE62_ESCAPE_VALUE + (high >> BASE62_ESCAPE_BITS)) | *chunk = (BASE62_ESCAPE_VALUE + (high >> BASE62_ESCAPE_BITS)) |
((uint64_t)(escaped & BASE62_ESCAPE_LOW_MASK) << BASE62_VALUE_BITS); ((uint64_t)(escaped & BASE62_ESCAPE_LOW_MASK) << BASE62_VALUE_BITS);
@@ -505,7 +499,7 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
int rc = decode_chunk(&input, &chunk, &width); int rc = decode_chunk(&input, &chunk, &width);
if (rc < 0) return rc; if (rc < 0) return rc;
if (rc == 0) return q <= BASE62_MAX_PADDING_BITS ? count : -10; if (rc == 0) return q <= BASE62_MAX_PADDING_BITS ? count : -EINVAL;
bits = chunk; bits = chunk;
filled = width; filled = width;
@@ -537,7 +531,7 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
unsigned width; unsigned width;
int rc = decode_chunk(&input, &chunk, &width); int rc = decode_chunk(&input, &chunk, &width);
if (rc < 0) return rc; if (rc < 0) return rc;
if (rc == 0) return -11; if (rc == 0) return -EINVAL;
bits |= chunk << filled; bits |= chunk << filled;
filled += width; filled += width;
} }
@@ -553,12 +547,12 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
} }
// Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates. // Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates.
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp); static int downsample_set(int hash_cnt, const unsigned* hash_pt, unsigned* ds_pt, int target_bpp);
static inline unsigned cache_bucket(uint32_t fingerprint, int target_bpp) { static inline unsigned cache_bucket(uint32_t fingerprint, int target_bpp) {
uint32_t mixed = fingerprint ^ ((uint32_t)target_bpp * CACHE_TARGET_BPP_MULTIPLIER); uint32_t mixed = fingerprint ^ ((uint32_t)target_bpp * UINT32_C(0x85ebca6b));
mixed ^= mixed >> 11; mixed ^= mixed >> 11;
mixed *= CACHE_FINGERPRINT_MULTIPLIER; mixed *= UINT32_C(0x9e3779b1);
mixed ^= mixed >> 16; mixed ^= mixed >> 16;
return mixed & (CACHE_BUCKETS - 1); return mixed & (CACHE_BUCKETS - 1);
@@ -609,7 +603,8 @@ static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigne
return ent->cnt; return ent->cnt;
} }
if (set_meta_fini(meta) < 0) return -4; int meta_status = set_meta_fini(meta);
if (meta_status < 0) return meta_status;
int len = (int)meta->len; int len = (int)meta->len;
int capacity = meta->value_capacity; int capacity = meta->value_capacity;
@@ -679,12 +674,12 @@ static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigne
} }
// Reduce a set of (bpp + 1) values to a set of bpp values. // Reduce a set of (bpp + 1) values to a set of bpp values.
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp) { static int downsample_set(int hash_cnt, const unsigned* hash_pt, unsigned* ds_pt, int target_bpp) {
unsigned mask = (1u << bpp) - 1; unsigned mask = (1u << target_bpp) - 1;
// find the first element with high bit set // find the first element with high bit set
int l = 0; int l = 0;
int u = cnt; int u = hash_cnt;
while (l < u) { while (l < u) {
int i = (l + u) / 2; int i = (l + u) / 2;
@@ -698,7 +693,7 @@ static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int
// initialize parts // initialize parts
const unsigned* ds_start = ds_pt; const unsigned* ds_start = ds_pt;
const unsigned *v1 = hash_pt + 0, *v1_end = hash_pt + u; const unsigned *v1 = hash_pt + 0, *v1_end = hash_pt + u;
const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + cnt; const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + hash_cnt;
// merge v1 and v2 into w // merge v1 and v2 into w
if (v1 < v1_end && v2 < v2_end) { if (v1 < v1_end && v2 < v2_end) {
@@ -971,10 +966,10 @@ static void test_metadata_and_chunks(void) {
{.input = "Zg", .rc = 1, .chunk = 62, .width = 10}, {.input = "Zg", .rc = 1, .chunk = 62, .width = 10},
{.input = "Zw", .rc = 1, .chunk = 63, .width = 10}, {.input = "Zw", .rc = 1, .chunk = 63, .width = 10},
{.input = "", .rc = 0, .chunk = 0, .width = 0}, {.input = "", .rc = 0, .chunk = 0, .width = 0},
{.input = "!", .rc = -1, .chunk = 0, .width = 0}, {.input = "!", .rc = -EINVAL, .chunk = 0, .width = 0},
{.input = "Z", .rc = -2, .chunk = 0, .width = 0}, {.input = "Z", .rc = -EINVAL, .chunk = 0, .width = 0},
{.input = "Z!", .rc = -3, .chunk = 0, .width = 0}, {.input = "Z!", .rc = -EINVAL, .chunk = 0, .width = 0},
{.input = "ZM", .rc = -4, .chunk = 0, .width = 0}, {.input = "ZM", .rc = -EINVAL, .chunk = 0, .width = 0},
}; };
for (size_t i = 0; i < sizeof(cases) / sizeof(*cases); ++i) { for (size_t i = 0; i < sizeof(cases) / sizeof(*cases); ++i) {
@@ -991,15 +986,15 @@ static void test_metadata_and_chunks(void) {
} }
struct set_meta meta; struct set_meta meta;
assert(set_meta_init("", &meta) == -4); // too short assert(set_meta_init("", &meta) == -EINVAL); // too short
assert(set_meta_init("da", &meta) == -4); assert(set_meta_init("da", &meta) == -EINVAL);
assert(set_meta_init("ca0", &meta) == -1); // incorrect bpp assert(set_meta_init("ca0", &meta) == -ERANGE); // incorrect bpp
assert(set_meta_init("{a0", &meta) == -1); assert(set_meta_init("{a0", &meta) == -ERANGE);
assert(set_meta_init("d`0", &meta) == -2); // incorrect Mshift assert(set_meta_init("d`0", &meta) == -ERANGE); // incorrect Mshift
assert(set_meta_init("dz0", &meta) == -2); assert(set_meta_init("dz0", &meta) == -ERANGE);
assert(set_meta_init("dd0", &meta) == -3); // Mshift == bpp assert(set_meta_init("dd0", &meta) == -EINVAL); // Mshift == bpp
assert(set_meta_init("da0", &meta) == 0); assert(set_meta_init("da0", &meta) == 0);
assert(set_meta_fini(&meta) == -4); // not enough data assert(set_meta_fini(&meta) == -EINVAL); // not enough data
assert(set_meta_init("da00", &meta) == 0); assert(set_meta_init("da00", &meta) == 0);
assert(set_meta_fini(&meta) == 0); // ok assert(set_meta_fini(&meta) == 0); // ok
assert(meta.len == 4); assert(meta.len == 4);