Compare commits
3
Commits
ebb00ee69f
...
8b65eb5e37
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b65eb5e37 | ||
|
|
39b4673421 | ||
|
|
4ed05f2061 |
+46
-51
@@ -2,6 +2,7 @@
|
||||
#undef NDEBUG
|
||||
#endif
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
@@ -15,26 +16,25 @@
|
||||
#include "set.h"
|
||||
#include "system.h"
|
||||
|
||||
#define CACHE_SIZE 512
|
||||
#define SET_HASH_SEED UINT32_C(0x9e3779b9)
|
||||
|
||||
#define CACHE_FINGERPRINT_MULTIPLIER UINT32_C(0x9e3779b1)
|
||||
#define CACHE_TARGET_BPP_MULTIPLIER UINT32_C(0x85ebca6b)
|
||||
enum {
|
||||
CACHE_SIZE = 512,
|
||||
CACHE_BUCKETS = 1024,
|
||||
};
|
||||
|
||||
enum {
|
||||
CACHE_BUCKETS = 1024,
|
||||
|
||||
SET_HEADER_SIZE = 2,
|
||||
SET_PARAM_CHAR_OFFSET = 7,
|
||||
SET_BPP_MIN = 10,
|
||||
SET_BPP_MAX = 32,
|
||||
SET_MSHIFT_MIN = 7,
|
||||
SET_MSHIFT_MAX = 31,
|
||||
};
|
||||
|
||||
enum {
|
||||
BASE62_ESCAPE_BITS = 4,
|
||||
BASE62_VALUE_BITS = 6,
|
||||
BASE62_ESCAPE_CHUNK_BITS = BASE62_ESCAPE_BITS + BASE62_VALUE_BITS,
|
||||
BASE62_MIN_BITS_PER_CHAR = BASE62_VALUE_BITS - 1,
|
||||
BASE62_MIN_BITS_PER_CHAR = BASE62_ESCAPE_CHUNK_BITS / 2,
|
||||
BASE62_MAX_PADDING_BITS = BASE62_VALUE_BITS - 1,
|
||||
BASE62_LOWERCASE_OFFSET = 10,
|
||||
BASE62_UPPERCASE_OFFSET = 36,
|
||||
@@ -42,12 +42,14 @@ enum {
|
||||
BASE62_VALUE_COUNT = 62,
|
||||
BASE62_ESCAPE_LOW_MASK = (1u << BASE62_ESCAPE_BITS) - 1,
|
||||
BASE62_ESCAPE_HIGH_MASK = 3u << BASE62_ESCAPE_BITS,
|
||||
};
|
||||
|
||||
enum {
|
||||
BASE62_INVALID = 0xee,
|
||||
BASE62_END = UCHAR_MAX,
|
||||
};
|
||||
|
||||
_Static_assert(CHAR_BIT == 8, "set:version requires 8-bit bytes");
|
||||
_Static_assert(CHAR_BIT == 8, "set:version relies on 8-bit");
|
||||
|
||||
struct set {
|
||||
size_t cnt;
|
||||
@@ -82,7 +84,7 @@ void set_add(struct set* set, const char* sym) {
|
||||
size_t length = strlen(sym) + 1;
|
||||
size_t required = set->strings_len + length;
|
||||
if (required > set->strings_cap) {
|
||||
size_t capacity = set->strings_cap ? set->strings_cap : 4096;
|
||||
size_t capacity = set->strings_cap ?: 4096;
|
||||
while (capacity < required) capacity *= 2;
|
||||
|
||||
set->strings = xrealloc(set->strings, capacity);
|
||||
@@ -94,8 +96,6 @@ void set_add(struct set* set, const char* sym) {
|
||||
memcpy(set->strings + set->strings_len, sym, length);
|
||||
set->strings_len = required;
|
||||
set->cnt++;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
struct set* set_free(struct set* set) {
|
||||
@@ -111,7 +111,7 @@ struct set* set_free(struct set* set) {
|
||||
// ---
|
||||
|
||||
static unsigned hash(const char* str) {
|
||||
unsigned hash = SET_HASH_SEED;
|
||||
unsigned hash = UINT32_C(0x9e3779b9);
|
||||
const unsigned char* p = (const unsigned char*)str;
|
||||
|
||||
while (*p) {
|
||||
@@ -171,8 +171,6 @@ static void sort_symbols(struct symbols* values, size_t count, int bpp) {
|
||||
}
|
||||
|
||||
if (source != values) memcpy(values, source, count * sizeof(*values));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// ---
|
||||
@@ -292,16 +290,12 @@ static inline void encode_writer_zeros(struct encode_writer* writer, unsigned co
|
||||
|
||||
encode_writer_flush(writer);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
static inline void encode_writer_put(struct encode_writer* writer, uint64_t value, unsigned width) {
|
||||
writer->bits |= value << writer->filled;
|
||||
writer->filled += width;
|
||||
encode_writer_flush(writer);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
static int encode_set(int cnt, const unsigned* hash_arr, int bpp, char* base62_str) {
|
||||
@@ -358,7 +352,7 @@ const char* set_fini(struct set* set, int bpp) {
|
||||
const char* right = set->strings + set->symbols_v[i + 1].offset;
|
||||
if (!strcmp(left, right)) continue;
|
||||
|
||||
fprintf(stderr, "warning: hash collision: %s %s\n", left, right);
|
||||
fprintf(stderr, "warning: set-version hash collision: %s %s\n", left, right);
|
||||
}
|
||||
|
||||
unsigned unique_hash[set->cnt];
|
||||
@@ -393,14 +387,14 @@ struct set_meta {
|
||||
|
||||
static int set_meta_init(const char* str, struct set_meta* meta) {
|
||||
// The header must be followed by at least one payload character.
|
||||
if (!str[0] || !str[1] || !str[SET_HEADER_SIZE]) return -4;
|
||||
if (!str[0] || !str[1] || !str[SET_HEADER_SIZE]) return -EINVAL;
|
||||
|
||||
int bpp = str[0] + SET_PARAM_CHAR_OFFSET - 'a';
|
||||
if (bpp < SET_BPP_MIN || bpp > SET_BPP_MAX) return -1;
|
||||
if (bpp < SET_BPP_MIN || bpp > SET_BPP_MAX) return -ERANGE;
|
||||
|
||||
int Mshift = str[1] + SET_PARAM_CHAR_OFFSET - 'a';
|
||||
if (Mshift < SET_MSHIFT_MIN || Mshift > SET_MSHIFT_MAX) return -2;
|
||||
if (Mshift >= bpp) return -3;
|
||||
if (Mshift < SET_MSHIFT_MIN || Mshift > SET_MSHIFT_MAX) return -ERANGE;
|
||||
if (Mshift >= bpp) return -EINVAL;
|
||||
|
||||
*meta = (struct set_meta){
|
||||
.str = str,
|
||||
@@ -420,7 +414,7 @@ static int set_meta_fini(struct set_meta* meta) {
|
||||
int bit_capacity = (int)payload_len * BASE62_VALUE_BITS;
|
||||
int value_capacity = bit_capacity / (meta->Mshift + 1);
|
||||
|
||||
if (value_capacity < 1) return -4;
|
||||
if (value_capacity < 1) return -EINVAL;
|
||||
|
||||
meta->len = len;
|
||||
meta->payload_len = payload_len;
|
||||
@@ -470,14 +464,14 @@ static inline int decode_chunk(const unsigned char** input, uint64_t* chunk, uns
|
||||
return 1;
|
||||
}
|
||||
if (value == BASE62_END) return 0;
|
||||
if (value == BASE62_INVALID) return -1;
|
||||
if (value == BASE62_INVALID) return -EINVAL;
|
||||
|
||||
unsigned escaped = char_to_num[*(*input)++];
|
||||
if (escaped == BASE62_END) return -2;
|
||||
if (escaped == BASE62_INVALID) return -3;
|
||||
if (escaped == BASE62_END) return -EINVAL;
|
||||
if (escaped == BASE62_INVALID) return -EINVAL;
|
||||
|
||||
unsigned high = escaped & BASE62_ESCAPE_HIGH_MASK;
|
||||
if (high == BASE62_ESCAPE_HIGH_MASK) return -4;
|
||||
if (high == BASE62_ESCAPE_HIGH_MASK) return -EINVAL;
|
||||
|
||||
*chunk = (BASE62_ESCAPE_VALUE + (high >> BASE62_ESCAPE_BITS)) |
|
||||
((uint64_t)(escaped & BASE62_ESCAPE_LOW_MASK) << BASE62_VALUE_BITS);
|
||||
@@ -505,7 +499,7 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
|
||||
int rc = decode_chunk(&input, &chunk, &width);
|
||||
|
||||
if (rc < 0) return rc;
|
||||
if (rc == 0) return q <= BASE62_MAX_PADDING_BITS ? count : -10;
|
||||
if (rc == 0) return q <= BASE62_MAX_PADDING_BITS ? count : -EINVAL;
|
||||
|
||||
bits = chunk;
|
||||
filled = width;
|
||||
@@ -537,7 +531,7 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
|
||||
unsigned width;
|
||||
int rc = decode_chunk(&input, &chunk, &width);
|
||||
if (rc < 0) return rc;
|
||||
if (rc == 0) return -11;
|
||||
if (rc == 0) return -EINVAL;
|
||||
bits |= chunk << filled;
|
||||
filled += width;
|
||||
}
|
||||
@@ -553,12 +547,12 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
|
||||
}
|
||||
|
||||
// Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates.
|
||||
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp);
|
||||
static int downsample_set(int hash_cnt, const unsigned* hash_pt, unsigned* ds_pt, int target_bpp);
|
||||
|
||||
static inline unsigned cache_bucket(uint32_t fingerprint, int target_bpp) {
|
||||
uint32_t mixed = fingerprint ^ ((uint32_t)target_bpp * CACHE_TARGET_BPP_MULTIPLIER);
|
||||
uint32_t mixed = fingerprint ^ ((uint32_t)target_bpp * UINT32_C(0x85ebca6b));
|
||||
mixed ^= mixed >> 11;
|
||||
mixed *= CACHE_FINGERPRINT_MULTIPLIER;
|
||||
mixed *= UINT32_C(0x9e3779b1);
|
||||
mixed ^= mixed >> 16;
|
||||
|
||||
return mixed & (CACHE_BUCKETS - 1);
|
||||
@@ -609,7 +603,8 @@ static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigne
|
||||
return ent->cnt;
|
||||
}
|
||||
|
||||
if (set_meta_fini(meta) < 0) return -4;
|
||||
int meta_status = set_meta_fini(meta);
|
||||
if (meta_status < 0) return meta_status;
|
||||
|
||||
int len = (int)meta->len;
|
||||
int capacity = meta->value_capacity;
|
||||
@@ -679,12 +674,12 @@ static int cache_decode_set(struct set_meta* meta, int target_bpp, const unsigne
|
||||
}
|
||||
|
||||
// Reduce a set of (bpp + 1) values to a set of bpp values.
|
||||
static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int bpp) {
|
||||
unsigned mask = (1u << bpp) - 1;
|
||||
static int downsample_set(int hash_cnt, const unsigned* hash_pt, unsigned* ds_pt, int target_bpp) {
|
||||
unsigned mask = (1u << target_bpp) - 1;
|
||||
|
||||
// find the first element with high bit set
|
||||
int l = 0;
|
||||
int u = cnt;
|
||||
int u = hash_cnt;
|
||||
while (l < u) {
|
||||
int i = (l + u) / 2;
|
||||
|
||||
@@ -698,7 +693,7 @@ static int downsample_set(int cnt, const unsigned* hash_pt, unsigned* ds_pt, int
|
||||
// initialize parts
|
||||
const unsigned* ds_start = ds_pt;
|
||||
const unsigned *v1 = hash_pt + 0, *v1_end = hash_pt + u;
|
||||
const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + cnt;
|
||||
const unsigned *v2 = hash_pt + u, *v2_end = hash_pt + hash_cnt;
|
||||
|
||||
// merge v1 and v2 into w
|
||||
if (v1 < v1_end && v2 < v2_end) {
|
||||
@@ -971,10 +966,10 @@ static void test_metadata_and_chunks(void) {
|
||||
{.input = "Zg", .rc = 1, .chunk = 62, .width = 10},
|
||||
{.input = "Zw", .rc = 1, .chunk = 63, .width = 10},
|
||||
{.input = "", .rc = 0, .chunk = 0, .width = 0},
|
||||
{.input = "!", .rc = -1, .chunk = 0, .width = 0},
|
||||
{.input = "Z", .rc = -2, .chunk = 0, .width = 0},
|
||||
{.input = "Z!", .rc = -3, .chunk = 0, .width = 0},
|
||||
{.input = "ZM", .rc = -4, .chunk = 0, .width = 0},
|
||||
{.input = "!", .rc = -EINVAL, .chunk = 0, .width = 0},
|
||||
{.input = "Z", .rc = -EINVAL, .chunk = 0, .width = 0},
|
||||
{.input = "Z!", .rc = -EINVAL, .chunk = 0, .width = 0},
|
||||
{.input = "ZM", .rc = -EINVAL, .chunk = 0, .width = 0},
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(*cases); ++i) {
|
||||
@@ -991,15 +986,15 @@ static void test_metadata_and_chunks(void) {
|
||||
}
|
||||
|
||||
struct set_meta meta;
|
||||
assert(set_meta_init("", &meta) == -4); // too short
|
||||
assert(set_meta_init("da", &meta) == -4);
|
||||
assert(set_meta_init("ca0", &meta) == -1); // incorrect bpp
|
||||
assert(set_meta_init("{a0", &meta) == -1);
|
||||
assert(set_meta_init("d`0", &meta) == -2); // incorrect Mshift
|
||||
assert(set_meta_init("dz0", &meta) == -2);
|
||||
assert(set_meta_init("dd0", &meta) == -3); // Mshift == bpp
|
||||
assert(set_meta_init("", &meta) == -EINVAL); // too short
|
||||
assert(set_meta_init("da", &meta) == -EINVAL);
|
||||
assert(set_meta_init("ca0", &meta) == -ERANGE); // incorrect bpp
|
||||
assert(set_meta_init("{a0", &meta) == -ERANGE);
|
||||
assert(set_meta_init("d`0", &meta) == -ERANGE); // incorrect Mshift
|
||||
assert(set_meta_init("dz0", &meta) == -ERANGE);
|
||||
assert(set_meta_init("dd0", &meta) == -EINVAL); // Mshift == bpp
|
||||
assert(set_meta_init("da0", &meta) == 0);
|
||||
assert(set_meta_fini(&meta) == -4); // not enough data
|
||||
assert(set_meta_fini(&meta) == -EINVAL); // not enough data
|
||||
assert(set_meta_init("da00", &meta) == 0);
|
||||
assert(set_meta_fini(&meta) == 0); // ok
|
||||
assert(meta.len == 4);
|
||||
|
||||
Reference in New Issue
Block a user