diff --git a/reimplement/newset.c b/reimplement/newset.c index 9dec8f1..f21048d 100644 --- a/reimplement/newset.c +++ b/reimplement/newset.c @@ -11,7 +11,6 @@ struct set { }; struct set* set_new() { - // should we use x___ funcs? struct set* set = xmalloc(sizeof *set); set->cnt = 0; set->symbols_v = NULL; @@ -48,17 +47,20 @@ struct set* set_free(struct set* set) { // --- -static unsigned int hash(const char* str) { - unsigned int hash = 0x9e3779b9; +static unsigned hash(const char* str) { + unsigned hash = 0x9e3779b9; const unsigned char* p = (const unsigned char*)str; + while (*p) { hash += *p++; hash += (hash << 10); hash ^= (hash >> 6); } + hash += (hash << 3); hash ^= (hash >> 11); hash += (hash << 15); + return hash; } @@ -72,6 +74,171 @@ int cmp(const void* arg1, const void* arg2) { return 0; } +// --- + +static int log2i(int n) { + int m = 0; + while (n / 2) m++; + + return m; +} + +// Calculate Mshift paramter for encoding. +static int encode_golomb_Mshift(int cnt, int bpp) { + // XXX Slightly better Mshift estimations are probably possible. + // Recheck "Compression and coding algorithms" by Moffat & Turpin. + int Mshift = bpp - log2i(cnt) - 1; + + // Adjust out-of-range values. + Mshift = (Mshift < 7) ? 7 : Mshift; + Mshift = (Mshift > 31) ? 31 : Mshift; + assert(Mshift < bpp); + + return Mshift; +} + +// Estimate how many bits can be filled up. +static inline int encode_golomb_size(int cnt, int Mshift) { + // XXX No precise estimation. However, we do not expect unary-encoded bits + // to take more than binary-encoded Mshift bits. + return Mshift * 2 * cnt + 16; +} + +// Estimate base62 buffer size required to encode a given number of bits. +static inline int encode_base62_size(int bit_cnt) { + // In the worst case, which is ZxZxZx..., five bits can make a character; + // the remaining bits can make a character, too. And the string must be + // null-terminated. + return bit_cnt / 5 + 2; +} + +static int encode_set_size(int cnt, int bpp) { + int Mshift = encode_golomb_Mshift(cnt, bpp); + int bit_cnt = encode_golomb_size(cnt, Mshift); + // two leading characters are special + return 2 + encode_base62_size(bit_cnt); +} + +// --- + +static void encode_delta(int cnt, unsigned* hash_arr_pt) { + assert(cnt > 0); + + unsigned* end_pt = hash_arr_pt + cnt; + unsigned prev_hash = *hash_arr_pt++; + + while (hash_arr_pt < end_pt) { + *hash_arr_pt -= prev_hash; + prev_hash += *hash_arr_pt++; + } + + return; +} + +// Main golomb encoding routine: package integers into bits. +// http://algo2.iti.uni-karlsruhe.de/singler/publications/cacheefficientbloomfilters-wea2007.pdf +// The first integer is then stored in unary coding (which is a variable-length sequence of '0' +// followed by a terminating '1'); the second part is stored in normal binary coding (using Mshift +// bits). +static int encode_golomb(int cnt, const unsigned* delta_arr_pt, int Mshift, char* bit_arr_pt) { + char* start_pt = bit_arr_pt; + const unsigned mask = (1 << Mshift) - 1; + + for (int i = 0; i < cnt; ++i) { + unsigned elem = *delta_arr_pt++; + + // first part: variable-length sequence + unsigned q = elem >> Mshift; + for (int j = 0; j < (int)q; ++j) { + *bit_arr_pt++ = 0; + } + + *bit_arr_pt++ = 1; + + // second part: lower Mshift bits + unsigned r = elem & mask; + for (int j = 0; j < Mshift; ++j) { + *bit_arr_pt++ = r & 1; + r >>= 1; + } + } + + return bit_arr_pt - start_pt; +} + +// Main base62 encoding routine: pack bit_arr into base62 string. +/* + * Base62 routines - encode bits with alnum characters. + * + * This is a base64-based base62 implementation. Values 0..61 are encoded + * with '0'..'9', 'a'..'z', and 'A'..'Z'. However, 'Z' is special: it will + * also encode 62 and 63. To achieve this, 'Z' will occupy two high bits in + * the next character. Thus 'Z' can be interpreted as an escape character + * (which indicates that the next character must be handled specially). + * Note that setting high bits to "00", "01" or "10" cannot contribute + * to another 'Z' (which would require high bits set to "11"). This is + * how multiple escapes are avoided. + */ +static int encode_base62(int bit_cnt, const char* bit_arr_pt, char* base62_str_pt) { + char* base62_start = base62_str_pt; + + // int bits_Z = 0; // bits from Z-escape; + int bits_cnt = 0; + unsigned bits_buf = 1; // 1 bit as marker + + while (bit_cnt-- > 0) { + bits_buf <<= 1; + bits_buf |= *bit_arr_pt++; + + if (!(bits_buf & (1 << 6))) continue; + + bits_buf &= ~(1 << 6); // remove flag + + if (bits_buf >= 61) { // 61 62 63 cases + base62_str_pt = bits_to_char(61, base62_str_pt); + bits_buf = (1 << 2) | (bits_buf - 61); // 1 (00|01|10) + } else { + base62_str_pt = bits_to_char(bits_buf, base62_str_pt); + bits_buf = 1; + } + } + + // flush buffer + if (bits_buf != 1) { + unsigned sliding_one = 1 << 6; + while (sliding_one > bits_buf) sliding_one >>= 1; + bits_buf &= ~sliding_one; // remove flag + + assert(bits_buf < 61); // should not be 61 62 63 cases + base62_str_pt = bits_to_char(bits_buf, base62_str_pt); + } + + *base62_str_pt = '\0'; + return base62_str_pt - base62_start; +} + +// --- + +static int encode_set(int cnt, unsigned* hash_arr, int bpp, char* base62_str) { + int Mshift = encode_golomb_Mshift(cnt, bpp); + int bit_cnt = encode_golomb_size(cnt, Mshift); + + char bit_arr[bit_cnt]; + + *base62_str++ = bpp - 7 + 'a'; + *base62_str++ = Mshift - 7 + 'a'; + + // hash_arr -> delta_arr + encode_delta(cnt, hash_arr); + bit_cnt = encode_golomb(cnt, hash_arr, Mshift, bit_arr); + assert(bit_cnt >= 0); + + size_t base62_len = encode_base62(bit_cnt, bit_arr, base62_str); + assert(base62_len > 0); + + return 2 + base62_len; +} + const char* set_fini(struct set* set, int bpp) { // Implementation for finalizing the set @@ -107,5 +274,19 @@ const char* set_fini(struct set* set, int bpp) { unique_hash[unique_cnt++] = set->symbols_v[i].hash; } - return NULL; + char base62_str[encode_set_size(unique_cnt, bpp)]; + encode_set(unique_cnt, unique_hash, bpp, base62_str); + + return xstrdup(base62_str); +} + +int main(void) { + struct set* set1 = set_new(); + set_add(set1, "mama"); + set_add(set1, "myla"); + set_add(set1, "ramu"); + const char* str10 = set_fini(set1, 16); + fprintf(stderr, "set10=%s\n", str10); + + return 0; }