Compare commits
5
Commits
main
...
da79041e88
| Author | SHA1 | Date | |
|---|---|---|---|
 krosh
|
da79041e88 | ||
|
krosh
|
f472bf6e36 | ||
|
krosh
|
9f17383626 | ||
|
krosh
|
3027f36307 | ||
|
krosh
|
373c5d71dc |
@@ -0,0 +1,296 @@
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifdef ARSV_WITH_RPM
|
||||
#include <rpm/header.h>
|
||||
#include <rpm/rpmio.h>
|
||||
#include <rpm/rpmlib.h>
|
||||
#include <rpm/rpmtag.h>
|
||||
#include <rpm/rpmtd.h>
|
||||
#endif
|
||||
|
||||
int arsv_set9_decode(const char* source, unsigned** hashes, size_t* count, unsigned* bpp);
|
||||
|
||||
#define D1_PREFIX "set:D1"
|
||||
#define D1_HEADER_LEN 8
|
||||
|
||||
static const char base64_alphabet[] =
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
static size_t base64_encoded_size(size_t byte_count) {
|
||||
if (byte_count > SIZE_MAX - 2) return SIZE_MAX;
|
||||
size_t groups = (byte_count + 2) / 3;
|
||||
if (groups > SIZE_MAX / 4) return SIZE_MAX;
|
||||
size_t size = groups * 4;
|
||||
size_t remainder = byte_count % 3;
|
||||
if (remainder) size -= 3 - remainder;
|
||||
return size;
|
||||
}
|
||||
|
||||
static void base64_encode(const unsigned char* input, size_t input_len, char* output) {
|
||||
while (input_len >= 3) {
|
||||
uint32_t value = ((uint32_t)input[0] << 16) | ((uint32_t)input[1] << 8) | input[2];
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
output[2] = base64_alphabet[(value >> 6) & 0x3f];
|
||||
output[3] = base64_alphabet[value & 0x3f];
|
||||
input += 3;
|
||||
input_len -= 3;
|
||||
output += 4;
|
||||
}
|
||||
if (input_len == 1) {
|
||||
uint32_t value = (uint32_t)input[0] << 16;
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
} else if (input_len == 2) {
|
||||
uint32_t value = ((uint32_t)input[0] << 16) | ((uint32_t)input[1] << 8);
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
output[2] = base64_alphabet[(value >> 6) & 0x3f];
|
||||
}
|
||||
}
|
||||
|
||||
static int encode_d1(const unsigned* hashes, size_t count, unsigned bpp, char** result) {
|
||||
if (!hashes || !count || !result || bpp < 10 || bpp > 32) return -EINVAL;
|
||||
if (count > (SIZE_MAX - 7) / bpp) return -EOVERFLOW;
|
||||
|
||||
size_t bit_count = count * bpp;
|
||||
size_t byte_count = (bit_count + 7) / 8;
|
||||
unsigned char* bytes = calloc(byte_count, 1);
|
||||
if (!bytes) return -ENOMEM;
|
||||
|
||||
unsigned char* output = bytes;
|
||||
uint64_t bits = 0;
|
||||
unsigned filled = 0;
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
bits |= (uint64_t)hashes[i] << filled;
|
||||
filled += bpp;
|
||||
while (filled >= 8) {
|
||||
*output++ = (unsigned char)bits;
|
||||
bits >>= 8;
|
||||
filled -= 8;
|
||||
}
|
||||
}
|
||||
if (filled) *output++ = (unsigned char)bits;
|
||||
if ((size_t)(output - bytes) != byte_count) {
|
||||
free(bytes);
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
size_t payload_len = base64_encoded_size(byte_count);
|
||||
if (payload_len == SIZE_MAX || payload_len > SIZE_MAX - D1_HEADER_LEN - 1) {
|
||||
free(bytes);
|
||||
return -EOVERFLOW;
|
||||
}
|
||||
char* encoded = malloc(D1_HEADER_LEN + payload_len + 1);
|
||||
if (!encoded) {
|
||||
free(bytes);
|
||||
return -ENOMEM;
|
||||
}
|
||||
memcpy(encoded, D1_PREFIX, sizeof(D1_PREFIX) - 1);
|
||||
encoded[6] = (char)('0' + bpp / 10);
|
||||
encoded[7] = (char)('0' + bpp % 10);
|
||||
base64_encode(bytes, byte_count, encoded + D1_HEADER_LEN);
|
||||
encoded[D1_HEADER_LEN + payload_len] = '\0';
|
||||
free(bytes);
|
||||
*result = encoded;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int convert_set(const char* source, char** result, size_t* hash_count, unsigned* bpp) {
|
||||
if (!source || strncmp(source, "set:", 4) != 0 || strncmp(source, D1_PREFIX, 6) == 0)
|
||||
return -EINVAL;
|
||||
|
||||
unsigned* hashes = NULL;
|
||||
size_t count = 0;
|
||||
unsigned precision = 0;
|
||||
int rc = arsv_set9_decode(source, &hashes, &count, &precision);
|
||||
if (rc == 0) rc = encode_d1(hashes, count, precision, result);
|
||||
free(hashes);
|
||||
if (rc < 0) return rc;
|
||||
if (hash_count) *hash_count = count;
|
||||
if (bpp) *bpp = precision;
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef ARSV_WITH_RPM
|
||||
struct statistics {
|
||||
unsigned long headers;
|
||||
unsigned long set_occurrences;
|
||||
unsigned long set_bytes_old;
|
||||
unsigned long set_bytes_new;
|
||||
unsigned long hashes;
|
||||
};
|
||||
|
||||
static int rewrite_version_tag(Header header, rpmTagVal tag, const char* package,
|
||||
struct statistics* stats) {
|
||||
struct rpmtd_s values;
|
||||
memset(&values, 0, sizeof(values));
|
||||
if (headerGet(header, tag, &values, HEADERGET_MINMEM) != 1) return 0;
|
||||
if (rpmtdType(&values) != RPM_STRING_ARRAY_TYPE) {
|
||||
rpmtdFreeData(&values);
|
||||
fprintf(stderr, "%s: tag %d is not a string array\n", package, (int)tag);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rpm_count_t count = rpmtdCount(&values);
|
||||
const char** rewritten = calloc((size_t)count, sizeof(*rewritten));
|
||||
if (!rewritten) {
|
||||
rpmtdFreeData(&values);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
int rc = 0;
|
||||
int changed = 0;
|
||||
rpmtdInit(&values);
|
||||
for (rpm_count_t i = 0; i < count; ++i) {
|
||||
const char* value = rpmtdNextString(&values);
|
||||
if (!value) {
|
||||
rc = -EINVAL;
|
||||
break;
|
||||
}
|
||||
if (strncmp(value, "set:", 4) == 0) {
|
||||
char* converted = NULL;
|
||||
size_t hashes = 0;
|
||||
unsigned bpp = 0;
|
||||
rc = convert_set(value, &converted, &hashes, &bpp);
|
||||
if (rc < 0) {
|
||||
fprintf(stderr, "%s: cannot convert tag %d index %u (rc=%d)\n", package,
|
||||
(int)tag, (unsigned)i, rc);
|
||||
break;
|
||||
}
|
||||
rewritten[i] = converted;
|
||||
changed = 1;
|
||||
++stats->set_occurrences;
|
||||
stats->set_bytes_old += strlen(value);
|
||||
stats->set_bytes_new += strlen(converted);
|
||||
stats->hashes += hashes;
|
||||
} else {
|
||||
rewritten[i] = strdup(value);
|
||||
if (!rewritten[i]) {
|
||||
rc = -ENOMEM;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rc == 0 && changed) {
|
||||
headerDel(header, tag);
|
||||
if (!headerPutStringArray(header, tag, rewritten, count)) rc = -EIO;
|
||||
}
|
||||
for (rpm_count_t i = 0; i < count; ++i) free((void*)rewritten[i]);
|
||||
free(rewritten);
|
||||
rpmtdFreeData(&values);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int rewrite_pkglist(const char* input_path, const char* output_path, unsigned long limit) {
|
||||
FD_t input = Fopen(input_path, "r.ufdio");
|
||||
if (!input || Ferror(input)) {
|
||||
fprintf(stderr, "%s: %s\n", input_path, input ? Fstrerror(input) : "cannot open");
|
||||
return 1;
|
||||
}
|
||||
FD_t output = Fopen(output_path, "w.ufdio");
|
||||
if (!output || Ferror(output)) {
|
||||
fprintf(stderr, "%s: %s\n", output_path, output ? Fstrerror(output) : "cannot open");
|
||||
Fclose(input);
|
||||
return 1;
|
||||
}
|
||||
|
||||
struct statistics stats = {0};
|
||||
Header header;
|
||||
int failed = 0;
|
||||
const rpmTagVal tags[] = {
|
||||
RPMTAG_REQUIREVERSION,
|
||||
RPMTAG_PROVIDEVERSION,
|
||||
RPMTAG_CONFLICTVERSION,
|
||||
RPMTAG_OBSOLETEVERSION,
|
||||
RPMTAG_RECOMMENDVERSION,
|
||||
RPMTAG_SUGGESTVERSION,
|
||||
RPMTAG_SUPPLEMENTVERSION,
|
||||
RPMTAG_ENHANCEVERSION,
|
||||
};
|
||||
while ((!limit || stats.headers < limit) &&
|
||||
(header = headerRead(input, HEADER_MAGIC_YES)) != NULL) {
|
||||
const char* package = headerGetString(header, RPMTAG_NAME);
|
||||
if (!package) package = "<unknown>";
|
||||
for (size_t i = 0; i < sizeof(tags) / sizeof(tags[0]); ++i) {
|
||||
int rc = rewrite_version_tag(header, tags[i], package, &stats);
|
||||
if (rc < 0) {
|
||||
failed = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!failed && headerWrite(output, header, HEADER_MAGIC_YES) != 0) {
|
||||
fprintf(stderr, "%s: failed to write header for %s\n", output_path, package);
|
||||
failed = 1;
|
||||
}
|
||||
headerFree(header);
|
||||
if (failed) break;
|
||||
++stats.headers;
|
||||
}
|
||||
if (!failed && Ferror(input)) {
|
||||
fprintf(stderr, "%s: read error: %s\n", input_path, Fstrerror(input));
|
||||
failed = 1;
|
||||
}
|
||||
if (Fclose(output) != 0) failed = 1;
|
||||
Fclose(input);
|
||||
|
||||
if (failed) {
|
||||
remove(output_path);
|
||||
return 1;
|
||||
}
|
||||
fprintf(stderr,
|
||||
"headers=%lu set_occurrences=%lu hashes=%lu old_set_bytes=%lu "
|
||||
"new_set_bytes=%lu\n",
|
||||
stats.headers, stats.set_occurrences, stats.hashes, stats.set_bytes_old,
|
||||
stats.set_bytes_new);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
static void usage(const char* program) {
|
||||
fprintf(stderr, "usage: %s --convert-set set:VALUE\n", program);
|
||||
#ifdef ARSV_WITH_RPM
|
||||
fprintf(stderr, " %s --rewrite INPUT OUTPUT [--max-headers N]\n", program);
|
||||
#endif
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc == 3 && strcmp(argv[1], "--convert-set") == 0) {
|
||||
char* converted = NULL;
|
||||
int rc = convert_set(argv[2], &converted, NULL, NULL);
|
||||
if (rc < 0) {
|
||||
fprintf(stderr, "cannot convert set value (rc=%d)\n", rc);
|
||||
return 1;
|
||||
}
|
||||
puts(converted);
|
||||
free(converted);
|
||||
return 0;
|
||||
}
|
||||
#ifdef ARSV_WITH_RPM
|
||||
if ((argc == 4 || argc == 6) && strcmp(argv[1], "--rewrite") == 0) {
|
||||
unsigned long limit = 0;
|
||||
if (argc == 6) {
|
||||
if (strcmp(argv[4], "--max-headers") != 0) {
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
char* end = NULL;
|
||||
errno = 0;
|
||||
limit = strtoul(argv[5], &end, 10);
|
||||
if (errno || !end || *end || !limit) {
|
||||
fprintf(stderr, "invalid --max-headers value: %s\n", argv[5]);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
return rewrite_pkglist(argv[2], argv[3], limit);
|
||||
}
|
||||
#endif
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create D1 copies of the real Sisyphus x86_64 and noarch pkglist files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[2]
|
||||
ARCHITECTURES = ("x86_64", "noarch")
|
||||
SUMMARY_RE = re.compile(
|
||||
r"headers=(?P<headers>\d+) set_occurrences=(?P<set_occurrences>\d+) "
|
||||
r"hashes=(?P<hashes>\d+) old_set_bytes=(?P<old_set_bytes>\d+) "
|
||||
r"new_set_bytes=(?P<new_set_bytes>\d+)"
|
||||
)
|
||||
VERSION_FORMAT = (
|
||||
"[%{REQUIREVERSION}\\n]"
|
||||
"[%{PROVIDEVERSION}\\n]"
|
||||
"[%{CONFLICTVERSION}\\n]"
|
||||
"[%{OBSOLETEVERSION}\\n]"
|
||||
"[%{RECOMMENDVERSION}\\n]"
|
||||
"[%{SUGGESTVERSION}\\n]"
|
||||
"[%{SUPPLEMENTVERSION}\\n]"
|
||||
"[%{ENHANCEVERSION}\\n]"
|
||||
)
|
||||
|
||||
|
||||
def classify_pkglist(path: Path) -> str | None:
|
||||
name = path.name
|
||||
if not name.endswith("_base_pkglist.classic"):
|
||||
return None
|
||||
if "_Sisyphus_x86%5f64_" in name or "_Sisyphus_x86_64_" in name:
|
||||
return "x86_64"
|
||||
if "_Sisyphus_noarch_" in name:
|
||||
return "noarch"
|
||||
return None
|
||||
|
||||
|
||||
def validate_output(path: Path) -> Path:
|
||||
resolved = path.expanduser().resolve()
|
||||
home = Path.home().resolve()
|
||||
source = ROOT.resolve()
|
||||
if resolved == home or resolved == source or source in resolved.parents:
|
||||
raise ValueError(f"refusing protected output path: {resolved}")
|
||||
if resolved == Path("/"):
|
||||
raise ValueError("refusing filesystem root as output")
|
||||
return resolved
|
||||
|
||||
|
||||
def run(
|
||||
command: list[str],
|
||||
*,
|
||||
check: bool = True,
|
||||
cwd: Path | None = None,
|
||||
stdout: int | None = None,
|
||||
) -> subprocess.CompletedProcess[bytes]:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
stdout=stdout if stdout is not None else subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode != 0:
|
||||
stderr = result.stderr.decode(errors="replace")
|
||||
raise RuntimeError(f"command failed ({result.returncode}): {' '.join(command)}\n{stderr}")
|
||||
return result
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def compile_converter(destination: Path) -> None:
|
||||
include = destination.parent / "compat-include"
|
||||
include.mkdir()
|
||||
for name in ("rpmlib.h", "system.h", "set.h"):
|
||||
(include / name).touch()
|
||||
run(
|
||||
[
|
||||
"cc",
|
||||
"-O2",
|
||||
"-std=gnu11",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
"-D_GNU_SOURCE",
|
||||
"-DARSV_SET9_EXPORT",
|
||||
"-DARSV_WITH_RPM",
|
||||
"-I",
|
||||
str(include),
|
||||
"-include",
|
||||
str(ROOT / "scripts/rpmsetcmp/newset_compat.h"),
|
||||
str(ROOT / "reimplement/set9.c"),
|
||||
str(HERE / "rewrite_sisyphus_pkglist.c"),
|
||||
"-lrpm",
|
||||
"-lrpmio",
|
||||
"-o",
|
||||
str(destination),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def find_pkglists(lists_dir: Path) -> dict[str, Path]:
|
||||
found: dict[str, Path] = {}
|
||||
for path in lists_dir.glob("*_base_pkglist.classic"):
|
||||
architecture = classify_pkglist(path)
|
||||
if architecture is None:
|
||||
continue
|
||||
if architecture in found:
|
||||
raise RuntimeError(f"multiple Sisyphus {architecture} pkglist files")
|
||||
found[architecture] = path
|
||||
missing = set(ARCHITECTURES) - set(found)
|
||||
if missing:
|
||||
raise RuntimeError(f"missing Sisyphus pkglist for: {', '.join(sorted(missing))}")
|
||||
return found
|
||||
|
||||
|
||||
def query_header_count(path: Path) -> int:
|
||||
process = subprocess.Popen(
|
||||
["pkglist-query", "%{NAME}\\n", str(path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
assert process.stdout is not None
|
||||
count = sum(1 for _ in process.stdout)
|
||||
stderr = process.stderr.read() if process.stderr else b""
|
||||
status = process.wait()
|
||||
if status != 0:
|
||||
raise RuntimeError(f"pkglist-query failed for {path}: {stderr.decode(errors='replace')}")
|
||||
return count
|
||||
|
||||
|
||||
def query_set_counts(path: Path) -> dict[str, int]:
|
||||
process = subprocess.Popen(
|
||||
["pkglist-query", VERSION_FORMAT, str(path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
assert process.stdout is not None
|
||||
old = direct = 0
|
||||
for raw_line in process.stdout:
|
||||
value = raw_line.strip()
|
||||
if value.startswith(b"set:D1"):
|
||||
direct += 1
|
||||
elif value.startswith(b"set:"):
|
||||
old += 1
|
||||
stderr = process.stderr.read() if process.stderr else b""
|
||||
status = process.wait()
|
||||
if status != 0:
|
||||
raise RuntimeError(f"pkglist-query failed for {path}: {stderr.decode(errors='replace')}")
|
||||
return {"set9": old, "d1": direct}
|
||||
|
||||
|
||||
def conversion_summary(stderr: bytes) -> dict[str, int]:
|
||||
text = stderr.decode(errors="replace")
|
||||
match = SUMMARY_RE.search(text)
|
||||
if not match:
|
||||
raise RuntimeError(f"converter did not report summary:\n{text}")
|
||||
return {name: int(value) for name, value in match.groupdict().items()}
|
||||
|
||||
|
||||
def convert_one(converter: Path, source: Path, destination: Path) -> dict[str, object]:
|
||||
result = run([str(converter), "--rewrite", str(source), str(destination)])
|
||||
summary = conversion_summary(result.stderr)
|
||||
source_headers = query_header_count(source)
|
||||
output_headers = query_header_count(destination)
|
||||
source_sets = query_set_counts(source)
|
||||
output_sets = query_set_counts(destination)
|
||||
if source_headers != output_headers or output_headers != summary["headers"]:
|
||||
raise RuntimeError(
|
||||
f"header count mismatch: source={source_headers} output={output_headers} "
|
||||
f"converter={summary['headers']}"
|
||||
)
|
||||
if source_sets["d1"] != 0:
|
||||
raise RuntimeError(f"source already contains {source_sets['d1']} D1 values")
|
||||
if output_sets["set9"] != 0:
|
||||
raise RuntimeError(f"output still contains {output_sets['set9']} set9 values")
|
||||
if source_sets["set9"] != output_sets["d1"] or output_sets["d1"] != summary["set_occurrences"]:
|
||||
raise RuntimeError(
|
||||
"set occurrence mismatch: "
|
||||
f"source={source_sets['set9']} output={output_sets['d1']} "
|
||||
f"converter={summary['set_occurrences']}"
|
||||
)
|
||||
if summary["set_occurrences"] == 0:
|
||||
raise RuntimeError("source pkglist contains no set versions")
|
||||
return {
|
||||
**summary,
|
||||
"source": {
|
||||
"path": source.name,
|
||||
"size": source.stat().st_size,
|
||||
"sha256": sha256_file(source),
|
||||
},
|
||||
"output": {
|
||||
"path": destination.name,
|
||||
"size": destination.stat().st_size,
|
||||
"sha256": sha256_file(destination),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def convert_local(output: Path, lists_dir: Path) -> None:
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
if any(output.iterdir()):
|
||||
raise RuntimeError(f"output directory is not empty: {output}")
|
||||
|
||||
lists_dir = lists_dir.expanduser().resolve()
|
||||
if not lists_dir.is_dir():
|
||||
raise RuntimeError(f"APT lists directory does not exist: {lists_dir}")
|
||||
pkglists = find_pkglists(lists_dir)
|
||||
staging = Path(tempfile.mkdtemp(prefix=".d1-staging-", dir=output))
|
||||
try:
|
||||
converter = staging / "rewrite-sisyphus-pkglist"
|
||||
compile_converter(converter)
|
||||
architectures: dict[str, object] = {}
|
||||
for architecture in ARCHITECTURES:
|
||||
destination = staging / f"Sisyphus.{architecture}.pkglist.classic"
|
||||
architectures[architecture] = convert_one(
|
||||
converter, pkglists[architecture], destination
|
||||
)
|
||||
converter.unlink()
|
||||
shutil.rmtree(staging / "compat-include")
|
||||
manifest = {
|
||||
"format": 1,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"input": {"lists_dir": str(lists_dir)},
|
||||
"rpm": run(["rpm", "--version"]).stdout.decode().strip(),
|
||||
"apt": run(["rpmquery", "--qf", "%{VERSION}-%{RELEASE}", "apt"])
|
||||
.stdout.decode()
|
||||
.strip(),
|
||||
"converter_source": {
|
||||
"set9_sha256": sha256_file(ROOT / "reimplement/set9.c"),
|
||||
"rewrite_sha256": sha256_file(HERE / "rewrite_sisyphus_pkglist.c"),
|
||||
},
|
||||
"architectures": architectures,
|
||||
}
|
||||
(staging / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
final = output / "d1-pkglists"
|
||||
staging.replace(final)
|
||||
print(final)
|
||||
for architecture in ARCHITECTURES:
|
||||
data = architectures[architecture]
|
||||
assert isinstance(data, dict)
|
||||
print(
|
||||
f"{architecture}: headers={data['headers']} "
|
||||
f"set_occurrences={data['set_occurrences']} "
|
||||
f"output={final / f'Sisyphus.{architecture}.pkglist.classic'}"
|
||||
)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Create D1 copies of local Sisyphus x86_64/noarch pkglist files"
|
||||
)
|
||||
parser.add_argument("output", type=Path, help="new or empty output directory")
|
||||
parser.add_argument(
|
||||
"--lists-dir",
|
||||
type=Path,
|
||||
default=Path("/var/lib/apt/lists"),
|
||||
help="APT lists snapshot to convert (default: /var/lib/apt/lists)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
output = validate_output(args.output)
|
||||
convert_local(output, args.lists_dir)
|
||||
except (OSError, RuntimeError, ValueError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -546,6 +546,37 @@ static int decode_set(const struct set_meta* meta, unsigned* hash_arr) {
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef ARSV_SET9_EXPORT
|
||||
/* Test-only bridge used to translate existing repository metadata without
|
||||
* reconstructing unavailable symbol names. The caller owns *hashes. */
|
||||
int arsv_set9_decode(const char* source, unsigned** hashes, size_t* count, unsigned* bpp) {
|
||||
if (!source || !hashes || !count || !bpp) return -EINVAL;
|
||||
|
||||
const char* str = source;
|
||||
if (strncmp(str, "set:", 4) == 0) str += 4;
|
||||
|
||||
struct set_meta meta;
|
||||
int rc = set_meta_init(str, &meta);
|
||||
if (rc < 0) return rc;
|
||||
rc = set_meta_fini(&meta);
|
||||
if (rc < 0) return rc;
|
||||
|
||||
unsigned* values = malloc((size_t)meta.value_capacity * sizeof(*values));
|
||||
if (!values) return -ENOMEM;
|
||||
|
||||
int decoded = decode_set(&meta, values);
|
||||
if (decoded <= 0) {
|
||||
free(values);
|
||||
return decoded < 0 ? decoded : -EINVAL;
|
||||
}
|
||||
|
||||
*hashes = values;
|
||||
*count = (size_t)decoded;
|
||||
*bpp = (unsigned)meta.bpp;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Bounded decoded-set cache: bucketed lookup plus O(1) LRU updates. */
|
||||
static int downsample_set(const unsigned* hash_pt, size_t hash_cnt, unsigned* dest_pt,
|
||||
int target_bpp);
|
||||
|
||||
Executable
+791
@@ -0,0 +1,791 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
export LC_ALL=C
|
||||
|
||||
# A/B benchmark:
|
||||
# set9: librpm из Sisyphus с reimplement/set9.c и исходными pkglist Sisyphus;
|
||||
# d1: librpm из того же commit с direct_hash/hash_set.c и теми же pkglist,
|
||||
# заранее преобразованными run_sisyphus_pkglist.py в формат set:D1.
|
||||
#
|
||||
# Все измеряемые команды работают без сети. Конвертация, сборка и gencaches
|
||||
# выполняются до таймера. Порядок замеров в каждом раунде: set9/d1/d1/set9.
|
||||
|
||||
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
||||
REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd)
|
||||
SET9_C=${SET9_C:-$REPO_ROOT/reimplement/set9.c}
|
||||
D1_C=${D1_C:-$REPO_ROOT/new_version/direct_hash/hash_set.c}
|
||||
PKGLIST_CONVERTER=${PKGLIST_CONVERTER:-$REPO_ROOT/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py}
|
||||
|
||||
WORK_ROOT=${WORK_ROOT:-$HOME/sisyphus-set9-d1-bench}
|
||||
RESULT_DIR=${RESULT_DIR:-$WORK_ROOT/results}
|
||||
SISYPHUS_MIRROR=${SISYPHUS_MIRROR:-https://ftp.altlinux.org/pub/distributions/ALTLinux}
|
||||
RPM_GIT=${RPM_GIT:-https://git.altlinux.org/gears/r/rpm.git}
|
||||
RPM_BRANCH=${RPM_BRANCH:-sisyphus}
|
||||
PACKAGER=${PACKAGER:-krosh <gudovdo@my.msu.ru>}
|
||||
CPU=${CPU:-0}
|
||||
ROUNDS=${ROUNDS:-2}
|
||||
RESET_WORK=${RESET_WORK:-1} # 0 — продолжить подготовку/сборки, 1 — начать заново.
|
||||
OPERATIONS=${OPERATIONS:-unmet install-rpm-build install-openuds-server install-password-store}
|
||||
APT_GET=${APT_GET:-/usr/lib/apt/apt-get}
|
||||
APT_CACHE=${APT_CACHE:-$(command -v apt-cache 2>/dev/null || true)}
|
||||
|
||||
usage()
|
||||
{
|
||||
cat <<'EOF'
|
||||
Usage: scripts/run-sisyphus-set9-d1-bench.sh
|
||||
|
||||
The script has no positional arguments. Configuration is passed through env:
|
||||
CPU=2 ROUNDS=3 RESET_WORK=1 ./scripts/run-sisyphus-set9-d1-bench.sh
|
||||
RESET_WORK=0 OPERATIONS='unmet check' ./scripts/run-sisyphus-set9-d1-bench.sh
|
||||
|
||||
Main variables:
|
||||
WORK_ROOT, RESULT_DIR, SISYPHUS_MIRROR, RPM_GIT, RPM_BRANCH, PACKAGER,
|
||||
SET9_C, D1_C, PKGLIST_CONVERTER, CPU, ROUNDS, RESET_WORK, OPERATIONS.
|
||||
|
||||
RESET_WORK=1 removes WORK_ROOT after running hsh --cleanup-only for old hasher
|
||||
workdirs. Timed runs never update repositories and never install packages.
|
||||
Snapshot preparation downloads x86_64 and noarch metadata from SISYPHUS_MIRROR
|
||||
into an isolated APT directory under WORK_ROOT; host APT configuration is untouched.
|
||||
EOF
|
||||
}
|
||||
|
||||
fail()
|
||||
{
|
||||
printf 'error: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
safe_remove_work_root()
|
||||
{
|
||||
local marker="$WORK_ROOT/.arsv-sisyphus-set-bench-root"
|
||||
|
||||
[[ -n $WORK_ROOT && $WORK_ROOT == /* && $WORK_ROOT != / &&
|
||||
$WORK_ROOT != "$HOME_REAL" && $HOME_REAL != "$WORK_ROOT/"* ]] ||
|
||||
fail "unsafe WORK_ROOT for removal: $WORK_ROOT"
|
||||
[[ $WORK_ROOT != "$REPO_ROOT" && $REPO_ROOT != "$WORK_ROOT/"* &&
|
||||
$WORK_ROOT != "$REPO_ROOT/"* && $WORK_ROOT != "$CWD_REAL" &&
|
||||
$CWD_REAL != "$WORK_ROOT/"* && $WORK_ROOT != "$CWD_REAL/"* ]] ||
|
||||
fail "WORK_ROOT overlaps the source repository or current directory: $WORK_ROOT"
|
||||
[[ ! -e $WORK_ROOT || (-f $marker && ! -L $marker) ]] ||
|
||||
fail "refusing to remove unmarked WORK_ROOT: $WORK_ROOT"
|
||||
if [[ -f $marker ]]; then
|
||||
grep -Fx 'ARSV Sisyphus set9/D1 benchmark work root' "$marker" >/dev/null ||
|
||||
fail "invalid WORK_ROOT ownership marker: $marker"
|
||||
fi
|
||||
}
|
||||
|
||||
canonical_work_subdir()
|
||||
{
|
||||
local path=$1 label=$2 resolved
|
||||
[[ ! -L $path ]] || fail "$label must not be a symlink: $path"
|
||||
resolved=$(realpath -e -- "$path") || fail "$label does not resolve: $path"
|
||||
[[ $resolved == "$WORK_ROOT/"* ]] ||
|
||||
fail "$label resolves outside WORK_ROOT: $path -> $resolved"
|
||||
printf '%s' "$resolved"
|
||||
}
|
||||
|
||||
ensure_work_subdir()
|
||||
{
|
||||
local path=$1 label=$2 resolved
|
||||
[[ ! -L $path ]] || fail "$label must not be a symlink: $path"
|
||||
resolved=$(realpath -m -- "$path")
|
||||
[[ $resolved == "$WORK_ROOT/"* ]] ||
|
||||
fail "$label resolves outside WORK_ROOT: $path -> $resolved"
|
||||
mkdir -p -- "$path"
|
||||
canonical_work_subdir "$path" "$label" >/dev/null
|
||||
}
|
||||
|
||||
write_snapshot_fingerprint()
|
||||
{
|
||||
{
|
||||
printf 'sisyphus_mirror=%s\n' "$SISYPHUS_MIRROR"
|
||||
printf 'converter=%s\n' "$(sha256sum "$PKGLIST_CONVERTER" | awk '{print $1}')"
|
||||
printf 'set9=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
|
||||
printf 'rewrite=%s\n' "$(sha256sum "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" | awk '{print $1}')"
|
||||
printf 'compat=%s\n' "$(sha256sum "$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h" | awk '{print $1}')"
|
||||
}
|
||||
}
|
||||
|
||||
validate_snapshot_reuse()
|
||||
{
|
||||
local current
|
||||
[[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt ]] || return 1
|
||||
current=$(mktemp)
|
||||
write_snapshot_fingerprint >"$current"
|
||||
if ! cmp -s "$current" "$SNAPSHOT/input-fingerprint.txt"; then
|
||||
rm -f "$current"
|
||||
fail 'snapshot inputs changed; use RESET_WORK=1'
|
||||
fi
|
||||
rm -f "$current"
|
||||
return 0
|
||||
}
|
||||
|
||||
write_source_fingerprint()
|
||||
{
|
||||
{
|
||||
printf 'rpm_git=%s\n' "$RPM_GIT"
|
||||
printf 'rpm_branch=%s\n' "$RPM_BRANCH"
|
||||
printf 'rpm_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
|
||||
}
|
||||
}
|
||||
|
||||
validate_source_reuse()
|
||||
{
|
||||
local current
|
||||
[[ -f $SOURCE_FINGERPRINT ]] ||
|
||||
fail 'source fingerprint missing; use RESET_WORK=1'
|
||||
[[ -z $(git -C "$SOURCE_BASE" status --porcelain) ]] ||
|
||||
fail 'RPM base source has local changes; use RESET_WORK=1'
|
||||
current=$(mktemp)
|
||||
write_source_fingerprint >"$current"
|
||||
if ! cmp -s "$current" "$SOURCE_FINGERPRINT"; then
|
||||
rm -f "$current"
|
||||
fail 'RPM source URL, branch, or commit changed; use RESET_WORK=1'
|
||||
fi
|
||||
rm -f "$current"
|
||||
}
|
||||
|
||||
prepare_spec()
|
||||
{
|
||||
local spec=$1 suffix=$2 version release new_release date
|
||||
|
||||
version=$(sed -n 's/^Version:[[:space:]]*//p' "$spec" | sed -n '1p')
|
||||
release=$(sed -n 's/^Release:[[:space:]]*//p' "$spec" | sed -n '1p')
|
||||
[[ -n $version && -n $release ]] || fail "cannot read Version/Release from $spec"
|
||||
|
||||
new_release="$release.$suffix"
|
||||
sed -i "0,/^Release:[[:space:]]*$release$/s//Release: $new_release/" "$spec"
|
||||
|
||||
date=$(date '+%a %b %d %Y')
|
||||
sed -i "/^%changelog/a\\
|
||||
* $date $PACKAGER $version-$new_release\\
|
||||
- Local Sisyphus set format benchmark build.\\
|
||||
" "$spec"
|
||||
}
|
||||
|
||||
apt_options()
|
||||
{
|
||||
local variant=$1
|
||||
APT_OPTIONS=(
|
||||
-o 'Dir::Etc::main=-'
|
||||
-o 'Dir::Etc::parts=-'
|
||||
-o "Dir::Etc::sourcelist=$SNAPSHOT/etc-apt/sources.list"
|
||||
-o "Dir::Etc::sourceparts=$SNAPSHOT/etc-apt/sources.list.d"
|
||||
-o 'Dir::Etc::preferences=-'
|
||||
-o 'Dir::Etc::preferencesparts=-'
|
||||
-o "Dir::State::lists=$variant/apt/lists/"
|
||||
-o "Dir::State::status=$COMMON/status"
|
||||
-o "Dir::Cache=$variant/apt/cache/"
|
||||
-o "Dir::Cache::archives=$variant/apt/cache/archives"
|
||||
-o "Dir::Cache::pkgcache=$variant/apt/cache/pkgcache.bin"
|
||||
-o "Dir::Cache::srcpkgcache=$variant/apt/cache/srcpkgcache.bin"
|
||||
-o "RPM::RootDir=$COMMON/root"
|
||||
)
|
||||
}
|
||||
|
||||
operation_command()
|
||||
{
|
||||
local operation=$1 variant=$2
|
||||
apt_options "$variant"
|
||||
|
||||
case $operation in
|
||||
unmet)
|
||||
COMMAND=("$APT_CACHE" -q "${APT_OPTIONS[@]}" unmet)
|
||||
;;
|
||||
check)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s check)
|
||||
;;
|
||||
autoremove)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s autoremove)
|
||||
;;
|
||||
install-rpm-build)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s install rpm-build)
|
||||
;;
|
||||
install-openuds-server)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s install openuds-server)
|
||||
;;
|
||||
install-password-store)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s install password-store)
|
||||
;;
|
||||
upgrade)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -o APT::Get::EnableUpgrade=true -s upgrade)
|
||||
;;
|
||||
dist-upgrade)
|
||||
COMMAND=("$APT_GET" -qq "${APT_OPTIONS[@]}" -s dist-upgrade)
|
||||
;;
|
||||
*)
|
||||
fail "unknown operation: $operation"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
variant_dir()
|
||||
{
|
||||
case $1 in
|
||||
set9) printf '%s' "$SET9_VARIANT" ;;
|
||||
d1) printf '%s' "$D1_VARIANT" ;;
|
||||
*) fail "unknown variant: $1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
variant_libdir()
|
||||
{
|
||||
local variant
|
||||
variant=$(variant_dir "$1")
|
||||
printf '%s' "$variant/lib/usr/lib64"
|
||||
}
|
||||
|
||||
prepare_command()
|
||||
{
|
||||
local operation=$1 variant_name=$2 variant
|
||||
variant=$(variant_dir "$variant_name")
|
||||
RUN_LIBDIR=$(variant_libdir "$variant_name")
|
||||
operation_command "$operation" "$variant"
|
||||
}
|
||||
|
||||
execute_command()
|
||||
{
|
||||
local stdout=$1 stderr=$2 status
|
||||
|
||||
if env LC_ALL=C APT_CONFIG="$COMMON/apt.conf" LD_LIBRARY_PATH="$RUN_LIBDIR" \
|
||||
taskset -c "$CPU" "${COMMAND[@]}" >"$stdout" 2>"$stderr"; then
|
||||
status=0
|
||||
else
|
||||
status=$?
|
||||
fi
|
||||
printf '%s' "$status"
|
||||
}
|
||||
|
||||
run_once()
|
||||
{
|
||||
local operation=$1 variant_name=$2 sequence=$3 sample=$4
|
||||
local variant raw stdout stderr start end status elapsed
|
||||
local stdout_sha stderr_sha stdout_bytes stderr_bytes
|
||||
variant=$(variant_dir "$variant_name")
|
||||
raw="$RESULT_DIR/raw/$operation/$variant_name"
|
||||
mkdir -p "$raw"
|
||||
stdout="$raw/$sample.stdout"
|
||||
stderr="$raw/$sample.stderr"
|
||||
|
||||
prepare_command "$operation" "$variant_name"
|
||||
start=$(date +%s%N)
|
||||
status=$(execute_command "$stdout" "$stderr")
|
||||
end=$(date +%s%N)
|
||||
elapsed=$(awk -v start="$start" -v end="$end" \
|
||||
'BEGIN { printf "%.6f", (end - start) / 1000000000 }')
|
||||
|
||||
stdout_sha=$(sha256sum "$stdout" | awk '{print $1}')
|
||||
stderr_sha=$(sha256sum "$stderr" | awk '{print $1}')
|
||||
stdout_bytes=$(stat -c %s "$stdout")
|
||||
stderr_bytes=$(stat -c %s "$stderr")
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$operation" "$sequence" "$variant_name" "$sample" "$elapsed" "$status" \
|
||||
"$stdout_sha" "$stdout_bytes" "$stderr_sha" "$stderr_bytes" >>"$RAW_RESULTS"
|
||||
printf '%-28s sequence=%-2s %-4s sample=%-2s %ss status=%s\n' \
|
||||
"$operation" "$sequence" "$variant_name" "$sample" "$elapsed" "$status"
|
||||
}
|
||||
|
||||
prepare_snapshot()
|
||||
{
|
||||
local converter_output
|
||||
local -a update_options
|
||||
validate_snapshot_reuse && return
|
||||
|
||||
printf '\n===== Preparing one immutable Sisyphus metadata snapshot =====\n'
|
||||
rm -rf "$SNAPSHOT"
|
||||
mkdir -p "$SNAPSHOT/lists/partial" \
|
||||
"$SNAPSHOT/etc-apt/sources.list.d" \
|
||||
"$SNAPSHOT/download-cache/archives/partial"
|
||||
printf '%s\n' \
|
||||
"rpm [alt] $SISYPHUS_MIRROR Sisyphus/x86_64 classic" \
|
||||
"rpm [alt] $SISYPHUS_MIRROR Sisyphus/noarch classic" \
|
||||
>"$SNAPSHOT/etc-apt/sources.list"
|
||||
|
||||
update_options=(
|
||||
-o 'Dir::Etc::main=-'
|
||||
-o 'Dir::Etc::parts=-'
|
||||
-o "Dir::Etc::sourcelist=$SNAPSHOT/etc-apt/sources.list"
|
||||
-o "Dir::Etc::sourceparts=$SNAPSHOT/etc-apt/sources.list.d"
|
||||
-o 'Dir::Etc::preferences=-'
|
||||
-o 'Dir::Etc::preferencesparts=-'
|
||||
-o "Dir::State::lists=$SNAPSHOT/lists"
|
||||
-o "Dir::State::status=$COMMON/status"
|
||||
-o "Dir::Cache=$SNAPSHOT/download-cache"
|
||||
-o "Dir::Cache::archives=$SNAPSHOT/download-cache/archives"
|
||||
-o "Dir::Cache::pkgcache=$SNAPSHOT/download-cache/pkgcache.bin"
|
||||
-o "Dir::Cache::srcpkgcache=$SNAPSHOT/download-cache/srcpkgcache.bin"
|
||||
-o "RPM::RootDir=$COMMON/root"
|
||||
)
|
||||
APT_CONFIG="$COMMON/apt.conf" "$APT_GET" -qq \
|
||||
"${update_options[@]}" update
|
||||
|
||||
converter_output="$SNAPSHOT/conversion"
|
||||
python3 "$PKGLIST_CONVERTER" "$converter_output" \
|
||||
--lists-dir "$SNAPSHOT/lists"
|
||||
mv "$converter_output/d1-pkglists/manifest.json" "$SNAPSHOT/manifest.json"
|
||||
mkdir -p "$SNAPSHOT/d1-pkglists"
|
||||
mv "$converter_output/d1-pkglists/"*.classic "$SNAPSHOT/d1-pkglists/"
|
||||
rmdir "$converter_output/d1-pkglists" "$converter_output"
|
||||
rm -rf "$SNAPSHOT/download-cache"
|
||||
rm -f "$SNAPSHOT/lists/lock"
|
||||
rm -rf "$SNAPSHOT/lists/partial"
|
||||
mkdir -p "$SNAPSHOT/lists/partial"
|
||||
write_snapshot_fingerprint >"$SNAPSHOT/input-fingerprint.txt"
|
||||
: >"$SNAPSHOT/.complete"
|
||||
}
|
||||
|
||||
prepare_variant_lists()
|
||||
{
|
||||
local variant=$1 format=$2 variant_apt
|
||||
if [[ -d $variant/apt ]]; then
|
||||
variant_apt=$(canonical_work_subdir "$variant/apt" 'APT variant directory')
|
||||
chmod -R u+w "$variant_apt"
|
||||
fi
|
||||
rm -rf "$variant/apt"
|
||||
mkdir -p "$variant/apt/lists" "$variant/apt/cache/archives/partial"
|
||||
cp -a "$SNAPSHOT/lists/." "$variant/apt/lists/"
|
||||
|
||||
if [[ $format == d1 ]]; then
|
||||
python3 - "$SNAPSHOT/manifest.json" "$SNAPSHOT/lists" \
|
||||
"$SNAPSHOT/d1-pkglists" "$variant/apt/lists" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
manifest_path, original_dir, d1_dir, target_dir = map(Path, sys.argv[1:])
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
for architecture, data in manifest["architectures"].items():
|
||||
source = original_dir / data["source"]["path"]
|
||||
converted = d1_dir / data["output"]["path"]
|
||||
target = target_dir / data["source"]["path"]
|
||||
if hashlib.sha256(source.read_bytes()).hexdigest() != data["source"]["sha256"]:
|
||||
raise SystemExit(f"source checksum mismatch for {architecture}: {source}")
|
||||
if hashlib.sha256(converted.read_bytes()).hexdigest() != data["output"]["sha256"]:
|
||||
raise SystemExit(f"D1 checksum mismatch for {architecture}: {converted}")
|
||||
shutil.copyfile(converted, target)
|
||||
PY
|
||||
fi
|
||||
}
|
||||
|
||||
build_variant()
|
||||
{
|
||||
local name=$1 source_c=$2 suffix=$3 variant source hasher repo spec
|
||||
local expected_fingerprint artifact_fingerprint
|
||||
local -a librpm_rpms
|
||||
variant=$(variant_dir "$name")
|
||||
source="$variant/src/rpm"
|
||||
hasher="$variant/hasher"
|
||||
repo="$hasher/repo/x86_64/RPMS.hasher"
|
||||
spec="$source/alt/rpm.spec"
|
||||
|
||||
printf '\n===== Building %s with %s =====\n' "$name" "$source_c"
|
||||
mkdir -p "$variant/src" "$variant/logs" "$hasher"
|
||||
shopt -s nullglob
|
||||
librpm_rpms=("$repo"/librpm7-[0-9]*".$suffix".x86_64.rpm)
|
||||
expected_fingerprint=$(mktemp)
|
||||
{
|
||||
printf 'base_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
|
||||
printf 'set_source=%s\n' "$(sha256sum "$source_c" | awk '{print $1}')"
|
||||
printf 'suffix=%s\n' "$suffix"
|
||||
printf 'packager=%s\n' "$PACKAGER"
|
||||
} >"$expected_fingerprint"
|
||||
if [[ -f $variant/input-fingerprint.txt ]]; then
|
||||
if ! cmp -s "$expected_fingerprint" "$variant/input-fingerprint.txt"; then
|
||||
rm -f "$expected_fingerprint"
|
||||
fail "$name build inputs changed; use RESET_WORK=1"
|
||||
fi
|
||||
elif ((${#librpm_rpms[@]} > 0)); then
|
||||
rm -f "$expected_fingerprint"
|
||||
fail "$name RPM exists without its input fingerprint; use RESET_WORK=1"
|
||||
fi
|
||||
if ((${#librpm_rpms[@]} == 0)) && [[ -d $source ]]; then
|
||||
rm -rf "$source"
|
||||
fi
|
||||
if [[ ! -d $source/.git ]]; then
|
||||
git clone --local "$SOURCE_BASE" "$source"
|
||||
cp "$source_c" "$source/lib/set.c"
|
||||
prepare_spec "$spec" "$suffix"
|
||||
cp "$expected_fingerprint" "$variant/input-fingerprint.txt"
|
||||
else
|
||||
cmp -s "$source_c" "$source/lib/set.c" ||
|
||||
fail "$source_c changed; use RESET_WORK=1"
|
||||
fi
|
||||
rm -f "$expected_fingerprint"
|
||||
|
||||
librpm_rpms=("$repo"/librpm7-[0-9]*".$suffix".x86_64.rpm)
|
||||
if ((${#librpm_rpms[@]} == 0)); then
|
||||
(
|
||||
cd "$source"
|
||||
gear-hsh \
|
||||
--commit \
|
||||
--with-stuff \
|
||||
--packager="$PACKAGER" \
|
||||
--no-sisyphus-check=changelog,packager,gpg \
|
||||
--mountpoints=/proc \
|
||||
--workdir="$hasher" \
|
||||
--target=x86_64
|
||||
) 2>&1 | tee "$variant/logs/rpm.log"
|
||||
else
|
||||
printf 'already built: %s\n' "${librpm_rpms[0]}"
|
||||
fi
|
||||
|
||||
librpm_rpms=("$repo"/librpm7-[0-9]*".$suffix".x86_64.rpm)
|
||||
((${#librpm_rpms[@]} == 1)) ||
|
||||
fail "expected one librpm7 package for $name, got ${#librpm_rpms[@]}"
|
||||
artifact_fingerprint=$(sha256sum "${librpm_rpms[0]}")
|
||||
if [[ -f $variant/librpm-artifact.sha256 ]]; then
|
||||
[[ $artifact_fingerprint == "$(<"$variant/librpm-artifact.sha256")" ]] ||
|
||||
fail "$name librpm artifact changed; use RESET_WORK=1"
|
||||
else
|
||||
printf '%s\n' "$artifact_fingerprint" >"$variant/librpm-artifact.sha256"
|
||||
fi
|
||||
|
||||
rm -rf "$variant/lib"
|
||||
mkdir -p "$variant/lib"
|
||||
(
|
||||
cd "$variant/lib"
|
||||
rpm2cpio "${librpm_rpms[0]}" | cpio -idm --quiet
|
||||
)
|
||||
[[ -e $variant/lib/usr/lib64/librpm.so.7 && -e $variant/lib/usr/lib64/librpmio.so.7 ]] ||
|
||||
fail "librpm libraries were not extracted for $name"
|
||||
for apt_binary in "$APT_GET" "$APT_CACHE"; do
|
||||
LD_LIBRARY_PATH="$variant/lib/usr/lib64" ldd "$apt_binary" |
|
||||
awk -v expected="$variant/lib/usr/lib64/librpm.so.7" '
|
||||
index($0, expected) { found = 1 }
|
||||
END { exit found ? 0 : 1 }
|
||||
' || fail "$apt_binary does not load the built librpm for $name"
|
||||
done
|
||||
}
|
||||
|
||||
build_cache()
|
||||
{
|
||||
local name=$1 variant libdir start end elapsed stdout stderr status
|
||||
variant=$(variant_dir "$name")
|
||||
libdir=$(variant_libdir "$name")
|
||||
apt_options "$variant"
|
||||
stdout="$variant/logs/gencaches.stdout"
|
||||
stderr="$variant/logs/gencaches.stderr"
|
||||
|
||||
chmod -R u+w "$variant/apt/cache"
|
||||
rm -f "$variant/apt/cache/pkgcache.bin" "$variant/apt/cache/srcpkgcache.bin"
|
||||
start=$(date +%s%N)
|
||||
if env LC_ALL=C APT_CONFIG="$COMMON/apt.conf" LD_LIBRARY_PATH="$libdir" \
|
||||
"$APT_CACHE" -q "${APT_OPTIONS[@]}" gencaches >"$stdout" 2>"$stderr"; then
|
||||
status=0
|
||||
else
|
||||
status=$?
|
||||
fi
|
||||
end=$(date +%s%N)
|
||||
elapsed=$(awk -v start="$start" -v end="$end" \
|
||||
'BEGIN { printf "%.6f", (end - start) / 1000000000 }')
|
||||
printf '%s\t%s\t%s\n' "$name" "$elapsed" "$status" >>"$CACHE_RESULTS"
|
||||
[[ $status -eq 0 && -s $variant/apt/cache/pkgcache.bin ]] ||
|
||||
fail "gencaches failed for $name; see $stderr"
|
||||
chmod -R a-w "$variant/apt/cache"
|
||||
}
|
||||
|
||||
record_cache_checksums()
|
||||
{
|
||||
local name variant
|
||||
: >"$RESULT_DIR/cache-files.before.txt"
|
||||
for name in set9 d1; do
|
||||
variant=$(variant_dir "$name")
|
||||
for cache_file in "$variant/apt/cache/pkgcache.bin" \
|
||||
"$variant/apt/cache/srcpkgcache.bin"; do
|
||||
stat -c '%n\t%D\t%i\t%s\t%Y\t%A' "$cache_file"
|
||||
sha256sum "$cache_file"
|
||||
done >>"$RESULT_DIR/cache-files.before.txt"
|
||||
done
|
||||
}
|
||||
|
||||
verify_cache_checksums()
|
||||
{
|
||||
local name variant
|
||||
: >"$RESULT_DIR/cache-files.after.txt"
|
||||
for name in set9 d1; do
|
||||
variant=$(variant_dir "$name")
|
||||
for cache_file in "$variant/apt/cache/pkgcache.bin" \
|
||||
"$variant/apt/cache/srcpkgcache.bin"; do
|
||||
stat -c '%n\t%D\t%i\t%s\t%Y\t%A' "$cache_file"
|
||||
sha256sum "$cache_file"
|
||||
done >>"$RESULT_DIR/cache-files.after.txt"
|
||||
done
|
||||
cmp -s "$RESULT_DIR/cache-files.before.txt" \
|
||||
"$RESULT_DIR/cache-files.after.txt" ||
|
||||
fail "APT cache changed during timed runs"
|
||||
}
|
||||
|
||||
write_provenance()
|
||||
{
|
||||
{
|
||||
printf 'rpm_git=%s\n' "$RPM_GIT"
|
||||
printf 'rpm_branch=%s\n' "$RPM_BRANCH"
|
||||
printf 'rpm_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
|
||||
printf 'sisyphus_mirror=%s\n' "$SISYPHUS_MIRROR"
|
||||
printf 'apt=%s\n' "$(rpmquery --qf '%{VERSION}-%{RELEASE}' apt)"
|
||||
printf 'rpm=%s\n' "$(rpm --version)"
|
||||
printf 'cpu=%s\n' "$CPU"
|
||||
printf 'rounds=%s\n' "$ROUNDS"
|
||||
printf 'operations=%s\n' "$OPERATIONS"
|
||||
sha256sum "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \
|
||||
"$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" \
|
||||
"$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h"
|
||||
printf 'set9_librpm='; cat "$SET9_VARIANT/librpm-artifact.sha256"
|
||||
printf 'd1_librpm='; cat "$D1_VARIANT/librpm-artifact.sha256"
|
||||
} >"$RESULT_DIR/provenance.txt"
|
||||
cp "$SNAPSHOT/manifest.json" "$RESULT_DIR/pkglist-manifest.json"
|
||||
}
|
||||
|
||||
summarize_results()
|
||||
{
|
||||
python3 - "$RAW_RESULTS" "$RESULT_DIR/summary.tsv" "$RESULT_DIR/summary.md" <<'PY'
|
||||
import csv
|
||||
import statistics
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
raw_path, tsv_path, markdown_path = map(Path, sys.argv[1:])
|
||||
with raw_path.open(newline="") as stream:
|
||||
rows = list(csv.DictReader(stream, delimiter="\t"))
|
||||
|
||||
groups = defaultdict(list)
|
||||
for row in rows:
|
||||
groups[row["operation"]].append(row)
|
||||
|
||||
summary = []
|
||||
all_equivalent = True
|
||||
for operation, items in groups.items():
|
||||
by_variant = defaultdict(list)
|
||||
signatures = set()
|
||||
statuses = set()
|
||||
for item in items:
|
||||
by_variant[item["variant"]].append(float(item["seconds"]))
|
||||
statuses.add(item["status"])
|
||||
signatures.add(
|
||||
(
|
||||
item["status"],
|
||||
item["stdout_sha256"],
|
||||
item["stdout_bytes"],
|
||||
item["stderr_sha256"],
|
||||
item["stderr_bytes"],
|
||||
)
|
||||
)
|
||||
if set(by_variant) != {"set9", "d1"}:
|
||||
raise SystemExit(f"missing variant samples for {operation}")
|
||||
if len(by_variant["set9"]) != len(by_variant["d1"]):
|
||||
raise SystemExit(f"unbalanced variant samples for {operation}")
|
||||
set9 = statistics.median(by_variant["set9"])
|
||||
d1 = statistics.median(by_variant["d1"])
|
||||
equivalent = len(signatures) == 1
|
||||
all_equivalent &= equivalent
|
||||
status = next(iter(statuses)) if len(statuses) == 1 else "mixed:" + ",".join(sorted(statuses))
|
||||
path = "normal" if status == "0" else "failure-path"
|
||||
summary.append(
|
||||
(
|
||||
operation,
|
||||
len(by_variant["set9"]),
|
||||
set9,
|
||||
min(by_variant["set9"]),
|
||||
max(by_variant["set9"]),
|
||||
d1,
|
||||
min(by_variant["d1"]),
|
||||
max(by_variant["d1"]),
|
||||
d1 / set9,
|
||||
status,
|
||||
path,
|
||||
equivalent,
|
||||
)
|
||||
)
|
||||
|
||||
with tsv_path.open("w", newline="") as stream:
|
||||
writer = csv.writer(stream, delimiter="\t", lineterminator="\n")
|
||||
writer.writerow(
|
||||
[
|
||||
"operation", "runs_per_variant", "set9_median_seconds", "set9_min_seconds",
|
||||
"set9_max_seconds", "d1_median_seconds", "d1_min_seconds", "d1_max_seconds",
|
||||
"d1/set9", "exit_status", "benchmark_path", "outputs_equal"
|
||||
]
|
||||
)
|
||||
for operation, count, set9, set9_min, set9_max, d1, d1_min, d1_max, ratio, status, path, equivalent in summary:
|
||||
writer.writerow(
|
||||
[
|
||||
operation, count, f"{set9:.6f}", f"{set9_min:.6f}", f"{set9_max:.6f}",
|
||||
f"{d1:.6f}", f"{d1_min:.6f}", f"{d1_max:.6f}", f"{ratio:.4f}",
|
||||
status, path, "yes" if equivalent else "NO"
|
||||
]
|
||||
)
|
||||
|
||||
lines = [
|
||||
"# Sisyphus set9 vs D1 benchmark",
|
||||
"",
|
||||
"Timed scope: resolver commands only; conversion, builds and `gencaches` are excluded.",
|
||||
"Each round uses the balanced order `set9 / d1 / d1 / set9` after one warm-up per variant.",
|
||||
"",
|
||||
"| operation | runs/variant | set9 median [min–max], s | D1 median [min–max], s | D1/set9 | status/path | output equivalence |",
|
||||
"|---|---:|---:|---:|---:|:---:|:---:|",
|
||||
]
|
||||
for operation, count, set9, set9_min, set9_max, d1, d1_min, d1_max, ratio, status, path, equivalent in summary:
|
||||
lines.append(
|
||||
f"| `{operation}` | {count} | {set9:.6f} [{set9_min:.6f}–{set9_max:.6f}] | "
|
||||
f"{d1:.6f} [{d1_min:.6f}–{d1_max:.6f}] | {ratio:.4f} | "
|
||||
f"{status}/{path} | {'yes' if equivalent else '**NO**'} |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"`D1/set9 < 1` means that the D1 variant was faster.",
|
||||
"Rows with non-zero status are explicitly labelled `failure-path`; do not treat them as successful resolver workloads.",
|
||||
"Output equivalence includes exit status plus exact stdout and stderr hashes for every repeat and both variants.",
|
||||
]
|
||||
)
|
||||
markdown_path.write_text("\n".join(lines) + "\n")
|
||||
if not all_equivalent:
|
||||
raise SystemExit(2)
|
||||
PY
|
||||
}
|
||||
|
||||
if [[ ${1:-} == --help || ${1:-} == -h ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
(($# == 0)) || fail "unexpected arguments; use --help"
|
||||
|
||||
for command in git gear-hsh hsh rpm rpmquery rpm2cpio cpio taskset awk sed cc \
|
||||
pkglist-query date sha256sum stat ldd python3 cmp cp mv tee realpath mktemp grep; do
|
||||
command -v "$command" >/dev/null || fail "required command not found: $command"
|
||||
done
|
||||
[[ -n $APT_CACHE && -x $APT_CACHE ]] || fail "apt-cache not found: $APT_CACHE"
|
||||
[[ -x $APT_GET ]] || fail "apt-get executable not found: $APT_GET"
|
||||
[[ -f $SET9_C ]] || fail "set9 source not found: $SET9_C"
|
||||
[[ -f $D1_C ]] || fail "D1 source not found: $D1_C"
|
||||
[[ -f $PKGLIST_CONVERTER ]] || fail "pkglist converter not found: $PKGLIST_CONVERTER"
|
||||
[[ $ROUNDS =~ ^[1-9][0-9]*$ ]] || fail "ROUNDS must be a positive integer"
|
||||
[[ $RESET_WORK =~ ^[01]$ ]] || fail "RESET_WORK must be 0 or 1"
|
||||
taskset -c "$CPU" true >/dev/null 2>&1 || fail "CPU $CPU is unavailable to taskset"
|
||||
read -r -a OPERATION_LIST <<<"$OPERATIONS"
|
||||
((${#OPERATION_LIST[@]} > 0)) || fail "OPERATIONS is empty"
|
||||
for operation in "${OPERATION_LIST[@]}"; do
|
||||
case $operation in
|
||||
unmet|check|autoremove|install-rpm-build|install-openuds-server|install-password-store|upgrade|dist-upgrade) ;;
|
||||
*) fail "unknown operation in OPERATIONS: $operation" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
HOME_REAL=$(realpath -m "$HOME")
|
||||
CWD_REAL=$(realpath -m "$PWD")
|
||||
REPO_ROOT=$(realpath -m "$REPO_ROOT")
|
||||
SET9_C=$(realpath -m "$SET9_C")
|
||||
D1_C=$(realpath -m "$D1_C")
|
||||
PKGLIST_CONVERTER=$(realpath -m "$PKGLIST_CONVERTER")
|
||||
WORK_ROOT=$(realpath -m "$WORK_ROOT")
|
||||
RESULT_DIR=$(realpath -m "$RESULT_DIR")
|
||||
[[ $RESULT_DIR == "$WORK_ROOT"/* ]] ||
|
||||
fail "RESULT_DIR must be inside WORK_ROOT: $RESULT_DIR"
|
||||
|
||||
COMMON="$WORK_ROOT/common"
|
||||
SNAPSHOT="$COMMON/snapshot"
|
||||
SOURCE_BASE="$WORK_ROOT/src/rpm-base"
|
||||
SOURCE_FINGERPRINT="$WORK_ROOT/src/rpm-base.fingerprint.txt"
|
||||
SET9_VARIANT="$WORK_ROOT/variants/set9"
|
||||
D1_VARIANT="$WORK_ROOT/variants/d1"
|
||||
|
||||
WORK_ROOT_EXISTED=0
|
||||
[[ -e $WORK_ROOT ]] && WORK_ROOT_EXISTED=1
|
||||
if ((RESET_WORK)); then
|
||||
safe_remove_work_root
|
||||
for old_hasher in "$SET9_VARIANT/hasher" "$D1_VARIANT/hasher"; do
|
||||
[[ -d $old_hasher ]] || continue
|
||||
old_hasher=$(canonical_work_subdir "$old_hasher" 'hasher workdir')
|
||||
hsh --cleanup-only --workdir="$old_hasher" ||
|
||||
fail "hasher cleanup failed: $old_hasher"
|
||||
done
|
||||
for old_apt in "$SET9_VARIANT/apt" "$D1_VARIANT/apt"; do
|
||||
[[ -d $old_apt ]] || continue
|
||||
old_apt=$(canonical_work_subdir "$old_apt" 'APT workdir')
|
||||
chmod -R u+w "$old_apt"
|
||||
done
|
||||
rm -rf "$WORK_ROOT"
|
||||
WORK_ROOT_EXISTED=0
|
||||
elif ((WORK_ROOT_EXISTED)); then
|
||||
marker="$WORK_ROOT/.arsv-sisyphus-set-bench-root"
|
||||
[[ -f $marker && ! -L $marker ]] ||
|
||||
fail "refusing unmarked existing WORK_ROOT: $WORK_ROOT"
|
||||
grep -Fx 'ARSV Sisyphus set9/D1 benchmark work root' "$marker" >/dev/null ||
|
||||
fail "invalid WORK_ROOT ownership marker: $marker"
|
||||
fi
|
||||
mkdir -p "$WORK_ROOT"
|
||||
MARKER="$WORK_ROOT/.arsv-sisyphus-set-bench-root"
|
||||
if [[ -e $MARKER ]]; then
|
||||
grep -Fx 'ARSV Sisyphus set9/D1 benchmark work root' "$MARKER" >/dev/null ||
|
||||
fail "invalid WORK_ROOT ownership marker: $MARKER"
|
||||
else
|
||||
printf '%s\n' 'ARSV Sisyphus set9/D1 benchmark work root' >"$MARKER"
|
||||
fi
|
||||
ensure_work_subdir "$WORK_ROOT/src" 'source directory'
|
||||
ensure_work_subdir "$WORK_ROOT/variants" 'variants directory'
|
||||
ensure_work_subdir "$COMMON/root/var/lib/rpm" 'isolated RPM database directory'
|
||||
ensure_work_subdir "$RESULT_DIR" 'results directory'
|
||||
: >"$COMMON/apt.conf"
|
||||
: >"$COMMON/status"
|
||||
|
||||
prepare_snapshot
|
||||
if [[ ! -d $SOURCE_BASE/.git ]]; then
|
||||
git clone --branch "$RPM_BRANCH" --single-branch "$RPM_GIT" "$SOURCE_BASE"
|
||||
write_source_fingerprint >"$SOURCE_FINGERPRINT"
|
||||
else
|
||||
validate_source_reuse
|
||||
fi
|
||||
|
||||
build_variant set9 "$SET9_C" arsvset9
|
||||
build_variant d1 "$D1_C" arsvd1
|
||||
prepare_variant_lists "$SET9_VARIANT" set9
|
||||
prepare_variant_lists "$D1_VARIANT" d1
|
||||
|
||||
mkdir -p "$RESULT_DIR/raw"
|
||||
CACHE_RESULTS="$RESULT_DIR/cache-build.tsv"
|
||||
printf 'variant\tseconds\texit_status\n' >"$CACHE_RESULTS"
|
||||
build_cache set9
|
||||
build_cache d1
|
||||
record_cache_checksums
|
||||
write_provenance
|
||||
|
||||
RAW_RESULTS="$RESULT_DIR/raw.tsv"
|
||||
printf 'operation\tsequence\tvariant\tsample\tseconds\tstatus\tstdout_sha256\tstdout_bytes\tstderr_sha256\tstderr_bytes\n' >"$RAW_RESULTS"
|
||||
declare -A SAMPLE_COUNTS=([set9]=0 [d1]=0)
|
||||
sequence=0
|
||||
|
||||
for operation in "${OPERATION_LIST[@]}"; do
|
||||
printf '\n===== Warm-up: %s =====\n' "$operation"
|
||||
mkdir -p "$RESULT_DIR/warmup/$operation"
|
||||
for variant_name in set9 d1; do
|
||||
prepare_command "$operation" "$variant_name"
|
||||
warm_status=$(execute_command \
|
||||
"$RESULT_DIR/warmup/$operation/$variant_name.stdout" \
|
||||
"$RESULT_DIR/warmup/$operation/$variant_name.stderr")
|
||||
printf '%s\n' "$warm_status" >"$RESULT_DIR/warmup/$operation/$variant_name.status"
|
||||
done
|
||||
|
||||
printf '===== Timed ABBA: %s =====\n' "$operation"
|
||||
for ((round = 1; round <= ROUNDS; ++round)); do
|
||||
for variant_name in set9 d1 d1 set9; do
|
||||
sequence=$((sequence + 1))
|
||||
SAMPLE_COUNTS[$variant_name]=$((SAMPLE_COUNTS[$variant_name] + 1))
|
||||
run_once "$operation" "$variant_name" "$sequence" "${SAMPLE_COUNTS[$variant_name]}"
|
||||
done
|
||||
done
|
||||
SAMPLE_COUNTS[set9]=0
|
||||
SAMPLE_COUNTS[d1]=0
|
||||
done
|
||||
|
||||
verify_cache_checksums
|
||||
if ! summarize_results; then
|
||||
fail "variant/repeat outputs differ; inspect $RESULT_DIR/raw and $RESULT_DIR/summary.md"
|
||||
fi
|
||||
|
||||
printf '\nDone. Results:\n'
|
||||
printf ' %s\n' "$RESULT_DIR/summary.md" "$RESULT_DIR/summary.tsv" \
|
||||
"$RESULT_DIR/raw.tsv" "$RESULT_DIR/cache-build.tsv" "$RESULT_DIR/provenance.txt" \
|
||||
"$RESULT_DIR/pkglist-manifest.json"
|
||||
Reference in New Issue
Block a user