test D1 realization
This commit is contained in:
@@ -0,0 +1,296 @@
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifdef ARSV_WITH_RPM
|
||||
#include <rpm/header.h>
|
||||
#include <rpm/rpmio.h>
|
||||
#include <rpm/rpmlib.h>
|
||||
#include <rpm/rpmtag.h>
|
||||
#include <rpm/rpmtd.h>
|
||||
#endif
|
||||
|
||||
int arsv_set9_decode(const char* source, unsigned** hashes, size_t* count, unsigned* bpp);
|
||||
|
||||
#define D1_PREFIX "set:D1"
|
||||
#define D1_HEADER_LEN 8
|
||||
|
||||
static const char base64_alphabet[] =
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
static size_t base64_encoded_size(size_t byte_count) {
|
||||
if (byte_count > SIZE_MAX - 2) return SIZE_MAX;
|
||||
size_t groups = (byte_count + 2) / 3;
|
||||
if (groups > SIZE_MAX / 4) return SIZE_MAX;
|
||||
size_t size = groups * 4;
|
||||
size_t remainder = byte_count % 3;
|
||||
if (remainder) size -= 3 - remainder;
|
||||
return size;
|
||||
}
|
||||
|
||||
static void base64_encode(const unsigned char* input, size_t input_len, char* output) {
|
||||
while (input_len >= 3) {
|
||||
uint32_t value = ((uint32_t)input[0] << 16) | ((uint32_t)input[1] << 8) | input[2];
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
output[2] = base64_alphabet[(value >> 6) & 0x3f];
|
||||
output[3] = base64_alphabet[value & 0x3f];
|
||||
input += 3;
|
||||
input_len -= 3;
|
||||
output += 4;
|
||||
}
|
||||
if (input_len == 1) {
|
||||
uint32_t value = (uint32_t)input[0] << 16;
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
} else if (input_len == 2) {
|
||||
uint32_t value = ((uint32_t)input[0] << 16) | ((uint32_t)input[1] << 8);
|
||||
output[0] = base64_alphabet[(value >> 18) & 0x3f];
|
||||
output[1] = base64_alphabet[(value >> 12) & 0x3f];
|
||||
output[2] = base64_alphabet[(value >> 6) & 0x3f];
|
||||
}
|
||||
}
|
||||
|
||||
static int encode_d1(const unsigned* hashes, size_t count, unsigned bpp, char** result) {
|
||||
if (!hashes || !count || !result || bpp < 10 || bpp > 32) return -EINVAL;
|
||||
if (count > (SIZE_MAX - 7) / bpp) return -EOVERFLOW;
|
||||
|
||||
size_t bit_count = count * bpp;
|
||||
size_t byte_count = (bit_count + 7) / 8;
|
||||
unsigned char* bytes = calloc(byte_count, 1);
|
||||
if (!bytes) return -ENOMEM;
|
||||
|
||||
unsigned char* output = bytes;
|
||||
uint64_t bits = 0;
|
||||
unsigned filled = 0;
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
bits |= (uint64_t)hashes[i] << filled;
|
||||
filled += bpp;
|
||||
while (filled >= 8) {
|
||||
*output++ = (unsigned char)bits;
|
||||
bits >>= 8;
|
||||
filled -= 8;
|
||||
}
|
||||
}
|
||||
if (filled) *output++ = (unsigned char)bits;
|
||||
if ((size_t)(output - bytes) != byte_count) {
|
||||
free(bytes);
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
size_t payload_len = base64_encoded_size(byte_count);
|
||||
if (payload_len == SIZE_MAX || payload_len > SIZE_MAX - D1_HEADER_LEN - 1) {
|
||||
free(bytes);
|
||||
return -EOVERFLOW;
|
||||
}
|
||||
char* encoded = malloc(D1_HEADER_LEN + payload_len + 1);
|
||||
if (!encoded) {
|
||||
free(bytes);
|
||||
return -ENOMEM;
|
||||
}
|
||||
memcpy(encoded, D1_PREFIX, sizeof(D1_PREFIX) - 1);
|
||||
encoded[6] = (char)('0' + bpp / 10);
|
||||
encoded[7] = (char)('0' + bpp % 10);
|
||||
base64_encode(bytes, byte_count, encoded + D1_HEADER_LEN);
|
||||
encoded[D1_HEADER_LEN + payload_len] = '\0';
|
||||
free(bytes);
|
||||
*result = encoded;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int convert_set(const char* source, char** result, size_t* hash_count, unsigned* bpp) {
|
||||
if (!source || strncmp(source, "set:", 4) != 0 || strncmp(source, D1_PREFIX, 6) == 0)
|
||||
return -EINVAL;
|
||||
|
||||
unsigned* hashes = NULL;
|
||||
size_t count = 0;
|
||||
unsigned precision = 0;
|
||||
int rc = arsv_set9_decode(source, &hashes, &count, &precision);
|
||||
if (rc == 0) rc = encode_d1(hashes, count, precision, result);
|
||||
free(hashes);
|
||||
if (rc < 0) return rc;
|
||||
if (hash_count) *hash_count = count;
|
||||
if (bpp) *bpp = precision;
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef ARSV_WITH_RPM
|
||||
struct statistics {
|
||||
unsigned long headers;
|
||||
unsigned long set_occurrences;
|
||||
unsigned long set_bytes_old;
|
||||
unsigned long set_bytes_new;
|
||||
unsigned long hashes;
|
||||
};
|
||||
|
||||
static int rewrite_version_tag(Header header, rpmTagVal tag, const char* package,
|
||||
struct statistics* stats) {
|
||||
struct rpmtd_s values;
|
||||
memset(&values, 0, sizeof(values));
|
||||
if (headerGet(header, tag, &values, HEADERGET_MINMEM) != 1) return 0;
|
||||
if (rpmtdType(&values) != RPM_STRING_ARRAY_TYPE) {
|
||||
rpmtdFreeData(&values);
|
||||
fprintf(stderr, "%s: tag %d is not a string array\n", package, (int)tag);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rpm_count_t count = rpmtdCount(&values);
|
||||
const char** rewritten = calloc((size_t)count, sizeof(*rewritten));
|
||||
if (!rewritten) {
|
||||
rpmtdFreeData(&values);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
int rc = 0;
|
||||
int changed = 0;
|
||||
rpmtdInit(&values);
|
||||
for (rpm_count_t i = 0; i < count; ++i) {
|
||||
const char* value = rpmtdNextString(&values);
|
||||
if (!value) {
|
||||
rc = -EINVAL;
|
||||
break;
|
||||
}
|
||||
if (strncmp(value, "set:", 4) == 0) {
|
||||
char* converted = NULL;
|
||||
size_t hashes = 0;
|
||||
unsigned bpp = 0;
|
||||
rc = convert_set(value, &converted, &hashes, &bpp);
|
||||
if (rc < 0) {
|
||||
fprintf(stderr, "%s: cannot convert tag %d index %u (rc=%d)\n", package,
|
||||
(int)tag, (unsigned)i, rc);
|
||||
break;
|
||||
}
|
||||
rewritten[i] = converted;
|
||||
changed = 1;
|
||||
++stats->set_occurrences;
|
||||
stats->set_bytes_old += strlen(value);
|
||||
stats->set_bytes_new += strlen(converted);
|
||||
stats->hashes += hashes;
|
||||
} else {
|
||||
rewritten[i] = strdup(value);
|
||||
if (!rewritten[i]) {
|
||||
rc = -ENOMEM;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rc == 0 && changed) {
|
||||
headerDel(header, tag);
|
||||
if (!headerPutStringArray(header, tag, rewritten, count)) rc = -EIO;
|
||||
}
|
||||
for (rpm_count_t i = 0; i < count; ++i) free((void*)rewritten[i]);
|
||||
free(rewritten);
|
||||
rpmtdFreeData(&values);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int rewrite_pkglist(const char* input_path, const char* output_path, unsigned long limit) {
|
||||
FD_t input = Fopen(input_path, "r.ufdio");
|
||||
if (!input || Ferror(input)) {
|
||||
fprintf(stderr, "%s: %s\n", input_path, input ? Fstrerror(input) : "cannot open");
|
||||
return 1;
|
||||
}
|
||||
FD_t output = Fopen(output_path, "w.ufdio");
|
||||
if (!output || Ferror(output)) {
|
||||
fprintf(stderr, "%s: %s\n", output_path, output ? Fstrerror(output) : "cannot open");
|
||||
Fclose(input);
|
||||
return 1;
|
||||
}
|
||||
|
||||
struct statistics stats = {0};
|
||||
Header header;
|
||||
int failed = 0;
|
||||
const rpmTagVal tags[] = {
|
||||
RPMTAG_REQUIREVERSION,
|
||||
RPMTAG_PROVIDEVERSION,
|
||||
RPMTAG_CONFLICTVERSION,
|
||||
RPMTAG_OBSOLETEVERSION,
|
||||
RPMTAG_RECOMMENDVERSION,
|
||||
RPMTAG_SUGGESTVERSION,
|
||||
RPMTAG_SUPPLEMENTVERSION,
|
||||
RPMTAG_ENHANCEVERSION,
|
||||
};
|
||||
while ((!limit || stats.headers < limit) &&
|
||||
(header = headerRead(input, HEADER_MAGIC_YES)) != NULL) {
|
||||
const char* package = headerGetString(header, RPMTAG_NAME);
|
||||
if (!package) package = "<unknown>";
|
||||
for (size_t i = 0; i < sizeof(tags) / sizeof(tags[0]); ++i) {
|
||||
int rc = rewrite_version_tag(header, tags[i], package, &stats);
|
||||
if (rc < 0) {
|
||||
failed = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!failed && headerWrite(output, header, HEADER_MAGIC_YES) != 0) {
|
||||
fprintf(stderr, "%s: failed to write header for %s\n", output_path, package);
|
||||
failed = 1;
|
||||
}
|
||||
headerFree(header);
|
||||
if (failed) break;
|
||||
++stats.headers;
|
||||
}
|
||||
if (!failed && Ferror(input)) {
|
||||
fprintf(stderr, "%s: read error: %s\n", input_path, Fstrerror(input));
|
||||
failed = 1;
|
||||
}
|
||||
if (Fclose(output) != 0) failed = 1;
|
||||
Fclose(input);
|
||||
|
||||
if (failed) {
|
||||
remove(output_path);
|
||||
return 1;
|
||||
}
|
||||
fprintf(stderr,
|
||||
"headers=%lu set_occurrences=%lu hashes=%lu old_set_bytes=%lu "
|
||||
"new_set_bytes=%lu\n",
|
||||
stats.headers, stats.set_occurrences, stats.hashes, stats.set_bytes_old,
|
||||
stats.set_bytes_new);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
static void usage(const char* program) {
|
||||
fprintf(stderr, "usage: %s --convert-set set:VALUE\n", program);
|
||||
#ifdef ARSV_WITH_RPM
|
||||
fprintf(stderr, " %s --rewrite INPUT OUTPUT [--max-headers N]\n", program);
|
||||
#endif
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc == 3 && strcmp(argv[1], "--convert-set") == 0) {
|
||||
char* converted = NULL;
|
||||
int rc = convert_set(argv[2], &converted, NULL, NULL);
|
||||
if (rc < 0) {
|
||||
fprintf(stderr, "cannot convert set value (rc=%d)\n", rc);
|
||||
return 1;
|
||||
}
|
||||
puts(converted);
|
||||
free(converted);
|
||||
return 0;
|
||||
}
|
||||
#ifdef ARSV_WITH_RPM
|
||||
if ((argc == 4 || argc == 6) && strcmp(argv[1], "--rewrite") == 0) {
|
||||
unsigned long limit = 0;
|
||||
if (argc == 6) {
|
||||
if (strcmp(argv[4], "--max-headers") != 0) {
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
char* end = NULL;
|
||||
errno = 0;
|
||||
limit = strtoul(argv[5], &end, 10);
|
||||
if (errno || !end || *end || !limit) {
|
||||
fprintf(stderr, "invalid --max-headers value: %s\n", argv[5]);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
return rewrite_pkglist(argv[2], argv[3], limit);
|
||||
}
|
||||
#endif
|
||||
usage(argv[0]);
|
||||
return 2;
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create D1 copies of the real Sisyphus x86_64 and noarch pkglist files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[2]
|
||||
DEFAULT_IMAGE = "registry.altlinux.org/sisyphus/alt:latest"
|
||||
ARCHITECTURES = ("x86_64", "noarch")
|
||||
SUMMARY_RE = re.compile(
|
||||
r"headers=(?P<headers>\d+) set_occurrences=(?P<set_occurrences>\d+) "
|
||||
r"hashes=(?P<hashes>\d+) old_set_bytes=(?P<old_set_bytes>\d+) "
|
||||
r"new_set_bytes=(?P<new_set_bytes>\d+)"
|
||||
)
|
||||
VERSION_FORMAT = (
|
||||
"[%{REQUIREVERSION}\\n]"
|
||||
"[%{PROVIDEVERSION}\\n]"
|
||||
"[%{CONFLICTVERSION}\\n]"
|
||||
"[%{OBSOLETEVERSION}\\n]"
|
||||
"[%{RECOMMENDVERSION}\\n]"
|
||||
"[%{SUGGESTVERSION}\\n]"
|
||||
"[%{SUPPLEMENTVERSION}\\n]"
|
||||
"[%{ENHANCEVERSION}\\n]"
|
||||
)
|
||||
|
||||
|
||||
def classify_pkglist(path: Path) -> str | None:
|
||||
name = path.name
|
||||
if not name.endswith("_base_pkglist.classic"):
|
||||
return None
|
||||
if "_Sisyphus_x86%5f64_" in name or "_Sisyphus_x86_64_" in name:
|
||||
return "x86_64"
|
||||
if "_Sisyphus_noarch_" in name:
|
||||
return "noarch"
|
||||
return None
|
||||
|
||||
|
||||
def validate_output(path: Path) -> Path:
|
||||
resolved = path.expanduser().resolve()
|
||||
home = Path.home().resolve()
|
||||
source = ROOT.resolve()
|
||||
if resolved == home or resolved == source or source in resolved.parents:
|
||||
raise ValueError(f"refusing protected output path: {resolved}")
|
||||
if resolved == Path("/"):
|
||||
raise ValueError("refusing filesystem root as output")
|
||||
return resolved
|
||||
|
||||
|
||||
def run(
|
||||
command: list[str],
|
||||
*,
|
||||
check: bool = True,
|
||||
cwd: Path | None = None,
|
||||
stdout: int | None = None,
|
||||
) -> subprocess.CompletedProcess[bytes]:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
stdout=stdout if stdout is not None else subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode != 0:
|
||||
stderr = result.stderr.decode(errors="replace")
|
||||
raise RuntimeError(f"command failed ({result.returncode}): {' '.join(command)}\n{stderr}")
|
||||
return result
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def compile_converter(destination: Path) -> None:
|
||||
include = destination.parent / "compat-include"
|
||||
include.mkdir()
|
||||
for name in ("rpmlib.h", "system.h", "set.h"):
|
||||
(include / name).touch()
|
||||
run(
|
||||
[
|
||||
"cc",
|
||||
"-O2",
|
||||
"-std=gnu11",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
"-D_GNU_SOURCE",
|
||||
"-DARSV_SET9_EXPORT",
|
||||
"-DARSV_WITH_RPM",
|
||||
"-I",
|
||||
str(include),
|
||||
"-include",
|
||||
str(ROOT / "scripts/rpmsetcmp/newset_compat.h"),
|
||||
str(ROOT / "reimplement/set9.c"),
|
||||
str(HERE / "rewrite_sisyphus_pkglist.c"),
|
||||
"-lrpm",
|
||||
"-lrpmio",
|
||||
"-o",
|
||||
str(destination),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def find_pkglists(lists_dir: Path) -> dict[str, Path]:
|
||||
found: dict[str, Path] = {}
|
||||
for path in lists_dir.glob("*_base_pkglist.classic"):
|
||||
architecture = classify_pkglist(path)
|
||||
if architecture is None:
|
||||
continue
|
||||
if architecture in found:
|
||||
raise RuntimeError(f"multiple Sisyphus {architecture} pkglist files")
|
||||
found[architecture] = path
|
||||
missing = set(ARCHITECTURES) - set(found)
|
||||
if missing:
|
||||
raise RuntimeError(f"missing Sisyphus pkglist for: {', '.join(sorted(missing))}")
|
||||
return found
|
||||
|
||||
|
||||
def query_header_count(path: Path) -> int:
|
||||
process = subprocess.Popen(
|
||||
["pkglist-query", "%{NAME}\\n", str(path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
assert process.stdout is not None
|
||||
count = sum(1 for _ in process.stdout)
|
||||
stderr = process.stderr.read() if process.stderr else b""
|
||||
status = process.wait()
|
||||
if status != 0:
|
||||
raise RuntimeError(f"pkglist-query failed for {path}: {stderr.decode(errors='replace')}")
|
||||
return count
|
||||
|
||||
|
||||
def query_set_counts(path: Path) -> dict[str, int]:
|
||||
process = subprocess.Popen(
|
||||
["pkglist-query", VERSION_FORMAT, str(path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
assert process.stdout is not None
|
||||
old = direct = 0
|
||||
for raw_line in process.stdout:
|
||||
value = raw_line.strip()
|
||||
if value.startswith(b"set:D1"):
|
||||
direct += 1
|
||||
elif value.startswith(b"set:"):
|
||||
old += 1
|
||||
stderr = process.stderr.read() if process.stderr else b""
|
||||
status = process.wait()
|
||||
if status != 0:
|
||||
raise RuntimeError(f"pkglist-query failed for {path}: {stderr.decode(errors='replace')}")
|
||||
return {"set9": old, "d1": direct}
|
||||
|
||||
|
||||
def conversion_summary(stderr: bytes) -> dict[str, int]:
|
||||
text = stderr.decode(errors="replace")
|
||||
match = SUMMARY_RE.search(text)
|
||||
if not match:
|
||||
raise RuntimeError(f"converter did not report summary:\n{text}")
|
||||
return {name: int(value) for name, value in match.groupdict().items()}
|
||||
|
||||
|
||||
def convert_one(converter: Path, source: Path, destination: Path) -> dict[str, object]:
|
||||
result = run([str(converter), "--rewrite", str(source), str(destination)])
|
||||
summary = conversion_summary(result.stderr)
|
||||
source_headers = query_header_count(source)
|
||||
output_headers = query_header_count(destination)
|
||||
source_sets = query_set_counts(source)
|
||||
output_sets = query_set_counts(destination)
|
||||
if source_headers != output_headers or output_headers != summary["headers"]:
|
||||
raise RuntimeError(
|
||||
f"header count mismatch: source={source_headers} output={output_headers} "
|
||||
f"converter={summary['headers']}"
|
||||
)
|
||||
if source_sets["d1"] != 0:
|
||||
raise RuntimeError(f"source already contains {source_sets['d1']} D1 values")
|
||||
if output_sets["set9"] != 0:
|
||||
raise RuntimeError(f"output still contains {output_sets['set9']} set9 values")
|
||||
if source_sets["set9"] != output_sets["d1"] or output_sets["d1"] != summary["set_occurrences"]:
|
||||
raise RuntimeError(
|
||||
"set occurrence mismatch: "
|
||||
f"source={source_sets['set9']} output={output_sets['d1']} "
|
||||
f"converter={summary['set_occurrences']}"
|
||||
)
|
||||
if summary["set_occurrences"] == 0:
|
||||
raise RuntimeError("source pkglist contains no set versions")
|
||||
return {
|
||||
**summary,
|
||||
"source": {
|
||||
"path": source.name,
|
||||
"size": source.stat().st_size,
|
||||
"sha256": sha256_file(source),
|
||||
},
|
||||
"output": {
|
||||
"path": destination.name,
|
||||
"size": destination.stat().st_size,
|
||||
"sha256": sha256_file(destination),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def image_metadata(image: str) -> dict[str, str]:
|
||||
result = run(
|
||||
[
|
||||
"podman",
|
||||
"image",
|
||||
"inspect",
|
||||
image,
|
||||
"--format",
|
||||
"{{.Digest}}|{{.Id}}",
|
||||
]
|
||||
)
|
||||
digest, image_id = result.stdout.decode().strip().split("|", 1)
|
||||
return {"name": image, "digest": digest, "id": image_id}
|
||||
|
||||
|
||||
def inner(output: Path, image: str, image_digest: str, image_id: str) -> None:
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
if any(output.iterdir()):
|
||||
raise RuntimeError(f"output directory is not empty: {output}")
|
||||
|
||||
pkglists = find_pkglists(Path("/var/lib/apt/lists"))
|
||||
staging = Path(tempfile.mkdtemp(prefix=".d1-staging-", dir=output))
|
||||
try:
|
||||
converter = staging / "rewrite-sisyphus-pkglist"
|
||||
compile_converter(converter)
|
||||
architectures: dict[str, object] = {}
|
||||
for architecture in ARCHITECTURES:
|
||||
destination = staging / f"Sisyphus.{architecture}.pkglist.classic"
|
||||
architectures[architecture] = convert_one(
|
||||
converter, pkglists[architecture], destination
|
||||
)
|
||||
converter.unlink()
|
||||
shutil.rmtree(staging / "compat-include")
|
||||
manifest = {
|
||||
"format": 1,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"image": {"name": image, "digest": image_digest, "id": image_id},
|
||||
"rpm": run(["rpm", "--version"]).stdout.decode().strip(),
|
||||
"apt": run(["rpmquery", "--qf", "%{VERSION}-%{RELEASE}", "apt"])
|
||||
.stdout.decode()
|
||||
.strip(),
|
||||
"converter_source": {
|
||||
"set9_sha256": sha256_file(ROOT / "reimplement/set9.c"),
|
||||
"rewrite_sha256": sha256_file(HERE / "rewrite_sisyphus_pkglist.c"),
|
||||
},
|
||||
"architectures": architectures,
|
||||
}
|
||||
(staging / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
final = output / "d1-pkglists"
|
||||
staging.replace(final)
|
||||
print(final)
|
||||
for architecture in ARCHITECTURES:
|
||||
data = architectures[architecture]
|
||||
assert isinstance(data, dict)
|
||||
print(
|
||||
f"{architecture}: headers={data['headers']} "
|
||||
f"set_occurrences={data['set_occurrences']} "
|
||||
f"output={final / f'Sisyphus.{architecture}.pkglist.classic'}"
|
||||
)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
|
||||
|
||||
def outer(output: Path, image: str) -> None:
|
||||
if output.exists() and any(output.iterdir()):
|
||||
raise RuntimeError(f"output directory is not empty: {output}")
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
run(["podman", "pull", image], stdout=None)
|
||||
metadata = image_metadata(image)
|
||||
inner_command = [
|
||||
"python3",
|
||||
"/src/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py",
|
||||
"--inner",
|
||||
"/out",
|
||||
"--image",
|
||||
image,
|
||||
"--image-digest",
|
||||
metadata["digest"],
|
||||
"--image-id",
|
||||
metadata["id"],
|
||||
]
|
||||
command = [
|
||||
"podman",
|
||||
"run",
|
||||
"--rm",
|
||||
"-v",
|
||||
f"{ROOT}:/src:ro",
|
||||
"-v",
|
||||
f"{output}:/out:rw",
|
||||
image,
|
||||
"sh",
|
||||
"-lc",
|
||||
(
|
||||
"apt-get update >/dev/null && "
|
||||
"apt-get install -y gcc librpm-devel apt-repo-tools python3 >/dev/null && "
|
||||
+ " ".join(shlex.quote(argument) for argument in inner_command)
|
||||
),
|
||||
]
|
||||
result = subprocess.run(command, check=False)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"container conversion failed with status {result.returncode}")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Create D1 copies of current Sisyphus x86_64/noarch pkglist files"
|
||||
)
|
||||
parser.add_argument("output", type=Path, help="new or empty output directory")
|
||||
parser.add_argument("--image", default=DEFAULT_IMAGE)
|
||||
parser.add_argument("--image-digest", default="", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--image-id", default="", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--inner", action="store_true", help=argparse.SUPPRESS)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
output = validate_output(args.output)
|
||||
if args.inner:
|
||||
inner(output, args.image, args.image_digest, args.image_id)
|
||||
else:
|
||||
outer(output, args.image)
|
||||
except (OSError, RuntimeError, ValueError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user