diff --git a/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py b/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py index 1b048ba..d05141d 100644 --- a/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py +++ b/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py @@ -84,13 +84,16 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() -def compile_converter(destination: Path) -> None: +def compile_converter( + destination: Path, + rpm_include_dir: Path | None = None, + rpm_libraries: tuple[Path, ...] = (), +) -> None: include = destination.parent / "compat-include" include.mkdir() for name in ("rpmlib.h", "system.h", "set.h"): (include / name).touch() - run( - [ + command = [ "cc", "-O2", "-std=gnu11", @@ -102,16 +105,23 @@ def compile_converter(destination: Path) -> None: "-DARSV_WITH_RPM", "-I", str(include), + ] + if rpm_include_dir is not None: + command.extend(["-I", str(rpm_include_dir)]) + command.extend( + [ "-include", str(ROOT / "scripts/rpmsetcmp/newset_compat.h"), str(ROOT / "reimplement/set9.c"), str(HERE / "rewrite_sisyphus_pkglist.c"), - "-lrpm", - "-lrpmio", - "-o", - str(destination), ] ) + if rpm_libraries: + command.extend(str(path) for path in rpm_libraries) + else: + command.extend(["-lrpm", "-lrpmio"]) + command.extend(["-o", str(destination)]) + run(command) def find_pkglists(lists_dir: Path) -> dict[str, Path]: @@ -212,7 +222,12 @@ def convert_one(converter: Path, source: Path, destination: Path) -> dict[str, o } -def convert_local(output: Path, lists_dir: Path) -> None: +def convert_local( + output: Path, + lists_dir: Path, + rpm_include_dir: Path | None = None, + rpm_libraries: tuple[Path, ...] = (), +) -> None: output.mkdir(parents=True, exist_ok=True) if any(output.iterdir()): raise RuntimeError(f"output directory is not empty: {output}") @@ -224,7 +239,7 @@ def convert_local(output: Path, lists_dir: Path) -> None: staging = Path(tempfile.mkdtemp(prefix=".d1-staging-", dir=output)) try: converter = staging / "rewrite-sisyphus-pkglist" - compile_converter(converter) + compile_converter(converter, rpm_include_dir, rpm_libraries) architectures: dict[str, object] = {} for architecture in ARCHITECTURES: destination = staging / f"Sisyphus.{architecture}.pkglist.classic" @@ -277,6 +292,18 @@ def parse_args() -> argparse.Namespace: default=Path("/var/lib/apt/lists"), help="APT lists snapshot to convert (default: /var/lib/apt/lists)", ) + parser.add_argument( + "--rpm-include-dir", + type=Path, + help="directory containing rpm/header.h (for an unpacked librpm-devel)", + ) + parser.add_argument( + "--rpm-library", + type=Path, + action="append", + default=[], + help="versioned RPM library to link; repeat for librpm and librpmio", + ) return parser.parse_args() @@ -284,7 +311,20 @@ def main() -> int: args = parse_args() try: output = validate_output(args.output) - convert_local(output, args.lists_dir) + rpm_include_dir = ( + args.rpm_include_dir.expanduser().resolve() + if args.rpm_include_dir is not None + else None + ) + rpm_libraries = tuple(path.expanduser().resolve() for path in args.rpm_library) + if rpm_include_dir is not None and not (rpm_include_dir / "rpm/header.h").is_file(): + raise ValueError(f"rpm/header.h not found below: {rpm_include_dir}") + if rpm_libraries and len(rpm_libraries) != 2: + raise ValueError("pass exactly two --rpm-library values: librpm and librpmio") + for library in rpm_libraries: + if not library.is_file(): + raise ValueError(f"RPM library not found: {library}") + convert_local(output, args.lists_dir, rpm_include_dir, rpm_libraries) except (OSError, RuntimeError, ValueError) as error: print(f"error: {error}", file=sys.stderr) return 1 diff --git a/scripts/run-sisyphus-set9-d1-bench.sh b/scripts/run-sisyphus-set9-d1-bench.sh index 181a100..c1ccc0c 100755 --- a/scripts/run-sisyphus-set9-d1-bench.sh +++ b/scripts/run-sisyphus-set9-d1-bench.sh @@ -15,6 +15,9 @@ REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd) SET9_C=${SET9_C:-$REPO_ROOT/reimplement/set9.c} D1_C=${D1_C:-$REPO_ROOT/new_version/direct_hash/hash_set.c} PKGLIST_CONVERTER=${PKGLIST_CONVERTER:-$REPO_ROOT/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py} +SET_REWRITER_C=$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c +SET_COMPAT_H=$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h +ORCHESTRATOR=$(realpath -e -- "${BASH_SOURCE[0]}") WORK_ROOT=${WORK_ROOT:-$HOME/sisyphus-set9-d1-bench} RESULT_DIR=${RESULT_DIR:-$WORK_ROOT/results} @@ -99,17 +102,38 @@ write_snapshot_fingerprint() { { printf 'sisyphus_mirror=%s\n' "$SISYPHUS_MIRROR" + printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')" printf 'converter=%s\n' "$(sha256sum "$PKGLIST_CONVERTER" | awk '{print $1}')" printf 'set9=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')" + printf 'd1=%s\n' "$(sha256sum "$D1_C" | awk '{print $1}')" printf 'rewrite=%s\n' "$(sha256sum "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" | awk '{print $1}')" printf 'compat=%s\n' "$(sha256sum "$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h" | awk '{print $1}')" + [[ ! -f $SNAPSHOT/devel-input-fingerprint.txt ]] || + cat "$SNAPSHOT/devel-input-fingerprint.txt" + write_runtime_fingerprint } } +write_runtime_fingerprint() +{ + local rpm_libdir library resolved owner + rpm_libdir=$(rpm --eval '%{_libdir}') + for library in librpm.so.7 librpmio.so.7; do + resolved=$(realpath -e "$rpm_libdir/$library") || + fail "installed $library not found" + owner=$(rpm -qf --qf '%{NAME}|%{SOURCERPM}|%{DISTTAG}' "$resolved") || + fail "cannot identify package owning $resolved" + printf 'runtime_%s_owner=%s\n' "$library" "$owner" + printf 'runtime_%s_sha256=%s\n' "$library" \ + "$(sha256sum "$resolved" | awk '{print $1}')" + done +} + validate_snapshot_reuse() { local current - [[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt ]] || return 1 + [[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt && + -f $SNAPSHOT/devel-input-fingerprint.txt ]] || return 1 current=$(mktemp) write_snapshot_fingerprint >"$current" if ! cmp -s "$current" "$SNAPSHOT/input-fingerprint.txt"; then @@ -163,10 +187,52 @@ prepare_spec() " "$spec" } +prepare_d1_build_corpus() +{ + local source=$1 spec helper_dir + spec=$source/alt/rpm.spec + helper_dir=$source/alt/arsv-d1-build + mkdir -p "$helper_dir" + cp "$SET9_C" "$helper_dir/set9.c" + cp "$SET_REWRITER_C" "$helper_dir/rewrite_sisyphus_pkglist.c" + cp "$SET_COMPAT_H" "$helper_dir/newset_compat.h" + + python3 - "$spec" <<'PY' +import sys +from pathlib import Path + +spec = Path(sys.argv[1]) +text = spec.read_text() +needle = "join -o 1.3,2.3 P R |shuf >setcmp-data\n" +replacement = needle + r'''# The buildroot RPM database contains legacy set9 values. This D1 build +# preserves the same relation corpus, but converts both operands before the +# format-specific setcmp/profile checks instead of feeding incompatible input. +mkdir -p arsv-d1-compat +touch arsv-d1-compat/rpmlib.h arsv-d1-compat/system.h arsv-d1-compat/set.h +%__cc %optflags -std=gnu11 -Wall -Wextra -Werror -D_GNU_SOURCE -DARSV_SET9_EXPORT \ + -I arsv-d1-compat -include alt/arsv-d1-build/newset_compat.h \ + alt/arsv-d1-build/set9.c alt/arsv-d1-build/rewrite_sisyphus_pkglist.c \ + -o arsv-convert-set +while read -r set1 set2; do + d1_set1=$(./arsv-convert-set --convert-set "$set1") + d1_set2=$(./arsv-convert-set --convert-set "$set2") + printf '%%s %%s\n' "$d1_set1" "$d1_set2" +done setcmp-data.d1 +test "$(wc -l "$SNAPSHOT/devel-input-fingerprint.txt" + converter_output="$SNAPSHOT/conversion" python3 "$PKGLIST_CONVERTER" "$converter_output" \ - --lists-dir "$SNAPSHOT/lists" + --lists-dir "$SNAPSHOT/lists" \ + --rpm-include-dir "$devel_root/usr/include" \ + --rpm-library "$rpm_library" \ + --rpm-library "$rpmio_library" mv "$converter_output/d1-pkglists/manifest.json" "$SNAPSHOT/manifest.json" mkdir -p "$SNAPSHOT/d1-pkglists" mv "$converter_output/d1-pkglists/"*.classic "$SNAPSHOT/d1-pkglists/" rmdir "$converter_output/d1-pkglists" "$converter_output" rm -rf "$SNAPSHOT/download-cache" + rm -rf "$devel_cache" "$devel_root" rm -f "$SNAPSHOT/lists/lock" rm -rf "$SNAPSHOT/lists/partial" mkdir -p "$SNAPSHOT/lists/partial" @@ -387,9 +510,15 @@ build_variant() expected_fingerprint=$(mktemp) { printf 'base_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)" + printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')" printf 'set_source=%s\n' "$(sha256sum "$source_c" | awk '{print $1}')" printf 'suffix=%s\n' "$suffix" printf 'packager=%s\n' "$PACKAGER" + if [[ $name == d1 ]]; then + printf 'set9_decoder=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')" + printf 'set_rewriter=%s\n' "$(sha256sum "$SET_REWRITER_C" | awk '{print $1}')" + printf 'set_compat=%s\n' "$(sha256sum "$SET_COMPAT_H" | awk '{print $1}')" + fi } >"$expected_fingerprint" if [[ -f $variant/input-fingerprint.txt ]]; then if ! cmp -s "$expected_fingerprint" "$variant/input-fingerprint.txt"; then @@ -407,10 +536,24 @@ build_variant() git clone --local "$SOURCE_BASE" "$source" cp "$source_c" "$source/lib/set.c" prepare_spec "$spec" "$suffix" + if [[ $name == d1 ]]; then + prepare_d1_build_corpus "$source" + fi + sha256sum "$spec" >"$variant/prepared-spec.sha256" cp "$expected_fingerprint" "$variant/input-fingerprint.txt" else + [[ -f $variant/prepared-spec.sha256 ]] && + sha256sum -c "$variant/prepared-spec.sha256" >/dev/null || + fail "$name prepared RPM spec changed; use RESET_WORK=1" cmp -s "$source_c" "$source/lib/set.c" || fail "$source_c changed; use RESET_WORK=1" + if [[ $name == d1 ]]; then + cmp -s "$SET9_C" "$source/alt/arsv-d1-build/set9.c" && + cmp -s "$SET_REWRITER_C" \ + "$source/alt/arsv-d1-build/rewrite_sisyphus_pkglist.c" && + cmp -s "$SET_COMPAT_H" "$source/alt/arsv-d1-build/newset_compat.h" || + fail 'D1 build corpus converter changed; use RESET_WORK=1' + fi fi rm -f "$expected_fingerprint" @@ -529,9 +672,11 @@ write_provenance() printf 'cpu=%s\n' "$CPU" printf 'rounds=%s\n' "$ROUNDS" printf 'operations=%s\n' "$OPERATIONS" - sha256sum "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \ + sha256sum "$ORCHESTRATOR" "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \ "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" \ "$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h" + cat "$SNAPSHOT/devel-input-fingerprint.txt" + write_runtime_fingerprint printf 'set9_librpm='; cat "$SET9_VARIANT/librpm-artifact.sha256" printf 'd1_librpm='; cat "$D1_VARIANT/librpm-artifact.sha256" } >"$RESULT_DIR/provenance.txt"