latest version

new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py

@@ -84,13 +84,16 @@ def sha256_file(path: Path) -> str:
     return digest.hexdigest()
 
 
-def compile_converter(destination: Path) -> None:
+def compile_converter(
+    destination: Path,
+    rpm_include_dir: Path | None = None,
+    rpm_libraries: tuple[Path, ...] = (),
+) -> None:
     include = destination.parent / "compat-include"
     include.mkdir()
     for name in ("rpmlib.h", "system.h", "set.h"):
         (include / name).touch()
-    run(
+    command = [
-        [
             "cc",
             "-O2",
             "-std=gnu11",
@@ -102,16 +105,23 @@ def compile_converter(destination: Path) -> None:
             "-DARSV_WITH_RPM",
             "-I",
             str(include),
+    ]
+    if rpm_include_dir is not None:
+        command.extend(["-I", str(rpm_include_dir)])
+    command.extend(
+        [
             "-include",
             str(ROOT / "scripts/rpmsetcmp/newset_compat.h"),
             str(ROOT / "reimplement/set9.c"),
             str(HERE / "rewrite_sisyphus_pkglist.c"),
-            "-lrpm",
-            "-lrpmio",
-            "-o",
-            str(destination),
         ]
     )
+    if rpm_libraries:
+        command.extend(str(path) for path in rpm_libraries)
+    else:
+        command.extend(["-lrpm", "-lrpmio"])
+    command.extend(["-o", str(destination)])
+    run(command)
 
 
 def find_pkglists(lists_dir: Path) -> dict[str, Path]:
@@ -212,7 +222,12 @@ def convert_one(converter: Path, source: Path, destination: Path) -> dict[str, o
     }
 
 
-def convert_local(output: Path, lists_dir: Path) -> None:
+def convert_local(
+    output: Path,
+    lists_dir: Path,
+    rpm_include_dir: Path | None = None,
+    rpm_libraries: tuple[Path, ...] = (),
+) -> None:
     output.mkdir(parents=True, exist_ok=True)
     if any(output.iterdir()):
         raise RuntimeError(f"output directory is not empty: {output}")
@@ -224,7 +239,7 @@ def convert_local(output: Path, lists_dir: Path) -> None:
     staging = Path(tempfile.mkdtemp(prefix=".d1-staging-", dir=output))
     try:
         converter = staging / "rewrite-sisyphus-pkglist"
-        compile_converter(converter)
+        compile_converter(converter, rpm_include_dir, rpm_libraries)
         architectures: dict[str, object] = {}
         for architecture in ARCHITECTURES:
             destination = staging / f"Sisyphus.{architecture}.pkglist.classic"
@@ -277,6 +292,18 @@ def parse_args() -> argparse.Namespace:
         default=Path("/var/lib/apt/lists"),
         help="APT lists snapshot to convert (default: /var/lib/apt/lists)",
     )
+    parser.add_argument(
+        "--rpm-include-dir",
+        type=Path,
+        help="directory containing rpm/header.h (for an unpacked librpm-devel)",
+    )
+    parser.add_argument(
+        "--rpm-library",
+        type=Path,
+        action="append",
+        default=[],
+        help="versioned RPM library to link; repeat for librpm and librpmio",
+    )
     return parser.parse_args()
 
 
@@ -284,7 +311,20 @@ def main() -> int:
     args = parse_args()
     try:
         output = validate_output(args.output)
-        convert_local(output, args.lists_dir)
+        rpm_include_dir = (
+            args.rpm_include_dir.expanduser().resolve()
+            if args.rpm_include_dir is not None
+            else None
+        )
+        rpm_libraries = tuple(path.expanduser().resolve() for path in args.rpm_library)
+        if rpm_include_dir is not None and not (rpm_include_dir / "rpm/header.h").is_file():
+            raise ValueError(f"rpm/header.h not found below: {rpm_include_dir}")
+        if rpm_libraries and len(rpm_libraries) != 2:
+            raise ValueError("pass exactly two --rpm-library values: librpm and librpmio")
+        for library in rpm_libraries:
+            if not library.is_file():
+                raise ValueError(f"RPM library not found: {library}")
+        convert_local(output, args.lists_dir, rpm_include_dir, rpm_libraries)
     except (OSError, RuntimeError, ValueError) as error:
         print(f"error: {error}", file=sys.stderr)
         return 1

scripts/run-sisyphus-set9-d1-bench.sh

@@ -15,6 +15,9 @@ REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd)
 SET9_C=${SET9_C:-$REPO_ROOT/reimplement/set9.c}
 D1_C=${D1_C:-$REPO_ROOT/new_version/direct_hash/hash_set.c}
 PKGLIST_CONVERTER=${PKGLIST_CONVERTER:-$REPO_ROOT/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py}
+SET_REWRITER_C=$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c
+SET_COMPAT_H=$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h
+ORCHESTRATOR=$(realpath -e -- "${BASH_SOURCE[0]}")
 
 WORK_ROOT=${WORK_ROOT:-$HOME/sisyphus-set9-d1-bench}
 RESULT_DIR=${RESULT_DIR:-$WORK_ROOT/results}
@@ -99,17 +102,38 @@ write_snapshot_fingerprint()
 {
     {
         printf 'sisyphus_mirror=%s\n' "$SISYPHUS_MIRROR"
+        printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')"
         printf 'converter=%s\n' "$(sha256sum "$PKGLIST_CONVERTER" | awk '{print $1}')"
         printf 'set9=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
+        printf 'd1=%s\n' "$(sha256sum "$D1_C" | awk '{print $1}')"
         printf 'rewrite=%s\n' "$(sha256sum "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" | awk '{print $1}')"
         printf 'compat=%s\n' "$(sha256sum "$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h" | awk '{print $1}')"
+        [[ ! -f $SNAPSHOT/devel-input-fingerprint.txt ]] ||
+            cat "$SNAPSHOT/devel-input-fingerprint.txt"
+        write_runtime_fingerprint
     }
 }
 
+write_runtime_fingerprint()
+{
+    local rpm_libdir library resolved owner
+    rpm_libdir=$(rpm --eval '%{_libdir}')
+    for library in librpm.so.7 librpmio.so.7; do
+        resolved=$(realpath -e "$rpm_libdir/$library") ||
+            fail "installed $library not found"
+        owner=$(rpm -qf --qf '%{NAME}|%{SOURCERPM}|%{DISTTAG}' "$resolved") ||
+            fail "cannot identify package owning $resolved"
+        printf 'runtime_%s_owner=%s\n' "$library" "$owner"
+        printf 'runtime_%s_sha256=%s\n' "$library" \
+            "$(sha256sum "$resolved" | awk '{print $1}')"
+    done
+}
+
 validate_snapshot_reuse()
 {
     local current
-    [[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt ]] || return 1
+    [[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt &&
+       -f $SNAPSHOT/devel-input-fingerprint.txt ]] || return 1
     current=$(mktemp)
     write_snapshot_fingerprint >"$current"
     if ! cmp -s "$current" "$SNAPSHOT/input-fingerprint.txt"; then
@@ -163,10 +187,52 @@ prepare_spec()
 " "$spec"
 }
 
+prepare_d1_build_corpus()
+{
+    local source=$1 spec helper_dir
+    spec=$source/alt/rpm.spec
+    helper_dir=$source/alt/arsv-d1-build
+    mkdir -p "$helper_dir"
+    cp "$SET9_C" "$helper_dir/set9.c"
+    cp "$SET_REWRITER_C" "$helper_dir/rewrite_sisyphus_pkglist.c"
+    cp "$SET_COMPAT_H" "$helper_dir/newset_compat.h"
+
+    python3 - "$spec" <<'PY'
+import sys
+from pathlib import Path
+
+spec = Path(sys.argv[1])
+text = spec.read_text()
+needle = "join -o 1.3,2.3 P R |shuf >setcmp-data\n"
+replacement = needle + r'''# The buildroot RPM database contains legacy set9 values.  This D1 build
+# preserves the same relation corpus, but converts both operands before the
+# format-specific setcmp/profile checks instead of feeding incompatible input.
+mkdir -p arsv-d1-compat
+touch arsv-d1-compat/rpmlib.h arsv-d1-compat/system.h arsv-d1-compat/set.h
+%__cc %optflags -std=gnu11 -Wall -Wextra -Werror -D_GNU_SOURCE -DARSV_SET9_EXPORT \
+    -I arsv-d1-compat -include alt/arsv-d1-build/newset_compat.h \
+    alt/arsv-d1-build/set9.c alt/arsv-d1-build/rewrite_sisyphus_pkglist.c \
+    -o arsv-convert-set
+while read -r set1 set2; do
+    d1_set1=$(./arsv-convert-set --convert-set "$set1")
+    d1_set2=$(./arsv-convert-set --convert-set "$set2")
+    printf '%%s %%s\n' "$d1_set1" "$d1_set2"
+done <setcmp-data >setcmp-data.d1
+test "$(wc -l <setcmp-data.d1)" -eq "$(wc -l <setcmp-data)"
+mv setcmp-data.d1 setcmp-data
+'''
+if text.count(needle) != 1:
+    raise SystemExit(f"expected exactly one setcmp corpus creation in {spec}")
+spec.write_text(text.replace(needle, replacement))
+PY
+    git -C "$source" add alt/arsv-d1-build
+}
+
 apt_options()
 {
     local variant=$1
     APT_OPTIONS=(
+        -o 'Debug::NoLocking=true'
         -o 'Dir::Etc::main=-'
         -o 'Dir::Etc::parts=-'
         -o "Dir::Etc::sourcelist=$SNAPSHOT/etc-apt/sources.list"
@@ -287,8 +353,10 @@ run_once()
 
 prepare_snapshot()
 {
-    local converter_output
+    local converter_output devel_cache devel_root rpm_libdir rpm_library rpmio_library
+    local devel_source runtime_source rpmfile
     local -a update_options
+    local -a rpm_devel_rpms popt_devel_rpms
     validate_snapshot_reuse && return
 
     printf '\n===== Preparing one immutable Sisyphus metadata snapshot =====\n'
@@ -319,14 +387,69 @@ prepare_snapshot()
     APT_CONFIG="$COMMON/apt.conf" "$APT_GET" -qq \
         "${update_options[@]}" update
 
+    # The converter needs C headers, but the benchmark host intentionally need
+    # not have development packages installed. Download the p11 packages that
+    # match the host's installed librpm into a private cache and extract only
+    # their headers; no system package is installed or changed.
+    devel_cache="$SNAPSHOT/devel-cache"
+    devel_root="$SNAPSHOT/devel-root"
+    mkdir -p "$devel_cache/archives/partial" "$devel_root"
+    env -u APT_CONFIG "$APT_GET" -qq -y -d \
+        -o "Dir::Cache=$devel_cache" \
+        -o "Dir::Cache::archives=$devel_cache/archives" \
+        -o "Dir::Cache::pkgcache=$devel_cache/pkgcache.bin" \
+        -o "Dir::Cache::srcpkgcache=$devel_cache/srcpkgcache.bin" \
+        install librpm-devel
+    shopt -s nullglob
+    rpm_devel_rpms=("$devel_cache"/archives/librpm-devel_*.rpm)
+    popt_devel_rpms=("$devel_cache"/archives/libpopt-devel_*.rpm)
+    shopt -u nullglob
+    ((${#rpm_devel_rpms[@]} == 1)) ||
+        fail "expected one downloaded librpm-devel RPM, got ${#rpm_devel_rpms[@]}"
+    ((${#popt_devel_rpms[@]} == 1)) ||
+        fail "expected one downloaded libpopt-devel RPM, got ${#popt_devel_rpms[@]}"
+    devel_source=$(rpm -qp --qf '%{SOURCERPM}|%{DISTTAG}' "${rpm_devel_rpms[0]}") ||
+        fail 'cannot identify downloaded librpm-devel source package'
+    for rpmfile in "${rpm_devel_rpms[@]}" "${popt_devel_rpms[@]}"; do
+        (cd "$devel_root" && rpm2cpio "$rpmfile" | cpio -idm --quiet './usr/include/*')
+    done
+    [[ -f $devel_root/usr/include/rpm/header.h && -f $devel_root/usr/include/popt.h ]] ||
+        fail 'failed to extract RPM development headers'
+    rpm_libdir=$(rpm --eval '%{_libdir}')
+    rpm_library=$(realpath -e "$rpm_libdir/librpm.so.7") ||
+        fail 'installed librpm.so.7 not found'
+    rpmio_library=$(realpath -e "$rpm_libdir/librpmio.so.7") ||
+        fail 'installed librpmio.so.7 not found'
+    for rpmfile in "$rpm_library" "$rpmio_library"; do
+        runtime_source=$(rpm -qf --qf '%{SOURCERPM}|%{DISTTAG}' "$rpmfile") ||
+            fail "cannot identify runtime package owning $rpmfile"
+        [[ $runtime_source == "$devel_source" ]] ||
+            fail "downloaded librpm-devel ($devel_source) does not match $rpmfile ($runtime_source)"
+    done
+    {
+        printf 'devel_librpm_identity=%s|%s\n' \
+            "$(rpm -qp --qf '%{NAME}|%{EVR}|%{DISTTAG}|%{ARCH}' "${rpm_devel_rpms[0]}")" \
+            "$devel_source"
+        printf 'devel_librpm_sha256=%s\n' \
+            "$(sha256sum "${rpm_devel_rpms[0]}" | awk '{print $1}')"
+        printf 'devel_popt_identity=%s\n' \
+            "$(rpm -qp --qf '%{NAME}|%{EVR}|%{DISTTAG}|%{ARCH}' "${popt_devel_rpms[0]}")"
+        printf 'devel_popt_sha256=%s\n' \
+            "$(sha256sum "${popt_devel_rpms[0]}" | awk '{print $1}')"
+    } >"$SNAPSHOT/devel-input-fingerprint.txt"
+
     converter_output="$SNAPSHOT/conversion"
     python3 "$PKGLIST_CONVERTER" "$converter_output" \
-        --lists-dir "$SNAPSHOT/lists"
+        --lists-dir "$SNAPSHOT/lists" \
+        --rpm-include-dir "$devel_root/usr/include" \
+        --rpm-library "$rpm_library" \
+        --rpm-library "$rpmio_library"
     mv "$converter_output/d1-pkglists/manifest.json" "$SNAPSHOT/manifest.json"
     mkdir -p "$SNAPSHOT/d1-pkglists"
     mv "$converter_output/d1-pkglists/"*.classic "$SNAPSHOT/d1-pkglists/"
     rmdir "$converter_output/d1-pkglists" "$converter_output"
     rm -rf "$SNAPSHOT/download-cache"
+    rm -rf "$devel_cache" "$devel_root"
     rm -f "$SNAPSHOT/lists/lock"
     rm -rf "$SNAPSHOT/lists/partial"
     mkdir -p "$SNAPSHOT/lists/partial"
@@ -387,9 +510,15 @@ build_variant()
     expected_fingerprint=$(mktemp)
     {
         printf 'base_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
+        printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')"
         printf 'set_source=%s\n' "$(sha256sum "$source_c" | awk '{print $1}')"
         printf 'suffix=%s\n' "$suffix"
         printf 'packager=%s\n' "$PACKAGER"
+        if [[ $name == d1 ]]; then
+            printf 'set9_decoder=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
+            printf 'set_rewriter=%s\n' "$(sha256sum "$SET_REWRITER_C" | awk '{print $1}')"
+            printf 'set_compat=%s\n' "$(sha256sum "$SET_COMPAT_H" | awk '{print $1}')"
+        fi
     } >"$expected_fingerprint"
     if [[ -f $variant/input-fingerprint.txt ]]; then
         if ! cmp -s "$expected_fingerprint" "$variant/input-fingerprint.txt"; then
@@ -407,10 +536,24 @@ build_variant()
         git clone --local "$SOURCE_BASE" "$source"
         cp "$source_c" "$source/lib/set.c"
         prepare_spec "$spec" "$suffix"
+        if [[ $name == d1 ]]; then
+            prepare_d1_build_corpus "$source"
+        fi
+        sha256sum "$spec" >"$variant/prepared-spec.sha256"
         cp "$expected_fingerprint" "$variant/input-fingerprint.txt"
     else
+        [[ -f $variant/prepared-spec.sha256 ]] &&
+            sha256sum -c "$variant/prepared-spec.sha256" >/dev/null ||
+            fail "$name prepared RPM spec changed; use RESET_WORK=1"
         cmp -s "$source_c" "$source/lib/set.c" ||
             fail "$source_c changed; use RESET_WORK=1"
+        if [[ $name == d1 ]]; then
+            cmp -s "$SET9_C" "$source/alt/arsv-d1-build/set9.c" &&
+                cmp -s "$SET_REWRITER_C" \
+                    "$source/alt/arsv-d1-build/rewrite_sisyphus_pkglist.c" &&
+                cmp -s "$SET_COMPAT_H" "$source/alt/arsv-d1-build/newset_compat.h" ||
+                fail 'D1 build corpus converter changed; use RESET_WORK=1'
+        fi
     fi
     rm -f "$expected_fingerprint"
 
@@ -529,9 +672,11 @@ write_provenance()
         printf 'cpu=%s\n' "$CPU"
         printf 'rounds=%s\n' "$ROUNDS"
         printf 'operations=%s\n' "$OPERATIONS"
-        sha256sum "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \
+        sha256sum "$ORCHESTRATOR" "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \
             "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" \
             "$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h"
+        cat "$SNAPSHOT/devel-input-fingerprint.txt"
+        write_runtime_fingerprint
         printf 'set9_librpm='; cat "$SET9_VARIANT/librpm-artifact.sha256"
         printf 'd1_librpm='; cat "$D1_VARIANT/librpm-artifact.sha256"
     } >"$RESULT_DIR/provenance.txt"