krosh/ARSV
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

latest version

Browse Source
This commit is contained in:
krosh
2026-08-14 01:52:11 +03:00
parent da79041e88
commit 2b30d26a56
2 changed files with 199 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py
+50 -10
View File
@@ -84,13 +84,16 @@ def sha256_file(path: Path) -> str:
return digest.hexdigest()
def compile_converter(destination: Path) -> None:
def compile_converter(
destination: Path,
rpm_include_dir: Path | None = None,
rpm_libraries: tuple[Path, ...] = (),
) -> None:
include = destination.parent / "compat-include"
include.mkdir()
for name in ("rpmlib.h", "system.h", "set.h"):
(include / name).touch()
run(
[
command = [
"cc",
"-O2",
"-std=gnu11",
@@ -102,16 +105,23 @@ def compile_converter(destination: Path) -> None:
"-DARSV_WITH_RPM",
"-I",
str(include),
]
if rpm_include_dir is not None:
command.extend(["-I", str(rpm_include_dir)])
command.extend(
[
"-include",
str(ROOT / "scripts/rpmsetcmp/newset_compat.h"),
str(ROOT / "reimplement/set9.c"),
str(HERE / "rewrite_sisyphus_pkglist.c"),
"-lrpm",
"-lrpmio",
"-o",
str(destination),
]
)
if rpm_libraries:
command.extend(str(path) for path in rpm_libraries)
else:
command.extend(["-lrpm", "-lrpmio"])
command.extend(["-o", str(destination)])
run(command)
def find_pkglists(lists_dir: Path) -> dict[str, Path]:
@@ -212,7 +222,12 @@ def convert_one(converter: Path, source: Path, destination: Path) -> dict[str, o
}
def convert_local(output: Path, lists_dir: Path) -> None:
def convert_local(
output: Path,
lists_dir: Path,
rpm_include_dir: Path | None = None,
rpm_libraries: tuple[Path, ...] = (),
) -> None:
output.mkdir(parents=True, exist_ok=True)
if any(output.iterdir()):
raise RuntimeError(f"output directory is not empty: {output}")
@@ -224,7 +239,7 @@ def convert_local(output: Path, lists_dir: Path) -> None:
staging = Path(tempfile.mkdtemp(prefix=".d1-staging-", dir=output))
try:
converter = staging / "rewrite-sisyphus-pkglist"
compile_converter(converter)
compile_converter(converter, rpm_include_dir, rpm_libraries)
architectures: dict[str, object] = {}
for architecture in ARCHITECTURES:
destination = staging / f"Sisyphus.{architecture}.pkglist.classic"
@@ -277,6 +292,18 @@ def parse_args() -> argparse.Namespace:
default=Path("/var/lib/apt/lists"),
help="APT lists snapshot to convert (default: /var/lib/apt/lists)",
)
parser.add_argument(
"--rpm-include-dir",
type=Path,
help="directory containing rpm/header.h (for an unpacked librpm-devel)",
)
parser.add_argument(
"--rpm-library",
type=Path,
action="append",
default=[],
help="versioned RPM library to link; repeat for librpm and librpmio",
)
return parser.parse_args()
@@ -284,7 +311,20 @@ def main() -> int:
args = parse_args()
try:
output = validate_output(args.output)
convert_local(output, args.lists_dir)
rpm_include_dir = (
args.rpm_include_dir.expanduser().resolve()
if args.rpm_include_dir is not None
else None
)
rpm_libraries = tuple(path.expanduser().resolve() for path in args.rpm_library)
if rpm_include_dir is not None and not (rpm_include_dir / "rpm/header.h").is_file():
raise ValueError(f"rpm/header.h not found below: {rpm_include_dir}")
if rpm_libraries and len(rpm_libraries) != 2:
raise ValueError("pass exactly two --rpm-library values: librpm and librpmio")
for library in rpm_libraries:
if not library.is_file():
raise ValueError(f"RPM library not found: {library}")
convert_local(output, args.lists_dir, rpm_include_dir, rpm_libraries)
except (OSError, RuntimeError, ValueError) as error:
print(f"error: {error}", file=sys.stderr)
return 1
scripts/run-sisyphus-set9-d1-bench.sh
+149 -4
View File
@@ -15,6 +15,9 @@ REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd)
SET9_C=${SET9_C:-$REPO_ROOT/reimplement/set9.c}
D1_C=${D1_C:-$REPO_ROOT/new_version/direct_hash/hash_set.c}
PKGLIST_CONVERTER=${PKGLIST_CONVERTER:-$REPO_ROOT/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py}
SET_REWRITER_C=$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c
SET_COMPAT_H=$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h
ORCHESTRATOR=$(realpath -e -- "${BASH_SOURCE[0]}")
WORK_ROOT=${WORK_ROOT:-$HOME/sisyphus-set9-d1-bench}
RESULT_DIR=${RESULT_DIR:-$WORK_ROOT/results}
@@ -99,17 +102,38 @@ write_snapshot_fingerprint()
{
{
printf 'sisyphus_mirror=%s\n' "$SISYPHUS_MIRROR"
printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')"
printf 'converter=%s\n' "$(sha256sum "$PKGLIST_CONVERTER" | awk '{print $1}')"
printf 'set9=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
printf 'd1=%s\n' "$(sha256sum "$D1_C" | awk '{print $1}')"
printf 'rewrite=%s\n' "$(sha256sum "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" | awk '{print $1}')"
printf 'compat=%s\n' "$(sha256sum "$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h" | awk '{print $1}')"
[[ ! -f $SNAPSHOT/devel-input-fingerprint.txt ]] ||
cat "$SNAPSHOT/devel-input-fingerprint.txt"
write_runtime_fingerprint
}
}
write_runtime_fingerprint()
{
local rpm_libdir library resolved owner
rpm_libdir=$(rpm --eval '%{_libdir}')
for library in librpm.so.7 librpmio.so.7; do
resolved=$(realpath -e "$rpm_libdir/$library") ||
fail "installed $library not found"
owner=$(rpm -qf --qf '%{NAME}|%{SOURCERPM}|%{DISTTAG}' "$resolved") ||
fail "cannot identify package owning $resolved"
printf 'runtime_%s_owner=%s\n' "$library" "$owner"
printf 'runtime_%s_sha256=%s\n' "$library" \
"$(sha256sum "$resolved" | awk '{print $1}')"
done
}
validate_snapshot_reuse()
{
local current
[[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt ]] || return 1
[[ -f $SNAPSHOT/.complete && -f $SNAPSHOT/input-fingerprint.txt &&
-f $SNAPSHOT/devel-input-fingerprint.txt ]] || return 1
current=$(mktemp)
write_snapshot_fingerprint >"$current"
if ! cmp -s "$current" "$SNAPSHOT/input-fingerprint.txt"; then
@@ -163,10 +187,52 @@ prepare_spec()
" "$spec"
}
prepare_d1_build_corpus()
{
local source=$1 spec helper_dir
spec=$source/alt/rpm.spec
helper_dir=$source/alt/arsv-d1-build
mkdir -p "$helper_dir"
cp "$SET9_C" "$helper_dir/set9.c"
cp "$SET_REWRITER_C" "$helper_dir/rewrite_sisyphus_pkglist.c"
cp "$SET_COMPAT_H" "$helper_dir/newset_compat.h"
python3 - "$spec" <<'PY'
import sys
from pathlib import Path
spec = Path(sys.argv[1])
text = spec.read_text()
needle = "join -o 1.3,2.3 P R |shuf >setcmp-data\n"
replacement = needle + r'''# The buildroot RPM database contains legacy set9 values. This D1 build
# preserves the same relation corpus, but converts both operands before the
# format-specific setcmp/profile checks instead of feeding incompatible input.
mkdir -p arsv-d1-compat
touch arsv-d1-compat/rpmlib.h arsv-d1-compat/system.h arsv-d1-compat/set.h
%__cc %optflags -std=gnu11 -Wall -Wextra -Werror -D_GNU_SOURCE -DARSV_SET9_EXPORT \
-I arsv-d1-compat -include alt/arsv-d1-build/newset_compat.h \
alt/arsv-d1-build/set9.c alt/arsv-d1-build/rewrite_sisyphus_pkglist.c \
-o arsv-convert-set
while read -r set1 set2; do
d1_set1=$(./arsv-convert-set --convert-set "$set1")
d1_set2=$(./arsv-convert-set --convert-set "$set2")
printf '%%s %%s\n' "$d1_set1" "$d1_set2"
done <setcmp-data >setcmp-data.d1
test "$(wc -l <setcmp-data.d1)" -eq "$(wc -l <setcmp-data)"
mv setcmp-data.d1 setcmp-data
'''
if text.count(needle) != 1:
raise SystemExit(f"expected exactly one setcmp corpus creation in {spec}")
spec.write_text(text.replace(needle, replacement))
PY
git -C "$source" add alt/arsv-d1-build
}
apt_options()
{
local variant=$1
APT_OPTIONS=(
-o 'Debug::NoLocking=true'
-o 'Dir::Etc::main=-'
-o 'Dir::Etc::parts=-'
-o "Dir::Etc::sourcelist=$SNAPSHOT/etc-apt/sources.list"
@@ -287,8 +353,10 @@ run_once()
prepare_snapshot()
{
local converter_output
local converter_output devel_cache devel_root rpm_libdir rpm_library rpmio_library
local devel_source runtime_source rpmfile
local -a update_options
local -a rpm_devel_rpms popt_devel_rpms
validate_snapshot_reuse && return
printf '\n===== Preparing one immutable Sisyphus metadata snapshot =====\n'
@@ -319,14 +387,69 @@ prepare_snapshot()
APT_CONFIG="$COMMON/apt.conf" "$APT_GET" -qq \
"${update_options[@]}" update
# The converter needs C headers, but the benchmark host intentionally need
# not have development packages installed. Download the p11 packages that
# match the host's installed librpm into a private cache and extract only
# their headers; no system package is installed or changed.
devel_cache="$SNAPSHOT/devel-cache"
devel_root="$SNAPSHOT/devel-root"
mkdir -p "$devel_cache/archives/partial" "$devel_root"
env -u APT_CONFIG "$APT_GET" -qq -y -d \
-o "Dir::Cache=$devel_cache" \
-o "Dir::Cache::archives=$devel_cache/archives" \
-o "Dir::Cache::pkgcache=$devel_cache/pkgcache.bin" \
-o "Dir::Cache::srcpkgcache=$devel_cache/srcpkgcache.bin" \
install librpm-devel
shopt -s nullglob
rpm_devel_rpms=("$devel_cache"/archives/librpm-devel_*.rpm)
popt_devel_rpms=("$devel_cache"/archives/libpopt-devel_*.rpm)
shopt -u nullglob
((${#rpm_devel_rpms[@]} == 1)) ||
fail "expected one downloaded librpm-devel RPM, got ${#rpm_devel_rpms[@]}"
((${#popt_devel_rpms[@]} == 1)) ||
fail "expected one downloaded libpopt-devel RPM, got ${#popt_devel_rpms[@]}"
devel_source=$(rpm -qp --qf '%{SOURCERPM}|%{DISTTAG}' "${rpm_devel_rpms[0]}") ||
fail 'cannot identify downloaded librpm-devel source package'
for rpmfile in "${rpm_devel_rpms[@]}" "${popt_devel_rpms[@]}"; do
(cd "$devel_root" && rpm2cpio "$rpmfile" | cpio -idm --quiet './usr/include/*')
done
[[ -f $devel_root/usr/include/rpm/header.h && -f $devel_root/usr/include/popt.h ]] ||
fail 'failed to extract RPM development headers'
rpm_libdir=$(rpm --eval '%{_libdir}')
rpm_library=$(realpath -e "$rpm_libdir/librpm.so.7") ||
fail 'installed librpm.so.7 not found'
rpmio_library=$(realpath -e "$rpm_libdir/librpmio.so.7") ||
fail 'installed librpmio.so.7 not found'
for rpmfile in "$rpm_library" "$rpmio_library"; do
runtime_source=$(rpm -qf --qf '%{SOURCERPM}|%{DISTTAG}' "$rpmfile") ||
fail "cannot identify runtime package owning $rpmfile"
[[ $runtime_source == "$devel_source" ]] ||
fail "downloaded librpm-devel ($devel_source) does not match $rpmfile ($runtime_source)"
done
{
printf 'devel_librpm_identity=%s|%s\n' \
"$(rpm -qp --qf '%{NAME}|%{EVR}|%{DISTTAG}|%{ARCH}' "${rpm_devel_rpms[0]}")" \
"$devel_source"
printf 'devel_librpm_sha256=%s\n' \
"$(sha256sum "${rpm_devel_rpms[0]}" | awk '{print $1}')"
printf 'devel_popt_identity=%s\n' \
"$(rpm -qp --qf '%{NAME}|%{EVR}|%{DISTTAG}|%{ARCH}' "${popt_devel_rpms[0]}")"
printf 'devel_popt_sha256=%s\n' \
"$(sha256sum "${popt_devel_rpms[0]}" | awk '{print $1}')"
} >"$SNAPSHOT/devel-input-fingerprint.txt"
converter_output="$SNAPSHOT/conversion"
python3 "$PKGLIST_CONVERTER" "$converter_output" \
--lists-dir "$SNAPSHOT/lists"
--lists-dir "$SNAPSHOT/lists" \
--rpm-include-dir "$devel_root/usr/include" \
--rpm-library "$rpm_library" \
--rpm-library "$rpmio_library"
mv "$converter_output/d1-pkglists/manifest.json" "$SNAPSHOT/manifest.json"
mkdir -p "$SNAPSHOT/d1-pkglists"
mv "$converter_output/d1-pkglists/"*.classic "$SNAPSHOT/d1-pkglists/"
rmdir "$converter_output/d1-pkglists" "$converter_output"
rm -rf "$SNAPSHOT/download-cache"
rm -rf "$devel_cache" "$devel_root"
rm -f "$SNAPSHOT/lists/lock"
rm -rf "$SNAPSHOT/lists/partial"
mkdir -p "$SNAPSHOT/lists/partial"
@@ -387,9 +510,15 @@ build_variant()
expected_fingerprint=$(mktemp)
{
printf 'base_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
printf 'orchestrator=%s\n' "$(sha256sum "$ORCHESTRATOR" | awk '{print $1}')"
printf 'set_source=%s\n' "$(sha256sum "$source_c" | awk '{print $1}')"
printf 'suffix=%s\n' "$suffix"
printf 'packager=%s\n' "$PACKAGER"
if [[ $name == d1 ]]; then
printf 'set9_decoder=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
printf 'set_rewriter=%s\n' "$(sha256sum "$SET_REWRITER_C" | awk '{print $1}')"
printf 'set_compat=%s\n' "$(sha256sum "$SET_COMPAT_H" | awk '{print $1}')"
fi
} >"$expected_fingerprint"
if [[ -f $variant/input-fingerprint.txt ]]; then
if ! cmp -s "$expected_fingerprint" "$variant/input-fingerprint.txt"; then
@@ -407,10 +536,24 @@ build_variant()
git clone --local "$SOURCE_BASE" "$source"
cp "$source_c" "$source/lib/set.c"
prepare_spec "$spec" "$suffix"
if [[ $name == d1 ]]; then
prepare_d1_build_corpus "$source"
fi
sha256sum "$spec" >"$variant/prepared-spec.sha256"
cp "$expected_fingerprint" "$variant/input-fingerprint.txt"
else
[[ -f $variant/prepared-spec.sha256 ]] &&
sha256sum -c "$variant/prepared-spec.sha256" >/dev/null ||
fail "$name prepared RPM spec changed; use RESET_WORK=1"
cmp -s "$source_c" "$source/lib/set.c" ||
fail "$source_c changed; use RESET_WORK=1"
if [[ $name == d1 ]]; then
cmp -s "$SET9_C" "$source/alt/arsv-d1-build/set9.c" &&
cmp -s "$SET_REWRITER_C" \
"$source/alt/arsv-d1-build/rewrite_sisyphus_pkglist.c" &&
cmp -s "$SET_COMPAT_H" "$source/alt/arsv-d1-build/newset_compat.h" ||
fail 'D1 build corpus converter changed; use RESET_WORK=1'
fi
fi
rm -f "$expected_fingerprint"
@@ -529,9 +672,11 @@ write_provenance()
printf 'cpu=%s\n' "$CPU"
printf 'rounds=%s\n' "$ROUNDS"
printf 'operations=%s\n' "$OPERATIONS"
sha256sum "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \
sha256sum "$ORCHESTRATOR" "$SET9_C" "$D1_C" "$PKGLIST_CONVERTER" \
"$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" \
"$REPO_ROOT/scripts/rpmsetcmp/newset_compat.h"
cat "$SNAPSHOT/devel-input-fingerprint.txt"
write_runtime_fingerprint
printf 'set9_librpm='; cat "$SET9_VARIANT/librpm-artifact.sha256"
printf 'd1_librpm='; cat "$D1_VARIANT/librpm-artifact.sha256"
} >"$RESULT_DIR/provenance.txt"