krosh/ARSV
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rm podman

Browse Source
This commit is contained in:
krosh
2026-08-13 03:37:48 +03:00
parent 3027f36307
commit 9f17383626
2 changed files with 86 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py
+14 -69
View File
@@ -7,7 +7,6 @@ import argparse
import hashlib
import json
import re
import shlex
import shutil
import subprocess
import sys
@@ -17,7 +16,6 @@ from pathlib import Path
HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[2]
DEFAULT_IMAGE = "registry.altlinux.org/sisyphus/alt:latest"
ARCHITECTURES = ("x86_64", "noarch")
SUMMARY_RE = re.compile(
r"headers=(?P<headers>\d+) set_occurrences=(?P<set_occurrences>\d+) "
@@ -214,27 +212,15 @@ def convert_one(converter: Path, source: Path, destination: Path) -> dict[str, o
}
def image_metadata(image: str) -> dict[str, str]:
result = run(
[
"podman",
"image",
"inspect",
image,
"--format",
"{{.Digest}}|{{.Id}}",
]
)
digest, image_id = result.stdout.decode().strip().split("|", 1)
return {"name": image, "digest": digest, "id": image_id}
def inner(output: Path, image: str, image_digest: str, image_id: str) -> None:
def convert_local(output: Path, lists_dir: Path) -> None:
output.mkdir(parents=True, exist_ok=True)
if any(output.iterdir()):
raise RuntimeError(f"output directory is not empty: {output}")
pkglists = find_pkglists(Path("/var/lib/apt/lists"))
lists_dir = lists_dir.expanduser().resolve()
if not lists_dir.is_dir():
raise RuntimeError(f"APT lists directory does not exist: {lists_dir}")
pkglists = find_pkglists(lists_dir)
staging = Path(tempfile.mkdtemp(prefix=".d1-staging-", dir=output))
try:
converter = staging / "rewrite-sisyphus-pkglist"
@@ -250,7 +236,7 @@ def inner(output: Path, image: str, image_digest: str, image_id: str) -> None:
manifest = {
"format": 1,
"created_at": datetime.now(timezone.utc).isoformat(),
"image": {"name": image, "digest": image_digest, "id": image_id},
"input": {"lists_dir": str(lists_dir)},
"rpm": run(["rpm", "--version"]).stdout.decode().strip(),
"apt": run(["rpmquery", "--qf", "%{VERSION}-%{RELEASE}", "apt"])
.stdout.decode()
@@ -280,55 +266,17 @@ def inner(output: Path, image: str, image_digest: str, image_id: str) -> None:
raise
def outer(output: Path, image: str) -> None:
if output.exists() and any(output.iterdir()):
raise RuntimeError(f"output directory is not empty: {output}")
output.mkdir(parents=True, exist_ok=True)
run(["podman", "pull", image], stdout=None)
metadata = image_metadata(image)
inner_command = [
"python3",
"/src/new_version/direct_hash/apt_benchmark/run_sisyphus_pkglist.py",
"--inner",
"/out",
"--image",
image,
"--image-digest",
metadata["digest"],
"--image-id",
metadata["id"],
]
command = [
"podman",
"run",
"--rm",
"-v",
f"{ROOT}:/src:ro",
"-v",
f"{output}:/out:rw",
image,
"sh",
"-lc",
(
"apt-get update >/dev/null && "
"apt-get install -y gcc librpm-devel apt-repo-tools python3 >/dev/null && "
+ " ".join(shlex.quote(argument) for argument in inner_command)
),
]
result = subprocess.run(command, check=False)
if result.returncode != 0:
raise RuntimeError(f"container conversion failed with status {result.returncode}")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create D1 copies of current Sisyphus x86_64/noarch pkglist files"
description="Create D1 copies of local Sisyphus x86_64/noarch pkglist files"
)
parser.add_argument("output", type=Path, help="new or empty output directory")
parser.add_argument("--image", default=DEFAULT_IMAGE)
parser.add_argument("--image-digest", default="", help=argparse.SUPPRESS)
parser.add_argument("--image-id", default="", help=argparse.SUPPRESS)
parser.add_argument("--inner", action="store_true", help=argparse.SUPPRESS)
parser.add_argument(
"--lists-dir",
type=Path,
default=Path("/var/lib/apt/lists"),
help="APT lists snapshot to convert (default: /var/lib/apt/lists)",
)
return parser.parse_args()
@@ -336,10 +284,7 @@ def main() -> int:
args = parse_args()
try:
output = validate_output(args.output)
if args.inner:
inner(output, args.image, args.image_digest, args.image_id)
else:
outer(output, args.image)
convert_local(output, args.lists_dir)
except (OSError, RuntimeError, ValueError) as error:
print(f"error: {error}", file=sys.stderr)
return 1
scripts/run-sisyphus-set9-d1-bench.sh
+72 -52
View File
@@ -18,7 +18,8 @@ PKGLIST_CONVERTER=${PKGLIST_CONVERTER:-$REPO_ROOT/new_version/direct_hash/apt_be
WORK_ROOT=${WORK_ROOT:-$HOME/sisyphus-set9-d1-bench}
RESULT_DIR=${RESULT_DIR:-$WORK_ROOT/results}
IMAGE=${IMAGE:-registry.altlinux.org/sisyphus/alt:latest}
APT_LISTS_SOURCE=${APT_LISTS_SOURCE:-/var/lib/apt/lists}
APT_ETC_SOURCE=${APT_ETC_SOURCE:-/etc/apt}
RPM_GIT=${RPM_GIT:-https://git.altlinux.org/gears/r/rpm.git}
RPM_BRANCH=${RPM_BRANCH:-sisyphus}
PACKAGER=${PACKAGER:-krosh <gudovdo@my.msu.ru>}
@@ -39,11 +40,14 @@ The script has no positional arguments. Configuration is passed through env:
RESET_WORK=0 OPERATIONS='unmet check' ./scripts/run-sisyphus-set9-d1-bench.sh
Main variables:
WORK_ROOT, RESULT_DIR, IMAGE, RPM_GIT, RPM_BRANCH, PACKAGER,
WORK_ROOT, RESULT_DIR, APT_LISTS_SOURCE, APT_ETC_SOURCE,
RPM_GIT, RPM_BRANCH, PACKAGER,
SET9_C, D1_C, PKGLIST_CONVERTER, CPU, ROUNDS, RESET_WORK, OPERATIONS.
RESET_WORK=1 removes WORK_ROOT after running hsh --cleanup-only for old hasher
workdirs. Timed runs never update repositories and never install packages.
Run apt-get update on the Sisyphus host before RESET_WORK=1; the script snapshots
the existing local APT lists and performs no metadata download itself.
EOF
}
@@ -72,12 +76,32 @@ safe_remove_work_root()
fi
}
canonical_work_subdir()
{
local path=$1 label=$2 resolved
[[ ! -L $path ]] || fail "$label must not be a symlink: $path"
resolved=$(realpath -e -- "$path") || fail "$label does not resolve: $path"
[[ $resolved == "$WORK_ROOT/"* ]] ||
fail "$label resolves outside WORK_ROOT: $path -> $resolved"
printf '%s' "$resolved"
}
ensure_work_subdir()
{
local path=$1 label=$2 resolved
[[ ! -L $path ]] || fail "$label must not be a symlink: $path"
resolved=$(realpath -m -- "$path")
[[ $resolved == "$WORK_ROOT/"* ]] ||
fail "$label resolves outside WORK_ROOT: $path -> $resolved"
mkdir -p -- "$path"
canonical_work_subdir "$path" "$label" >/dev/null
}
write_snapshot_fingerprint()
{
{
printf 'image=%s\n' "$IMAGE"
printf 'image_digest=%s\n' "$(podman image inspect "$IMAGE" --format '{{.Digest}}')"
printf 'image_id=%s\n' "$(podman image inspect "$IMAGE" --format '{{.Id}}')"
printf 'apt_lists_source=%s\n' "$APT_LISTS_SOURCE"
printf 'apt_etc_source=%s\n' "$APT_ETC_SOURCE"
printf 'converter=%s\n' "$(sha256sum "$PKGLIST_CONVERTER" | awk '{print $1}')"
printf 'set9=%s\n' "$(sha256sum "$SET9_C" | awk '{print $1}')"
printf 'rewrite=%s\n' "$(sha256sum "$REPO_ROOT/new_version/direct_hash/apt_benchmark/rewrite_sisyphus_pkglist.c" | awk '{print $1}')"
@@ -266,59 +290,42 @@ run_once()
prepare_snapshot()
{
local image_digest image_id converter_rel
local converter_output
validate_snapshot_reuse && return
case $PKGLIST_CONVERTER in
"$REPO_ROOT"/*) converter_rel=${PKGLIST_CONVERTER#"$REPO_ROOT/"} ;;
*) fail "PKGLIST_CONVERTER must be inside REPO_ROOT: $PKGLIST_CONVERTER" ;;
esac
printf '\n===== Preparing one immutable Sisyphus metadata snapshot =====\n'
printf '\n===== Preparing one immutable local Sisyphus metadata snapshot =====\n'
rm -rf "$SNAPSHOT"
mkdir -p "$SNAPSHOT/container-output"
podman pull "$IMAGE"
image_digest=$(podman image inspect "$IMAGE" --format '{{.Digest}}')
image_id=$(podman image inspect "$IMAGE" --format '{{.Id}}')
podman run --rm \
-e "ARSV_IMAGE=$IMAGE" \
-e "ARSV_IMAGE_DIGEST=$image_digest" \
-e "ARSV_IMAGE_ID=$image_id" \
-e "ARSV_CONVERTER_REL=$converter_rel" \
-v "$REPO_ROOT:/src:ro" \
-v "$SNAPSHOT/container-output:/out:rw" \
"$IMAGE" sh -euc '
apt-get update >/dev/null
apt-get install -y gcc librpm-devel apt-repo-tools python3 >/dev/null
python3 "/src/$ARSV_CONVERTER_REL" \
--inner /out/conversion \
--image "$ARSV_IMAGE" \
--image-digest "$ARSV_IMAGE_DIGEST" \
--image-id "$ARSV_IMAGE_ID"
cp -a /var/lib/apt/lists /out/original-lists
cp -a /etc/apt /out/etc-apt
'
mv "$SNAPSHOT/container-output/original-lists" "$SNAPSHOT/lists"
mv "$SNAPSHOT/container-output/etc-apt" "$SNAPSHOT/etc-apt"
mv "$SNAPSHOT/container-output/conversion/d1-pkglists/manifest.json" "$SNAPSHOT/manifest.json"
mkdir -p "$SNAPSHOT/d1-pkglists"
mv "$SNAPSHOT/container-output/conversion/d1-pkglists/"*.classic "$SNAPSHOT/d1-pkglists/"
rm -rf "$SNAPSHOT/container-output"
mkdir -p "$SNAPSHOT/lists" "$SNAPSHOT/etc-apt/sources.list.d"
cp -a "$APT_LISTS_SOURCE/." "$SNAPSHOT/lists/"
if [[ -f $APT_ETC_SOURCE/sources.list ]]; then
cp -L "$APT_ETC_SOURCE/sources.list" "$SNAPSHOT/etc-apt/sources.list"
else
: >"$SNAPSHOT/etc-apt/sources.list"
fi
if [[ -d $APT_ETC_SOURCE/sources.list.d ]]; then
cp -LR "$APT_ETC_SOURCE/sources.list.d/." "$SNAPSHOT/etc-apt/sources.list.d/"
fi
rm -f "$SNAPSHOT/lists/lock"
rm -rf "$SNAPSHOT/lists/partial"
mkdir -p "$SNAPSHOT/lists/partial" "$SNAPSHOT/etc-apt/sources.list.d"
[[ -e $SNAPSHOT/etc-apt/sources.list ]] || : >"$SNAPSHOT/etc-apt/sources.list"
mkdir -p "$SNAPSHOT/lists/partial"
converter_output="$SNAPSHOT/conversion"
python3 "$PKGLIST_CONVERTER" "$converter_output" \
--lists-dir "$SNAPSHOT/lists"
mv "$converter_output/d1-pkglists/manifest.json" "$SNAPSHOT/manifest.json"
mkdir -p "$SNAPSHOT/d1-pkglists"
mv "$converter_output/d1-pkglists/"*.classic "$SNAPSHOT/d1-pkglists/"
rmdir "$converter_output/d1-pkglists" "$converter_output"
write_snapshot_fingerprint >"$SNAPSHOT/input-fingerprint.txt"
: >"$SNAPSHOT/.complete"
}
prepare_variant_lists()
{
local variant=$1 format=$2
local variant=$1 format=$2 variant_apt
if [[ -d $variant/apt ]]; then
chmod -R u+w "$variant/apt"
variant_apt=$(canonical_work_subdir "$variant/apt" 'APT variant directory')
chmod -R u+w "$variant_apt"
fi
rm -rf "$variant/apt"
mkdir -p "$variant/apt/lists" "$variant/apt/cache/archives/partial"
@@ -502,9 +509,8 @@ write_provenance()
printf 'rpm_git=%s\n' "$RPM_GIT"
printf 'rpm_branch=%s\n' "$RPM_BRANCH"
printf 'rpm_commit=%s\n' "$(git -C "$SOURCE_BASE" rev-parse HEAD)"
printf 'image=%s\n' "$IMAGE"
printf 'image_digest=%s\n' "$(podman image inspect "$IMAGE" --format '{{.Digest}}')"
printf 'image_id=%s\n' "$(podman image inspect "$IMAGE" --format '{{.Id}}')"
printf 'apt_lists_source=%s\n' "$APT_LISTS_SOURCE"
printf 'apt_etc_source=%s\n' "$APT_ETC_SOURCE"
printf 'apt=%s\n' "$(rpmquery --qf '%{VERSION}-%{RELEASE}' apt)"
printf 'rpm=%s\n' "$(rpm --version)"
printf 'cpu=%s\n' "$CPU"
@@ -634,8 +640,8 @@ if [[ ${1:-} == --help || ${1:-} == -h ]]; then
fi
(($# == 0)) || fail "unexpected arguments; use --help"
for command in git podman gear-hsh hsh rpm rpmquery rpm2cpio cpio taskset awk sed \
date sha256sum stat ldd python3 cmp cp mv tee realpath mktemp grep; do
for command in git gear-hsh hsh rpm rpmquery rpm2cpio cpio taskset awk sed cc \
pkglist-query date sha256sum stat ldd python3 cmp cp mv tee realpath mktemp grep; do
command -v "$command" >/dev/null || fail "required command not found: $command"
done
[[ -n $APT_CACHE && -x $APT_CACHE ]] || fail "apt-cache not found: $APT_CACHE"
@@ -644,6 +650,7 @@ done
[[ -f $D1_C ]] || fail "D1 source not found: $D1_C"
[[ -f $PKGLIST_CONVERTER ]] || fail "pkglist converter not found: $PKGLIST_CONVERTER"
[[ $ROUNDS =~ ^[1-9][0-9]*$ ]] || fail "ROUNDS must be a positive integer"
[[ $RESET_WORK =~ ^[01]$ ]] || fail "RESET_WORK must be 0 or 1"
taskset -c "$CPU" true >/dev/null 2>&1 || fail "CPU $CPU is unavailable to taskset"
read -r -a OPERATION_LIST <<<"$OPERATIONS"
((${#OPERATION_LIST[@]} > 0)) || fail "OPERATIONS is empty"
@@ -657,8 +664,15 @@ done
HOME_REAL=$(realpath -m "$HOME")
CWD_REAL=$(realpath -m "$PWD")
REPO_ROOT=$(realpath -m "$REPO_ROOT")
SET9_C=$(realpath -m "$SET9_C")
D1_C=$(realpath -m "$D1_C")
PKGLIST_CONVERTER=$(realpath -m "$PKGLIST_CONVERTER")
APT_LISTS_SOURCE=$(realpath -m "$APT_LISTS_SOURCE")
APT_ETC_SOURCE=$(realpath -m "$APT_ETC_SOURCE")
WORK_ROOT=$(realpath -m "$WORK_ROOT")
RESULT_DIR=$(realpath -m "$RESULT_DIR")
[[ -d $APT_LISTS_SOURCE ]] || fail "APT lists directory not found: $APT_LISTS_SOURCE"
[[ -d $APT_ETC_SOURCE ]] || fail "APT configuration directory not found: $APT_ETC_SOURCE"
[[ $RESULT_DIR == "$WORK_ROOT"/* ]] ||
fail "RESULT_DIR must be inside WORK_ROOT: $RESULT_DIR"
@@ -675,11 +689,13 @@ if ((RESET_WORK)); then
safe_remove_work_root
for old_hasher in "$SET9_VARIANT/hasher" "$D1_VARIANT/hasher"; do
[[ -d $old_hasher ]] || continue
old_hasher=$(canonical_work_subdir "$old_hasher" 'hasher workdir')
hsh --cleanup-only --workdir="$old_hasher" ||
fail "hasher cleanup failed: $old_hasher"
done
for old_apt in "$SET9_VARIANT/apt" "$D1_VARIANT/apt"; do
[[ -d $old_apt ]] || continue
old_apt=$(canonical_work_subdir "$old_apt" 'APT workdir')
chmod -R u+w "$old_apt"
done
rm -rf "$WORK_ROOT"
@@ -691,7 +707,7 @@ elif ((WORK_ROOT_EXISTED)); then
grep -Fx 'ARSV Sisyphus set9/D1 benchmark work root' "$marker" >/dev/null ||
fail "invalid WORK_ROOT ownership marker: $marker"
fi
mkdir -p "$WORK_ROOT/src" "$WORK_ROOT/variants" "$COMMON/root/var/lib/rpm" "$RESULT_DIR"
mkdir -p "$WORK_ROOT"
MARKER="$WORK_ROOT/.arsv-sisyphus-set-bench-root"
if [[ -e $MARKER ]]; then
grep -Fx 'ARSV Sisyphus set9/D1 benchmark work root' "$MARKER" >/dev/null ||
@@ -699,6 +715,10 @@ if [[ -e $MARKER ]]; then
else
printf '%s\n' 'ARSV Sisyphus set9/D1 benchmark work root' >"$MARKER"
fi
ensure_work_subdir "$WORK_ROOT/src" 'source directory'
ensure_work_subdir "$WORK_ROOT/variants" 'variants directory'
ensure_work_subdir "$COMMON/root/var/lib/rpm" 'isolated RPM database directory'
ensure_work_subdir "$RESULT_DIR" 'results directory'
: >"$COMMON/apt.conf"
: >"$COMMON/status"