From fc0730c7514b0c81cddc155ed5ddd28466c0dcf9 Mon Sep 17 00:00:00 2001 From: Dmitrii Krosh Date: Mon, 20 Jul 2026 19:23:23 +0300 Subject: [PATCH] WIP --- scripts/check_sisyphus_set_relations.sh | 225 +++++++++++++++++++++ tests/test_check_sisyphus_set_relations.py | 132 ++++++++++++ tt | 6 - 3 files changed, 357 insertions(+), 6 deletions(-) create mode 100755 scripts/check_sisyphus_set_relations.sh create mode 100644 tests/test_check_sisyphus_set_relations.py diff --git a/scripts/check_sisyphus_set_relations.sh b/scripts/check_sisyphus_set_relations.sh new file mode 100755 index 0000000..60858b1 --- /dev/null +++ b/scripts/check_sisyphus_set_relations.sh @@ -0,0 +1,225 @@ +#!/bin/bash +set -euo pipefail +export LC_ALL=C + +MIRROR=https://ftp.altlinux.org/pub/distributions/ALTLinux +REQUIRE_LIMIT=100 + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd) +WORK=$(mktemp -d "${TMPDIR:-/tmp}/arsv-set-check.XXXXXX") +trap 'rm -rf "$WORK"' EXIT + +for command in apt-get apt-cache awk sort join comm paste cc; do + command -v "$command" >/dev/null || { + printf 'required command not found: %s\n' "$command" >&2 + exit 2 + } +done + +build_comparator() +{ + : >"$WORK/rpmlib.h" + : >"$WORK/system.h" + cat >"$WORK/compare.c" <<'EOF' +#include +#include +#include + +int rpmsetcmp(const char *, const char *); + +int main(void) +{ + char *provided = NULL; + char *required = NULL; + size_t provided_size = 0; + size_t required_size = 0; + ssize_t provided_length; + + while ((provided_length = getline(&provided, &provided_size, stdin)) >= 0) { + ssize_t required_length = getline(&required, &required_size, stdin); + if (provided_length < 1 || required_length < 1) { + free(provided); + free(required); + return 2; + } + if (provided[provided_length - 1] == '\n') + provided[provided_length - 1] = '\0'; + if (required[required_length - 1] == '\n') + required[required_length - 1] = '\0'; + + printf("%d\n", rpmsetcmp(provided, required)); + } + free(provided); + free(required); + return 0; +} +EOF + + cc -O2 -std=gnu11 -D_GNU_SOURCE -I"$WORK" \ + -include "$SCRIPT_DIR/newset_compat.h" \ + "$REPO_ROOT/reimplement/newset.c" "$WORK/compare.c" \ + -o "$WORK/compare" +} + +fetch_metadata() +{ + mkdir -p "$WORK/apt/lists/partial" "$WORK/apt/archives/partial" + : >"$WORK/apt/apt.conf" + : >"$WORK/apt/status" + printf '%s\n' \ + "rpm [alt] $MIRROR Sisyphus/x86_64 classic" \ + "rpm [alt] $MIRROR Sisyphus/noarch classic" \ + >"$WORK/apt/sources.list" + + local options=( + -o 'Dir::Etc::main=-' + -o 'Dir::Etc::parts=-' + -o "Dir::Etc::sourcelist=$WORK/apt/sources.list" + -o 'Dir::Etc::sourceparts=-' + -o "Dir::State::lists=$WORK/apt/lists" + -o "Dir::State::status=$WORK/apt/status" + -o "Dir::Cache::archives=$WORK/apt/archives" + -o "Dir::Cache::pkgcache=$WORK/apt/pkgcache.bin" + -o "Dir::Cache::srcpkgcache=$WORK/apt/srcpkgcache.bin" + ) + + APT_CONFIG="$WORK/apt/apt.conf" apt-get "${options[@]}" -qq update + APT_CONFIG="$WORK/apt/apt.conf" apt-cache "${options[@]}" dumpavail \ + >"$WORK/available.txt" +} + +extract_relations() +{ + awk ' + function emit(kind, text, count, parts, i, item, capability, encoded) { + count = split(text, parts, /,[[:space:]]*/) + for (i = 1; i <= count; i++) { + item = parts[i] + if (kind == "P" && item !~ /[[:space:]]+\(?=[[:space:]]+set:[[:alnum:]]+\)?$/) + continue + if (kind == "R" && item !~ /[[:space:]]+\(?>=[[:space:]]+set:[[:alnum:]]+\)?$/) + continue + + capability = item + sub(/[[:space:]]+\(?>?=[[:space:]]+set:[[:alnum:]]+\)?$/, "", capability) + encoded = item + sub(/^.*[[:space:]]+\(?>?=[[:space:]]+/, "", encoded) + sub(/\)$/, "", encoded) + printf "%s\t%s\t%s\t%s\n", kind, package, capability, encoded + } + } + function flush() { + if (field != "") + emit(field, value) + field = "" + value = "" + } + { + if ($0 ~ /^[[:space:]]/ && field != "") { + line = $0 + sub(/^[[:space:]]+/, "", line) + value = value " " line + next + } + + flush() + if ($0 ~ /^Package: /) + package = substr($0, 10) + else if ($0 ~ /^Provides: /) { + field = "P" + value = substr($0, 11) + } else if ($0 ~ /^(Depends|Pre-Depends): /) { + field = "R" + value = substr($0, index($0, ":") + 2) + } else if ($0 == "") + package = "" + } + END { flush() } + ' "$WORK/available.txt" >"$WORK/relations.tsv" +} + +check_relations() +{ + local tab=$'\t' result + + awk -F '\t' '$1 == "P" { print $3 "\t" $2 "\t" $4 }' \ + "$WORK/relations.tsv" | LC_ALL=C sort -t "$tab" -k1,1 \ + >"$WORK/providers.tsv" + awk -F '\t' -v limit="$REQUIRE_LIMIT" \ + '$1 == "R" { print $3 "\t" $2 "\t" $4; if (++n == limit) exit }' \ + "$WORK/relations.tsv" | LC_ALL=C sort -u \ + >"$WORK/requirements.tsv" + + join -t "$tab" "$WORK/providers.tsv" "$WORK/requirements.tsv" \ + >"$WORK/pairs.tsv" + : >"$WORK/paired.tsv" + : >"$WORK/compatible.tsv" + : >"$WORK/decode-errors.tsv" + + awk -F '\t' '{ print $3; print $5 }' "$WORK/pairs.tsv" \ + | "$WORK/compare" >"$WORK/results.txt" + paste "$WORK/pairs.tsv" "$WORK/results.txt" >"$WORK/compared.tsv" + + while IFS=$'\t' read -r capability provider provided consumer required result; do + printf '%s\t%s\t%s\n' "$capability" "$consumer" "$required" \ + >>"$WORK/paired.tsv" + case $result in + 0|1) + printf '%s\t%s\t%s\n' "$capability" "$consumer" "$required" \ + >>"$WORK/compatible.tsv" + ;; + -3|-4) + printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$consumer" "$provider" "$capability" "$provided" "$required" "$result" \ + >>"$WORK/decode-errors.tsv" + ;; + esac + done <"$WORK/compared.tsv" + + for file in requirements paired compatible; do + LC_ALL=C sort -u "$WORK/$file.tsv" -o "$WORK/$file.tsv" + done + comm -23 "$WORK/requirements.tsv" "$WORK/paired.tsv" >"$WORK/no-provider.tsv" + comm -23 "$WORK/paired.tsv" "$WORK/compatible.tsv" >"$WORK/incompatible.tsv" +} + +count_lines() +{ + awk 'END { print NR + 0 }' "$1" +} + +build_comparator +printf 'Updating Sisyphus metadata...\n' +fetch_metadata +extract_relations +check_relations + +requirements=$(count_lines "$WORK/requirements.tsv") +compatible=$(count_lines "$WORK/compatible.tsv") +no_provider=$(count_lines "$WORK/no-provider.tsv") +incompatible=$(count_lines "$WORK/incompatible.tsv") +decode_errors=$(count_lines "$WORK/decode-errors.tsv") +comparisons=$(count_lines "$WORK/pairs.tsv") + +printf '\nrequirements: %s\n' "$requirements" +printf 'comparisons: %s\n' "$comparisons" +printf 'compatible: %s\n' "$compatible" +printf 'no provider: %s\n' "$no_provider" +printf 'incompatible: %s\n' "$incompatible" +printf 'decode errors:%s\n' "$decode_errors" + +if ((no_provider)); then + printf '\nRequirements without a provider (first 10):\n' + sed -n '1,10p' "$WORK/no-provider.tsv" +fi +if ((incompatible)); then + printf '\nUnsatisfied set requirements (first 10):\n' + sed -n '1,10p' "$WORK/incompatible.tsv" +fi + +((requirements > 0)) || { + printf 'no set requirements found\n' >&2 + exit 2 +} +((no_provider == 0 && incompatible == 0 && decode_errors == 0)) diff --git a/tests/test_check_sisyphus_set_relations.py b/tests/test_check_sisyphus_set_relations.py new file mode 100644 index 0000000..c3ac7d7 --- /dev/null +++ b/tests/test_check_sisyphus_set_relations.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "check_sisyphus_set_relations.sh" +PROVIDER_SET = "set:jg2ZwFjdTD2B0" +REQUIRED_SET = "set:jiXMb" + + +def run_with_metadata( + tmp_path: Path, metadata: str, *, locale: str | None = None +) -> subprocess.CompletedProcess[str]: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + metadata_file = tmp_path / "available.txt" + metadata_file.write_text(metadata, encoding="utf-8") + + (fake_bin / "apt-get").write_text( + """#!/bin/sh +archive= +for argument in "$@"; do + case $argument in + Dir::Cache::archives=*) archive=${argument#*=} ;; + esac +done +[ -n "$archive" ] && [ -d "$archive/partial" ] +""", + encoding="utf-8", + ) + (fake_bin / "apt-cache").write_text( + '#!/bin/sh\ncat "$FAKE_APT_DUMP"\n', encoding="utf-8" + ) + (fake_bin / "apt-get").chmod(0o755) + (fake_bin / "apt-cache").chmod(0o755) + + environment = os.environ.copy() + environment["PATH"] = f"{fake_bin}:{environment['PATH']}" + environment["FAKE_APT_DUMP"] = str(metadata_file) + if locale is not None: + environment["LC_ALL"] = locale + return subprocess.run( + ["bash", str(SCRIPT)], + cwd=REPO_ROOT, + text=True, + capture_output=True, + env=environment, + ) + + +def test_accepts_a_real_provider_subset_relation_and_continuation_line(tmp_path: Path) -> None: + metadata = f"""\ +Package: libexample +Provides: libexample.so.1()(64bit) (= {PROVIDER_SET}) + +Package: example +Depends: plain-dependency, + libexample.so.1()(64bit) (>= {REQUIRED_SET}) + +""" + + completed = run_with_metadata(tmp_path, metadata) + + assert completed.returncode == 0, completed.stderr + assert "requirements: 1" in completed.stdout + assert "compatible: 1" in completed.stdout + assert "incompatible: 0" in completed.stdout + + +def test_rejects_a_provider_that_does_not_contain_the_required_set(tmp_path: Path) -> None: + metadata = f"""\ +Package: libexample +Provides: libexample.so.1()(64bit) (= {REQUIRED_SET}) + +Package: example +Depends: libexample.so.1()(64bit) (>= {PROVIDER_SET}) + +""" + + completed = run_with_metadata(tmp_path, metadata) + + assert completed.returncode == 1 + assert "requirements: 1" in completed.stdout + assert "compatible: 0" in completed.stdout + assert "incompatible: 1" in completed.stdout + assert "example" in completed.stdout + + +def test_uses_one_collation_order_for_sort_and_join(tmp_path: Path) -> None: + metadata = f"""\ +Package: userdb-provider +Provides: /usr/lib64/courier-authlib/libauthuserdb.so.0()(64bit) (= {PROVIDER_SET}) + +Package: auth-provider +Provides: /usr/lib64/courier-authlib/libcourierauth.so.0()(64bit) (= {PROVIDER_SET}) + +Package: auth-consumer +Depends: /usr/lib64/courier-authlib/libcourierauth.so.0()(64bit) (>= {REQUIRED_SET}) + +Package: common-consumer +Depends: /usr/lib64/courier-authlib/libcourierauthcommon.so.0()(64bit) (>= {REQUIRED_SET}) + +""" + + completed = run_with_metadata(tmp_path, metadata, locale="ru_RU.utf8") + + assert completed.returncode == 1 + assert "is not sorted" not in completed.stderr + assert "requirements: 2" in completed.stdout + assert "compatible: 1" in completed.stdout + assert "no provider: 1" in completed.stdout + + +def test_large_set_is_not_passed_as_a_command_line_argument(tmp_path: Path) -> None: + oversized_set = "set:" + "a" * 140_000 + metadata = f"""\ +Package: libexample +Provides: libexample.so.1()(64bit) (= {oversized_set}) + +Package: example +Depends: libexample.so.1()(64bit) (>= {REQUIRED_SET}) + +""" + + completed = run_with_metadata(tmp_path, metadata) + + assert completed.returncode == 1 + assert "Argument list too long" not in completed.stderr + assert "decode errors:1" in completed.stdout diff --git a/tt b/tt index 10fdfa9..ee71842 100644 --- a/tt +++ b/tt @@ -5,13 +5,7 @@ 2026-07-19T03:48:54Z DEPENDENCY SILLY x86_64 mismatch requires libjpeg.so.62()(64bit) >= set:jfvZkgcUQqm1WcNFYdT6Zh set:kgvLPFlp5EP4DOJVCPR1WC6 - -2026-07-19T03:48:54Z DEPENDENCY SILLY x86_64 match requires libpng16.so.16()(64bit) >= set:lhkC1fmC6EFQ5Klh4dezRCyvQFOXIZ3uwhXdUhBK49 set:lhkC1fmC6EFQ5Klh4dezRCyvQFOXIZ3uwhXdUhBK49 - -2026-07-19T03:58:11Z START abuse x86_64 processing - - - - - 0.8-alt2@1308659965 -2026-07-19T03:58:33Z DEPENDENCY abuse x86_64 match requires libSDL-1.2.so.0()(64bit) >= set:lgkZiCZcFvTDDX6Y1PaRgCdFuVAwizcZfaTezfxZFUxcDOoVbbKRxY6ahoMjPQkxX5fdZtjAKGEY4 set:lgkZiCZcFvTDDX6Y1PaRgCdFuVAwizcZfaTezfxZFUxcDOoVbbKRxY6ahoMjPQkxX5fdZtjAKGEY4 - 2026-07-19T03:58:33Z DEPENDENCY abuse x86_64 mismatch requires libSDL_mixer-1.2.so.0()(64bit) >= set:lgIdCOqw7JFqcojADYwCZ7pQY3CJ1LnAqVzI3Z0QXq0pLy1 set:kfMSoaHVyGBhV7CJ1YuVyx95uAdQPsq0b5zW6rUlsY3 - -2026-07-19T03:58:33Z SUMMARY abuse x86_64 mismatch - - - - - requires expected=2 generated=2 differences=1 extras=0; complete=1 2026-07-19T04:08:20Z DEPENDENCY airstrike x86_64 mismatch requires libSDL_mixer-1.2.so.0()(64bit) >= set:lirzppdVhrMiF4ChTZdaR8 set:khS6jpdPZrxpQlg2V9Bq4 - - -