#!/usr/bin/env python3 """Benchmark rpmsetcmp from a selected set.c on hardcoded ELF pairs.""" from __future__ import annotations import argparse import csv import os import shlex import shutil import statistics import subprocess import sys from dataclasses import dataclass from pathlib import Path from typing import Sequence import compare_realization as common REPO_ROOT = Path(__file__).resolve().parents[1] RUN_CMP_SOURCE = REPO_ROOT / "scripts" / "run_cmp.cpp" DEFAULT_RPM_ROOT = REPO_ROOT / "rpm-build" DEFAULT_RESULT_ROOT = REPO_ROOT / "res_cmp" LIBC_CANDIDATES = ( "/usr/lib/libc.so.6", "/lib64/libc.so.6", "/usr/lib64/libc.so.6", "/lib/libc.so.6", "/lib/x86_64-linux-gnu/libc.so.6", ) @dataclass(frozen=True) class ComparisonCase: name: str binary_candidates: tuple[str, ...] library_candidates: tuple[str, ...] # The benchmark corpus is intentionally fixed here rather than supplied via CLI. HARDCODED_CASES = ( ComparisonCase("true-libc", ("/usr/bin/true", "/bin/true"), LIBC_CANDIDATES), ComparisonCase("ls-libc", ("/usr/bin/ls", "/bin/ls"), LIBC_CANDIDATES), ComparisonCase("bash-libc", ("/usr/bin/bash", "/bin/bash"), LIBC_CANDIDATES), ComparisonCase( "python-libc", ("/usr/bin/python3", "/usr/local/bin/python3"), LIBC_CANDIDATES, ), ) SUMMARY_FIELDS = ( "case", "binary", "library", "runs", "cmp_result", "outputs_consistent", "median_rpmsetcmp_ns", ) @dataclass(frozen=True) class ResolvedCase: name: str binary: Path library: Path @dataclass(frozen=True) class PreparedCase: case: ResolvedCase provider_set: str required_set: str @dataclass(frozen=True) class ComparisonSummary: prepared: PreparedCase runs: int cmp_result: int outputs_consistent: bool median_ns: int | float def as_row(self) -> dict[str, str]: return { "case": self.prepared.case.name, "binary": str(self.prepared.case.binary), "library": str(self.prepared.case.library), "runs": str(self.runs), "cmp_result": str(self.cmp_result), "outputs_consistent": "yes" if self.outputs_consistent else "no", "median_rpmsetcmp_ns": common.format_number(self.median_ns), } def parse_args(argv: Sequence[str]) -> argparse.Namespace: parser = argparse.ArgumentParser( description=( "Compile a selected set.c and benchmark only rpmsetcmp on the " "hardcoded binary/library corpus." ) ) parser.add_argument("set_c", type=Path, help="path to the set.c implementation") parser.add_argument( "-n", "--runs", type=common.positive_int, default=10, help="rpmsetcmp process runs per hardcoded case (default: 10)", ) parser.add_argument( "--rpm-root", type=Path, default=DEFAULT_RPM_ROOT, help="ALT rpm source root", ) parser.add_argument( "--res-dir", type=Path, default=DEFAULT_RESULT_ROOT, help="result root (default: repository res_cmp/)", ) parser.add_argument("--name", help="implementation directory name under res_cmp/") parser.add_argument( "--bpp", type=common.bpp_value, help="force one bpp value while preparing provider/required sets", ) parser.add_argument("--cc", default="cc", help="C compiler (default: cc)") parser.add_argument("--cxx", default="g++", help="C++ compiler (default: g++)") return parser.parse_args(argv) def first_existing(candidates: Sequence[str]) -> Path | None: for candidate in candidates: path = Path(candidate) if path.is_file(): return path.resolve() return None def resolve_cases(cases: Sequence[ComparisonCase]) -> list[ResolvedCase]: resolved: list[ResolvedCase] = [] for case in cases: binary = first_existing(case.binary_candidates) library = first_existing(case.library_candidates) if binary is None or library is None: missing = "binary" if binary is None else "library" print(f"warning: skipping {case.name}: no hardcoded {missing} path exists", file=sys.stderr) continue resolved.append(ResolvedCase(common.safe_name(case.name), binary, library)) if not resolved: raise RuntimeError("none of the hardcoded binary/library cases is available") return resolved def compile_cmp_runner(build_dir: Path, rpm_root: Path, cxx: str) -> Path: runner = build_dir / "run_cmp" command = [ cxx, "-O2", "-std=c++17", "-Wall", "-Wextra", "-Werror", str(RUN_CMP_SOURCE), str(build_dir / "set.o"), str(build_dir / "rpmmalloc.o"), "-o", str(runner), ] log: list[str] = ["\n[rpmsetcmp runner]", "$ " + shlex.join(command)] completed = subprocess.run(command, cwd=rpm_root, text=True, capture_output=True) if completed.stdout: log.append(completed.stdout.rstrip()) if completed.stderr: log.append(completed.stderr.rstrip()) log.append(f"[exit {completed.returncode}]") build_log = build_dir / "build.log" with build_log.open("a", encoding="utf-8") as stream: stream.write("\n".join(log) + "\n") if completed.returncode != 0: raise RuntimeError(f"run_cmp compilation failed; see {build_log}") return runner def run_set_once( runner: Path, target: Path, sets_dir: Path, slug: str, bpp: int | None, ) -> common.ParsedRun: command = [str(runner)] if bpp is not None: command.extend(("--bpp", str(bpp))) command.append(str(target)) completed = subprocess.run(command, text=True, capture_output=True) stdout_path = sets_dir / f"{slug}.tsv" stderr_path = sets_dir / f"{slug}.stderr" stdout_path.write_text(completed.stdout, encoding="utf-8") stderr_path.write_text(completed.stderr, encoding="utf-8") if completed.returncode != 0: raise RuntimeError(f"run_set failed for {target}; see {stderr_path}") return common.parse_run_set_output(completed.stdout) def same_file(left: str, right: Path) -> bool: try: return os.path.samefile(left, right) except OSError: return Path(left).resolve() == right.resolve() def prepare_cases( cases: Sequence[ResolvedCase], run_set: Path, sets_dir: Path, bpp: int | None, ) -> list[PreparedCase]: sets_dir.mkdir(parents=True) cache: dict[Path, common.ParsedRun] = {} used_slugs: set[str] = set() def inspect(target: Path) -> common.ParsedRun: parsed = cache.get(target) if parsed is not None: return parsed slug = common.target_slug(target, used_slugs) parsed = run_set_once(run_set, target, sets_dir, slug, bpp) cache[target] = parsed return parsed prepared: list[PreparedCase] = [] for case in cases: library_run = inspect(case.library) provider_rows = [row for row in library_run.rows if row.get("role") == "provided"] if len(provider_rows) != 1: raise RuntimeError( f"expected one provider set for {case.library}, got {len(provider_rows)}" ) binary_run = inspect(case.binary) required_rows = [ row for row in binary_run.rows if row.get("role") == "required" and row.get("object") and same_file(row["object"], case.library) ] if len(required_rows) != 1: raise RuntimeError( f"expected one {case.library.name} requirement from {case.binary}, " f"got {len(required_rows)}" ) provider_row = provider_rows[0] required_row = required_rows[0] if provider_row["bpp"] != required_row["bpp"]: raise RuntimeError( f"bpp mismatch for {case.name}: provider={provider_row['bpp']} " f"required={required_row['bpp']}" ) prepared.append( PreparedCase( case=case, provider_set=provider_row["set"], required_set=required_row["set"], ) ) return prepared def parse_cmp_output(output: str) -> tuple[int, int]: values: dict[str, str] = {} for line in output.splitlines(): if "\t" not in line: continue key, value = line.split("\t", 1) values[key] = value try: result = int(values["cmp_result"]) elapsed = int(values["rpmsetcmp_ns"]) except (KeyError, ValueError) as exception: raise RuntimeError("invalid run_cmp output") from exception if elapsed < 0: raise RuntimeError("run_cmp returned a negative duration") return result, elapsed def benchmark_case( runner: Path, prepared: PreparedCase, runs: int, output_dir: Path, ) -> ComparisonSummary: output_dir.mkdir(parents=True) results: list[int] = [] timings: list[int] = [] for run_number in range(1, runs + 1): completed = subprocess.run( [str(runner), prepared.provider_set, prepared.required_set], text=True, capture_output=True, ) stem = f"run-{run_number:03d}" stdout_path = output_dir / f"{stem}.tsv" stderr_path = output_dir / f"{stem}.stderr" stdout_path.write_text(completed.stdout, encoding="utf-8") stderr_path.write_text(completed.stderr, encoding="utf-8") if completed.returncode != 0: raise RuntimeError( f"run_cmp failed for {prepared.case.name} on run {run_number}; " f"see {stderr_path}" ) result, elapsed = parse_cmp_output(completed.stdout) results.append(result) timings.append(elapsed) return ComparisonSummary( prepared=prepared, runs=runs, cmp_result=results[0], outputs_consistent=len(set(results)) == 1, median_ns=statistics.median(timings), ) def write_summary(path: Path, summaries: Sequence[ComparisonSummary]) -> None: with path.open("w", newline="", encoding="utf-8") as stream: writer = csv.DictWriter(stream, fieldnames=list(SUMMARY_FIELDS), delimiter="\t") writer.writeheader() for summary in summaries: writer.writerow(summary.as_row()) def print_summary(summaries: Sequence[ComparisonSummary]) -> None: print("\t".join(SUMMARY_FIELDS)) for summary in summaries: row = summary.as_row() print("\t".join(row[field] for field in SUMMARY_FIELDS)) def write_metadata( path: Path, set_source: Path, rpm_root: Path, runs: int, cases: Sequence[ResolvedCase], ) -> None: lines = [ f"set_c\t{set_source}", f"rpm_root\t{rpm_root}", f"runs\t{runs}", ] for case in cases: lines.append(f"case\t{case.name}\t{case.binary}\t{case.library}") path.write_text("\n".join(lines) + "\n", encoding="utf-8") def main(argv: Sequence[str]) -> int: args = parse_args(argv) try: common.require_commands((args.cc, args.cxx, "eu-readelf", "eu-elfclassify", "nm")) set_source = common.resolve_file(args.set_c, "set.c") rpm_root = common.resolve_directory(args.rpm_root, "rpm root") common.resolve_file(rpm_root / "system.h", "system.h") cases = resolve_cases(HARDCODED_CASES) name = common.safe_name(args.name) if args.name else common.default_name(set_source) result_dir = args.res_dir.expanduser().resolve() / name if result_dir.exists(): shutil.rmtree(result_dir) build_dir = result_dir / "build" sets_dir = result_dir / "sets" runs_dir = result_dir / "runs" build_dir.mkdir(parents=True) runs_dir.mkdir() run_set = common.compile_runner(set_source, rpm_root, build_dir, args.cc, args.cxx) run_cmp = compile_cmp_runner(build_dir, rpm_root, args.cxx) prepared = prepare_cases(cases, run_set, sets_dir, args.bpp) summaries = [ benchmark_case(run_cmp, item, args.runs, runs_dir / item.case.name) for item in prepared ] write_summary(result_dir / "summary.tsv", summaries) write_metadata(result_dir / "metadata.tsv", set_source, rpm_root, args.runs, cases) print_summary(summaries) print(f"\nresults\t{result_dir}") return 0 except (OSError, RuntimeError, ValueError) as exception: print(f"compare_cmp_realization: {exception}", file=sys.stderr) return 2 if __name__ == "__main__": raise SystemExit(main(sys.argv[1:]))