add script for test on random data
This commit is contained in:
Executable
+414
@@ -0,0 +1,414 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Continuously compare rpmsetcmp results from set.c and newset.c."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import shlex
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
MAX_N = 1000 # max words in file
|
||||
MAX_M = 100 # max len for word
|
||||
MIN_BPP = 10
|
||||
MAX_BPP = 32
|
||||
ALPHABET = ".0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
|
||||
LARGE_ALPHABET = "".join(chr(code) for code in range(33, 127))
|
||||
CASES = (1, 0, -2, -3)
|
||||
CC = "cc"
|
||||
CFLAGS = ("-O2", "-std=gnu11", "-D_GNU_SOURCE")
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
REPO_ROOT = SCRIPT_DIR.parent.parent
|
||||
OLD_SET_SOURCE = REPO_ROOT / "set.c"
|
||||
NEW_SET_SOURCE = REPO_ROOT / "reimplement" / "newset.c"
|
||||
MKSET_SOURCE = SCRIPT_DIR / "mkset.c"
|
||||
SETCMP_SOURCE = SCRIPT_DIR / "setcmp.c"
|
||||
COMPAT_HEADER = SCRIPT_DIR / "newset_compat.h"
|
||||
ERROR_DIR = SCRIPT_DIR / "error"
|
||||
|
||||
SET_HEADER = """\
|
||||
#ifndef ARSV_SET_H
|
||||
#define ARSV_SET_H
|
||||
|
||||
struct set;
|
||||
struct set *set_new(void);
|
||||
void set_add(struct set *set, const char *symbol);
|
||||
const char *set_fini(struct set *set, int bpp);
|
||||
int rpmsetcmp(const char *set1, const char *set2);
|
||||
|
||||
#endif
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Outcome:
|
||||
result: int | None
|
||||
returncode: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
def comparison_key(self) -> tuple[object, ...]:
|
||||
if self.result is not None:
|
||||
return ("result", self.result)
|
||||
return ("process", self.returncode, self.stdout, self.stderr)
|
||||
|
||||
def describe(self) -> str:
|
||||
if self.result is not None:
|
||||
return str(self.result)
|
||||
return (
|
||||
f"exit={self.returncode}, stdout={self.stdout!r}, "
|
||||
f"stderr={self.stderr!r}"
|
||||
)
|
||||
|
||||
|
||||
def build_binary(
|
||||
source: Path, wrapper: Path, output: Path, build_dir: Path
|
||||
) -> None:
|
||||
command = [
|
||||
CC,
|
||||
*CFLAGS,
|
||||
f"-I{build_dir}",
|
||||
"-include",
|
||||
str(COMPAT_HEADER),
|
||||
str(source),
|
||||
str(wrapper),
|
||||
"-o",
|
||||
str(output),
|
||||
]
|
||||
completed = subprocess.run(command, text=True, capture_output=True)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Compilation failed: {shlex.join(command)}\n{completed.stderr}"
|
||||
)
|
||||
|
||||
|
||||
def run_mkset(binary: Path, input_path: Path, bpp: int) -> str:
|
||||
with input_path.open("r", encoding="ascii") as input_file:
|
||||
completed = subprocess.run(
|
||||
[str(binary), str(bpp)],
|
||||
stdin=input_file,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"{binary.name} failed with code {completed.returncode}:\n"
|
||||
f"{completed.stderr}"
|
||||
)
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
def run_setcmp(binary: Path, set1: str, set2: str) -> Outcome:
|
||||
completed = subprocess.run(
|
||||
[str(binary), set1, set2],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
stdout = completed.stdout.strip()
|
||||
stderr = completed.stderr.strip()
|
||||
result: int | None = None
|
||||
|
||||
if completed.returncode == 0:
|
||||
try:
|
||||
result = int(stdout)
|
||||
except ValueError:
|
||||
pass
|
||||
elif completed.returncode == 1:
|
||||
if "set1 error" in stderr:
|
||||
result = -3
|
||||
elif "set2 error" in stderr:
|
||||
result = -4
|
||||
|
||||
return Outcome(result, completed.returncode, stdout, stderr)
|
||||
|
||||
|
||||
def random_word(alphabet: str = ALPHABET) -> str:
|
||||
length = random.randint(1, MAX_M)
|
||||
return "".join(random.choices(alphabet, k=length))
|
||||
|
||||
|
||||
def generate_words(n: int) -> list[str]:
|
||||
words: list[str] = []
|
||||
used: set[str] = set()
|
||||
while len(words) < n:
|
||||
word = random_word()
|
||||
if word not in used:
|
||||
used.add(word)
|
||||
words.append(word)
|
||||
return words
|
||||
|
||||
|
||||
def write_words(path: Path, words: list[str]) -> None:
|
||||
path.write_text("".join(f"{word}\n" for word in words), encoding="ascii")
|
||||
|
||||
|
||||
def generate_second_words(case: int, words: list[str]) -> list[str]:
|
||||
if case == 0:
|
||||
return words.copy()
|
||||
|
||||
remove_count = random.randint(1, len(words) - 1)
|
||||
removed_indices = set(random.sample(range(len(words)), remove_count))
|
||||
second_words = [
|
||||
word for index, word in enumerate(words) if index not in removed_indices
|
||||
]
|
||||
|
||||
if case == -2:
|
||||
used = set(words)
|
||||
add_count = random.randint(1, remove_count)
|
||||
while add_count > 0:
|
||||
word = random_word()
|
||||
if word not in used:
|
||||
used.add(word)
|
||||
second_words.append(word)
|
||||
add_count -= 1
|
||||
random.shuffle(second_words)
|
||||
|
||||
return second_words
|
||||
|
||||
|
||||
def generate_invalid_set(alphabet: str) -> str:
|
||||
invalid_first_chars = [character for character in alphabet if character not in "cdefghijklmnopqrstuvwxyz"]
|
||||
first = random.choice(invalid_first_chars)
|
||||
length = random.randint(1, MAX_M)
|
||||
tail = "".join(random.choices(alphabet, k=length - 1))
|
||||
return f"set:{first}{tail}"
|
||||
|
||||
|
||||
def compare_pair(
|
||||
label: str,
|
||||
setcmp: Path,
|
||||
setcmp_new: Path,
|
||||
old_set1: str,
|
||||
old_set2: str,
|
||||
new_set1: str,
|
||||
new_set2: str,
|
||||
) -> tuple[str, Outcome, Outcome]:
|
||||
old_outcome = run_setcmp(setcmp, old_set1, old_set2)
|
||||
new_outcome = run_setcmp(setcmp_new, new_set1, new_set2)
|
||||
return label, old_outcome, new_outcome
|
||||
|
||||
|
||||
def save_error(
|
||||
test_number: int,
|
||||
case: int,
|
||||
bpp: int,
|
||||
artifacts: dict[str, bytes],
|
||||
comparisons: list[tuple[str, Outcome, Outcome]],
|
||||
) -> None:
|
||||
timestamp = time.time_ns()
|
||||
case_name = str(case).replace("-", "minus")
|
||||
stem = f"test_{test_number}_case_{case_name}_bpp{bpp}_{timestamp}"
|
||||
|
||||
for suffix, content in artifacts.items():
|
||||
(ERROR_DIR / f"{stem}_{suffix}").write_bytes(content)
|
||||
|
||||
details = []
|
||||
for label, old_outcome, new_outcome in comparisons:
|
||||
details.append(
|
||||
f"{label}: setcmp={old_outcome.describe()} "
|
||||
f"setcmp_new={new_outcome.describe()}"
|
||||
)
|
||||
(ERROR_DIR / f"{stem}_results.txt").write_text(
|
||||
"\n".join(details) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(f"mismatch: saved inputs as {ERROR_DIR / stem}_*", flush=True)
|
||||
|
||||
|
||||
def run_relation_case(
|
||||
case: int,
|
||||
test_number: int,
|
||||
n: int,
|
||||
bpp: int,
|
||||
input1_path: Path,
|
||||
input2_path: Path,
|
||||
mkset: Path,
|
||||
mkset_new: Path,
|
||||
setcmp: Path,
|
||||
setcmp_new: Path,
|
||||
) -> None:
|
||||
words1 = generate_words(n)
|
||||
words2 = generate_second_words(case, words1)
|
||||
write_words(input1_path, words1)
|
||||
write_words(input2_path, words2)
|
||||
|
||||
old_set1 = run_mkset(mkset, input1_path, bpp)
|
||||
new_set1 = run_mkset(mkset_new, input1_path, bpp)
|
||||
old_set2 = run_mkset(mkset, input2_path, bpp)
|
||||
new_set2 = run_mkset(mkset_new, input2_path, bpp)
|
||||
|
||||
comparisons = [
|
||||
compare_pair(
|
||||
str(case),
|
||||
setcmp,
|
||||
setcmp_new,
|
||||
old_set1,
|
||||
old_set2,
|
||||
new_set1,
|
||||
new_set2,
|
||||
)
|
||||
]
|
||||
if case == 1:
|
||||
comparisons.append(
|
||||
compare_pair(
|
||||
"-1 (swapped)",
|
||||
setcmp,
|
||||
setcmp_new,
|
||||
old_set2,
|
||||
old_set1,
|
||||
new_set2,
|
||||
new_set1,
|
||||
)
|
||||
)
|
||||
|
||||
mismatches = [
|
||||
comparison
|
||||
for comparison in comparisons
|
||||
if comparison[1].comparison_key() != comparison[2].comparison_key()
|
||||
]
|
||||
if mismatches:
|
||||
save_error(
|
||||
test_number,
|
||||
case,
|
||||
bpp,
|
||||
{
|
||||
"input1.txt": input1_path.read_bytes(),
|
||||
"input2.txt": input2_path.read_bytes(),
|
||||
},
|
||||
comparisons,
|
||||
)
|
||||
|
||||
|
||||
def run_invalid_case(
|
||||
test_number: int,
|
||||
n: int,
|
||||
bpp: int,
|
||||
input_path: Path,
|
||||
mkset: Path,
|
||||
mkset_new: Path,
|
||||
setcmp: Path,
|
||||
setcmp_new: Path,
|
||||
invalid_alphabet: str | None = None,
|
||||
) -> None:
|
||||
words = generate_words(n)
|
||||
write_words(input_path, words)
|
||||
old_valid_set = run_mkset(mkset, input_path, bpp)
|
||||
new_valid_set = run_mkset(mkset_new, input_path, bpp)
|
||||
|
||||
if invalid_alphabet is None:
|
||||
invalid_alphabet = random.choice((ALPHABET, LARGE_ALPHABET))
|
||||
invalid_set = generate_invalid_set(invalid_alphabet)
|
||||
alphabet_name = "same" if invalid_alphabet == ALPHABET else "large"
|
||||
|
||||
comparisons = [
|
||||
compare_pair(
|
||||
f"-3 ({alphabet_name} alphabet)",
|
||||
setcmp,
|
||||
setcmp_new,
|
||||
invalid_set,
|
||||
old_valid_set,
|
||||
invalid_set,
|
||||
new_valid_set,
|
||||
),
|
||||
compare_pair(
|
||||
f"-4 ({alphabet_name} alphabet, swapped)",
|
||||
setcmp,
|
||||
setcmp_new,
|
||||
old_valid_set,
|
||||
invalid_set,
|
||||
new_valid_set,
|
||||
invalid_set,
|
||||
),
|
||||
]
|
||||
|
||||
mismatches = [
|
||||
comparison
|
||||
for comparison in comparisons
|
||||
if comparison[1].comparison_key() != comparison[2].comparison_key()
|
||||
]
|
||||
if mismatches:
|
||||
save_error(
|
||||
test_number,
|
||||
-3,
|
||||
bpp,
|
||||
{
|
||||
"input.txt": input_path.read_bytes(),
|
||||
"invalid_set.txt": f"{invalid_set}\n".encode("ascii"),
|
||||
},
|
||||
comparisons,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if MAX_N < 2 or MAX_M < 1:
|
||||
raise ValueError("MAX_N must be at least 2 and MAX_M at least 1")
|
||||
if not 10 <= MIN_BPP <= MAX_BPP <= 32:
|
||||
raise ValueError("MIN_BPP and MAX_BPP must be in the range 10..32")
|
||||
|
||||
ERROR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="arsv-rpmsetcmp-") as temporary:
|
||||
build_dir = Path(temporary)
|
||||
(build_dir / "rpmlib.h").touch()
|
||||
(build_dir / "system.h").touch()
|
||||
(build_dir / "set.h").write_text(SET_HEADER, encoding="ascii")
|
||||
|
||||
mkset = build_dir / "mkset"
|
||||
mkset_new = build_dir / "mkset_new"
|
||||
setcmp = build_dir / "setcmp"
|
||||
setcmp_new = build_dir / "setcmp_new"
|
||||
input1_path = build_dir / "input1.txt"
|
||||
input2_path = build_dir / "input2.txt"
|
||||
|
||||
build_binary(OLD_SET_SOURCE, MKSET_SOURCE, mkset, build_dir)
|
||||
build_binary(NEW_SET_SOURCE, MKSET_SOURCE, mkset_new, build_dir)
|
||||
build_binary(OLD_SET_SOURCE, SETCMP_SOURCE, setcmp, build_dir)
|
||||
build_binary(NEW_SET_SOURCE, SETCMP_SOURCE, setcmp_new, build_dir)
|
||||
|
||||
tests_completed = 0
|
||||
while True:
|
||||
case = random.choice(CASES)
|
||||
min_n = 2 if case in (1, -2) else 1
|
||||
n = random.randint(min_n, MAX_N)
|
||||
bpp = random.randint(MIN_BPP, MAX_BPP)
|
||||
current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
print(
|
||||
f"{current_time} tests={tests_completed} case={case} "
|
||||
f"bpp={bpp} n={n}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if case == -3:
|
||||
run_invalid_case(
|
||||
tests_completed + 1,
|
||||
n,
|
||||
bpp,
|
||||
input1_path,
|
||||
mkset,
|
||||
mkset_new,
|
||||
setcmp,
|
||||
setcmp_new,
|
||||
)
|
||||
else:
|
||||
run_relation_case(
|
||||
case,
|
||||
tests_completed + 1,
|
||||
n,
|
||||
bpp,
|
||||
input1_path,
|
||||
input2_path,
|
||||
mkset,
|
||||
mkset_new,
|
||||
setcmp,
|
||||
setcmp_new,
|
||||
)
|
||||
|
||||
tests_completed += 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user