one more test on real sysiphus packets
This commit is contained in:
@@ -0,0 +1,385 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure P(hash output bit = 1) over a line-oriented ASCII corpus."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from statistics import fmean
|
||||
from typing import TextIO
|
||||
|
||||
from plot_probability_map import ProbabilityRow, render_bit_probabilities
|
||||
from probability_map import HASHES, HASH_FUNCS_DIR
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
DEFAULT_OUTPUT_DIR = ROOT / "corpus_bit_distribution"
|
||||
|
||||
|
||||
class DistributionError(RuntimeError):
|
||||
"""Raised when a corpus or hash adapter cannot be processed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HashSpec:
|
||||
bits: int
|
||||
expression: str
|
||||
|
||||
|
||||
HASH_SPECS = {
|
||||
"jenkinsOAAT": HashSpec(32, "jenkins_oaat(word, length)"),
|
||||
"xxh64": HashSpec(64, "xxh64(word, length, XXH64_SEED)"),
|
||||
"t1ha2": HashSpec(64, "t1ha2_atonce(word, length, T1HA2_SEED)"),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Distribution:
|
||||
hash_name: str
|
||||
samples: int
|
||||
bits: int
|
||||
ones: tuple[int, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.samples < 1:
|
||||
raise ValueError("число образцов должно быть положительным")
|
||||
if self.bits < 1 or len(self.ones) != self.bits:
|
||||
raise ValueError("число счётчиков должно совпадать с шириной хэша")
|
||||
if any(count < 0 or count > self.samples for count in self.ones):
|
||||
raise ValueError("счётчик единиц должен находиться между 0 и samples")
|
||||
|
||||
@property
|
||||
def probabilities(self) -> tuple[float, ...]:
|
||||
return tuple(count / self.samples for count in self.ones)
|
||||
|
||||
|
||||
def c_include_path(path: Path) -> str:
|
||||
"""Return a C string-safe absolute include path."""
|
||||
return str(path.resolve()).replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
|
||||
def make_adapter_source(source: Path, spec: HashSpec) -> str:
|
||||
"""Build a C program that aggregates bit counts using an existing hash source."""
|
||||
return f'''#define _POSIX_C_SOURCE 200809L
|
||||
#define main arsv_original_hash_cli_main
|
||||
#include "{c_include_path(source)}"
|
||||
#undef main
|
||||
#include <inttypes.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
int main(void)
|
||||
{{
|
||||
char *buffer = NULL;
|
||||
size_t capacity = 0;
|
||||
uint64_t samples = UINT64_C(0);
|
||||
uint64_t ones[{spec.bits}] = {{UINT64_C(0)}};
|
||||
ssize_t bytes_read;
|
||||
|
||||
while ((bytes_read = getline(&buffer, &capacity, stdin)) >= 0) {{
|
||||
size_t length = (size_t)bytes_read;
|
||||
const unsigned char *word = (const unsigned char *)buffer;
|
||||
|
||||
while (length > 0U && is_ascii_trailing_space(word[length - 1U])) {{
|
||||
--length;
|
||||
}}
|
||||
if (length == 0U) {{
|
||||
fprintf(stderr, "corpus contains an empty line\\n");
|
||||
free(buffer);
|
||||
return EXIT_FAILURE;
|
||||
}}
|
||||
for (size_t index = 0; index < length; ++index) {{
|
||||
if (word[index] > 0x7fU) {{
|
||||
fprintf(stderr, "corpus must contain ASCII characters only\\n");
|
||||
free(buffer);
|
||||
return EXIT_FAILURE;
|
||||
}}
|
||||
}}
|
||||
if (samples == UINT64_MAX) {{
|
||||
fprintf(stderr, "sample counter overflow\\n");
|
||||
free(buffer);
|
||||
return EXIT_FAILURE;
|
||||
}}
|
||||
|
||||
uint64_t hash = (uint64_t)({spec.expression});
|
||||
for (unsigned int bit = 0U; bit < {spec.bits}U; ++bit) {{
|
||||
ones[bit] += (hash >> bit) & UINT64_C(1);
|
||||
}}
|
||||
++samples;
|
||||
}}
|
||||
if (ferror(stdin)) {{
|
||||
fprintf(stderr, "failed to read corpus\\n");
|
||||
free(buffer);
|
||||
return EXIT_FAILURE;
|
||||
}}
|
||||
free(buffer);
|
||||
|
||||
printf("%" PRIu64 ",{spec.bits}", samples);
|
||||
for (unsigned int bit = 0U; bit < {spec.bits}U; ++bit) {{
|
||||
printf(",%" PRIu64, ones[bit]);
|
||||
}}
|
||||
putchar('\\n');
|
||||
return EXIT_SUCCESS;
|
||||
}}
|
||||
'''
|
||||
|
||||
|
||||
def measure_hash(
|
||||
hash_name: str,
|
||||
corpus: Path,
|
||||
hash_funcs_dir: Path = HASH_FUNCS_DIR,
|
||||
) -> Distribution:
|
||||
"""Compile a batch adapter and count set output bits for every corpus line."""
|
||||
try:
|
||||
spec = HASH_SPECS[hash_name]
|
||||
except KeyError as error:
|
||||
supported = ", ".join(HASH_SPECS)
|
||||
raise DistributionError(
|
||||
f"нет batch-адаптера для {hash_name!r}; доступны: {supported}"
|
||||
) from error
|
||||
|
||||
source = hash_funcs_dir / hash_name / "bin_hash.c"
|
||||
if not source.is_file():
|
||||
raise DistributionError(f"не найден исходник хэша: {source}")
|
||||
if not corpus.is_file():
|
||||
raise DistributionError(f"не найден corpus: {corpus}")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix=f"arsv-{hash_name}-") as temporary:
|
||||
root = Path(temporary)
|
||||
adapter_source = root / "batch_counts.c"
|
||||
adapter = root / "batch_counts"
|
||||
adapter_source.write_text(make_adapter_source(source, spec), encoding="utf-8")
|
||||
command = [
|
||||
"cc",
|
||||
"-std=c11",
|
||||
"-O3",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Wpedantic",
|
||||
"-Werror",
|
||||
str(adapter_source),
|
||||
"-o",
|
||||
str(adapter),
|
||||
]
|
||||
compiled = subprocess.run(command, text=True, capture_output=True, check=False)
|
||||
if compiled.returncode != 0:
|
||||
details = compiled.stderr.strip() or compiled.stdout.strip()
|
||||
raise DistributionError(f"не удалось собрать адаптер {hash_name}: {details}")
|
||||
|
||||
try:
|
||||
with corpus.open("rb") as stream:
|
||||
measured = subprocess.run(
|
||||
[str(adapter)],
|
||||
stdin=stream,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError as error:
|
||||
raise DistributionError(f"не удалось прочитать {corpus}: {error}") from error
|
||||
if measured.returncode != 0:
|
||||
details = measured.stderr.strip() or measured.stdout.strip()
|
||||
raise DistributionError(f"batch-прогон {hash_name} завершился ошибкой: {details}")
|
||||
|
||||
fields = measured.stdout.strip().split(",")
|
||||
if len(fields) != spec.bits + 2:
|
||||
raise DistributionError(
|
||||
f"batch-прогон {hash_name} вернул {len(fields)} полей вместо {spec.bits + 2}"
|
||||
)
|
||||
try:
|
||||
samples = int(fields[0])
|
||||
bits = int(fields[1])
|
||||
ones = tuple(int(value) for value in fields[2:])
|
||||
except ValueError as error:
|
||||
raise DistributionError(
|
||||
f"batch-прогон {hash_name} вернул некорректные счётчики"
|
||||
) from error
|
||||
if bits != spec.bits:
|
||||
raise DistributionError(
|
||||
f"batch-прогон {hash_name} сообщил ширину {bits} вместо {spec.bits}"
|
||||
)
|
||||
try:
|
||||
return Distribution(hash_name, samples, bits, ones)
|
||||
except ValueError as error:
|
||||
raise DistributionError(f"некорректный результат {hash_name}: {error}") from error
|
||||
|
||||
|
||||
def write_distribution_csv(stream: TextIO, distribution: Distribution) -> None:
|
||||
"""Write bit counts and probabilities from LSB to MSB."""
|
||||
writer = csv.writer(stream, lineterminator="\n")
|
||||
writer.writerow(["bit", "ones", "zeros", "probability", "deviation_from_0.5"])
|
||||
for bit, (ones, probability) in enumerate(
|
||||
zip(distribution.ones, distribution.probabilities, strict=True)
|
||||
):
|
||||
writer.writerow(
|
||||
[
|
||||
bit,
|
||||
ones,
|
||||
distribution.samples - ones,
|
||||
f"{probability:.9f}",
|
||||
f"{abs(probability - 0.5):.9f}",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def write_summary_csv(stream: TextIO, distributions: Sequence[Distribution]) -> None:
|
||||
writer = csv.writer(stream, lineterminator="\n")
|
||||
writer.writerow(
|
||||
[
|
||||
"hash",
|
||||
"samples",
|
||||
"bits",
|
||||
"mean_probability",
|
||||
"mean_abs_deviation",
|
||||
"max_abs_deviation",
|
||||
"min_probability",
|
||||
"max_probability",
|
||||
]
|
||||
)
|
||||
for distribution in distributions:
|
||||
probabilities = distribution.probabilities
|
||||
deviations = [abs(value - 0.5) for value in probabilities]
|
||||
writer.writerow(
|
||||
[
|
||||
distribution.hash_name,
|
||||
distribution.samples,
|
||||
distribution.bits,
|
||||
f"{fmean(probabilities):.9f}",
|
||||
f"{fmean(deviations):.9f}",
|
||||
f"{max(deviations):.9f}",
|
||||
f"{min(probabilities):.9f}",
|
||||
f"{max(probabilities):.9f}",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_manifest(
|
||||
path: Path,
|
||||
corpus: Path,
|
||||
distributions: Sequence[Distribution],
|
||||
) -> None:
|
||||
samples = {distribution.samples for distribution in distributions}
|
||||
if len(samples) != 1:
|
||||
raise DistributionError("хэши обработали разное число строк corpus")
|
||||
content = "\n".join(
|
||||
[
|
||||
f"corpus={corpus}",
|
||||
f"corpus_bytes={corpus.stat().st_size}",
|
||||
f"corpus_sha256={sha256_file(corpus)}",
|
||||
f"samples={samples.pop()}",
|
||||
f"hashes={','.join(item.hash_name for item in distributions)}",
|
||||
"metric=P(hash_output_bit=1)",
|
||||
"bit_order=bit_0_is_LSB",
|
||||
"input_format=one_ASCII_symbol_per_line",
|
||||
"",
|
||||
]
|
||||
)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def run(
|
||||
corpus: Path,
|
||||
output_directory: Path,
|
||||
hash_names: Sequence[str],
|
||||
) -> list[Distribution]:
|
||||
"""Measure all requested hashes and create CSV, manifest, and one PNG."""
|
||||
names = list(dict.fromkeys(hash_names))
|
||||
if not names:
|
||||
raise DistributionError("нужно указать хотя бы один хэш")
|
||||
|
||||
distributions = [measure_hash(name, corpus) for name in names]
|
||||
sample_counts = {item.samples for item in distributions}
|
||||
if len(sample_counts) != 1:
|
||||
raise DistributionError("хэши обработали разное число строк corpus")
|
||||
|
||||
output_directory.mkdir(parents=True, exist_ok=True)
|
||||
for distribution in distributions:
|
||||
path = output_directory / f"{distribution.hash_name}.csv"
|
||||
with path.open("w", encoding="utf-8", newline="") as stream:
|
||||
write_distribution_csv(stream, distribution)
|
||||
|
||||
with (output_directory / "summary.csv").open(
|
||||
"w", encoding="utf-8", newline=""
|
||||
) as stream:
|
||||
write_summary_csv(stream, distributions)
|
||||
write_manifest(output_directory / "manifest.txt", corpus, distributions)
|
||||
|
||||
rows = [
|
||||
ProbabilityRow(
|
||||
operation=item.hash_name,
|
||||
pairs=item.samples,
|
||||
probabilities=list(item.probabilities),
|
||||
)
|
||||
for item in distributions
|
||||
]
|
||||
render_bit_probabilities(
|
||||
rows,
|
||||
output_directory / "bit_distribution.png",
|
||||
"C++ Provides corpus: P(hash output bit = 1)",
|
||||
)
|
||||
return distributions
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Считает прямое распределение единиц по выходным битам хэша "
|
||||
"для ASCII corpus (одна строка — одно значение). bit_0 — младший бит."
|
||||
)
|
||||
)
|
||||
parser.add_argument("corpus", type=Path, help="текстовый corpus")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=DEFAULT_OUTPUT_DIR,
|
||||
help=f"отдельный каталог результатов (по умолчанию: {DEFAULT_OUTPUT_DIR})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hash",
|
||||
dest="hashes",
|
||||
action="append",
|
||||
help="проверить указанный хэш; можно повторять (по умолчанию все)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
arguments = parser.parse_args(argv)
|
||||
try:
|
||||
distributions = run(
|
||||
corpus=arguments.corpus,
|
||||
output_directory=arguments.output,
|
||||
hash_names=arguments.hashes or HASHES,
|
||||
)
|
||||
except (DistributionError, OSError, ValueError) as error:
|
||||
parser.error(str(error))
|
||||
|
||||
for distribution in distributions:
|
||||
probabilities = distribution.probabilities
|
||||
mean_deviation = fmean(abs(value - 0.5) for value in probabilities)
|
||||
print(
|
||||
f"hash={distribution.hash_name} samples={distribution.samples} "
|
||||
f"bits={distribution.bits} mean_abs_deviation={mean_deviation:.9f}"
|
||||
)
|
||||
print(f"wrote {arguments.output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+426
@@ -0,0 +1,426 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract ALT RPM Provides symbol names into a hash-testing corpus.
|
||||
|
||||
Local RPM paths are processed directly. Other positional arguments are resolved
|
||||
as binary package names through the ALT Repository Database (RDB).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO, Sequence, cast
|
||||
|
||||
|
||||
RDB_BASE = "https://rdb.altlinux.org/api"
|
||||
NEWC_MAGICS = {b"070701", b"070702"}
|
||||
NEWC_HEADER_SIZE = 110
|
||||
COPY_CHUNK_SIZE = 1024 * 1024
|
||||
MAX_CPIO_NAME_SIZE = 1024 * 1024
|
||||
|
||||
|
||||
class CorpusError(RuntimeError):
|
||||
"""A user-facing extraction error."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackageInput:
|
||||
label: str
|
||||
path: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackageSymbols:
|
||||
label: str
|
||||
elf_files: int
|
||||
symbols: frozenset[str]
|
||||
|
||||
|
||||
def parse_arguments(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Extract canonical ALT rpm-build Provided ELF symbols from multiple "
|
||||
"RPM packages, one symbol per output line."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"packages",
|
||||
nargs="+",
|
||||
help="local .rpm path or ALT binary package name",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
default="-",
|
||||
help="output file (default: stdout)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cpp-only",
|
||||
action="store_true",
|
||||
help="keep only Itanium C++ ABI mangled names beginning with _Z",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--branch",
|
||||
default="p11",
|
||||
help="ALT branch for package-name resolution (default: p11)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--arch",
|
||||
default="x86_64",
|
||||
help="binary package architecture (default: x86_64)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rpm2cpio",
|
||||
default=shutil.which("rpm2cpio") or "rpm2cpio",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--provided-symbols",
|
||||
default=(
|
||||
"/usr/lib/rpm/provided_symbols"
|
||||
if Path("/usr/lib/rpm/provided_symbols").is_file()
|
||||
else shutil.which("provided_symbols") or "provided_symbols"
|
||||
),
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--file-command",
|
||||
default=shutil.which("file") or "file",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def read_json(url: str) -> dict[str, object]:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=60) as response:
|
||||
return json.load(response)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||
raise CorpusError(f"failed to read ALT RDB response from {url}: {error}") from error
|
||||
|
||||
|
||||
def download_package(name: str, branch: str, arch: str, directory: Path) -> PackageInput:
|
||||
query = urllib.parse.urlencode({"branch": branch, "name": name, "arch": arch})
|
||||
metadata_url = f"{RDB_BASE}/site/pkghash_by_binary_name?{query}"
|
||||
metadata = read_json(metadata_url)
|
||||
package_hash = metadata.get("pkghash")
|
||||
if not isinstance(package_hash, str) or not package_hash:
|
||||
raise CorpusError(f"ALT RDB did not return a package hash for {name!r}")
|
||||
|
||||
download_url = (
|
||||
f"{RDB_BASE}/site/package_downloads_bin/{package_hash}?"
|
||||
f"{urllib.parse.urlencode({'branch': branch, 'arch': arch})}"
|
||||
)
|
||||
download_data = read_json(download_url)
|
||||
downloads = download_data.get("downloads")
|
||||
if not isinstance(downloads, list):
|
||||
raise CorpusError(f"ALT RDB did not return downloads for {name!r}")
|
||||
|
||||
package_record: dict[str, object] | None = None
|
||||
for architecture_record in downloads:
|
||||
if not isinstance(architecture_record, dict):
|
||||
continue
|
||||
if architecture_record.get("arch") != arch:
|
||||
continue
|
||||
packages = architecture_record.get("packages")
|
||||
if isinstance(packages, list):
|
||||
for candidate in packages:
|
||||
if isinstance(candidate, dict):
|
||||
package_record = candidate
|
||||
break
|
||||
if package_record is not None:
|
||||
break
|
||||
|
||||
if package_record is None:
|
||||
raise CorpusError(f"ALT RDB has no {arch} RPM download for {name!r}")
|
||||
|
||||
filename = package_record.get("name")
|
||||
url = package_record.get("url")
|
||||
expected_md5 = package_record.get("md5")
|
||||
if not isinstance(filename, str) or not filename.endswith(".rpm"):
|
||||
raise CorpusError(f"ALT RDB returned an invalid RPM filename for {name!r}")
|
||||
if not isinstance(url, str) or not url.startswith(("https://", "http://")):
|
||||
raise CorpusError(f"ALT RDB returned an invalid RPM URL for {name!r}")
|
||||
|
||||
destination = directory / filename
|
||||
temporary = destination.with_suffix(destination.suffix + ".part")
|
||||
digest = hashlib.md5(usedforsecurity=False)
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=180) as response, temporary.open("wb") as output:
|
||||
while chunk := response.read(COPY_CHUNK_SIZE):
|
||||
output.write(chunk)
|
||||
digest.update(chunk)
|
||||
except OSError as error:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise CorpusError(f"failed to download {name!r} from {url}: {error}") from error
|
||||
|
||||
if isinstance(expected_md5, str) and digest.hexdigest().lower() != expected_md5.lower():
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise CorpusError(f"MD5 mismatch for downloaded package {name!r}")
|
||||
temporary.replace(destination)
|
||||
return PackageInput(f"{name} ({filename})", destination)
|
||||
|
||||
|
||||
def resolve_packages(
|
||||
specifications: Sequence[str], branch: str, arch: str, directory: Path
|
||||
) -> list[PackageInput]:
|
||||
resolved: list[PackageInput] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for specification in specifications:
|
||||
candidate = Path(specification).expanduser()
|
||||
if candidate.is_file():
|
||||
package = PackageInput(candidate.name, candidate.resolve())
|
||||
elif candidate.exists():
|
||||
raise CorpusError(f"package input is not a regular file: {candidate}")
|
||||
elif "/" in specification or specification.endswith(".rpm"):
|
||||
raise CorpusError(f"RPM file does not exist: {candidate}")
|
||||
else:
|
||||
package = download_package(specification, branch, arch, directory)
|
||||
|
||||
identity = (package.label, str(package.path))
|
||||
if identity not in seen:
|
||||
resolved.append(package)
|
||||
seen.add(identity)
|
||||
return resolved
|
||||
|
||||
|
||||
def read_exact(stream: BinaryIO, size: int, context: str) -> bytes:
|
||||
chunks: list[bytes] = []
|
||||
remaining = size
|
||||
while remaining:
|
||||
chunk = stream.read(remaining)
|
||||
if not chunk:
|
||||
raise CorpusError(f"truncated newc stream while reading {context}")
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def discard_exact(stream: BinaryIO, size: int, context: str) -> None:
|
||||
remaining = size
|
||||
while remaining:
|
||||
chunk = stream.read(min(remaining, COPY_CHUNK_SIZE))
|
||||
if not chunk:
|
||||
raise CorpusError(f"truncated newc stream while reading {context}")
|
||||
remaining -= len(chunk)
|
||||
|
||||
|
||||
def copy_exact(stream: BinaryIO, output: BinaryIO, size: int, context: str) -> None:
|
||||
remaining = size
|
||||
while remaining:
|
||||
chunk = stream.read(min(remaining, COPY_CHUNK_SIZE))
|
||||
if not chunk:
|
||||
raise CorpusError(f"truncated newc stream while reading {context}")
|
||||
output.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
|
||||
|
||||
def parse_newc_header(header: bytes) -> tuple[int, int, int]:
|
||||
if len(header) != NEWC_HEADER_SIZE or header[:6] not in NEWC_MAGICS:
|
||||
raise CorpusError("rpm2cpio output is not a valid newc archive")
|
||||
try:
|
||||
fields = [int(header[6 + index * 8 : 14 + index * 8], 16) for index in range(13)]
|
||||
except ValueError as error:
|
||||
raise CorpusError("newc header contains a non-hexadecimal field") from error
|
||||
mode = fields[1]
|
||||
file_size = fields[6]
|
||||
name_size = fields[11]
|
||||
if name_size < 1 or name_size > MAX_CPIO_NAME_SIZE:
|
||||
raise CorpusError(f"invalid newc pathname size: {name_size}")
|
||||
return mode, file_size, name_size
|
||||
|
||||
|
||||
def extract_elf_members(stream: BinaryIO, directory: Path) -> list[Path]:
|
||||
elf_files: list[Path] = []
|
||||
member_index = 0
|
||||
while True:
|
||||
header = read_exact(stream, NEWC_HEADER_SIZE, "header")
|
||||
mode, file_size, name_size = parse_newc_header(header)
|
||||
raw_name = read_exact(stream, name_size, "pathname")
|
||||
if raw_name[-1:] != b"\0":
|
||||
raise CorpusError("newc pathname is not NUL-terminated")
|
||||
discard_exact(stream, -(NEWC_HEADER_SIZE + name_size) % 4, "pathname padding")
|
||||
|
||||
if raw_name[:-1] == b"TRAILER!!!":
|
||||
discard_exact(stream, file_size, "trailer data")
|
||||
discard_exact(stream, -file_size % 4, "trailer padding")
|
||||
break
|
||||
|
||||
member_index += 1
|
||||
prefix_size = min(file_size, 4)
|
||||
prefix = read_exact(stream, prefix_size, "member data")
|
||||
remaining = file_size - prefix_size
|
||||
if stat.S_ISREG(mode) and prefix == b"\x7fELF":
|
||||
destination = directory / f"elf-{member_index:06d}"
|
||||
with destination.open("wb") as output:
|
||||
output.write(prefix)
|
||||
copy_exact(stream, output, remaining, "ELF member data")
|
||||
elf_files.append(destination)
|
||||
else:
|
||||
discard_exact(stream, remaining, "member data")
|
||||
discard_exact(stream, -file_size % 4, "member padding")
|
||||
return elf_files
|
||||
|
||||
|
||||
def extract_package_symbols(
|
||||
package: PackageInput,
|
||||
rpm2cpio: str,
|
||||
provided_symbols: str,
|
||||
file_command: str,
|
||||
directory: Path,
|
||||
) -> PackageSymbols:
|
||||
package_directory = directory / f"package-{len(list(directory.iterdir())):04d}"
|
||||
package_directory.mkdir()
|
||||
process = subprocess.Popen(
|
||||
[rpm2cpio, str(package.path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
assert process.stdout is not None
|
||||
assert process.stderr is not None
|
||||
try:
|
||||
extracted_elf_files = extract_elf_members(
|
||||
cast(BinaryIO, process.stdout), package_directory
|
||||
)
|
||||
except Exception:
|
||||
process.kill()
|
||||
process.communicate()
|
||||
raise
|
||||
finally:
|
||||
process.stdout.close()
|
||||
|
||||
stderr = process.stderr.read().decode(errors="replace")
|
||||
return_code = process.wait()
|
||||
if return_code != 0:
|
||||
raise CorpusError(
|
||||
f"rpm2cpio failed for {package.label} with status {return_code}: {stderr.strip()}"
|
||||
)
|
||||
|
||||
elf_files: list[Path] = []
|
||||
environment = os.environ.copy()
|
||||
environment["LC_ALL"] = "C"
|
||||
for path in extracted_elf_files:
|
||||
result = subprocess.run(
|
||||
[file_command, "--brief", "--", str(path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise CorpusError(
|
||||
f"file failed for {package.label} with status "
|
||||
f"{result.returncode}: {result.stderr.strip()}"
|
||||
)
|
||||
description = f" {result.stdout.strip()} "
|
||||
if (
|
||||
" ELF " in description
|
||||
and " shared object, " in description
|
||||
and " shared object, no machine, " not in description
|
||||
):
|
||||
elf_files.append(path)
|
||||
|
||||
if not elf_files:
|
||||
return PackageSymbols(package.label, 0, frozenset())
|
||||
|
||||
result = subprocess.run(
|
||||
[provided_symbols, *(str(path) for path in elf_files)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise CorpusError(
|
||||
f"provided_symbols failed for {package.label} with status "
|
||||
f"{result.returncode}: {result.stderr.strip()}"
|
||||
)
|
||||
symbols = frozenset(line for line in result.stdout.splitlines() if line)
|
||||
return PackageSymbols(package.label, len(elf_files), symbols)
|
||||
|
||||
|
||||
def render_symbols(symbols: set[str]) -> str:
|
||||
if not symbols:
|
||||
return ""
|
||||
return "\n".join(sorted(symbols)) + "\n"
|
||||
|
||||
|
||||
def write_output(path: str, content: str) -> None:
|
||||
if path == "-":
|
||||
sys.stdout.write(content)
|
||||
return
|
||||
destination = Path(path).expanduser()
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w", encoding="utf-8", dir=destination.parent, delete=False
|
||||
) as temporary:
|
||||
temporary.write(content)
|
||||
temporary_path = Path(temporary.name)
|
||||
temporary_path.replace(destination)
|
||||
|
||||
|
||||
def run(arguments: argparse.Namespace) -> int:
|
||||
with tempfile.TemporaryDirectory(prefix="provided-symbols-") as temporary_name:
|
||||
temporary = Path(temporary_name)
|
||||
download_directory = temporary / "downloads"
|
||||
extraction_directory = temporary / "extract"
|
||||
download_directory.mkdir()
|
||||
extraction_directory.mkdir()
|
||||
packages = resolve_packages(
|
||||
arguments.packages, arguments.branch, arguments.arch, download_directory
|
||||
)
|
||||
|
||||
combined: set[str] = set()
|
||||
total_elf_files = 0
|
||||
for package in packages:
|
||||
package_symbols = extract_package_symbols(
|
||||
package,
|
||||
arguments.rpm2cpio,
|
||||
arguments.provided_symbols,
|
||||
arguments.file_command,
|
||||
extraction_directory,
|
||||
)
|
||||
selected = {
|
||||
symbol
|
||||
for symbol in package_symbols.symbols
|
||||
if not arguments.cpp_only or symbol.startswith("_Z")
|
||||
}
|
||||
combined.update(selected)
|
||||
total_elf_files += package_symbols.elf_files
|
||||
print(
|
||||
f"package={package_symbols.label} "
|
||||
f"elf_files={package_symbols.elf_files} "
|
||||
f"symbols={len(package_symbols.symbols)} selected={len(selected)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
write_output(arguments.output, render_symbols(combined))
|
||||
print(
|
||||
f"packages={len(packages)} elf_files={total_elf_files} "
|
||||
f"unique_symbols={len(combined)} output={arguments.output}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
try:
|
||||
return run(parse_arguments(argv))
|
||||
except (CorpusError, OSError, subprocess.SubprocessError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,22 @@
|
||||
branch=p11
|
||||
arch=x86_64
|
||||
filter=Itanium C++ ABI mangled names beginning with _Z
|
||||
extractor=/usr/lib/rpm/provided_symbols
|
||||
selection=largest x86_64 C++ candidates found by Provides set-string length in apt-cache dumpavail
|
||||
candidate=slicer set_length=509743
|
||||
candidate=libmlir22.1 set_length=303303
|
||||
candidate=libmlir21.1 set_length=284616
|
||||
candidate=libmlir18.1 set_length=204305
|
||||
candidate=libmlir20.1 set_length=182470
|
||||
|
||||
package=slicer (slicer-5.10.0-alt1.x86_64.rpm) elf_files=227 symbols=380667 selected=380530
|
||||
package=libmlir22.1 (libmlir22.1-22.1.1-alt0.1.x86_64.rpm) elf_files=9 symbols=146036 selected=145767
|
||||
package=libmlir21.1 (libmlir21.1-21.1.5-alt0.1.x86_64.rpm) elf_files=8 symbols=136102 selected=135854
|
||||
package=libmlir20.1 (libmlir20.1-20.1.8-alt0.4.x86_64.rpm) elf_files=8 symbols=90904 selected=90656
|
||||
package=libmlir18.1 (libmlir18.1-18.1.8-alt0.2.x86_64.rpm) elf_files=7 symbols=103099 selected=102835
|
||||
packages=5 elf_files=259 unique_symbols=577509 output=cpp_provides_p11.txt
|
||||
similar_lcp12=575206 share=0.996012
|
||||
similar_lcp24=548917 share=0.950491
|
||||
median_max_lcp=61
|
||||
max_lcp=1018
|
||||
sha256=bf8e760bdf4d52b42040cff4625de73988d8dd6b517566efd186c71e3382753d
|
||||
File diff suppressed because it is too large
Load Diff
BIN
Binary file not shown.
|
After Width: | Height: | Size: 39 KiB |
@@ -0,0 +1,33 @@
|
||||
bit,ones,zeros,probability,deviation_from_0.5
|
||||
0,287790,289719,0.498329896,0.001670104
|
||||
1,288092,289417,0.498852832,0.001147168
|
||||
2,288750,288759,0.499992208,0.000007792
|
||||
3,288546,288963,0.499638967,0.000361033
|
||||
4,288778,288731,0.500040692,0.000040692
|
||||
5,289060,288449,0.500528996,0.000528996
|
||||
6,288814,288695,0.500103029,0.000103029
|
||||
7,288788,288721,0.500058008,0.000058008
|
||||
8,288712,288797,0.499926408,0.000073592
|
||||
9,288209,289300,0.499055426,0.000944574
|
||||
10,289080,288429,0.500563628,0.000563628
|
||||
11,288964,288545,0.500362765,0.000362765
|
||||
12,289040,288469,0.500494365,0.000494365
|
||||
13,289175,288334,0.500728127,0.000728127
|
||||
14,288883,288626,0.500222507,0.000222507
|
||||
15,288908,288601,0.500265797,0.000265797
|
||||
16,288259,289250,0.499142005,0.000857995
|
||||
17,288493,289016,0.499547193,0.000452807
|
||||
18,288101,289408,0.498868416,0.001131584
|
||||
19,289164,288345,0.500709080,0.000709080
|
||||
20,288817,288692,0.500108223,0.000108223
|
||||
21,288723,288786,0.499945455,0.000054545
|
||||
22,288577,288932,0.499692645,0.000307355
|
||||
23,288523,288986,0.499599140,0.000400860
|
||||
24,288625,288884,0.499775761,0.000224239
|
||||
25,288128,289381,0.498915168,0.001084832
|
||||
26,288821,288688,0.500115150,0.000115150
|
||||
27,288762,288747,0.500012987,0.000012987
|
||||
28,289539,287970,0.501358420,0.001358420
|
||||
29,288493,289016,0.499547193,0.000452807
|
||||
30,288614,288895,0.499756714,0.000243286
|
||||
31,288681,288828,0.499872729,0.000127271
|
||||
|
@@ -0,0 +1,8 @@
|
||||
corpus=input_symbols/cpp_provides_p11.txt
|
||||
corpus_bytes=54941449
|
||||
corpus_sha256=bf8e760bdf4d52b42040cff4625de73988d8dd6b517566efd186c71e3382753d
|
||||
samples=577509
|
||||
hashes=jenkinsOAAT,xxh64,t1ha2
|
||||
metric=P(hash_output_bit=1)
|
||||
bit_order=bit_0_is_LSB
|
||||
input_format=one_ASCII_symbol_per_line
|
||||
@@ -0,0 +1,4 @@
|
||||
hash,samples,bits,mean_probability,mean_abs_deviation,max_abs_deviation,min_probability,max_probability
|
||||
jenkinsOAAT,577509,32,0.499879060,0.000475425,0.001670104,0.498329896,0.501358420
|
||||
xxh64,577509,64,0.499871404,0.000497665,0.001704735,0.498295265,0.501221626
|
||||
t1ha2,577509,64,0.500008550,0.000532513,0.001876161,0.498123839,0.501256257
|
||||
|
@@ -0,0 +1,65 @@
|
||||
bit,ones,zeros,probability,deviation_from_0.5
|
||||
0,288524,288985,0.499600872,0.000399128
|
||||
1,289273,288236,0.500897822,0.000897822
|
||||
2,289241,288268,0.500842411,0.000842411
|
||||
3,289050,288459,0.500511680,0.000511680
|
||||
4,288827,288682,0.500125539,0.000125539
|
||||
5,289341,288168,0.501015569,0.001015569
|
||||
6,288570,288939,0.499680524,0.000319476
|
||||
7,289480,288029,0.501256257,0.001256257
|
||||
8,289469,288040,0.501237210,0.001237210
|
||||
9,289359,288150,0.501046737,0.001046737
|
||||
10,288340,289169,0.499282262,0.000717738
|
||||
11,288850,288659,0.500165365,0.000165365
|
||||
12,288401,289108,0.499387888,0.000612112
|
||||
13,288828,288681,0.500127271,0.000127271
|
||||
14,288809,288700,0.500094371,0.000094371
|
||||
15,288437,289072,0.499450225,0.000549775
|
||||
16,287671,289838,0.498123839,0.001876161
|
||||
17,288776,288733,0.500037229,0.000037229
|
||||
18,288058,289451,0.498793958,0.001206042
|
||||
19,289300,288209,0.500944574,0.000944574
|
||||
20,288721,288788,0.499941992,0.000058008
|
||||
21,288403,289106,0.499391351,0.000608649
|
||||
22,288749,288760,0.499990476,0.000009524
|
||||
23,289382,288127,0.501086563,0.001086563
|
||||
24,288503,289006,0.499564509,0.000435491
|
||||
25,289237,288272,0.500835485,0.000835485
|
||||
26,288918,288591,0.500283112,0.000283112
|
||||
27,288889,288620,0.500232897,0.000232897
|
||||
28,288205,289304,0.499048500,0.000951500
|
||||
29,288637,288872,0.499796540,0.000203460
|
||||
30,289047,288462,0.500506486,0.000506486
|
||||
31,288479,289030,0.499522951,0.000477049
|
||||
32,289147,288362,0.500679643,0.000679643
|
||||
33,288429,289080,0.499436372,0.000563628
|
||||
34,288675,288834,0.499862340,0.000137660
|
||||
35,288817,288692,0.500108223,0.000108223
|
||||
36,288302,289207,0.499216462,0.000783538
|
||||
37,288197,289312,0.499034647,0.000965353
|
||||
38,289472,288037,0.501242405,0.001242405
|
||||
39,289181,288328,0.500738517,0.000738517
|
||||
40,289046,288463,0.500504754,0.000504754
|
||||
41,288910,288599,0.500269260,0.000269260
|
||||
42,288608,288901,0.499746324,0.000253676
|
||||
43,288745,288764,0.499983550,0.000016450
|
||||
44,288429,289080,0.499436372,0.000563628
|
||||
45,288738,288771,0.499971429,0.000028571
|
||||
46,288425,289084,0.499429446,0.000570554
|
||||
47,288857,288652,0.500177486,0.000177486
|
||||
48,288795,288714,0.500070129,0.000070129
|
||||
49,288155,289354,0.498961921,0.001038079
|
||||
50,288639,288870,0.499800003,0.000199997
|
||||
51,289013,288496,0.500447612,0.000447612
|
||||
52,288717,288792,0.499935066,0.000064934
|
||||
53,289130,288379,0.500650206,0.000650206
|
||||
54,288936,288573,0.500314281,0.000314281
|
||||
55,288904,288605,0.500258870,0.000258870
|
||||
56,288616,288893,0.499760177,0.000239823
|
||||
57,288873,288636,0.500205192,0.000205192
|
||||
58,288340,289169,0.499282262,0.000717738
|
||||
59,288507,289002,0.499571435,0.000428565
|
||||
60,288986,288523,0.500400860,0.000400860
|
||||
61,288162,289347,0.498974042,0.001025958
|
||||
62,288354,289155,0.499306504,0.000693496
|
||||
63,288725,288784,0.499948919,0.000051081
|
||||
|
@@ -0,0 +1,65 @@
|
||||
bit,ones,zeros,probability,deviation_from_0.5
|
||||
0,288778,288731,0.500040692,0.000040692
|
||||
1,288503,289006,0.499564509,0.000435491
|
||||
2,288868,288641,0.500196534,0.000196534
|
||||
3,288638,288871,0.499798272,0.000201728
|
||||
4,288821,288688,0.500115150,0.000115150
|
||||
5,288588,288921,0.499711693,0.000288307
|
||||
6,288882,288627,0.500220776,0.000220776
|
||||
7,289460,288049,0.501221626,0.001221626
|
||||
8,288866,288643,0.500193071,0.000193071
|
||||
9,288162,289347,0.498974042,0.001025958
|
||||
10,289120,288389,0.500632891,0.000632891
|
||||
11,288842,288667,0.500151513,0.000151513
|
||||
12,288456,289053,0.499483125,0.000516875
|
||||
13,288410,289099,0.499403472,0.000596528
|
||||
14,289202,288307,0.500774880,0.000774880
|
||||
15,288850,288659,0.500165365,0.000165365
|
||||
16,288499,289010,0.499557583,0.000442417
|
||||
17,288163,289346,0.498975774,0.001024226
|
||||
18,288220,289289,0.499074473,0.000925527
|
||||
19,288892,288617,0.500238092,0.000238092
|
||||
20,288695,288814,0.499896971,0.000103029
|
||||
21,288797,288712,0.500073592,0.000073592
|
||||
22,288795,288714,0.500070129,0.000070129
|
||||
23,288188,289321,0.499019063,0.000980937
|
||||
24,288864,288645,0.500189607,0.000189607
|
||||
25,288653,288856,0.499824245,0.000175755
|
||||
26,289025,288484,0.500468391,0.000468391
|
||||
27,288750,288759,0.499992208,0.000007792
|
||||
28,289267,288242,0.500887432,0.000887432
|
||||
29,288448,289061,0.499469272,0.000530728
|
||||
30,288742,288767,0.499978355,0.000021645
|
||||
31,288247,289262,0.499121226,0.000878774
|
||||
32,288629,288880,0.499782687,0.000217313
|
||||
33,289449,288060,0.501202579,0.001202579
|
||||
34,288761,288748,0.500011255,0.000011255
|
||||
35,288208,289301,0.499053694,0.000946306
|
||||
36,288308,289201,0.499226852,0.000773148
|
||||
37,289073,288436,0.500551507,0.000551507
|
||||
38,288394,289115,0.499375767,0.000624233
|
||||
39,289142,288367,0.500670985,0.000670985
|
||||
40,288574,288935,0.499687451,0.000312549
|
||||
41,288209,289300,0.499055426,0.000944574
|
||||
42,288292,289217,0.499199147,0.000800853
|
||||
43,288781,288728,0.500045887,0.000045887
|
||||
44,287770,289739,0.498295265,0.001704735
|
||||
45,289014,288495,0.500449344,0.000449344
|
||||
46,289209,288300,0.500787001,0.000787001
|
||||
47,289366,288143,0.501058858,0.001058858
|
||||
48,288445,289064,0.499464078,0.000535922
|
||||
49,288384,289125,0.499358452,0.000641548
|
||||
50,288509,289000,0.499574898,0.000425102
|
||||
51,288599,288910,0.499730740,0.000269260
|
||||
52,288771,288738,0.500028571,0.000028571
|
||||
53,287887,289622,0.498497859,0.001502141
|
||||
54,288871,288638,0.500201728,0.000201728
|
||||
55,288294,289215,0.499202610,0.000797390
|
||||
56,288557,288952,0.499658014,0.000341986
|
||||
57,288416,289093,0.499413862,0.000586138
|
||||
58,288674,288835,0.499860608,0.000139392
|
||||
59,288644,288865,0.499808661,0.000191339
|
||||
60,288679,288830,0.499869266,0.000130734
|
||||
61,288828,288681,0.500127271,0.000127271
|
||||
62,288905,288604,0.500260602,0.000260602
|
||||
63,289202,288307,0.500774880,0.000774880
|
||||
|
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Behavior tests for corpus_bit_distribution.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
import corpus_bit_distribution
|
||||
import probability_map
|
||||
|
||||
|
||||
class CorpusBitDistributionTests(unittest.TestCase):
|
||||
def test_batch_counts_match_scalar_cli_for_every_hash(self) -> None:
|
||||
words = ["a", "abc", "HashWord", "_ZN4llvm3fooEv"]
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
corpus = Path(temporary_directory) / "corpus.txt"
|
||||
corpus.write_text("".join(f"{word}\n" for word in words), encoding="ascii")
|
||||
|
||||
for hash_name in probability_map.HASHES:
|
||||
with self.subTest(hash_name=hash_name):
|
||||
distribution = corpus_bit_distribution.measure_hash(
|
||||
hash_name,
|
||||
corpus,
|
||||
)
|
||||
executable = probability_map.prepare_hash(hash_name)
|
||||
scalar = [
|
||||
probability_map.run_hash(executable, word)[0] for word in words
|
||||
]
|
||||
expected_ones = tuple(
|
||||
sum((value >> bit) & 1 for value in scalar)
|
||||
for bit in range(distribution.bits)
|
||||
)
|
||||
|
||||
self.assertEqual(distribution.samples, len(words))
|
||||
self.assertEqual(distribution.ones, expected_ones)
|
||||
|
||||
def test_writes_one_csv_row_per_lsb_first_bit(self) -> None:
|
||||
distribution = corpus_bit_distribution.Distribution(
|
||||
hash_name="tiny",
|
||||
samples=4,
|
||||
bits=3,
|
||||
ones=(1, 2, 4),
|
||||
)
|
||||
stream = io.StringIO()
|
||||
|
||||
corpus_bit_distribution.write_distribution_csv(stream, distribution)
|
||||
|
||||
rows = list(csv.reader(io.StringIO(stream.getvalue())))
|
||||
self.assertEqual(
|
||||
rows[0],
|
||||
["bit", "ones", "zeros", "probability", "deviation_from_0.5"],
|
||||
)
|
||||
self.assertEqual(rows[1], ["0", "1", "3", "0.250000000", "0.250000000"])
|
||||
self.assertEqual(rows[2], ["1", "2", "2", "0.500000000", "0.000000000"])
|
||||
self.assertEqual(rows[3], ["2", "4", "0", "1.000000000", "0.500000000"])
|
||||
|
||||
def test_run_creates_separate_csv_summary_manifest_and_png(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
corpus = root / "corpus.txt"
|
||||
corpus.write_text("a\nabc\nHashWord\n", encoding="ascii")
|
||||
output = root / "results"
|
||||
|
||||
result = corpus_bit_distribution.run(
|
||||
corpus=corpus,
|
||||
output_directory=output,
|
||||
hash_names=["jenkinsOAAT"],
|
||||
)
|
||||
|
||||
self.assertEqual(result[0].samples, 3)
|
||||
self.assertTrue((output / "jenkinsOAAT.csv").is_file())
|
||||
self.assertTrue((output / "summary.csv").is_file())
|
||||
self.assertTrue((output / "manifest.txt").is_file())
|
||||
image_path = output / "bit_distribution.png"
|
||||
self.assertTrue(image_path.is_file())
|
||||
with Image.open(image_path) as image:
|
||||
self.assertEqual(image.format, "PNG")
|
||||
self.assertGreater(image.width, 300)
|
||||
self.assertGreater(image.height, 200)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Behavior tests for extract_provided_symbols.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "extract_provided_symbols.py"
|
||||
|
||||
|
||||
def newc_entry(name: str, data: bytes, mode: int = stat.S_IFREG | 0o644) -> bytes:
|
||||
encoded_name = name.encode("utf-8") + b"\0"
|
||||
fields = (
|
||||
1,
|
||||
mode,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
0,
|
||||
len(data),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
len(encoded_name),
|
||||
0,
|
||||
)
|
||||
header = b"070701" + b"".join(f"{value:08x}".encode() for value in fields)
|
||||
record = header + encoded_name
|
||||
record += b"\0" * (-len(record) % 4)
|
||||
record += data
|
||||
record += b"\0" * (-len(data) % 4)
|
||||
return record
|
||||
|
||||
|
||||
def newc_archive(entries: list[tuple[str, bytes, int]]) -> bytes:
|
||||
payload = b"".join(newc_entry(*entry) for entry in entries)
|
||||
return payload + newc_entry("TRAILER!!!", b"", 0)
|
||||
|
||||
|
||||
class ExtractProvidedSymbolsTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.work = Path(self.temporary_directory.name)
|
||||
self.rpm2cpio = self.work / "rpm2cpio"
|
||||
self.provided_symbols = self.work / "provided_symbols"
|
||||
self.file_command = self.work / "file"
|
||||
|
||||
self.rpm2cpio.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import pathlib, sys\n"
|
||||
"sys.stdout.buffer.write(pathlib.Path(sys.argv[1]).read_bytes())\n"
|
||||
)
|
||||
self.provided_symbols.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import pathlib, sys\n"
|
||||
"symbols = set()\n"
|
||||
"for name in sys.argv[1:]:\n"
|
||||
" symbols.update(pathlib.Path(name).read_bytes()[4:].decode().splitlines())\n"
|
||||
"print(*sorted(symbols), sep='\\n')\n"
|
||||
)
|
||||
self.file_command.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import pathlib, sys\n"
|
||||
"data = pathlib.Path(sys.argv[-1]).read_bytes()\n"
|
||||
"kind = 'pie executable' if b'EXECUTABLE' in data else 'shared object'\n"
|
||||
"print(f'ELF 64-bit LSB {kind}, x86-64, version 1 (SYSV)')\n"
|
||||
)
|
||||
self.rpm2cpio.chmod(0o755)
|
||||
self.provided_symbols.chmod(0o755)
|
||||
self.file_command.chmod(0o755)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.temporary_directory.cleanup()
|
||||
|
||||
def run_script(self, *arguments: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--rpm2cpio",
|
||||
str(self.rpm2cpio),
|
||||
"--provided-symbols",
|
||||
str(self.provided_symbols),
|
||||
"--file-command",
|
||||
str(self.file_command),
|
||||
*arguments,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_combines_unique_cpp_symbols_from_multiple_packages(self) -> None:
|
||||
first = self.work / "first.rpm"
|
||||
second = self.work / "second.rpm"
|
||||
output = self.work / "symbols.txt"
|
||||
first.write_bytes(
|
||||
newc_archive(
|
||||
[
|
||||
("./usr/lib64/libfirst.so", b"\x7fELF_ZN3Foo3barEv\nplain_c_symbol\n", stat.S_IFREG | 0o755),
|
||||
("./usr/share/doc/readme", b"not ELF", stat.S_IFREG | 0o644),
|
||||
("./usr/lib64/libalias.so", b"libfirst.so", stat.S_IFLNK | 0o777),
|
||||
]
|
||||
)
|
||||
)
|
||||
second.write_bytes(
|
||||
newc_archive(
|
||||
[
|
||||
(
|
||||
"./usr/lib64/libsecond.so",
|
||||
b"\x7fELF_ZN3Foo3barEv\n_ZN3Foo3bazEv\nanother_c_symbol\n",
|
||||
stat.S_IFREG | 0o755,
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
result = self.run_script(
|
||||
"--cpp-only", "--output", str(output), str(first), str(second)
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(
|
||||
output.read_text().splitlines(),
|
||||
["_ZN3Foo3barEv", "_ZN3Foo3bazEv"],
|
||||
)
|
||||
self.assertIn("package=first.rpm elf_files=1 symbols=2 selected=1", result.stderr)
|
||||
self.assertIn("package=second.rpm elf_files=1 symbols=3 selected=2", result.stderr)
|
||||
self.assertIn("unique_symbols=2", result.stderr)
|
||||
self.assertEqual(result.stdout, "")
|
||||
|
||||
def test_without_cpp_filter_preserves_all_provided_symbols(self) -> None:
|
||||
package = self.work / "all.rpm"
|
||||
package.write_bytes(
|
||||
newc_archive(
|
||||
[
|
||||
(
|
||||
"./usr/lib64/liball.so",
|
||||
b"\x7fELF_ZN3Foo3barEv\nplain_c_symbol\n",
|
||||
stat.S_IFREG | 0o755,
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
result = self.run_script(str(package))
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.splitlines(), ["_ZN3Foo3barEv", "plain_c_symbol"])
|
||||
|
||||
def test_excludes_elf_executables_that_do_not_generate_library_provides(self) -> None:
|
||||
package = self.work / "mixed.rpm"
|
||||
package.write_bytes(
|
||||
newc_archive(
|
||||
[
|
||||
(
|
||||
"./usr/lib64/libmixed.so",
|
||||
b"\x7fELF_ZN3Lib3runEv\n",
|
||||
stat.S_IFREG | 0o755,
|
||||
),
|
||||
(
|
||||
"./usr/bin/mixed",
|
||||
b"\x7fELFEXECUTABLE\n_ZN4Exec3runEv\n",
|
||||
stat.S_IFREG | 0o755,
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
result = self.run_script("--cpp-only", str(package))
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.splitlines(), ["_ZN3Lib3runEv"])
|
||||
self.assertIn("elf_files=1", result.stderr)
|
||||
|
||||
def test_rejects_malformed_cpio_stream(self) -> None:
|
||||
package = self.work / "broken.rpm"
|
||||
package.write_bytes(b"not a newc archive")
|
||||
|
||||
result = self.run_script(str(package))
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("newc", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user