misc/nac3/find_anomalies.py

75 lines
2.4 KiB
Python

#!/usr/bin/env python
# PLACE ME IN nac3standalone/demo/find_anomalies.py AND DO `chmod +x` ON ME!!
from pathlib import Path
from typing import *
import argparse
import subprocess
import itertools
def proc_interpret(py_path: Path) -> subprocess.CompletedProcess:
return subprocess.run(["./interpret_demo.py", str(py_path)], capture_output=True)
def proc_nac3(
py_path: Path,
size_t: int,
lli: bool,
nac3args: Optional[list[str]] = None
) -> subprocess.CompletedProcess:
# Prepare args
args = ["./run_demo.sh"]
if lli:
args.append("--lli")
args.extend(["--out", "/dev/stdout"])
if nac3args is not None:
args.extend(nac3args)
args.extend(["-s", str(size_t)])
args.append(str(py_path))
# Run
return subprocess.run(args, capture_output=True)
def main():
nac3args = ["--debug"] # Other confs
for py_path in Path.cwd().glob("src/*.py"):
for size_t, lli in itertools.product([32, 64], [False, True]):
errors = []
# Check NAC3
nac3result = proc_nac3(py_path, size_t=size_t, lli=lli, nac3args=nac3args)
def preview(content: bytes) -> str:
if len(content) == 0:
return "<empty>"
NCHARS = 120
if len(content) > NCHARS:
return repr(content[:NCHARS]) + "..."
else:
return repr(content)
# Check if the process ran successfully first
if nac3result.returncode != 0:
errors.append(f"- abnormal return code {nac3result.returncode}")
errors.append(f" - stdout: {preview(nac3result.stdout)}")
errors.append(f" - stderr: {preview(nac3result.stderr)}")
else:
# ... the process ran OK, what about its program outputs?
if nac3result.stderr:
errors.append(f"- stderr is non-empty: {preview(nac3result.stderr)}")
correct_output = proc_interpret(py_path)
if nac3result.stdout != correct_output.stdout:
errors.append(f"- stdout is incorrect: {preview(nac3result.stdout)}...")
if errors:
print(f"\n>>>>>> {py_path.name=} {size_t=} {lli=}")
for error in errors:
print(f"{error}")
if __name__ == "__main__":
main()