Add nac3/find_anomalies.py

nac3/find_anomalies.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+# PLACE ME IN nac3standalone/demo/find_anomalies.py AND DO `chmod +x` ON ME!!
+
+from pathlib import Path
+from typing import *
+import argparse
+import subprocess
+import itertools
+
+def proc_interpret(py_path: Path) -> subprocess.CompletedProcess:
+    return subprocess.run(["./interpret_demo.py", str(py_path)], capture_output=True)
+
+def proc_nac3(
+    py_path: Path,
+    size_t: int,
+    lli: bool,
+    nac3args: Optional[list[str]] = None
+) -> subprocess.CompletedProcess:
+    # Prepare args
+    args = ["./run_demo.sh"]
+    if lli:
+        args.append("--lli")
+
+    args.extend(["--out", "/dev/stdout"])
+
+    if nac3args is not None:
+        args.extend(nac3args)
+
+    args.extend(["-s", str(size_t)])
+    args.append(str(py_path))
+
+    # Run
+    return subprocess.run(args, capture_output=True)
+
+def main():
+    nac3args = ["--debug"] # Other confs
+    for py_path in Path.cwd().glob("src/*.py"):
+        for size_t, lli in itertools.product([32, 64], [False, True]):
+            errors = []
+
+            # Check NAC3
+            nac3result = proc_nac3(py_path, size_t=size_t, lli=lli, nac3args=nac3args)
+
+            def preview(content: bytes) -> str:
+                if len(content) == 0:
+                    return "<empty>"
+
+                NCHARS = 120
+                if len(content) > NCHARS:
+                    return repr(content[:NCHARS]) + "..."
+                else:
+                    return repr(content)
+
+            # Check if the process ran successfully first
+            if nac3result.returncode != 0:
+                errors.append(f"- abnormal return code {nac3result.returncode}")
+                errors.append(f"    - stdout: {preview(nac3result.stdout)}")
+                errors.append(f"    - stderr: {preview(nac3result.stderr)}")
+            else:
+                # ... the process ran OK, what about its program outputs?
+                if nac3result.stderr:
+                    errors.append(f"- stderr is non-empty: {preview(nac3result.stderr)}")
+
+                correct_output = proc_interpret(py_path)
+                if nac3result.stdout != correct_output.stdout:
+                    errors.append(f"- stdout is incorrect: {preview(nac3result.stdout)}...")
+            
+            if errors:
+                print(f"\n>>>>>> {py_path.name=} {size_t=} {lli=}")
+                for error in errors:
+                    print(f"{error}")
+
+if __name__ == "__main__":
+    main()