diff --git a/tools/test_budget_report.py b/tools/test_budget_report.py new file mode 100644 index 0000000..c4788e7 --- /dev/null +++ b/tools/test_budget_report.py @@ -0,0 +1,133 @@ +"""Offline failure-path tests for AirDef's opt-in WASM budget report.""" +import hashlib +import json +import os +import tempfile +import unittest +from pathlib import Path +from subprocess import CompletedProcess +from unittest import mock + +import airdef + + +class BudgetReportFailureTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="airdef-budget-report-test-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.fork = self.root / "horizon" + self.mod_root = self.root / "airdef-mod" + self.artifact = self.mod_root / "target" / "horizon-mod" / "airdef.hmod" + self.report = self.root / "report.json" + + (self.fork / "crates/horizon-mod-sdk/wit").mkdir(parents=True) + (self.fork / "crates/horizon-mods/src").mkdir(parents=True) + (self.mod_root / "tests/budgets").mkdir(parents=True) + (self.mod_root / "src").mkdir(parents=True) + self.artifact.parent.mkdir(parents=True) + (self.fork / "crates/horizon-mod-sdk/wit/horizon.wit").write_text("package test:mods;\n") + (self.fork / "crates/horizon-mods/src/runtime.rs").write_text( + "const MEMORY: usize = 64 << 20;\n" + "const FUEL_INIT: u64 = 2000000000;\n" + "const FUEL_TICK: u64 = 300000000;\n" + "const FUEL_FRAME: u64 = 10000000;\n" + "const FUEL_EVENT: u64 = 1000000000;\n" + "wasm_component_model(true); consume_fuel(true);\n" + ) + (self.fork / "Cargo.lock").write_text( + '[[package]]\nname = "wasmtime"\nversion = "48.0.7"\n' + ) + (self.mod_root / "src/layout.rs").write_text("pub const MAX_UNITS: usize = 2000;\n") + (self.mod_root / "tests/budgets/wasm_budget.toml").write_text( + '[[step]]\ncommand = "airdef test"\n' + ) + self.artifact.write_bytes(b"minimal synthetic hmod") + self.diff_sha = hashlib.sha256(b"synthetic diff").hexdigest() + self.wit_sha = hashlib.sha256((self.fork / "crates/horizon-mod-sdk/wit/horizon.wit").read_bytes()).hexdigest() + self.identity = { + "airdef": "airdef-sha", + "airdef_dirty": False, + "airdef_diff_sha256": self.diff_sha, + "engine": "engine-id", + "engine_pin": "engine-pin", + "wit_sha256": self.wit_sha, + "artifact_sha256": hashlib.sha256(self.artifact.read_bytes()).hexdigest(), + } + (self.artifact.parent / "build-identity.json").write_text(json.dumps(self.identity)) + + self.patches = [ + mock.patch.object(airdef, "ROOT", self.root), + mock.patch.object(airdef, "FORK", self.fork), + mock.patch.object(airdef, "engine_commit", return_value="engine-pin"), + mock.patch.object(airdef, "engine_identity", return_value="engine-id"), + mock.patch.object(airdef, "fork_api", return_value=(27, [])), + mock.patch.object(airdef, "hmod", return_value=self.artifact), + mock.patch.object(airdef, "cargo", return_value="cargo"), + mock.patch.dict(os.environ, {"AIRDEF_BUDGET_REPORT": str(self.report)}), + mock.patch.object(airdef.subprocess, "check_output", side_effect=self.fake_git), + ] + for patcher in self.patches: + patcher.start() + self.addCleanup(patcher.stop) + + @staticmethod + def fake_git(command, **kwargs): + if "rev-parse" in command: + result = "airdef-sha\n" + elif "status" in command: + result = "" + elif "diff" in command: + result = b"synthetic diff" if kwargs.get("text") is False else "synthetic diff" + else: + raise AssertionError(f"unexpected git command: {command}") + return result + + def read_report(self): + return json.loads(self.report.read_text()) + + def invoke_and_require_failure(self): + with self.assertRaises(SystemExit): + airdef.test_budgets() + self.assertTrue(self.report.is_file(), "failure path must persist its report") + report = self.read_report() + self.assertEqual(report["status"], "failed") + return report + + def test_early_system_exit_writes_report_and_skips_backend(self): + backend = mock.Mock() + with mock.patch.object(airdef, "mod", side_effect=SystemExit("synthetic build failure")), \ + mock.patch.object(airdef.subprocess, "run", backend): + report = self.invoke_and_require_failure() + + self.assertIn("SystemExit", report["error"]) + backend.assert_not_called() + + def test_stale_artifact_identity_blocks_backend(self): + stale = dict(self.identity, engine_pin="old-engine-pin") + (self.artifact.parent / "build-identity.json").write_text(json.dumps(stale)) + backend = mock.Mock() + with mock.patch.object(airdef, "mod"), \ + mock.patch.object(airdef.subprocess, "run", backend): + report = self.invoke_and_require_failure() + + self.assertFalse(report["build_identity"]["matches"]) + self.assertFalse(report["build_identity"]["checks"]["engine_pin"]) + self.assertIn("stale", report["error"]) + backend.assert_not_called() + + def test_runtime_fuel_trap_persists_nonzero_exit_and_output(self): + backend = mock.Mock(return_value=CompletedProcess( + args=["cargo", "run"], returncode=137, stdout="error: all fuel consumed (fuel trap)\n" + )) + with mock.patch.object(airdef, "mod"), \ + mock.patch.object(airdef.subprocess, "run", backend): + report = self.invoke_and_require_failure() + + backend.assert_called_once() + self.assertEqual(report["exit_code"], 137) + self.assertIn("fuel trap", report["output"]) + + +if __name__ == "__main__": + unittest.main()