diff --git a/tools/airdef.py b/tools/airdef.py index 449be7b..3d7a26b 100644 --- a/tools/airdef.py +++ b/tools/airdef.py @@ -157,6 +157,10 @@ def test_wasm(): cwd=ROOT / "airdef-mod", env=build_env) +BUDGET_SCENARIO = "tests/budgets/wasm_flat_budget.toml" +BUDGET_ASSERTED_SHA256 = "bcc0359289ab20ab3de608bcc130f72d2cf88f9f6fbc470d7b6846e9db5d11df" + + def test_budgets(): """Run the opt-in heavy scenario and always write a machine-readable report.""" report_path = Path(os.environ.get("AIRDEF_BUDGET_REPORT") or @@ -164,13 +168,15 @@ def test_budgets(): report = { "schema": 1, "command": "test-budgets", - "scenario": "tests/budgets/wasm_budget.toml", + "scenario": BUDGET_SCENARIO, + "scenario_render_world": "flat at y=64", "status": "failed", "exit_code": None, "scenario_counts": {"scripts": 1, "steps": None, "raid_requests": None, "raid_units_requested": None, "air_defense_units": None, "layout_unit_limit": None, "frames": None, "ticks": None, - "accepted_raids_observed": None, "peak_airborne_observed": None}, + "accepted_raids_asserted": None, "snapshot_airborne_asserted": None, + "snapshot_tick": None, "peak_airborne_observed": None}, "runtime_limits": {"source": "horizon/crates/horizon-mods/src/runtime.rs", "metering": "source_configured; measured_per_scenario: unavailable"}, } @@ -205,7 +211,7 @@ def test_budgets(): lock_text = (ROOT / "horizon" / "Cargo.lock").read_text() wasmtime = re.search(r'\[\[package\]\]\s*name = "wasmtime"\s*version = "([^"]+)"', lock_text) report["wasmtime_version"] = wasmtime.group(1) if wasmtime else None - fixture = ROOT / "airdef-mod" / "tests" / "budgets" / "wasm_budget.toml" + fixture = ROOT / "airdef-mod" / BUDGET_SCENARIO fixture_text = fixture.read_text() counts = report["scenario_counts"] counts["steps"] = len(re.findall(r"(?m)^\[\[step\]\]$", fixture_text)) @@ -238,7 +244,6 @@ def test_budgets(): ", ".join(k for k, ok in identity_checks.items() if not ok)) build_env = dict(os.environ) build_env["PATH"] = str(Path(cargo()).parent) + os.pathsep + build_env.get("PATH", "") - fixture = ROOT / "airdef-mod" / "tests" / "budgets" / "wasm_budget.toml" with tempfile.TemporaryDirectory(prefix="airdef-wasm-budget-") as temp: isolated = Path(temp) (isolated / "tests").mkdir() @@ -252,6 +257,11 @@ def test_budgets(): report["exit_code"] = result.returncode report["status"] = "passed" if result.returncode == 0 else "failed" report["output"] = result.stdout[-12000:] + # These assertions belong to this exact fixture, not to arbitrary edited scripts. + if result.returncode == 0 and report["scenario_sha256"] == BUDGET_ASSERTED_SHA256: + counts["accepted_raids_asserted"] = 2 + counts["snapshot_airborne_asserted"] = 184 + counts["snapshot_tick"] = 2102 except Exception as exc: report["error"] = f"{type(exc).__name__}: {exc}" except SystemExit as exc: diff --git a/tools/test_budget_report.py b/tools/test_budget_report.py index 9d7abd7..df4ddf1 100644 --- a/tools/test_budget_report.py +++ b/tools/test_budget_report.py @@ -39,7 +39,7 @@ class BudgetReportFailureTests(unittest.TestCase): '[[package]]\nname = "wasmtime"\nversion = "48.0.7"\n' ) (self.mod_root / "src/layout.rs").write_text("pub const MAX_UNITS: usize = 2000;\n") - (self.mod_root / "tests/budgets/wasm_budget.toml").write_text( + (self.mod_root / "tests/budgets/wasm_flat_budget.toml").write_text( '[[step]]\ncommand = "airdef test"\n' ) self.artifact.write_bytes(b"minimal synthetic hmod") @@ -95,7 +95,7 @@ class BudgetReportFailureTests(unittest.TestCase): return report def test_counts_are_requests_and_derived_from_fixture(self): - (self.mod_root / "tests/budgets/wasm_budget.toml").write_text( + (self.mod_root / "tests/budgets/wasm_flat_budget.toml").write_text( '[[step]]\ncommand = "airdef deploy s400"\n' '[[step]]\ncommand = "airdef wave kh101 80 4000"\n' '[[step]]\nticks = 450\nframes = 225\n' @@ -106,7 +106,22 @@ class BudgetReportFailureTests(unittest.TestCase): self.assertEqual((counts["steps"], counts["air_defense_units"]), (4, 1)) self.assertEqual((counts["raid_requests"], counts["raid_units_requested"]), (2, 120)) self.assertEqual((counts["ticks"], counts["frames"]), (450, 225)) - self.assertIsNone(counts["accepted_raids_observed"]) + self.assertIsNone(counts["accepted_raids_asserted"]) + self.assertIsNone(counts["peak_airborne_observed"]) + + def test_success_reports_only_assertions_of_exact_fixture(self): + source = Path(__file__).resolve().parent.parent / "airdef-mod" / airdef.BUDGET_SCENARIO + (self.mod_root / airdef.BUDGET_SCENARIO).write_bytes(source.read_bytes()) + def backend(command, cwd, **kwargs): + self.assertEqual((Path(cwd) / "tests/wasm_budget.toml").read_bytes(), source.read_bytes()) + return CompletedProcess(args=command, returncode=0, stdout="test passed") + with mock.patch.object(airdef, "mod"), mock.patch.object(airdef.subprocess, "run", side_effect=backend): + airdef.test_budgets() + report = self.read_report() + self.assertEqual(report["scenario"], airdef.BUDGET_SCENARIO) + self.assertEqual(report["scenario_render_world"], "flat at y=64") + counts = report["scenario_counts"] + self.assertEqual((counts["accepted_raids_asserted"], counts["snapshot_airborne_asserted"], counts["snapshot_tick"]), (2, 184, 2102)) self.assertIsNone(counts["peak_airborne_observed"]) def test_early_system_exit_writes_report_and_skips_backend(self):