diff --git a/docs/ENGINE.md b/docs/ENGINE.md index 987b4b1..ee80318 100644 --- a/docs/ENGINE.md +++ b/docs/ENGINE.md @@ -118,7 +118,13 @@ API/WIT/артефакт, проверку build identity, сценарий, Was записываются как failed и дают ненулевой exit code. Лимиты берутся из исходников: memory 64 MiB; fuel init/tick/frame/event — 2B/300M/10M/1B. -**Фактический расход fuel и peak memory не измеряются этим отчётом.** Сохранённый успешный прогон: +Начиная с продолжения 4 ноября `test-budgets` включает `--fuel-profile` и сохраняет +`measured_fuel.callbacks`: число вызовов, максимум guest fuel, бюджет и ошибки отдельно по callback. +Отсутствие запрошенных счётчиков делает проверку неуспешной, даже если сценарий завершился. +Это не время host/GPU и не peak memory; невызывавшиеся callbacks, включая init до включения профиля, +не получают выдуманного нуля. Первый прогон: tick 2102 вызова, max 106132646/300000000; +command 69 вызовов, max 1013370/1000000000; ошибок нет. +Исторический отчёт без измерения fuel: [API27 budget report с поправкой покрытия](measurements/api27-wasm-budget.json). Проверены реальный WASM guest с переходами состояний/владельца/мира, сборка клиента и два коротких diff --git a/tools/airdef.py b/tools/airdef.py index 441554c..0238756 100644 --- a/tools/airdef.py +++ b/tools/airdef.py @@ -161,6 +161,16 @@ BUDGET_SCENARIO = "tests/budgets/wasm_flat_budget.toml" BUDGET_ASSERTED_SHA256 = "bcc0359289ab20ab3de608bcc130f72d2cf88f9f6fbc470d7b6846e9db5d11df" +def fuel_measurements(output): + """Read the pinned host's callback counters; never infer them from source caps.""" + samples = [] + for mod_id, callback, calls, maximum, budget, failures in re.findall( + r"(?m)^fuel (\S+) (\S+): calls=(\d+) max=(\d+) budget=(\d+) failures=(\d+)\s*$", output): + samples.append({"mod": mod_id, "callback": callback, "calls": int(calls), + "max_used": int(maximum), "budget": int(budget), "failures": int(failures)}) + return samples + + def test_budgets(): """Run the opt-in heavy scenario and always write a machine-readable report.""" report_path = Path(os.environ.get("AIRDEF_BUDGET_REPORT") or @@ -178,7 +188,9 @@ def test_budgets(): "accepted_raids_asserted": None, "snapshot_airborne_asserted": None, "snapshot_tick": None, "peak_airborne_observed": None}, "runtime_limits": {"source": "horizon/crates/horizon-mods/src/runtime.rs", - "metering": "source_configured; measured_per_scenario: unavailable"}, + "metering": "source_configured"}, + "measured_fuel": {"available": False, "callbacks": [], + "scope": "guest callback fuel; excludes host CPU, memory and GPU time"}, } def git(*args, text=True): @@ -253,7 +265,8 @@ def test_budgets(): (isolated / "tests").mkdir() (isolated / "tests" / "wasm_budget.toml").write_bytes(fixture_bytes) command = [cargo(), "run", "--locked", "--manifest-path", FORK / "Cargo.toml", - "-p", "cargo-horizon-mod", "--", "horizon-mod", "test", "--hmod", artifact, "budget"] + "-p", "cargo-horizon-mod", "--", "horizon-mod", "test", "--hmod", artifact, + "--fuel-profile", "budget"] print("+", " ".join(str(c) for c in command), flush=True) result = subprocess.run([str(c) for c in command], cwd=isolated, env=build_env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -261,6 +274,11 @@ def test_budgets(): report["exit_code"] = result.returncode report["status"] = "passed" if result.returncode == 0 else "failed" report["output"] = result.stdout[-12000:] + samples = fuel_measurements(result.stdout) + report["measured_fuel"].update({"available": bool(samples), "callbacks": samples}) + if result.returncode == 0 and not samples: + report["status"] = "failed" + raise RuntimeError("pinned runtime did not return requested callback fuel counters") # These assertions belong to this exact fixture, not to arbitrary edited scripts. if result.returncode == 0 and report["scenario_sha256"] == BUDGET_ASSERTED_SHA256: counts["accepted_raids_asserted"] = 2 diff --git a/tools/test_budget_report.py b/tools/test_budget_report.py index df4ddf1..33b68f2 100644 --- a/tools/test_budget_report.py +++ b/tools/test_budget_report.py @@ -114,7 +114,9 @@ class BudgetReportFailureTests(unittest.TestCase): (self.mod_root / airdef.BUDGET_SCENARIO).write_bytes(source.read_bytes()) def backend(command, cwd, **kwargs): self.assertEqual((Path(cwd) / "tests/wasm_budget.toml").read_bytes(), source.read_bytes()) - return CompletedProcess(args=command, returncode=0, stdout="test passed") + self.assertIn("--fuel-profile", command) + return CompletedProcess(args=command, returncode=0, stdout=( + "test passed\nfuel airdef tick: calls=2102 max=123456 budget=300000000 failures=0\n")) with mock.patch.object(airdef, "mod"), mock.patch.object(airdef.subprocess, "run", side_effect=backend): airdef.test_budgets() report = self.read_report() @@ -123,6 +125,16 @@ class BudgetReportFailureTests(unittest.TestCase): counts = report["scenario_counts"] self.assertEqual((counts["accepted_raids_asserted"], counts["snapshot_airborne_asserted"], counts["snapshot_tick"]), (2, 184, 2102)) self.assertIsNone(counts["peak_airborne_observed"]) + self.assertEqual(report["measured_fuel"]["callbacks"], [{ + "mod": "airdef", "callback": "tick", "calls": 2102, + "max_used": 123456, "budget": 300000000, "failures": 0}]) + + def test_success_without_requested_measurements_is_not_a_budget_proof(self): + backend = mock.Mock(return_value=CompletedProcess(args=[], returncode=0, stdout="test passed\n")) + with mock.patch.object(airdef, "mod"), mock.patch.object(airdef.subprocess, "run", backend): + report = self.invoke_and_require_failure() + self.assertFalse(report["measured_fuel"]["available"]) + self.assertIn("did not return", report["error"]) def test_early_system_exit_writes_report_and_skips_backend(self): backend = mock.Mock() @@ -148,7 +160,8 @@ class BudgetReportFailureTests(unittest.TestCase): def test_runtime_fuel_trap_persists_nonzero_exit_and_output(self): backend = mock.Mock(return_value=CompletedProcess( - args=["cargo", "run"], returncode=137, stdout="error: all fuel consumed (fuel trap)\n" + args=["cargo", "run"], returncode=137, stdout=("error: all fuel consumed (fuel trap)\n" + "fuel airdef tick: calls=4 max=300000000 budget=300000000 failures=1\n") )) with mock.patch.object(airdef, "mod"), \ mock.patch.object(airdef.subprocess, "run", backend): @@ -157,6 +170,7 @@ class BudgetReportFailureTests(unittest.TestCase): backend.assert_called_once() self.assertEqual(report["exit_code"], 137) self.assertIn("fuel trap", report["output"]) + self.assertEqual(report["measured_fuel"]["callbacks"][0]["failures"], 1) if __name__ == "__main__":