test: record measured callback fuel in budget reports

Co-Authored-By: Codex <noreply@openai.com>

docs/ENGINE.md

@@ -118,7 +118,13 @@ API/WIT/артефакт, проверку build identity, сценарий, Was
 записываются как failed и дают ненулевой exit code.
 
 Лимиты берутся из исходников: memory 64 MiB; fuel init/tick/frame/event — 2B/300M/10M/1B.
-**Фактический расход fuel и peak memory не измеряются этим отчётом.** Сохранённый успешный прогон:
+Начиная с продолжения 4 ноября `test-budgets` включает `--fuel-profile` и сохраняет
+`measured_fuel.callbacks`: число вызовов, максимум guest fuel, бюджет и ошибки отдельно по callback.
+Отсутствие запрошенных счётчиков делает проверку неуспешной, даже если сценарий завершился.
+Это не время host/GPU и не peak memory; невызывавшиеся callbacks, включая init до включения профиля,
+не получают выдуманного нуля. Первый прогон: tick 2102 вызова, max 106132646/300000000;
+command 69 вызовов, max 1013370/1000000000; ошибок нет.
+Исторический отчёт без измерения fuel:
 [API27 budget report с поправкой покрытия](measurements/api27-wasm-budget.json).
 
 Проверены реальный WASM guest с переходами состояний/владельца/мира, сборка клиента и два коротких

tools/airdef.py

@@ -161,6 +161,16 @@ BUDGET_SCENARIO = "tests/budgets/wasm_flat_budget.toml"
 BUDGET_ASSERTED_SHA256 = "bcc0359289ab20ab3de608bcc130f72d2cf88f9f6fbc470d7b6846e9db5d11df"
 
 
+def fuel_measurements(output):
+    """Read the pinned host's callback counters; never infer them from source caps."""
+    samples = []
+    for mod_id, callback, calls, maximum, budget, failures in re.findall(
+            r"(?m)^fuel (\S+) (\S+): calls=(\d+) max=(\d+) budget=(\d+) failures=(\d+)\s*$", output):
+        samples.append({"mod": mod_id, "callback": callback, "calls": int(calls),
+                        "max_used": int(maximum), "budget": int(budget), "failures": int(failures)})
+    return samples
+
+
 def test_budgets():
     """Run the opt-in heavy scenario and always write a machine-readable report."""
     report_path = Path(os.environ.get("AIRDEF_BUDGET_REPORT") or
@@ -178,7 +188,9 @@ def test_budgets():
                              "accepted_raids_asserted": None, "snapshot_airborne_asserted": None,
                              "snapshot_tick": None, "peak_airborne_observed": None},
         "runtime_limits": {"source": "horizon/crates/horizon-mods/src/runtime.rs",
-                           "metering": "source_configured; measured_per_scenario: unavailable"},
+                           "metering": "source_configured"},
+        "measured_fuel": {"available": False, "callbacks": [],
+                          "scope": "guest callback fuel; excludes host CPU, memory and GPU time"},
     }
 
     def git(*args, text=True):
@@ -253,7 +265,8 @@ def test_budgets():
             (isolated / "tests").mkdir()
             (isolated / "tests" / "wasm_budget.toml").write_bytes(fixture_bytes)
             command = [cargo(), "run", "--locked", "--manifest-path", FORK / "Cargo.toml",
-                       "-p", "cargo-horizon-mod", "--", "horizon-mod", "test", "--hmod", artifact, "budget"]
+                       "-p", "cargo-horizon-mod", "--", "horizon-mod", "test", "--hmod", artifact,
+                       "--fuel-profile", "budget"]
             print("+", " ".join(str(c) for c in command), flush=True)
             result = subprocess.run([str(c) for c in command], cwd=isolated, env=build_env,
                                     text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -261,6 +274,11 @@ def test_budgets():
             report["exit_code"] = result.returncode
             report["status"] = "passed" if result.returncode == 0 else "failed"
             report["output"] = result.stdout[-12000:]
+            samples = fuel_measurements(result.stdout)
+            report["measured_fuel"].update({"available": bool(samples), "callbacks": samples})
+            if result.returncode == 0 and not samples:
+                report["status"] = "failed"
+                raise RuntimeError("pinned runtime did not return requested callback fuel counters")
             # These assertions belong to this exact fixture, not to arbitrary edited scripts.
             if result.returncode == 0 and report["scenario_sha256"] == BUDGET_ASSERTED_SHA256:
                 counts["accepted_raids_asserted"] = 2

tools/test_budget_report.py

@@ -114,7 +114,9 @@ class BudgetReportFailureTests(unittest.TestCase):
         (self.mod_root / airdef.BUDGET_SCENARIO).write_bytes(source.read_bytes())
         def backend(command, cwd, **kwargs):
             self.assertEqual((Path(cwd) / "tests/wasm_budget.toml").read_bytes(), source.read_bytes())
-            return CompletedProcess(args=command, returncode=0, stdout="test passed")
+            self.assertIn("--fuel-profile", command)
+            return CompletedProcess(args=command, returncode=0, stdout=(
+                "test passed\nfuel airdef tick: calls=2102 max=123456 budget=300000000 failures=0\n"))
         with mock.patch.object(airdef, "mod"), mock.patch.object(airdef.subprocess, "run", side_effect=backend):
             airdef.test_budgets()
         report = self.read_report()
@@ -123,6 +125,16 @@ class BudgetReportFailureTests(unittest.TestCase):
         counts = report["scenario_counts"]
         self.assertEqual((counts["accepted_raids_asserted"], counts["snapshot_airborne_asserted"], counts["snapshot_tick"]), (2, 184, 2102))
         self.assertIsNone(counts["peak_airborne_observed"])
+        self.assertEqual(report["measured_fuel"]["callbacks"], [{
+            "mod": "airdef", "callback": "tick", "calls": 2102,
+            "max_used": 123456, "budget": 300000000, "failures": 0}])
+
+    def test_success_without_requested_measurements_is_not_a_budget_proof(self):
+        backend = mock.Mock(return_value=CompletedProcess(args=[], returncode=0, stdout="test passed\n"))
+        with mock.patch.object(airdef, "mod"), mock.patch.object(airdef.subprocess, "run", backend):
+            report = self.invoke_and_require_failure()
+        self.assertFalse(report["measured_fuel"]["available"])
+        self.assertIn("did not return", report["error"])
 
     def test_early_system_exit_writes_report_and_skips_backend(self):
         backend = mock.Mock()
@@ -148,7 +160,8 @@ class BudgetReportFailureTests(unittest.TestCase):
 
     def test_runtime_fuel_trap_persists_nonzero_exit_and_output(self):
         backend = mock.Mock(return_value=CompletedProcess(
-            args=["cargo", "run"], returncode=137, stdout="error: all fuel consumed (fuel trap)\n"
+            args=["cargo", "run"], returncode=137, stdout=("error: all fuel consumed (fuel trap)\n"
+                "fuel airdef tick: calls=4 max=300000000 budget=300000000 failures=1\n")
         ))
         with mock.patch.object(airdef, "mod"), \
              mock.patch.object(airdef.subprocess, "run", backend):
@@ -157,6 +170,7 @@ class BudgetReportFailureTests(unittest.TestCase):
         backend.assert_called_once()
         self.assertEqual(report["exit_code"], 137)
         self.assertIn("fuel trap", report["output"])
+        self.assertEqual(report["measured_fuel"]["callbacks"][0]["failures"], 1)
 
 
 if __name__ == "__main__":