diff --git a/src/pipeline/gates.py b/src/pipeline/gates.py index a7bd8062..a55df090 100644 --- a/src/pipeline/gates.py +++ b/src/pipeline/gates.py @@ -252,9 +252,27 @@ def check(self, recipe: RecipeSpec, ctx: GateContext) -> GateResult: return GateResult(self.name, False, f"artifact {actual_mb:.1f} MB exceeds the " f"declared {float(size_mb):.1f} MB budget") - return GateResult(self.name, True, - f"{actual_mb:.1f} MB within the " - f"{float(size_mb):.1f} MB budget") + detail = (f"{actual_mb:.1f} MB within the " + f"{float(size_mb):.1f} MB budget") + + budget_ms = perf.get("decode_ms_per_row") + if budget_ms is not None: + measured = (ctx.extra or {}).get("measured") or {} + ms = measured.get("decode_ms_per_row") + if ms is None: + return GateResult( + self.name, False, + "perf.decode_ms_per_row declared but never assumed: " + "no measurement supplied — run tools/bench_decode.py " + "or the CI decode bench") + if float(ms) > float(budget_ms): + return GateResult( + self.name, False, + f"decode {float(ms):.1f} ms/row exceeds the declared " + f"{float(budget_ms):.1f} ms/row budget") + detail += (f"; decode {float(ms):.1f} ms/row within the " + f"{float(budget_ms):.1f} ms/row budget") + return GateResult(self.name, True, detail) register_gate(QualityGate()) diff --git a/tests/test_pipeline_latency_budget.py b/tests/test_pipeline_latency_budget.py new file mode 100644 index 00000000..93ddedf0 --- /dev/null +++ b/tests/test_pipeline_latency_budget.py @@ -0,0 +1,49 @@ +"""WO73/C3 — the decode-latency budget: perf.decode_ms_per_row declared +in the index entry vs the measured value (ctx.extra["measured"]), which +the bench tool (tools/bench_decode.py) or CI produces. +""" + +from __future__ import annotations + +from pipeline.gates import BudgetGate, GateContext + + +def _ctx(perf: dict, measured: dict | None) -> GateContext: + entry = {"size": 1_000_000, "perf": perf} + extra = {"index_entry": entry} + if measured is not None: + extra["measured"] = measured + return GateContext(extra=extra) + + +def test_latency_within_budget_passes() -> None: + result = BudgetGate().check( + recipe=None, + ctx=_ctx({"size_mb": 10, "decode_ms_per_row": 500}, + {"decode_ms_per_row": 321.5}), + ) + assert result.passed + assert "321.5" in result.detail + + +def test_latency_over_budget_fails() -> None: + result = BudgetGate().check( + recipe=None, + ctx=_ctx({"size_mb": 10, "decode_ms_per_row": 100}, + {"decode_ms_per_row": 140.0}), + ) + assert not result.passed + assert "exceeds" in result.detail + + +def test_declared_latency_without_measurement_fails() -> None: + result = BudgetGate().check( + recipe=None, ctx=_ctx({"size_mb": 10, "decode_ms_per_row": 100}, None)) + assert not result.passed + assert "never assumed" in result.detail + + +def test_no_budgets_passes_with_note() -> None: + result = BudgetGate().check(recipe=None, ctx=_ctx({}, None)) + assert result.passed + assert "no perf budgets declared" in result.detail diff --git a/tools/bench_decode.py b/tools/bench_decode.py new file mode 100644 index 00000000..d93211bf --- /dev/null +++ b/tools/bench_decode.py @@ -0,0 +1,68 @@ +"""WO73/C3 — decode-latency bench: the measured half of the +perf.decode_ms_per_row budget. + +Decodes a model's frozen conformance fixtures through the shipped +decode mode (single-row greedy) and reports wall-clock ms/row. The +number feeds BudgetGate via ctx.extra["measured"]; a budget is never +assumed without it. + +Usage: + python tools/bench_decode.py [--repeats 3] \ + [--vectors conformance/-vectors.json] + +Resolves the artifact through the index (sha256-verified), so the +measured number always corresponds to the shipped bytes. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import sys +import time +from pathlib import Path + +REPO = Path(__file__).parent.parent +sys.path.insert(0, str(REPO / "runtime" / "src")) + +from interscript_ml import Model # noqa: E402 + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("model_id") + ap.add_argument("--repeats", type=int, default=3) + ap.add_argument("--vectors", type=Path, default=None) + args = ap.parse_args() + + vectors_path = args.vectors or ( + REPO / "conformance" / f"{args.model_id}-vectors.json") + vectors = json.loads(vectors_path.read_text(encoding="utf-8")) + rows = vectors["vectors"] + if not rows: + sys.exit(f"no vectors in {vectors_path}") + + model = Model.load(args.model_id) # resolves via the index, verifies sha256 + + # warm-up (index fetch, ORT session init, first-graph allocation) + model.translate(rows[0]["src"]) + + per_row = [] + for _ in range(args.repeats): + t0 = time.perf_counter() + for row in rows: + model.translate(row["src"]) + per_row.append((time.perf_counter() - t0) * 1000 / len(rows)) + + print(json.dumps({ + "model": args.model_id, + "rows": len(rows), + "repeats": args.repeats, + "decode_ms_per_row_median": round(statistics.median(per_row), 2), + "decode_ms_per_row_max": round(max(per_row), 2), + })) + + +if __name__ == "__main__": + main()