Audit fixes: metric convention required, publish guard, CI contracts job

.github/workflows/test.yml

@@ -57,6 +57,16 @@ jobs:
       - name: IMF export + parity gate fixture tests (torch vs ORT)
         run: PYTHONPATH=src python -m pytest tests/test_imf_export.py tests/test_imf_parity.py -v
 
+  contracts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - run: pip install -e ".[dev]"
+      - name: Corpus/recipe contract validation (structure fails; license blocks report, publish-time enforced)
+        run: PYTHONPATH=src python -m pipeline.cli validate
+
   python-runtime:
     runs-on: ubuntu-latest
     steps:

recipes/ara-diac.yaml

@@ -14,6 +14,7 @@ stages:
     route: legacy:interscript-train/continue_oracle_train_hf.py
     params: {epochs: 2}
 gates:
+  convention: row-mean
   quality: {wer_max: 3.5607}  # run-047 first-alt line; stretch 2.70 (QCRI frontier parity)
   parity: {mode: single-row, agree_min: 0.98, three_way: true}
 negative_results:

recipes/fas-diac.yaml

@@ -9,6 +9,7 @@ stages:
     route: legacy:interscript-train/train_fas_diac_hf.py
     params: {epochs: 3, bs: 16, accum: 2, gate: 0.0052}
 gates:
+  convention: corpus-level
   quality: {cer_max: 0.0052}
   parity: {mode: single-row, agree_min: 0.98, three_way: true}
 negative_results:

recipes/urd-diac.yaml

@@ -11,6 +11,7 @@ stages:
   - stage: decode_policy
     params: {rep_penalty: 1.2, no_repeat_ngram: 3}
 gates:
+  convention: row-mean
   quality: {cer_max: 0.045}
   parity: {mode: single-row, agree_min: 0.98, three_way: true}
 negative_results:

recipes/urd-g2p.yaml

@@ -8,6 +8,7 @@ stages:
   - stage: sft
     route: legacy:interscript-train/train_urd_g2p_hf.py
 gates:
+  convention: row-mean
   quality: {cer_max: 0.17}  # torch greedy 0.16 (artifact ≡ torch under the WO51 single-row contract)
   parity: {mode: single-row, agree_min: 0.98, three_way: true}
 negative_results:

src/pipeline/cli.py

@@ -33,10 +33,12 @@ def _find_recipe(path: Path, recipe_id: str) -> Path:
 
 def cmd_validate(args: argparse.Namespace) -> int:
     problems = validate_recipe_dir(args.recipes, args.corpora)
+    hard = [p for p in problems if "owner review" not in p]
     for p in problems:
-        print(f"FAIL {p}", file=sys.stderr)
-    print(f"{len(problems)} problem(s)")
-    return 1 if problems else 0
+        mark = "FAIL" if "owner review" not in p else "BLOCKED"
+        print(f"{mark} {p}", file=sys.stderr)
+    print(f"{len(hard)} problem(s), {len(problems) - len(hard)} owner-blocked")
+    return 1 if hard else 0
 
 
 def cmd_plan(args: argparse.Namespace) -> int:

src/pipeline/contracts.py

@@ -77,6 +77,8 @@ class ParityGate(_Strict):
 
 class GateSpec(_Strict):
     quality: dict[str, float]
+    convention: Literal["row-mean", "corpus-level"]  # the fas scar: a gate
+    # number without its aggregation convention is not a gate
     parity: ParityGate
     license: bool = True
 

src/pipeline/publish.py

@@ -0,0 +1,63 @@
+"""WO52 publish last-mile — the owner-gated guard layer.
+
+Publishing refuses unless: the version is explicit (never a moving
+ref — versions are the owner's decision), every gate is green with
+licenses strictly enforced, and a conformance vectors file exists for
+the model. The actual artifact publication stays the proven legacy
+path (scripts/publish_model.py + the index-vN flow); this module is
+the gate that decides whether it may run at all.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from pathlib import Path
+
+from pipeline.contracts import RecipeSpec
+from pipeline.gates import evaluate
+
+VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$")
+
+
+class PublishRefusal(RuntimeError):
+    pass
+
+
+def check_publish_ready(
+    recipe: RecipeSpec,
+    corpora_dir: Path,
+    version: str,
+    metrics: dict | None,
+    parity: dict | None,
+    conformance_path: Path | None,
+) -> dict:
+    if not VERSION_RE.match(version or ""):
+        raise PublishRefusal(
+            f"version {version!r} is not an explicit semver — versions are "
+            "the owner's decision and never a moving ref"
+        )
+    results = evaluate(recipe, corpora_dir, metrics=metrics, parity=parity)
+    gate_map = {g.name: g.passed for g in results}
+    license_detail = next(
+        (g.detail for g in results if g.name == "license" and not g.passed), ""
+    )
+    if not gate_map.get("license", False):
+        raise PublishRefusal(f"license gate: {license_detail or 'blocked'}")
+    for name in ("quality", "parity"):
+        if not gate_map.get(name, False):
+            detail = next(g.detail for g in results if g.name == name)
+            raise PublishRefusal(f"{name} gate: {detail}")
+    if conformance_path is None or not Path(conformance_path).is_file():
+        raise PublishRefusal(
+            "no conformance vectors file — a model ships frozen fixtures or not at all"
+        )
+    return {"gates": gate_map, "conformance": True, "version": version}
+
+
+def run_publisher(
+    publisher: Callable[[list[str]], int], argv: list[str]
+) -> int:
+    """Invoke the injected publisher (legacy scripts/publish_model.py
+    adapter) after the guard has passed."""
+    return publisher(argv)

tests/fixtures/recipes/urd-diac.yaml

@@ -11,6 +11,7 @@ stages:
   - stage: decode_policy
     params: {rep_penalty: 1.2, no_repeat_ngram: 3}
 gates:
+  convention: row-mean
   quality: {cer_max: 0.045}
   parity: {mode: single-row, agree_min: 0.98, three_way: true}
 negative_results:

tests/test_audit_fixes.py

@@ -0,0 +1,152 @@
+"""Framework audit fixes (2026-10-18):
+
+G2 — metric convention is schema-required. The fas saga (five arms
+reproducing 0.60-0.81% against a recorded 0.52%) proved that a gate
+number without its aggregation convention is not a gate: row-mean and
+corpus-level CER differ by ~0.2 points on this data.
+
+G3 — the publish last-mile refuses to run without an explicit owner
+version, green gates (license strict), and conformance fixtures.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+import yaml
+
+from pipeline.contracts import load_recipe
+from pipeline.publish import PublishRefusal, check_publish_ready
+
+REPO = Path(__file__).parent.parent
+RECIPES = REPO / "recipes"
+CORPORA = REPO / "corpora"
+
+
+def test_gate_convention_required() -> None:
+    body = {
+        "id": "xx-test",
+        "task": "transliteration",
+        "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"},
+        "model": "byt5-small",
+        "stages": [{"stage": "sft", "params": {"epochs": 1}}],
+        "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}},
+    }
+    with pytest.raises(ValueError, match="convention"):
+        __import__("pipeline.contracts", fromlist=["RecipeSpec"]).RecipeSpec.model_validate(body)
+
+
+def test_shipped_recipes_declare_convention() -> None:
+    for rpath in sorted(RECIPES.glob("*.yaml")):
+        r = load_recipe(rpath)
+        assert r.gates.convention in ("row-mean", "corpus-level"), rpath
+
+
+def _tmp_registry(tmp_path: Path, license_value: str) -> Path:
+    corpus = {
+        "id": "xx-src",
+        "task": "transliteration",
+        "sources": [{"kind": "hf-dataset", "id": "org/ds", "license": license_value}],
+        "splits": {
+            "train": {"path": "train.jsonl", "rows": 10},
+            "test": {"path": "test.jsonl", "rows": 5},
+        },
+    }
+    recipe = {
+        "id": "xx-test",
+        "task": "transliteration",
+        "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"},
+        "model": "byt5-small",
+        "stages": [{"stage": "sft", "params": {"epochs": 1}}],
+        "gates": {
+            "quality": {"cer_max": 0.1},
+            "parity": {"agree_min": 0.98},
+            "convention": "row-mean",
+        },
+    }
+    (tmp_path / "corpora").mkdir()
+    (tmp_path / "recipes").mkdir()
+    (tmp_path / "corpora" / "xx-src.yaml").write_text(yaml.safe_dump(corpus))
+    (tmp_path / "recipes" / "xx-test.yaml").write_text(yaml.safe_dump(recipe))
+    return tmp_path
+
+
+def test_publish_refuses_moving_version(tmp_path: Path) -> None:
+    reg = _tmp_registry(tmp_path, "CC0-1.0")
+    recipe = load_recipe(reg / "recipes" / "xx-test.yaml")
+    with pytest.raises(PublishRefusal, match="version"):
+        check_publish_ready(
+            recipe,
+            reg / "corpora",
+            version="latest",
+            metrics={"cer": 0.05},
+            parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0,
+                    "agree_bf16_ort": 1.0},
+            conformance_path=reg / "vectors.json",
+        )
+
+
+def test_publish_refuses_blocked_license(tmp_path: Path) -> None:
+    reg = _tmp_registry(tmp_path, "LicenseRef-non-commercial")
+    recipe = load_recipe(reg / "recipes" / "xx-test.yaml")
+    with pytest.raises(PublishRefusal, match="owner review"):
+        check_publish_ready(
+            recipe,
+            reg / "corpora",
+            version="1.0.0",
+            metrics={"cer": 0.05},
+            parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0,
+                    "agree_bf16_ort": 1.0},
+            conformance_path=reg / "vectors.json",
+        )
+
+
+def test_publish_refuses_failing_quality(tmp_path: Path) -> None:
+    reg = _tmp_registry(tmp_path, "CC0-1.0")
+    recipe = load_recipe(reg / "recipes" / "xx-test.yaml")
+    with pytest.raises(PublishRefusal, match="quality"):
+        check_publish_ready(
+            recipe,
+            reg / "corpora",
+            version="1.0.0",
+            metrics={"cer": 0.5},
+            parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0,
+                    "agree_bf16_ort": 1.0},
+            conformance_path=reg / "vectors.json",
+        )
+
+
+def test_publish_refuses_missing_conformance(tmp_path: Path) -> None:
+    reg = _tmp_registry(tmp_path, "CC0-1.0")
+    recipe = load_recipe(reg / "recipes" / "xx-test.yaml")
+    with pytest.raises(PublishRefusal, match="conformance"):
+        check_publish_ready(
+            recipe,
+            reg / "corpora",
+            version="1.0.0",
+            metrics={"cer": 0.05},
+            parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0,
+                    "agree_bf16_ort": 1.0},
+            conformance_path=None,
+        )
+
+
+def test_publish_ready_happy_path(tmp_path: Path) -> None:
+    reg = _tmp_registry(tmp_path, "CC0-1.0")
+    (reg / "vectors.json").write_text(
+        json.dumps({"model": "xx", "vectors": [{"src": "a", "out": "b"}]})
+    )
+    recipe = load_recipe(reg / "recipes" / "xx-test.yaml")
+    checks = check_publish_ready(
+        recipe,
+        reg / "corpora",
+        version="1.0.0",
+        metrics={"cer": 0.05},
+        parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0,
+                "agree_bf16_ort": 1.0},
+        conformance_path=reg / "vectors.json",
+    )
+    assert checks["gates"]["license"] is True
+    assert checks["conformance"] is True

tests/test_pipeline_contracts.py

@@ -44,7 +44,8 @@ def test_stage_needs_route_or_params() -> None:
         "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"},
         "model": "byt5-small",
         "stages": [{"stage": "sft"}],
-        "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}},
+        "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
+                   "parity": {"agree_min": 0.98}},
     }
     with pytest.raises(ValueError, match="route"):
         RecipeSpec.model_validate(body)
@@ -57,7 +58,8 @@ def test_no_llm_teachers_for_diacritization() -> None:
         "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"},
         "model": "byt5-small",
         "stages": [{"stage": "distill", "route": "legacy:llm_teacher.py"}],
-        "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}},
+        "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
+                   "parity": {"agree_min": 0.98}},
     }
     with pytest.raises(ValueError, match="llm"):
         RecipeSpec.model_validate(body)
@@ -70,7 +72,8 @@ def test_corpus_ref_pin_format() -> None:
         "corpora": {"train": "xx-src@latest", "test": "xx-src@v1#test"},
         "model": "byt5-small",
         "stages": [{"stage": "sft", "params": {"epochs": 1}}],
-        "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}},
+        "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
+                   "parity": {"agree_min": 0.98}},
     }
     with pytest.raises(ValueError, match="pin"):
         RecipeSpec.model_validate(body)
@@ -109,7 +112,8 @@ def test_recipe_refs_resolve_into_registry(tmp_path: Path) -> None:
         "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"},
         "model": "byt5-small",
         "stages": [{"stage": "sft", "params": {"epochs": 1}}],
-        "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}},
+        "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
+                   "parity": {"agree_min": 0.98}},
     }
     import yaml  # noqa: PLC0415
 
@@ -134,7 +138,8 @@ def test_unverified_license_is_blocking(tmp_path: Path) -> None:
         "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"},
         "model": "byt5-small",
         "stages": [{"stage": "sft", "params": {"epochs": 1}}],
-        "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}},
+        "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
+                   "parity": {"agree_min": 0.98}},
     }
     import yaml  # noqa: PLC0415
 
@@ -186,7 +191,8 @@ def test_restricted_but_read_license_blocks(tmp_path: Path) -> None:
         "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"},
         "model": "byt5-small",
         "stages": [{"stage": "sft", "params": {"epochs": 1}}],
-        "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}},
+        "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
+                   "parity": {"agree_min": 0.98}},
     }
     (tmp_path / "corpora").mkdir()
     (tmp_path / "recipes").mkdir()