diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48e2cc78..eb91dc39 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,6 +57,16 @@ jobs: - name: IMF export + parity gate fixture tests (torch vs ORT) run: PYTHONPATH=src python -m pytest tests/test_imf_export.py tests/test_imf_parity.py -v + contracts: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: "3.11" } + - run: pip install -e ".[dev]" + - name: Corpus/recipe contract validation (structure fails; license blocks report, publish-time enforced) + run: PYTHONPATH=src python -m pipeline.cli validate + python-runtime: runs-on: ubuntu-latest steps: diff --git a/recipes/ara-diac.yaml b/recipes/ara-diac.yaml index 9a0b8e34..22bf3148 100644 --- a/recipes/ara-diac.yaml +++ b/recipes/ara-diac.yaml @@ -14,6 +14,7 @@ stages: route: legacy:interscript-train/continue_oracle_train_hf.py params: {epochs: 2} gates: + convention: row-mean quality: {wer_max: 3.5607} # run-047 first-alt line; stretch 2.70 (QCRI frontier parity) parity: {mode: single-row, agree_min: 0.98, three_way: true} negative_results: diff --git a/recipes/fas-diac.yaml b/recipes/fas-diac.yaml index 7cfe4430..83d722e2 100644 --- a/recipes/fas-diac.yaml +++ b/recipes/fas-diac.yaml @@ -9,6 +9,7 @@ stages: route: legacy:interscript-train/train_fas_diac_hf.py params: {epochs: 3, bs: 16, accum: 2, gate: 0.0052} gates: + convention: corpus-level quality: {cer_max: 0.0052} parity: {mode: single-row, agree_min: 0.98, three_way: true} negative_results: diff --git a/recipes/urd-diac.yaml b/recipes/urd-diac.yaml index 7ac82298..9f32e29f 100644 --- a/recipes/urd-diac.yaml +++ b/recipes/urd-diac.yaml @@ -11,6 +11,7 @@ stages: - stage: decode_policy params: {rep_penalty: 1.2, no_repeat_ngram: 3} gates: + convention: row-mean quality: {cer_max: 0.045} parity: {mode: single-row, agree_min: 0.98, three_way: true} negative_results: diff --git a/recipes/urd-g2p.yaml b/recipes/urd-g2p.yaml index 8248db9e..81556ee0 100644 --- a/recipes/urd-g2p.yaml +++ b/recipes/urd-g2p.yaml @@ -8,6 +8,7 @@ stages: - stage: sft route: legacy:interscript-train/train_urd_g2p_hf.py gates: + convention: row-mean quality: {cer_max: 0.17} # torch greedy 0.16 (artifact ≡ torch under the WO51 single-row contract) parity: {mode: single-row, agree_min: 0.98, three_way: true} negative_results: diff --git a/src/pipeline/cli.py b/src/pipeline/cli.py index c60183a8..7f5f81ab 100644 --- a/src/pipeline/cli.py +++ b/src/pipeline/cli.py @@ -33,10 +33,12 @@ def _find_recipe(path: Path, recipe_id: str) -> Path: def cmd_validate(args: argparse.Namespace) -> int: problems = validate_recipe_dir(args.recipes, args.corpora) + hard = [p for p in problems if "owner review" not in p] for p in problems: - print(f"FAIL {p}", file=sys.stderr) - print(f"{len(problems)} problem(s)") - return 1 if problems else 0 + mark = "FAIL" if "owner review" not in p else "BLOCKED" + print(f"{mark} {p}", file=sys.stderr) + print(f"{len(hard)} problem(s), {len(problems) - len(hard)} owner-blocked") + return 1 if hard else 0 def cmd_plan(args: argparse.Namespace) -> int: diff --git a/src/pipeline/contracts.py b/src/pipeline/contracts.py index b64a62aa..f441a7fb 100644 --- a/src/pipeline/contracts.py +++ b/src/pipeline/contracts.py @@ -77,6 +77,8 @@ class ParityGate(_Strict): class GateSpec(_Strict): quality: dict[str, float] + convention: Literal["row-mean", "corpus-level"] # the fas scar: a gate + # number without its aggregation convention is not a gate parity: ParityGate license: bool = True diff --git a/src/pipeline/publish.py b/src/pipeline/publish.py new file mode 100644 index 00000000..142cd765 --- /dev/null +++ b/src/pipeline/publish.py @@ -0,0 +1,63 @@ +"""WO52 publish last-mile — the owner-gated guard layer. + +Publishing refuses unless: the version is explicit (never a moving +ref — versions are the owner's decision), every gate is green with +licenses strictly enforced, and a conformance vectors file exists for +the model. The actual artifact publication stays the proven legacy +path (scripts/publish_model.py + the index-vN flow); this module is +the gate that decides whether it may run at all. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from pathlib import Path + +from pipeline.contracts import RecipeSpec +from pipeline.gates import evaluate + +VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") + + +class PublishRefusal(RuntimeError): + pass + + +def check_publish_ready( + recipe: RecipeSpec, + corpora_dir: Path, + version: str, + metrics: dict | None, + parity: dict | None, + conformance_path: Path | None, +) -> dict: + if not VERSION_RE.match(version or ""): + raise PublishRefusal( + f"version {version!r} is not an explicit semver — versions are " + "the owner's decision and never a moving ref" + ) + results = evaluate(recipe, corpora_dir, metrics=metrics, parity=parity) + gate_map = {g.name: g.passed for g in results} + license_detail = next( + (g.detail for g in results if g.name == "license" and not g.passed), "" + ) + if not gate_map.get("license", False): + raise PublishRefusal(f"license gate: {license_detail or 'blocked'}") + for name in ("quality", "parity"): + if not gate_map.get(name, False): + detail = next(g.detail for g in results if g.name == name) + raise PublishRefusal(f"{name} gate: {detail}") + if conformance_path is None or not Path(conformance_path).is_file(): + raise PublishRefusal( + "no conformance vectors file — a model ships frozen fixtures or not at all" + ) + return {"gates": gate_map, "conformance": True, "version": version} + + +def run_publisher( + publisher: Callable[[list[str]], int], argv: list[str] +) -> int: + """Invoke the injected publisher (legacy scripts/publish_model.py + adapter) after the guard has passed.""" + return publisher(argv) diff --git a/tests/fixtures/recipes/urd-diac.yaml b/tests/fixtures/recipes/urd-diac.yaml index 331b3ef2..f45d7a03 100644 --- a/tests/fixtures/recipes/urd-diac.yaml +++ b/tests/fixtures/recipes/urd-diac.yaml @@ -11,6 +11,7 @@ stages: - stage: decode_policy params: {rep_penalty: 1.2, no_repeat_ngram: 3} gates: + convention: row-mean quality: {cer_max: 0.045} parity: {mode: single-row, agree_min: 0.98, three_way: true} negative_results: diff --git a/tests/test_audit_fixes.py b/tests/test_audit_fixes.py new file mode 100644 index 00000000..26559ea5 --- /dev/null +++ b/tests/test_audit_fixes.py @@ -0,0 +1,152 @@ +"""Framework audit fixes (2026-10-18): + +G2 — metric convention is schema-required. The fas saga (five arms +reproducing 0.60-0.81% against a recorded 0.52%) proved that a gate +number without its aggregation convention is not a gate: row-mean and +corpus-level CER differ by ~0.2 points on this data. + +G3 — the publish last-mile refuses to run without an explicit owner +version, green gates (license strict), and conformance fixtures. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from pipeline.contracts import load_recipe +from pipeline.publish import PublishRefusal, check_publish_ready + +REPO = Path(__file__).parent.parent +RECIPES = REPO / "recipes" +CORPORA = REPO / "corpora" + + +def test_gate_convention_required() -> None: + body = { + "id": "xx-test", + "task": "transliteration", + "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"}, + "model": "byt5-small", + "stages": [{"stage": "sft", "params": {"epochs": 1}}], + "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}}, + } + with pytest.raises(ValueError, match="convention"): + __import__("pipeline.contracts", fromlist=["RecipeSpec"]).RecipeSpec.model_validate(body) + + +def test_shipped_recipes_declare_convention() -> None: + for rpath in sorted(RECIPES.glob("*.yaml")): + r = load_recipe(rpath) + assert r.gates.convention in ("row-mean", "corpus-level"), rpath + + +def _tmp_registry(tmp_path: Path, license_value: str) -> Path: + corpus = { + "id": "xx-src", + "task": "transliteration", + "sources": [{"kind": "hf-dataset", "id": "org/ds", "license": license_value}], + "splits": { + "train": {"path": "train.jsonl", "rows": 10}, + "test": {"path": "test.jsonl", "rows": 5}, + }, + } + recipe = { + "id": "xx-test", + "task": "transliteration", + "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"}, + "model": "byt5-small", + "stages": [{"stage": "sft", "params": {"epochs": 1}}], + "gates": { + "quality": {"cer_max": 0.1}, + "parity": {"agree_min": 0.98}, + "convention": "row-mean", + }, + } + (tmp_path / "corpora").mkdir() + (tmp_path / "recipes").mkdir() + (tmp_path / "corpora" / "xx-src.yaml").write_text(yaml.safe_dump(corpus)) + (tmp_path / "recipes" / "xx-test.yaml").write_text(yaml.safe_dump(recipe)) + return tmp_path + + +def test_publish_refuses_moving_version(tmp_path: Path) -> None: + reg = _tmp_registry(tmp_path, "CC0-1.0") + recipe = load_recipe(reg / "recipes" / "xx-test.yaml") + with pytest.raises(PublishRefusal, match="version"): + check_publish_ready( + recipe, + reg / "corpora", + version="latest", + metrics={"cer": 0.05}, + parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0, + "agree_bf16_ort": 1.0}, + conformance_path=reg / "vectors.json", + ) + + +def test_publish_refuses_blocked_license(tmp_path: Path) -> None: + reg = _tmp_registry(tmp_path, "LicenseRef-non-commercial") + recipe = load_recipe(reg / "recipes" / "xx-test.yaml") + with pytest.raises(PublishRefusal, match="owner review"): + check_publish_ready( + recipe, + reg / "corpora", + version="1.0.0", + metrics={"cer": 0.05}, + parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0, + "agree_bf16_ort": 1.0}, + conformance_path=reg / "vectors.json", + ) + + +def test_publish_refuses_failing_quality(tmp_path: Path) -> None: + reg = _tmp_registry(tmp_path, "CC0-1.0") + recipe = load_recipe(reg / "recipes" / "xx-test.yaml") + with pytest.raises(PublishRefusal, match="quality"): + check_publish_ready( + recipe, + reg / "corpora", + version="1.0.0", + metrics={"cer": 0.5}, + parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0, + "agree_bf16_ort": 1.0}, + conformance_path=reg / "vectors.json", + ) + + +def test_publish_refuses_missing_conformance(tmp_path: Path) -> None: + reg = _tmp_registry(tmp_path, "CC0-1.0") + recipe = load_recipe(reg / "recipes" / "xx-test.yaml") + with pytest.raises(PublishRefusal, match="conformance"): + check_publish_ready( + recipe, + reg / "corpora", + version="1.0.0", + metrics={"cer": 0.05}, + parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0, + "agree_bf16_ort": 1.0}, + conformance_path=None, + ) + + +def test_publish_ready_happy_path(tmp_path: Path) -> None: + reg = _tmp_registry(tmp_path, "CC0-1.0") + (reg / "vectors.json").write_text( + json.dumps({"model": "xx", "vectors": [{"src": "a", "out": "b"}]}) + ) + recipe = load_recipe(reg / "recipes" / "xx-test.yaml") + checks = check_publish_ready( + recipe, + reg / "corpora", + version="1.0.0", + metrics={"cer": 0.05}, + parity={"ort_single_cer": 0.05, "single_vs_batched_agree": 1.0, + "agree_bf16_ort": 1.0}, + conformance_path=reg / "vectors.json", + ) + assert checks["gates"]["license"] is True + assert checks["conformance"] is True diff --git a/tests/test_pipeline_contracts.py b/tests/test_pipeline_contracts.py index 4c8393d4..0fc7a7ab 100644 --- a/tests/test_pipeline_contracts.py +++ b/tests/test_pipeline_contracts.py @@ -44,7 +44,8 @@ def test_stage_needs_route_or_params() -> None: "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"}, "model": "byt5-small", "stages": [{"stage": "sft"}], - "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}}, + "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean", + "parity": {"agree_min": 0.98}}, } with pytest.raises(ValueError, match="route"): RecipeSpec.model_validate(body) @@ -57,7 +58,8 @@ def test_no_llm_teachers_for_diacritization() -> None: "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"}, "model": "byt5-small", "stages": [{"stage": "distill", "route": "legacy:llm_teacher.py"}], - "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}}, + "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean", + "parity": {"agree_min": 0.98}}, } with pytest.raises(ValueError, match="llm"): RecipeSpec.model_validate(body) @@ -70,7 +72,8 @@ def test_corpus_ref_pin_format() -> None: "corpora": {"train": "xx-src@latest", "test": "xx-src@v1#test"}, "model": "byt5-small", "stages": [{"stage": "sft", "params": {"epochs": 1}}], - "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}}, + "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean", + "parity": {"agree_min": 0.98}}, } with pytest.raises(ValueError, match="pin"): RecipeSpec.model_validate(body) @@ -109,7 +112,8 @@ def test_recipe_refs_resolve_into_registry(tmp_path: Path) -> None: "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"}, "model": "byt5-small", "stages": [{"stage": "sft", "params": {"epochs": 1}}], - "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}}, + "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean", + "parity": {"agree_min": 0.98}}, } import yaml # noqa: PLC0415 @@ -134,7 +138,8 @@ def test_unverified_license_is_blocking(tmp_path: Path) -> None: "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"}, "model": "byt5-small", "stages": [{"stage": "sft", "params": {"epochs": 1}}], - "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}}, + "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean", + "parity": {"agree_min": 0.98}}, } import yaml # noqa: PLC0415 @@ -186,7 +191,8 @@ def test_restricted_but_read_license_blocks(tmp_path: Path) -> None: "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"}, "model": "byt5-small", "stages": [{"stage": "sft", "params": {"epochs": 1}}], - "gates": {"quality": {"cer_max": 0.1}, "parity": {"agree_min": 0.98}}, + "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean", + "parity": {"agree_min": 0.98}}, } (tmp_path / "corpora").mkdir() (tmp_path / "recipes").mkdir()