Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions recipes/ara-diac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ stages:
route: legacy:interscript-train/continue_oracle_train_hf.py
params: {epochs: 2}
gates:
anchor: run-047 # byt5-large first-alt gate line
convention: row-mean
quality: {wer_max: 3.5607} # run-047 first-alt line; stretch 2.70 (QCRI frontier parity)
parity: {mode: single-row, agree_min: 0.98, three_way: true}
Expand Down
1 change: 1 addition & 0 deletions recipes/fas-diac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ stages:
route: legacy:interscript-train/train_fas_diac_hf.py
params: {epochs: 3, bs: 16, accum: 2, gate: 0.0052}
gates:
anchor: TODO.sota/47-fas-diac-release.md (docs split, corpus-level) # the 0.52% docs line
convention: corpus-level
quality: {cer_max: 0.0052}
parity: {mode: single-row, agree_min: 0.98, three_way: true}
Expand Down
1 change: 1 addition & 0 deletions recipes/urd-diac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ stages:
- stage: decode_policy
params: {rep_penalty: 1.2, no_repeat_ngram: 3}
gates:
anchor: run-066 # urd-diac-base gate pass
convention: row-mean
quality: {cer_max: 0.045}
parity: {mode: single-row, agree_min: 0.98, three_way: true}
Expand Down
1 change: 1 addition & 0 deletions recipes/urd-g2p.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ stages:
- stage: sft
route: legacy:interscript-train/train_urd_g2p_hf.py
gates:
anchor: run-059 # urd-g2p-base gate pass
convention: row-mean
quality: {cer_max: 0.17} # torch greedy 0.16 (artifact ≡ torch under the WO51 single-row contract)
parity: {mode: single-row, agree_min: 0.98, three_way: true}
Expand Down
9 changes: 9 additions & 0 deletions src/pipeline/contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,18 @@ class GateSpec(_Strict):
quality: dict[str, float]
convention: Literal["row-mean", "corpus-level"] # the fas scar: a gate
# number without its aggregation convention is not a gate
anchor: str # the protocol/ledger section this gate line is anchored to
parity: ParityGate
license: bool = True

@field_validator("anchor")
@classmethod
def _anchor_nonempty(cls, v: str) -> str:
if not v.strip():
raise ValueError("gates: anchor must name the protocol/ledger "
"section the gate line is anchored to")
return v

@field_validator("quality")
@classmethod
def _metrics(cls, v: dict[str, float]) -> dict[str, float]:
Expand Down
1 change: 1 addition & 0 deletions tests/fixtures/recipes/urd-diac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ stages:
- stage: decode_policy
params: {rep_penalty: 1.2, no_repeat_ngram: 3}
gates:
anchor: run-047 # fixture anchor
convention: row-mean
quality: {cer_max: 0.045}
parity: {mode: single-row, agree_min: 0.98, three_way: true}
Expand Down
2 changes: 1 addition & 1 deletion tests/test_audit_fixes.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ def _tmp_registry(tmp_path: Path, license_value: str) -> Path:
"gates": {
"quality": {"cer_max": 0.1},
"parity": {"agree_min": 0.98},
"convention": "row-mean",
"convention": "row-mean", "anchor": "run-047",
},
}
(tmp_path / "corpora").mkdir()
Expand Down
31 changes: 31 additions & 0 deletions tests/test_pipeline_contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ def test_stage_needs_route_or_params() -> None:
"model": "byt5-small",
"stages": [{"stage": "sft"}],
"gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
"anchor": "run-047",
"parity": {"agree_min": 0.98}},
}
with pytest.raises(ValueError, match="route"):
Expand All @@ -59,6 +60,7 @@ def test_no_llm_teachers_for_diacritization() -> None:
"model": "byt5-small",
"stages": [{"stage": "distill", "route": "legacy:llm_teacher.py"}],
"gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
"anchor": "run-047",
"parity": {"agree_min": 0.98}},
}
with pytest.raises(ValueError, match="llm"):
Expand All @@ -73,6 +75,7 @@ def test_corpus_ref_pin_format() -> None:
"model": "byt5-small",
"stages": [{"stage": "sft", "params": {"epochs": 1}}],
"gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
"anchor": "run-047",
"parity": {"agree_min": 0.98}},
}
with pytest.raises(ValueError, match="pin"):
Expand Down Expand Up @@ -113,6 +116,7 @@ def test_recipe_refs_resolve_into_registry(tmp_path: Path) -> None:
"model": "byt5-small",
"stages": [{"stage": "sft", "params": {"epochs": 1}}],
"gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
"anchor": "run-047",
"parity": {"agree_min": 0.98}},
}
import yaml # noqa: PLC0415
Expand All @@ -139,6 +143,7 @@ def test_unverified_license_is_blocking(tmp_path: Path) -> None:
"model": "byt5-small",
"stages": [{"stage": "sft", "params": {"epochs": 1}}],
"gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
"anchor": "run-047",
"parity": {"agree_min": 0.98}},
}
import yaml # noqa: PLC0415
Expand Down Expand Up @@ -192,6 +197,7 @@ def test_restricted_but_read_license_blocks(tmp_path: Path) -> None:
"model": "byt5-small",
"stages": [{"stage": "sft", "params": {"epochs": 1}}],
"gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
"anchor": "run-047",
"parity": {"agree_min": 0.98}},
}
(tmp_path / "corpora").mkdir()
Expand All @@ -206,3 +212,28 @@ def test_load_corpus_roundtrip() -> None:
c = load_corpus(FIXTURES / "corpora" / "urd-dia.yaml")
assert c.id == "urd-dia"
assert "train" in c.splits and "test" in c.splits


def test_gate_block_requires_anchor_and_convention() -> None:
base = {
"id": "xx-anchor",
"task": "transliteration",
"corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"},
"model": "byt5-small",
"stages": [{"stage": "sft", "params": {"epochs": 1}}],
}
no_anchor = {
**base,
"gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
"parity": {"agree_min": 0.98}},
}
with pytest.raises(ValueError, match="anchor"):
RecipeSpec.model_validate(no_anchor)

no_convention = {
**base,
"gates": {"quality": {"cer_max": 0.1}, "anchor": "run-047",
"parity": {"agree_min": 0.98}},
}
with pytest.raises(ValueError, match="convention"):
RecipeSpec.model_validate(no_convention)
1 change: 1 addition & 0 deletions tests/test_pipeline_plan_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ def test_license_gate_blocks_publish(tmp_path: Path) -> None:
"model": "byt5-small",
"stages": [{"stage": "sft", "params": {"epochs": 1}}],
"gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
"anchor": "run-047",
"parity": {"agree_min": 0.98}},
}
(tmp_path / "corpora").mkdir()
Expand Down
1 change: 1 addition & 0 deletions tests/test_pipeline_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ def _recipe_body() -> dict:
"gates": {
"quality": {"cer_max": 0.1},
"convention": "row-mean",
"anchor": "run-047",
"parity": {"agree_min": 0.98},
},
}
Expand Down
Loading