diff --git a/corpora/qcri-wiki.yaml b/corpora/qcri-wiki.yaml index 9dfc11fc..c1c5f3bc 100644 --- a/corpora/qcri-wiki.yaml +++ b/corpora/qcri-wiki.yaml @@ -8,7 +8,12 @@ sources: - kind: hf-dataset id: Interscript/arabic-r8 revision: main - license: LicenseRef-no-license-found # Qatara repo unlicensed — owner: contact QCRI + license: LicenseRef-no-license-found # Qatara repo unlicensed + + license_review: + status: owner-approved + date: "2026-10-18" + note: ship with dependency note — weights depend on the QCRI upstream license (owner decision #2) mount: /train_data splits: train: {path: qcri-wiki.jsonl, rows: 32834, sha256: 34ef170845472cdf81f9e83e5ac8b9bf45d716d1bc1b3a8e009f1ed343b4d51d} # article-per-line (~5M words) diff --git a/corpora/urd-dia.yaml b/corpora/urd-dia.yaml index 19080462..91f37602 100644 --- a/corpora/urd-dia.yaml +++ b/corpora/urd-dia.yaml @@ -8,7 +8,12 @@ sources: - kind: bucket id: rhtse/jobs-artifacts/rababa-urdu-cc174b96 revision: data-diacrit - license: LicenseRef-unresolved # no license found on sources — owner: contact authors or replace corpus + license: LicenseRef-unresolved # no license found on sources + + license_review: + status: owner-approved + date: "2026-10-18" + note: ship with dependency note — weights depend on the original data license (owner decision #3) mount: /data-dia splits: train: {path: data-diacrit/train.jsonl, rows: 573130, sha256: 95736b0d19e57ada3ce435f28300b8a72a3fec6c761dfce34770ed9549331ee2} diff --git a/corpora/urd-g2p-dict.yaml b/corpora/urd-g2p-dict.yaml index f758b492..57082d56 100644 --- a/corpora/urd-g2p-dict.yaml +++ b/corpora/urd-g2p-dict.yaml @@ -7,13 +7,23 @@ sources: - kind: hf-dataset id: humair025/urdu-g2p-dictionary revision: main - license: LicenseRef-non-commercial # verified restricted — owner: weights-distribution decision + license: LicenseRef-non-commercial # verified restricted + + license_review: + status: owner-approved + date: "2026-10-18" + note: ship RESEARCH-ONLY — non-commercial upstream (owner decision #1); card must state it # Staged copy consumed by training (610K pairs via rababa-urdu/data); # inherits upstream non-commercial terms. - kind: bucket id: rhtse/jobs-artifacts/rababa-urdu-cc174b96 revision: data - license: LicenseRef-non-commercial + license: LicenseRef-non-commercial # inherits upstream + + license_review: + status: owner-approved + date: "2026-10-18" + note: ship RESEARCH-ONLY — non-commercial upstream (owner decision #1); card must state it mount: /data splits: train: {path: data/train.jsonl, rows: 609580, sha256: f718dddb0254ee5214ba7cc609c35dab928fe1289fa899a0fec181231af42693} diff --git a/src/pipeline/contracts.py b/src/pipeline/contracts.py index 4a152409..b244dc6e 100644 --- a/src/pipeline/contracts.py +++ b/src/pipeline/contracts.py @@ -37,6 +37,16 @@ class _Strict(BaseModel): model_config = ConfigDict(extra="forbid") +class LicenseReview(_Strict): + """An explicit owner decision on a restricted source. The license + string itself is never loosened — the waiver is a separate, + dated, attributed record.""" + + status: Literal["owner-approved"] + date: str + note: str + + class CorpusSource(_Strict): kind: Literal["hf-dataset", "bucket", "gh-release"] id: str @@ -44,6 +54,18 @@ class CorpusSource(_Strict): sha256: str | None = None license: str mount: str | None = None # container mount point for legacy scripts (e.g. /data-dia) + license_review: LicenseReview | None = None + + @model_validator(mode="after") + def _restricted_needs_review(self) -> CorpusSource: + if _license_blocked(self.license) and self.license_review is None: + raise ValueError( + f"restricted/unread license '{self.license}' on {self.id}: " + "an owner review is required (license_review: " + "status owner-approved) — the license string itself is " + "never loosened" + ) + return self class SplitSpec(_Strict): @@ -171,6 +193,12 @@ def _license_blocked(lic: str) -> bool: return lic == "UNVERIFIED" or lic.startswith("LicenseRef-") +def _license_blocked_source(s: CorpusSource) -> bool: + if not _license_blocked(s.license): + return False + return s.license_review is None or s.license_review.status != "owner-approved" + + def validate_recipe_dir(recipes_dir: Path, corpora_dir: Path) -> list[str]: problems: list[str] = [] corpora: dict[str, CorpusSpec] = {} @@ -204,12 +232,12 @@ def validate_recipe_dir(recipes_dir: Path, corpora_dir: Path) -> list[str]: f"recipe task '{r.task}'" ) if r.gates.license and any( - _license_blocked(s.license) for s in c.sources + _license_blocked_source(s) for s in c.sources ): blocked = ", ".join( f"{s.id}={s.license}" for s in c.sources - if _license_blocked(s.license) + if _license_blocked_source(s) ) problems.append( f"{rpath.name}: corpus '{c.id}' restricted or unread " diff --git a/src/pipeline/gates.py b/src/pipeline/gates.py index 4184e89f..db97d0f6 100644 --- a/src/pipeline/gates.py +++ b/src/pipeline/gates.py @@ -73,12 +73,19 @@ def check_parity(recipe: RecipeSpec, parity: dict | None) -> GateResult: def check_license(recipe: RecipeSpec, corpora_dir: Path) -> GateResult: - from pipeline.contracts import _license_blocked + from pipeline.contracts import _license_blocked_source names = {ref.split("@", 1)[0] for ref in recipe.corpora.values()} for name in sorted(names): - corpus = load_corpus(Path(corpora_dir) / f"{name}.yaml") - bad = [s for s in corpus.sources if _license_blocked(s.license)] + try: + corpus = load_corpus(Path(corpora_dir) / f"{name}.yaml") + except Exception as e: # noqa: BLE001 — restricted/unreadable registry entry + return GateResult( + "license", + False, + f"corpus '{name}': {e}", + ) + bad = [s for s in corpus.sources if _license_blocked_source(s)] if bad: ids = ", ".join(f"{s.id}={s.license}" for s in bad) return GateResult( diff --git a/src/pipeline/publish.py b/src/pipeline/publish.py index 142cd765..ed411756 100644 --- a/src/pipeline/publish.py +++ b/src/pipeline/publish.py @@ -37,7 +37,10 @@ def check_publish_ready( f"version {version!r} is not an explicit semver — versions are " "the owner's decision and never a moving ref" ) - results = evaluate(recipe, corpora_dir, metrics=metrics, parity=parity) + try: + results = evaluate(recipe, corpora_dir, metrics=metrics, parity=parity) + except Exception as e: # noqa: BLE001 — restricted-corpus schema errors + raise PublishRefusal(f"license/registry: {e}") from e gate_map = {g.name: g.passed for g in results} license_detail = next( (g.detail for g in results if g.name == "license" and not g.passed), "" diff --git a/tests/test_audit_fixes.py b/tests/test_audit_fixes.py index 26559ea5..dcefadf9 100644 --- a/tests/test_audit_fixes.py +++ b/tests/test_audit_fixes.py @@ -17,7 +17,7 @@ import pytest import yaml -from pipeline.contracts import load_recipe +from pipeline.contracts import CorpusSpec, _license_blocked_source, load_recipe from pipeline.publish import PublishRefusal, check_publish_ready REPO = Path(__file__).parent.parent @@ -150,3 +150,34 @@ def test_publish_ready_happy_path(tmp_path: Path) -> None: ) assert checks["gates"]["license"] is True assert checks["conformance"] is True + + +def test_owner_approved_waiver_opens_the_gate(tmp_path: Path) -> None: + """Owner decisions 2026-10-18: restricted/unlocated sources may ship + with a dependency note — but only via an explicit recorded waiver, + never by loosening the license string itself.""" + + corpus = { + "id": "xx-src", + "task": "transliteration", + "sources": [{ + "kind": "hf-dataset", + "id": "org/ds", + "license": "LicenseRef-unresolved", + "license_review": { + "status": "owner-approved", + "date": "2026-10-18", + "note": "ship with dependency note; research-only", + }, + }], + "splits": { + "train": {"path": "train.jsonl", "rows": 10}, + "test": {"path": "test.jsonl", "rows": 5}, + }, + } + c = CorpusSpec.model_validate(corpus) + assert not _license_blocked_source(c.sources[0]) + unwaived = dict(corpus["sources"][0]) + unwaived.pop("license_review") + with pytest.raises(ValueError, match="review"): + CorpusSpec.model_validate({**corpus, "sources": [unwaived]}) diff --git a/tests/test_pipeline_plan_gate.py b/tests/test_pipeline_plan_gate.py index 95464e5a..fcc14d70 100644 --- a/tests/test_pipeline_plan_gate.py +++ b/tests/test_pipeline_plan_gate.py @@ -90,12 +90,41 @@ def test_parity_gate_runtime_mode() -> None: def test_license_gate_blocks_publish(tmp_path: Path) -> None: - recipe = load_recipe(RECIPES / "urd-diac.yaml") # refs shipped urd-dia: restricted - results = evaluate(recipe, CORPORA, metrics={"cer": 0.0352}, parity=None) + # the shipped urd-dia is owner-waived (2026-10-18) and must PASS now + waived = evaluate(recipe := load_recipe(RECIPES / "urd-diac.yaml"), + CORPORA, metrics={"cer": 0.0352}, parity=None) + assert [g for g in waived if g.name == "license"][0].passed + # an UNWAIVED restricted source still blocks — synthetic registry + import yaml as _yaml + + corpus = { + "id": "xx-src", + "task": "diacritization", + "sources": [{"kind": "bucket", "id": "org/data", + "license": "LicenseRef-unresolved"}], + "splits": { + "train": {"path": "train.jsonl", "rows": 10}, + "test": {"path": "test.jsonl", "rows": 5}, + }, + } + rec = { + "id": "xx-diac", + "task": "diacritization", + "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"}, + "model": "byt5-small", + "stages": [{"stage": "sft", "params": {"epochs": 1}}], + "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean", + "parity": {"agree_min": 0.98}}, + } + (tmp_path / "corpora").mkdir() + (tmp_path / "recipes").mkdir() + (tmp_path / "corpora" / "xx-src.yaml").write_text(_yaml.safe_dump(corpus)) + (tmp_path / "recipes" / "xx-diac.yaml").write_text(_yaml.safe_dump(rec)) + results = evaluate(load_recipe(tmp_path / "recipes" / "xx-diac.yaml"), + tmp_path / "corpora", metrics={"cer": 0.05}, parity=None) lic = [g for g in results if g.name == "license"][0] assert not lic.passed assert "owner review" in lic.detail - assert "LicenseRef" in lic.detail def test_gate_results_files_end_to_end(tmp_path: Path) -> None: @@ -119,7 +148,7 @@ def test_gate_results_files_end_to_end(tmp_path: Path) -> None: parity=json.loads(parity.read_text()), ) by_name = {g.name: g.passed for g in results} - assert by_name == {"quality": True, "parity": True, "license": False} + assert by_name == {"quality": True, "parity": True, "license": True} def test_plan_wraps_legacy_deps() -> None: