License waivers: owner decisions encoded (research-only / dependency notes)

corpora/qcri-wiki.yaml

@@ -8,7 +8,12 @@ sources:
   - kind: hf-dataset
     id: Interscript/arabic-r8
     revision: main
-    license: LicenseRef-no-license-found  # Qatara repo unlicensed — owner: contact QCRI
+    license: LicenseRef-no-license-found  # Qatara repo unlicensed
+
+    license_review:
+      status: owner-approved
+      date: "2026-10-18"
+      note: ship with dependency note — weights depend on the QCRI upstream license (owner decision #2)
     mount: /train_data
 splits:
   train: {path: qcri-wiki.jsonl, rows: 32834, sha256: 34ef170845472cdf81f9e83e5ac8b9bf45d716d1bc1b3a8e009f1ed343b4d51d}  # article-per-line (~5M words)

corpora/urd-dia.yaml

@@ -8,7 +8,12 @@ sources:
   - kind: bucket
     id: rhtse/jobs-artifacts/rababa-urdu-cc174b96
     revision: data-diacrit
-    license: LicenseRef-unresolved  # no license found on sources — owner: contact authors or replace corpus
+    license: LicenseRef-unresolved  # no license found on sources
+
+    license_review:
+      status: owner-approved
+      date: "2026-10-18"
+      note: ship with dependency note — weights depend on the original data license (owner decision #3)
     mount: /data-dia
 splits:
   train: {path: data-diacrit/train.jsonl, rows: 573130, sha256: 95736b0d19e57ada3ce435f28300b8a72a3fec6c761dfce34770ed9549331ee2}

corpora/urd-g2p-dict.yaml

@@ -7,13 +7,23 @@ sources:
   - kind: hf-dataset
     id: humair025/urdu-g2p-dictionary
     revision: main
-    license: LicenseRef-non-commercial  # verified restricted — owner: weights-distribution decision
+    license: LicenseRef-non-commercial  # verified restricted
+
+    license_review:
+      status: owner-approved
+      date: "2026-10-18"
+      note: ship RESEARCH-ONLY — non-commercial upstream (owner decision #1); card must state it
   # Staged copy consumed by training (610K pairs via rababa-urdu/data);
   # inherits upstream non-commercial terms.
   - kind: bucket
     id: rhtse/jobs-artifacts/rababa-urdu-cc174b96
     revision: data
-    license: LicenseRef-non-commercial
+    license: LicenseRef-non-commercial  # inherits upstream
+
+    license_review:
+      status: owner-approved
+      date: "2026-10-18"
+      note: ship RESEARCH-ONLY — non-commercial upstream (owner decision #1); card must state it
     mount: /data
 splits:
   train: {path: data/train.jsonl, rows: 609580, sha256: f718dddb0254ee5214ba7cc609c35dab928fe1289fa899a0fec181231af42693}

src/pipeline/contracts.py

@@ -37,13 +37,35 @@ class _Strict(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
 
+class LicenseReview(_Strict):
+    """An explicit owner decision on a restricted source. The license
+    string itself is never loosened — the waiver is a separate,
+    dated, attributed record."""
+
+    status: Literal["owner-approved"]
+    date: str
+    note: str
+
+
 class CorpusSource(_Strict):
     kind: Literal["hf-dataset", "bucket", "gh-release"]
     id: str
     revision: str | None = None
     sha256: str | None = None
     license: str
     mount: str | None = None  # container mount point for legacy scripts (e.g. /data-dia)
+    license_review: LicenseReview | None = None
+
+    @model_validator(mode="after")
+    def _restricted_needs_review(self) -> CorpusSource:
+        if _license_blocked(self.license) and self.license_review is None:
+            raise ValueError(
+                f"restricted/unread license '{self.license}' on {self.id}: "
+                "an owner review is required (license_review: "
+                "status owner-approved) — the license string itself is "
+                "never loosened"
+            )
+        return self
 
 
 class SplitSpec(_Strict):
@@ -171,6 +193,12 @@ def _license_blocked(lic: str) -> bool:
     return lic == "UNVERIFIED" or lic.startswith("LicenseRef-")
 
 
+def _license_blocked_source(s: CorpusSource) -> bool:
+    if not _license_blocked(s.license):
+        return False
+    return s.license_review is None or s.license_review.status != "owner-approved"
+
+
 def validate_recipe_dir(recipes_dir: Path, corpora_dir: Path) -> list[str]:
     problems: list[str] = []
     corpora: dict[str, CorpusSpec] = {}
@@ -204,12 +232,12 @@ def validate_recipe_dir(recipes_dir: Path, corpora_dir: Path) -> list[str]:
                     f"recipe task '{r.task}'"
                 )
             if r.gates.license and any(
-                _license_blocked(s.license) for s in c.sources
+                _license_blocked_source(s) for s in c.sources
             ):
                 blocked = ", ".join(
                     f"{s.id}={s.license}"
                     for s in c.sources
-                    if _license_blocked(s.license)
+                    if _license_blocked_source(s)
                 )
                 problems.append(
                     f"{rpath.name}: corpus '{c.id}' restricted or unread "

src/pipeline/gates.py

@@ -73,12 +73,19 @@ def check_parity(recipe: RecipeSpec, parity: dict | None) -> GateResult:
 
 
 def check_license(recipe: RecipeSpec, corpora_dir: Path) -> GateResult:
-    from pipeline.contracts import _license_blocked
+    from pipeline.contracts import _license_blocked_source
 
     names = {ref.split("@", 1)[0] for ref in recipe.corpora.values()}
     for name in sorted(names):
-        corpus = load_corpus(Path(corpora_dir) / f"{name}.yaml")
-        bad = [s for s in corpus.sources if _license_blocked(s.license)]
+        try:
+            corpus = load_corpus(Path(corpora_dir) / f"{name}.yaml")
+        except Exception as e:  # noqa: BLE001 — restricted/unreadable registry entry
+            return GateResult(
+                "license",
+                False,
+                f"corpus '{name}': {e}",
+            )
+        bad = [s for s in corpus.sources if _license_blocked_source(s)]
         if bad:
             ids = ", ".join(f"{s.id}={s.license}" for s in bad)
             return GateResult(

src/pipeline/publish.py

@@ -37,7 +37,10 @@ def check_publish_ready(
             f"version {version!r} is not an explicit semver — versions are "
             "the owner's decision and never a moving ref"
         )
-    results = evaluate(recipe, corpora_dir, metrics=metrics, parity=parity)
+    try:
+        results = evaluate(recipe, corpora_dir, metrics=metrics, parity=parity)
+    except Exception as e:  # noqa: BLE001 — restricted-corpus schema errors
+        raise PublishRefusal(f"license/registry: {e}") from e
     gate_map = {g.name: g.passed for g in results}
     license_detail = next(
         (g.detail for g in results if g.name == "license" and not g.passed), ""

tests/test_audit_fixes.py

@@ -17,7 +17,7 @@
 import pytest
 import yaml
 
-from pipeline.contracts import load_recipe
+from pipeline.contracts import CorpusSpec, _license_blocked_source, load_recipe
 from pipeline.publish import PublishRefusal, check_publish_ready
 
 REPO = Path(__file__).parent.parent
@@ -150,3 +150,34 @@ def test_publish_ready_happy_path(tmp_path: Path) -> None:
     )
     assert checks["gates"]["license"] is True
     assert checks["conformance"] is True
+
+
+def test_owner_approved_waiver_opens_the_gate(tmp_path: Path) -> None:
+    """Owner decisions 2026-10-18: restricted/unlocated sources may ship
+    with a dependency note — but only via an explicit recorded waiver,
+    never by loosening the license string itself."""
+
+    corpus = {
+        "id": "xx-src",
+        "task": "transliteration",
+        "sources": [{
+            "kind": "hf-dataset",
+            "id": "org/ds",
+            "license": "LicenseRef-unresolved",
+            "license_review": {
+                "status": "owner-approved",
+                "date": "2026-10-18",
+                "note": "ship with dependency note; research-only",
+            },
+        }],
+        "splits": {
+            "train": {"path": "train.jsonl", "rows": 10},
+            "test": {"path": "test.jsonl", "rows": 5},
+        },
+    }
+    c = CorpusSpec.model_validate(corpus)
+    assert not _license_blocked_source(c.sources[0])
+    unwaived = dict(corpus["sources"][0])
+    unwaived.pop("license_review")
+    with pytest.raises(ValueError, match="review"):
+        CorpusSpec.model_validate({**corpus, "sources": [unwaived]})

tests/test_pipeline_plan_gate.py

@@ -90,12 +90,41 @@ def test_parity_gate_runtime_mode() -> None:
 
 
 def test_license_gate_blocks_publish(tmp_path: Path) -> None:
-    recipe = load_recipe(RECIPES / "urd-diac.yaml")  # refs shipped urd-dia: restricted
-    results = evaluate(recipe, CORPORA, metrics={"cer": 0.0352}, parity=None)
+    # the shipped urd-dia is owner-waived (2026-10-18) and must PASS now
+    waived = evaluate(recipe := load_recipe(RECIPES / "urd-diac.yaml"),
+                      CORPORA, metrics={"cer": 0.0352}, parity=None)
+    assert [g for g in waived if g.name == "license"][0].passed
+    # an UNWAIVED restricted source still blocks — synthetic registry
+    import yaml as _yaml
+
+    corpus = {
+        "id": "xx-src",
+        "task": "diacritization",
+        "sources": [{"kind": "bucket", "id": "org/data",
+                     "license": "LicenseRef-unresolved"}],
+        "splits": {
+            "train": {"path": "train.jsonl", "rows": 10},
+            "test": {"path": "test.jsonl", "rows": 5},
+        },
+    }
+    rec = {
+        "id": "xx-diac",
+        "task": "diacritization",
+        "corpora": {"train": "xx-src@v1", "test": "xx-src@v1#test"},
+        "model": "byt5-small",
+        "stages": [{"stage": "sft", "params": {"epochs": 1}}],
+        "gates": {"quality": {"cer_max": 0.1}, "convention": "row-mean",
+                  "parity": {"agree_min": 0.98}},
+    }
+    (tmp_path / "corpora").mkdir()
+    (tmp_path / "recipes").mkdir()
+    (tmp_path / "corpora" / "xx-src.yaml").write_text(_yaml.safe_dump(corpus))
+    (tmp_path / "recipes" / "xx-diac.yaml").write_text(_yaml.safe_dump(rec))
+    results = evaluate(load_recipe(tmp_path / "recipes" / "xx-diac.yaml"),
+                       tmp_path / "corpora", metrics={"cer": 0.05}, parity=None)
     lic = [g for g in results if g.name == "license"][0]
     assert not lic.passed
     assert "owner review" in lic.detail
-    assert "LicenseRef" in lic.detail
 
 
 def test_gate_results_files_end_to_end(tmp_path: Path) -> None:
@@ -119,7 +148,7 @@ def test_gate_results_files_end_to_end(tmp_path: Path) -> None:
         parity=json.loads(parity.read_text()),
     )
     by_name = {g.name: g.passed for g in results}
-    assert by_name == {"quality": True, "parity": True, "license": False}
+    assert by_name == {"quality": True, "parity": True, "license": True}
 
 
 def test_plan_wraps_legacy_deps() -> None: