WO52 M3+M5: corpus pins + conformance fixtures

conformance/CONFORMANCE.md

@@ -0,0 +1,50 @@
+# IMF conformance — the frozen-fixture contract
+
+Every runtime that mounts an Interscript ML model (python, ruby,
+typescript, the API worker) must reproduce a model's conformance
+vectors **byte-exactly** under the declared decode mode. If a runtime
+change alters any fixture output, the change does not merge — same
+rule KLF runs on its tokenizers ("frozen fixtures are the conformance
+contract").
+
+## What ships here
+
+| file | model | decode mode | rows |
+|---|---|---|---|
+| `urd-diac-base-vectors.json` | urd-diac-base-1.0 | single-row-greedy | 12 |
+
+## Decode mode contract (byt5 IMF artifacts)
+
+**single-row-greedy** — WO51 proved this is the artifact contract:
+
+- encode: UTF-8 bytes, ids `byte + 3`, **trailing EOS (id 1)** appended
+  (raw `text.bytes` without the table/EOS is silent garbage — the
+  byt5-byte-table rule)
+- encoder: one row per call, no padding, no attention mask (the
+  exported encoder takes `input_ids` only)
+- decoder: greedy argmax until EOS, 512-step cap
+- output: tokens `t -> byte(t - 3)`, decoded UTF-8
+
+Batched decode without an attention mask corrupts encoder states
+(PAD_GAPS up to 0.13 observed) and is NOT part of the contract. A
+runtime may batch only against an artifact whose encoder declares an
+attention-mask input (future exporter option; gated by a batched
+conformance fixture set).
+
+## Running
+
+```sh
+isxml conformance --vectors conformance/urd-diac-base-vectors.json \
+                  --zip /path/to/urd-diac-base-1.0-fp16.zip   # exit 0 = conformant
+```
+
+The vectors file schema is validated (`pipeline.conformance.validate_vectors`);
+outputs compare byte-exact (no normalization, no CER tolerance).
+
+## Adding fixtures for a new model
+
+Generate vectors with the reference decoder against the shipped zip
+(see `interscript-train/gen_conformance_vectors.py` for the pattern),
+pin the models.yaml id + index_version, and note the source split +
+its sha256 in the file. Model release gate (WO52) requires a
+conformance file before a models.yaml entry ships.

conformance/urd-diac-base-vectors.json

@@ -0,0 +1,58 @@
+{
+ "model": "urd-diac-base-1.0",
+ "index_version": 1,
+ "decode": {
+  "mode": "single-row-greedy"
+ },
+ "source": "first 12 rows of urd-dia#test (sha256 6d822ae7…)",
+ "vectors": [
+  {
+   "src": "فاسد العقیدہ",
+   "out": "فْاسْدِ الْعُقْیدْہَ"
+  },
+  {
+   "src": "سربراہ",
+   "out": "سْرَبْرْاَہْ"
+  },
+  {
+   "src": "پیاریں",
+   "out": "پْیِارْیںْ"
+  },
+  {
+   "src": "ڈھوا",
+   "out": "ڈْھَوا"
+  },
+  {
+   "src": "بے کرایہ داری",
+   "out": "بْے کْرْاِیہْ دْارْی"
+  },
+  {
+   "src": "چونک لگانا",
+   "out": "چْونْکْ لْگْانْاَ"
+  },
+  {
+   "src": "پکہ",
+   "out": "پْکَہْ"
+  },
+  {
+   "src": "چھوٹا ٹیلہ",
+   "out": "چْھْوٹْا ٹْیلْہْ"
+  },
+  {
+   "src": "بے تشریفی",
+   "out": "بْے تْشْرَیفْی"
+  },
+  {
+   "src": "دروازہ نادانی",
+   "out": "دْرَوازْہْ نْادْاَنِی"
+  },
+  {
+   "src": "کرچا",
+   "out": "کْرَچْا"
+  },
+  {
+   "src": "لوک",
+   "out": "لْوکْ"
+  }
+ ]
+}

corpora/fas-dia.yaml

@@ -15,5 +15,5 @@ sources:
     revision: run-076
     license: CC0-1.0
 splits:
-  train: {path: train.jsonl}  # pin rows+sha256 from bucket manifest (M3)
-  test: {path: test.jsonl}
+  train: {path: train.jsonl, rows: 445869, sha256: 05ea4b33aaaec7fcddabd8d6bbefa56323efd6fac30b6ae242869372026eb845}
+  test: {path: test.jsonl, rows: 9288, sha256: 67513aa16ec142abde1df2d1782d548aa84f3841c171150b53a7ba82cb60b95f}

corpora/qcri-wiki.yaml

@@ -11,6 +11,6 @@ sources:
     license: LicenseRef-no-license-found  # Qatara repo unlicensed — owner: contact QCRI
     mount: /train_data
 splits:
-  train: {path: qcri-wiki.jsonl}  # pin rows+sha256 from dataset manifest (M3)
+  train: {path: qcri-wiki.jsonl, rows: 32834, sha256: 34ef170845472cdf81f9e83e5ac8b9bf45d716d1bc1b3a8e009f1ed343b4d51d}  # article-per-line (~5M words)
   test: {path: wikinews/}  # WikiNews-2024 multiref (OOD register gate)
   sadeed: {path: sadeeddiac-25.parquet}  # SadeedDiac-25 (identity gate)

corpora/urd-dia.yaml

@@ -11,5 +11,5 @@ sources:
     license: LicenseRef-unresolved  # no license found on sources — owner: contact authors or replace corpus
     mount: /data-dia
 splits:
-  train: {path: data-diacrit/train.jsonl}  # pin rows+sha256 from bucket manifest (M3)
-  test: {path: data-diacrit/test.jsonl}
+  train: {path: data-diacrit/train.jsonl, rows: 573130, sha256: 95736b0d19e57ada3ce435f28300b8a72a3fec6c761dfce34770ed9549331ee2}
+  test: {path: data-diacrit/test.jsonl, rows: 11940, sha256: 6d822ae70a383e5547ee82e841a65e83fac81ae47ff320999b8e6b101b1790bb}

corpora/urd-g2p-dict.yaml

@@ -16,5 +16,5 @@ sources:
     license: LicenseRef-non-commercial
     mount: /data
 splits:
-  train: {path: data/train.jsonl}  # pin rows+sha256 from bucket manifest (M3)
-  test: {path: data/test.jsonl}
+  train: {path: data/train.jsonl, rows: 609580, sha256: f718dddb0254ee5214ba7cc609c35dab928fe1289fa899a0fec181231af42693}
+  test: {path: data/test.jsonl, rows: 12699, sha256: 7f26bbd01b884e3fddef96c78f151344c0b167407e8e4bb60a6ee0d316a19b36}

src/pipeline/cli.py

@@ -80,6 +80,24 @@ def cmd_gate(args: argparse.Namespace) -> int:
     return 1 if failed else 0
 
 
+def cmd_conformance(args: argparse.Namespace) -> int:
+    from pipeline.conformance import load_vectors, ort_single_row_decoder, run_conformance
+
+    vectors = load_vectors(args.vectors)
+    res = run_conformance(vectors, ort_single_row_decoder(args.zip))
+    for m in res.mismatches[:10]:
+        print(
+            f"MISMATCH [{m['index']}] src={m['src']!r} "
+            f"expected={m['expected']!r} got={m['got']!r}",
+            file=sys.stderr,
+        )
+    print(
+        f"{'PASS' if res.passed else 'FAIL'} conformance {res.model}: "
+        f"{res.checked - len(res.mismatches)}/{res.checked} byte-exact"
+    )
+    return 0 if res.passed else 1
+
+
 def main(argv: list[str] | None = None) -> int:
     ap = argparse.ArgumentParser(prog="isxml")
     sub = ap.add_subparsers(dest="cmd", required=True)
@@ -105,6 +123,11 @@ def main(argv: list[str] | None = None) -> int:
     g.add_argument("--corpora", type=Path, default=DEFAULT_CORPORA_DIR)
     g.set_defaults(fn=cmd_gate)
 
+    c = sub.add_parser("conformance", help="run frozen vectors against an IMF zip")
+    c.add_argument("--vectors", required=True, type=Path)
+    c.add_argument("--zip", required=True, type=Path)
+    c.set_defaults(fn=cmd_conformance)
+
     args = ap.parse_args(argv)
     return args.fn(args)
 

src/pipeline/conformance.py

@@ -0,0 +1,114 @@
+"""WO52 M5 — conformance fixtures: the client-side IMF contract.
+
+A vectors file pins (model, decode mode, src -> exact output); every
+runtime must reproduce the outputs byte-exactly under the declared
+decode mode. The ONNX decoder implements the single-row greedy byt5
+path (ids byte+3 with trailing EOS) — the mode WO51 proved to be the
+artifact contract.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+def validate_vectors(v: dict[str, Any]) -> list[str]:
+    problems: list[str] = []
+    if not v.get("model"):
+        problems.append("missing 'model' (the models.yaml id)")
+    mode = (v.get("decode") or {}).get("mode")
+    if mode != "single-row-greedy":
+        problems.append(f"decode.mode must be 'single-row-greedy', got {mode!r}")
+    vectors = v.get("vectors")
+    if not vectors:
+        problems.append("'vectors' is empty — a fixture pins at least one row")
+        return problems
+    for i, row in enumerate(vectors):
+        if not row.get("src"):
+            problems.append(f"vectors[{i}]: missing 'src'")
+        if "out" not in row:
+            problems.append(f"vectors[{i}]: missing 'out' (expected output)")
+    return problems
+
+
+def load_vectors(path: Path) -> dict[str, Any]:
+    v = json.loads(Path(path).read_text(encoding="utf-8"))
+    problems = validate_vectors(v)
+    if problems:
+        raise ValueError(f"invalid conformance vectors: {'; '.join(problems)}")
+    return v
+
+
+@dataclass
+class ConformanceResult:
+    model: str
+    passed: bool
+    checked: int
+    mismatches: list[dict] = field(default_factory=list)
+
+
+def run_conformance(
+    vectors: dict[str, Any], decode: Callable[[str], str]
+) -> ConformanceResult:
+    rows = vectors["vectors"]
+    mismatches = []
+    for i, row in enumerate(rows):
+        got = decode(row["src"])
+        if got != row["out"]:
+            mismatches.append(
+                {"index": i, "src": row["src"], "expected": row["out"], "got": got}
+            )
+    return ConformanceResult(
+        model=vectors["model"],
+        passed=not mismatches,
+        checked=len(rows),
+        mismatches=mismatches,
+    )
+
+
+def ort_single_row_decoder(zip_path: Path) -> Callable[[str], str]:
+    """The reference decoder: IMF zip, single-row greedy byt5 decode.
+
+    Import-lazy so the pure logic above is testable without onnxruntime.
+    """
+    import tempfile
+    import zipfile
+
+    import numpy as np
+
+    tmp = Path(tempfile.mkdtemp())
+    with zipfile.ZipFile(zip_path) as z:
+        for name in ("encoder.onnx", "decoder.onnx"):
+            (tmp / name).write_bytes(z.read(name))
+    import onnxruntime as ort
+
+    opts = ort.SessionOptions()
+    opts.intra_op_num_threads = max(1, (os.cpu_count() or 1) // 2)
+    es = ort.InferenceSession(str(tmp / "encoder.onnx"), opts,
+                              providers=["CPUExecutionProvider"])
+    ds = ort.InferenceSession(str(tmp / "decoder.onnx"), opts,
+                              providers=["CPUExecutionProvider"])
+    enc_in = es.get_inputs()[0].name
+    dec_in = [i.name for i in ds.get_inputs()]
+
+    def decode(src: str) -> str:
+        ids = np.array([[b + 3 for b in src.encode("utf-8")] + [1]],
+                       dtype=np.int64)  # byte+3, trailing EOS
+        h = es.run(None, {enc_in: ids})[0]
+        cur = np.zeros((1, 1), dtype=np.int64)
+        seq: list[int] = []
+        for _ in range(512):
+            lg = ds.run(None, {dec_in[0]: cur, dec_in[1]: h})[0]
+            t = int(lg[:, -1, :].argmax(-1)[0])
+            if t == 1:
+                break
+            seq.append(t)
+            cur = np.concatenate([cur, [[t]]], 1)
+        return bytes(max(0, t - 3) for t in seq).decode("utf-8", "replace")
+
+    return decode

tests/test_pipeline_conformance.py

@@ -0,0 +1,68 @@
+"""WO52 M5 — conformance fixtures: the client-side IMF contract.
+
+A conformance vectors file pins (model id, decode mode, src -> exact
+output). Any runtime — python, ruby, typescript, the API worker — must
+reproduce the outputs byte-exactly under the declared decode mode or
+the change does not merge (the frozen-fixture rule).
+
+Decode mode for byt5 IMF artifacts is single-row greedy: ids are
+byte+3 with trailing EOS (the byt5 byte-table rule — raw text bytes
+are silent garbage).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from pipeline.conformance import (
+    load_vectors,
+    run_conformance,
+    validate_vectors,
+)
+
+VECTORS = {
+    "model": "urd-diac-base-1.0",
+    "decode": {"mode": "single-row-greedy"},
+    "vectors": [
+        {"src": "سلام", "out": "سَلَام"},
+        {"src": "کتاب", "out": "کِتَاب"},
+    ],
+}
+
+
+def test_validate_vectors_accepts_good(tmp_path: Path) -> None:
+    p = tmp_path / "v.json"
+    p.write_text(json.dumps(VECTORS), encoding="utf-8")
+    v = load_vectors(p)
+    assert v["model"] == "urd-diac-base-1.0"
+    assert validate_vectors(v) == []
+
+
+def test_validate_vectors_rejects_missing_fields(tmp_path: Path) -> None:
+    bad = {"vectors": [{"src": "x"}]}
+    p = tmp_path / "v.json"
+    p.write_text(json.dumps(bad), encoding="utf-8")
+    with pytest.raises(ValueError, match="model"):
+        load_vectors(p)
+    problems = validate_vectors(dict(VECTORS, vectors=[{"src": "x"}]))
+    assert any("out" in q for q in problems)
+    assert any("empty" in q for q in validate_vectors(dict(VECTORS, vectors=[])))
+
+
+def test_conformance_passes_byte_exact() -> None:
+    expected = {v["src"]: v["out"] for v in VECTORS["vectors"]}
+    res = run_conformance(VECTORS, decode=expected.__getitem__)
+    assert res.passed
+    assert res.checked == 2
+    assert res.mismatches == []
+
+
+def test_conformance_reports_mismatch_row() -> None:
+    res = run_conformance(VECTORS, decode=lambda s: "غلط" if s == "کتاب" else "سَلَام")
+    assert not res.passed
+    assert res.mismatches[0]["index"] == 1
+    assert res.mismatches[0]["expected"] == "کِتَاب"
+    assert res.mismatches[0]["got"] == "غلط"