From e2e8c48f865b12531e75d098f303fe0a3d5be58a Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 18 Oct 2026 08:32:35 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(corpora):=20M3=20=E2=80=94=20pin=20row?= =?UTF-8?q?s=20+=20sha256=20for=20every=20shipped=20split?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stream-hashed all seven splits (no local copies): urd-dia 573,130/ 11,940, fas-dia 445,869/9,288 (the docs split, sha 67513aa1), qcri- wiki 32,834 article-lines, urd-g2p 609,580/12,699. Corpora are now content-addressed: a rebuild that changes a byte fails the registry. --- corpora/fas-dia.yaml | 4 ++-- corpora/qcri-wiki.yaml | 2 +- corpora/urd-dia.yaml | 4 ++-- corpora/urd-g2p-dict.yaml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/corpora/fas-dia.yaml b/corpora/fas-dia.yaml index 4f661143..1f1a74b1 100644 --- a/corpora/fas-dia.yaml +++ b/corpora/fas-dia.yaml @@ -15,5 +15,5 @@ sources: revision: run-076 license: CC0-1.0 splits: - train: {path: train.jsonl} # pin rows+sha256 from bucket manifest (M3) - test: {path: test.jsonl} + train: {path: train.jsonl, rows: 445869, sha256: 05ea4b33aaaec7fcddabd8d6bbefa56323efd6fac30b6ae242869372026eb845} + test: {path: test.jsonl, rows: 9288, sha256: 67513aa16ec142abde1df2d1782d548aa84f3841c171150b53a7ba82cb60b95f} diff --git a/corpora/qcri-wiki.yaml b/corpora/qcri-wiki.yaml index 1accdbd2..9dfc11fc 100644 --- a/corpora/qcri-wiki.yaml +++ b/corpora/qcri-wiki.yaml @@ -11,6 +11,6 @@ sources: license: LicenseRef-no-license-found # Qatara repo unlicensed — owner: contact QCRI mount: /train_data splits: - train: {path: qcri-wiki.jsonl} # pin rows+sha256 from dataset manifest (M3) + train: {path: qcri-wiki.jsonl, rows: 32834, sha256: 34ef170845472cdf81f9e83e5ac8b9bf45d716d1bc1b3a8e009f1ed343b4d51d} # article-per-line (~5M words) test: {path: wikinews/} # WikiNews-2024 multiref (OOD register gate) sadeed: {path: sadeeddiac-25.parquet} # SadeedDiac-25 (identity gate) diff --git a/corpora/urd-dia.yaml b/corpora/urd-dia.yaml index bdec4144..19080462 100644 --- a/corpora/urd-dia.yaml +++ b/corpora/urd-dia.yaml @@ -11,5 +11,5 @@ sources: license: LicenseRef-unresolved # no license found on sources — owner: contact authors or replace corpus mount: /data-dia splits: - train: {path: data-diacrit/train.jsonl} # pin rows+sha256 from bucket manifest (M3) - test: {path: data-diacrit/test.jsonl} + train: {path: data-diacrit/train.jsonl, rows: 573130, sha256: 95736b0d19e57ada3ce435f28300b8a72a3fec6c761dfce34770ed9549331ee2} + test: {path: data-diacrit/test.jsonl, rows: 11940, sha256: 6d822ae70a383e5547ee82e841a65e83fac81ae47ff320999b8e6b101b1790bb} diff --git a/corpora/urd-g2p-dict.yaml b/corpora/urd-g2p-dict.yaml index b933ac25..f758b492 100644 --- a/corpora/urd-g2p-dict.yaml +++ b/corpora/urd-g2p-dict.yaml @@ -16,5 +16,5 @@ sources: license: LicenseRef-non-commercial mount: /data splits: - train: {path: data/train.jsonl} # pin rows+sha256 from bucket manifest (M3) - test: {path: data/test.jsonl} + train: {path: data/train.jsonl, rows: 609580, sha256: f718dddb0254ee5214ba7cc609c35dab928fe1289fa899a0fec181231af42693} + test: {path: data/test.jsonl, rows: 12699, sha256: 7f26bbd01b884e3fddef96c78f151344c0b167407e8e4bb60a6ee0d316a19b36} From 5f053abeec5747d476d09e6caaba66e030e8c62b Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 18 Oct 2026 08:36:52 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat(pipeline):=20WO52=20M5=20=E2=80=94=20c?= =?UTF-8?q?onformance=20fixtures=20+=20isxml=20conformance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frozen-fixture contract for every IMF runtime: a vectors file pins (model, single-row-greedy decode mode, src -> exact output); runtimes must reproduce outputs byte-exactly or the change does not merge. Includes the reference ONNX decoder (byt5 byte+3 with trailing EOS — the WO51 single-row contract) and the isxml conformance verb. 4 tests written first; suite 234 green. --- src/pipeline/cli.py | 23 ++++++ src/pipeline/conformance.py | 114 +++++++++++++++++++++++++++++ tests/test_pipeline_conformance.py | 68 +++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 src/pipeline/conformance.py create mode 100644 tests/test_pipeline_conformance.py diff --git a/src/pipeline/cli.py b/src/pipeline/cli.py index 830c0264..c60183a8 100644 --- a/src/pipeline/cli.py +++ b/src/pipeline/cli.py @@ -80,6 +80,24 @@ def cmd_gate(args: argparse.Namespace) -> int: return 1 if failed else 0 +def cmd_conformance(args: argparse.Namespace) -> int: + from pipeline.conformance import load_vectors, ort_single_row_decoder, run_conformance + + vectors = load_vectors(args.vectors) + res = run_conformance(vectors, ort_single_row_decoder(args.zip)) + for m in res.mismatches[:10]: + print( + f"MISMATCH [{m['index']}] src={m['src']!r} " + f"expected={m['expected']!r} got={m['got']!r}", + file=sys.stderr, + ) + print( + f"{'PASS' if res.passed else 'FAIL'} conformance {res.model}: " + f"{res.checked - len(res.mismatches)}/{res.checked} byte-exact" + ) + return 0 if res.passed else 1 + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="isxml") sub = ap.add_subparsers(dest="cmd", required=True) @@ -105,6 +123,11 @@ def main(argv: list[str] | None = None) -> int: g.add_argument("--corpora", type=Path, default=DEFAULT_CORPORA_DIR) g.set_defaults(fn=cmd_gate) + c = sub.add_parser("conformance", help="run frozen vectors against an IMF zip") + c.add_argument("--vectors", required=True, type=Path) + c.add_argument("--zip", required=True, type=Path) + c.set_defaults(fn=cmd_conformance) + args = ap.parse_args(argv) return args.fn(args) diff --git a/src/pipeline/conformance.py b/src/pipeline/conformance.py new file mode 100644 index 00000000..7b982f2d --- /dev/null +++ b/src/pipeline/conformance.py @@ -0,0 +1,114 @@ +"""WO52 M5 — conformance fixtures: the client-side IMF contract. + +A vectors file pins (model, decode mode, src -> exact output); every +runtime must reproduce the outputs byte-exactly under the declared +decode mode. The ONNX decoder implements the single-row greedy byt5 +path (ids byte+3 with trailing EOS) — the mode WO51 proved to be the +artifact contract. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +def validate_vectors(v: dict[str, Any]) -> list[str]: + problems: list[str] = [] + if not v.get("model"): + problems.append("missing 'model' (the models.yaml id)") + mode = (v.get("decode") or {}).get("mode") + if mode != "single-row-greedy": + problems.append(f"decode.mode must be 'single-row-greedy', got {mode!r}") + vectors = v.get("vectors") + if not vectors: + problems.append("'vectors' is empty — a fixture pins at least one row") + return problems + for i, row in enumerate(vectors): + if not row.get("src"): + problems.append(f"vectors[{i}]: missing 'src'") + if "out" not in row: + problems.append(f"vectors[{i}]: missing 'out' (expected output)") + return problems + + +def load_vectors(path: Path) -> dict[str, Any]: + v = json.loads(Path(path).read_text(encoding="utf-8")) + problems = validate_vectors(v) + if problems: + raise ValueError(f"invalid conformance vectors: {'; '.join(problems)}") + return v + + +@dataclass +class ConformanceResult: + model: str + passed: bool + checked: int + mismatches: list[dict] = field(default_factory=list) + + +def run_conformance( + vectors: dict[str, Any], decode: Callable[[str], str] +) -> ConformanceResult: + rows = vectors["vectors"] + mismatches = [] + for i, row in enumerate(rows): + got = decode(row["src"]) + if got != row["out"]: + mismatches.append( + {"index": i, "src": row["src"], "expected": row["out"], "got": got} + ) + return ConformanceResult( + model=vectors["model"], + passed=not mismatches, + checked=len(rows), + mismatches=mismatches, + ) + + +def ort_single_row_decoder(zip_path: Path) -> Callable[[str], str]: + """The reference decoder: IMF zip, single-row greedy byt5 decode. + + Import-lazy so the pure logic above is testable without onnxruntime. + """ + import tempfile + import zipfile + + import numpy as np + + tmp = Path(tempfile.mkdtemp()) + with zipfile.ZipFile(zip_path) as z: + for name in ("encoder.onnx", "decoder.onnx"): + (tmp / name).write_bytes(z.read(name)) + import onnxruntime as ort + + opts = ort.SessionOptions() + opts.intra_op_num_threads = max(1, (os.cpu_count() or 1) // 2) + es = ort.InferenceSession(str(tmp / "encoder.onnx"), opts, + providers=["CPUExecutionProvider"]) + ds = ort.InferenceSession(str(tmp / "decoder.onnx"), opts, + providers=["CPUExecutionProvider"]) + enc_in = es.get_inputs()[0].name + dec_in = [i.name for i in ds.get_inputs()] + + def decode(src: str) -> str: + ids = np.array([[b + 3 for b in src.encode("utf-8")] + [1]], + dtype=np.int64) # byte+3, trailing EOS + h = es.run(None, {enc_in: ids})[0] + cur = np.zeros((1, 1), dtype=np.int64) + seq: list[int] = [] + for _ in range(512): + lg = ds.run(None, {dec_in[0]: cur, dec_in[1]: h})[0] + t = int(lg[:, -1, :].argmax(-1)[0]) + if t == 1: + break + seq.append(t) + cur = np.concatenate([cur, [[t]]], 1) + return bytes(max(0, t - 3) for t in seq).decode("utf-8", "replace") + + return decode diff --git a/tests/test_pipeline_conformance.py b/tests/test_pipeline_conformance.py new file mode 100644 index 00000000..b35bd729 --- /dev/null +++ b/tests/test_pipeline_conformance.py @@ -0,0 +1,68 @@ +"""WO52 M5 — conformance fixtures: the client-side IMF contract. + +A conformance vectors file pins (model id, decode mode, src -> exact +output). Any runtime — python, ruby, typescript, the API worker — must +reproduce the outputs byte-exactly under the declared decode mode or +the change does not merge (the frozen-fixture rule). + +Decode mode for byt5 IMF artifacts is single-row greedy: ids are +byte+3 with trailing EOS (the byt5 byte-table rule — raw text bytes +are silent garbage). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from pipeline.conformance import ( + load_vectors, + run_conformance, + validate_vectors, +) + +VECTORS = { + "model": "urd-diac-base-1.0", + "decode": {"mode": "single-row-greedy"}, + "vectors": [ + {"src": "سلام", "out": "سَلَام"}, + {"src": "کتاب", "out": "کِتَاب"}, + ], +} + + +def test_validate_vectors_accepts_good(tmp_path: Path) -> None: + p = tmp_path / "v.json" + p.write_text(json.dumps(VECTORS), encoding="utf-8") + v = load_vectors(p) + assert v["model"] == "urd-diac-base-1.0" + assert validate_vectors(v) == [] + + +def test_validate_vectors_rejects_missing_fields(tmp_path: Path) -> None: + bad = {"vectors": [{"src": "x"}]} + p = tmp_path / "v.json" + p.write_text(json.dumps(bad), encoding="utf-8") + with pytest.raises(ValueError, match="model"): + load_vectors(p) + problems = validate_vectors(dict(VECTORS, vectors=[{"src": "x"}])) + assert any("out" in q for q in problems) + assert any("empty" in q for q in validate_vectors(dict(VECTORS, vectors=[]))) + + +def test_conformance_passes_byte_exact() -> None: + expected = {v["src"]: v["out"] for v in VECTORS["vectors"]} + res = run_conformance(VECTORS, decode=expected.__getitem__) + assert res.passed + assert res.checked == 2 + assert res.mismatches == [] + + +def test_conformance_reports_mismatch_row() -> None: + res = run_conformance(VECTORS, decode=lambda s: "غلط" if s == "کتاب" else "سَلَام") + assert not res.passed + assert res.mismatches[0]["index"] == 1 + assert res.mismatches[0]["expected"] == "کِتَاب" + assert res.mismatches[0]["got"] == "غلط" From 7cabe4ec4ad246d915dc5bbe59e613c63bd557c3 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 18 Oct 2026 08:39:09 +0800 Subject: [PATCH 3/3] feat(conformance): freeze urd-diac-base vectors + the decode contract 12 byte-exact vectors generated with the reference decoder from the shipped fp16 zip (first 12 rows of urd-dia#test); CONFORMANCE.md records the single-row-greedy decode contract and the WO51 rule that maskless batched decode is outside it. --- conformance/CONFORMANCE.md | 50 ++++++++++++++++++++++ conformance/urd-diac-base-vectors.json | 58 ++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 conformance/CONFORMANCE.md create mode 100644 conformance/urd-diac-base-vectors.json diff --git a/conformance/CONFORMANCE.md b/conformance/CONFORMANCE.md new file mode 100644 index 00000000..2cc150c1 --- /dev/null +++ b/conformance/CONFORMANCE.md @@ -0,0 +1,50 @@ +# IMF conformance — the frozen-fixture contract + +Every runtime that mounts an Interscript ML model (python, ruby, +typescript, the API worker) must reproduce a model's conformance +vectors **byte-exactly** under the declared decode mode. If a runtime +change alters any fixture output, the change does not merge — same +rule KLF runs on its tokenizers ("frozen fixtures are the conformance +contract"). + +## What ships here + +| file | model | decode mode | rows | +|---|---|---|---| +| `urd-diac-base-vectors.json` | urd-diac-base-1.0 | single-row-greedy | 12 | + +## Decode mode contract (byt5 IMF artifacts) + +**single-row-greedy** — WO51 proved this is the artifact contract: + +- encode: UTF-8 bytes, ids `byte + 3`, **trailing EOS (id 1)** appended + (raw `text.bytes` without the table/EOS is silent garbage — the + byt5-byte-table rule) +- encoder: one row per call, no padding, no attention mask (the + exported encoder takes `input_ids` only) +- decoder: greedy argmax until EOS, 512-step cap +- output: tokens `t -> byte(t - 3)`, decoded UTF-8 + +Batched decode without an attention mask corrupts encoder states +(PAD_GAPS up to 0.13 observed) and is NOT part of the contract. A +runtime may batch only against an artifact whose encoder declares an +attention-mask input (future exporter option; gated by a batched +conformance fixture set). + +## Running + +```sh +isxml conformance --vectors conformance/urd-diac-base-vectors.json \ + --zip /path/to/urd-diac-base-1.0-fp16.zip # exit 0 = conformant +``` + +The vectors file schema is validated (`pipeline.conformance.validate_vectors`); +outputs compare byte-exact (no normalization, no CER tolerance). + +## Adding fixtures for a new model + +Generate vectors with the reference decoder against the shipped zip +(see `interscript-train/gen_conformance_vectors.py` for the pattern), +pin the models.yaml id + index_version, and note the source split + +its sha256 in the file. Model release gate (WO52) requires a +conformance file before a models.yaml entry ships. diff --git a/conformance/urd-diac-base-vectors.json b/conformance/urd-diac-base-vectors.json new file mode 100644 index 00000000..ff37d8bd --- /dev/null +++ b/conformance/urd-diac-base-vectors.json @@ -0,0 +1,58 @@ +{ + "model": "urd-diac-base-1.0", + "index_version": 1, + "decode": { + "mode": "single-row-greedy" + }, + "source": "first 12 rows of urd-dia#test (sha256 6d822ae7…)", + "vectors": [ + { + "src": "فاسد العقیدہ", + "out": "فْاسْدِ الْعُقْیدْہَ" + }, + { + "src": "سربراہ", + "out": "سْرَبْرْاَہْ" + }, + { + "src": "پیاریں", + "out": "پْیِارْیںْ" + }, + { + "src": "ڈھوا", + "out": "ڈْھَوا" + }, + { + "src": "بے کرایہ داری", + "out": "بْے کْرْاِیہْ دْارْی" + }, + { + "src": "چونک لگانا", + "out": "چْونْکْ لْگْانْاَ" + }, + { + "src": "پکہ", + "out": "پْکَہْ" + }, + { + "src": "چھوٹا ٹیلہ", + "out": "چْھْوٹْا ٹْیلْہْ" + }, + { + "src": "بے تشریفی", + "out": "بْے تْشْرَیفْی" + }, + { + "src": "دروازہ نادانی", + "out": "دْرَوازْہْ نْادْاَنِی" + }, + { + "src": "کرچا", + "out": "کْرَچْا" + }, + { + "src": "لوک", + "out": "لْوکْ" + } + ] +} \ No newline at end of file