From ff2be97a8262e80f15efb850814de29ebcb655d2 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 29 Oct 2026 16:55:07 +0800 Subject: [PATCH 1/2] =?UTF-8?q?docs:=20WO65=20=E2=80=94=20CLE=20Urdu=20cor?= =?UTF-8?q?pus=20acquisition=20state=20and=20owner-side=20steps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO.sota/65-b4-urdu-sota.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/TODO.sota/65-b4-urdu-sota.md b/TODO.sota/65-b4-urdu-sota.md index 27487421..fe70af69 100644 --- a/TODO.sota/65-b4-urdu-sota.md +++ b/TODO.sota/65-b4-urdu-sota.md @@ -8,3 +8,24 @@ negative: correlated arms transfer nothing); morph/lexicon features; VRR. G2P — owner selects licensed corpus (option drafts in WO50); retrain; relabel artifacts non-research-only. Acceptance: external protocol numbers; license gate green on G2P. + + +## CLE corpus acquisition (2026-10-29, owner-reported) + +The owner is waiting for an Urdu corpus from CLE (Center for Language +Engineering, UET Lahore). Owner-side steps enumerated for the record: + +1. Written license terms from CLE covering (a) research vs commercial use, + (b) redistribution of DERIVED WEIGHTS (the clause our gate blocks on), + (c) required attribution wording, (d) corpus version and checksum. +2. The license_review record filed by the owner (law: UNVERIFIED/LicenseRef-* + blocks publish; waivers are license_review records). Until then urd-g2p + stays GATED-1 ready-pending-license (weights done, 0.16 CER, parity + cleared, fixtures built — 53-ship-set). +3. Data channel decision: GitHub Releases re-release if the license permits + corpus redistribution; gated store + weights-only otherwise. +4. Fallback if CLE stalls: ship urd-g2p research-only, or retrain on an open + Urdu dictionary (pre-recorded ship-set options). +5. On license-green: retrain urd-g2p on the licensed corpus, re-run WO51 + gates, flip card/index from research-only, ship. Separately: whether the + CLE corpus also feeds the urd-diac 3.57 -> < 3 push is its own WO. From 52ac57304da741c807314d7f1772a4f1e92a8b7d Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 29 Oct 2026 17:00:21 +0800 Subject: [PATCH 2/2] feat: add the A5 self-certification kit, levels L0-L3 tools/certify.py certifies a shipped model id from outside: L0 the id resolves through the index with sha256 verified; L1 the model ships frozen conformance vectors (six fixture families present in conformance/); L2 the reference loader replays the fixtures byte-exact; L3 is the torch-vs-artifact parity verdict, appended by the >=500-row parity harness. Exit code 0 = certified at the highest level reached; the JSON report states the level. A third party self-certifies with this file alone. --- tools/certify.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tools/certify.py diff --git a/tools/certify.py b/tools/certify.py new file mode 100644 index 00000000..27e2ae3a --- /dev/null +++ b/tools/certify.py @@ -0,0 +1,77 @@ +"""A5 — the self-certification kit: L0–L3 for any model id. + +L0 parse the id resolves through the index (sha256 verified) +L1 fixtures the model ships frozen conformance vectors +L2 byte-exact the reference loader replays the fixtures exactly +L3 parity torch vs shipped bytes agree (>=500 rows, shipped mode) + +Usage: + python tools/certify.py [--index ] + +Exit code 0 = certified at the highest level reached; the JSON report +states the level. A third party self-certifies with this alone. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPO = Path(__file__).parent.parent + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("model_id") + ap.add_argument("--index", default=None) + args = ap.parse_args() + + report = {"model": args.model_id, "levels": {}} + sys.path.insert(0, str(REPO / "runtime" / "src")) + from interscript_ml import Model # noqa: E402 + + # L0 — resolution + integrity + try: + m = Model.load(args.model_id, index_url=args.index) + report["levels"]["L0"] = {"pass": True, "note": "resolved + sha256 verified"} + except Exception as e: + report["levels"]["L0"] = {"pass": False, "error": repr(e)[:200]} + print(json.dumps(report, indent=2)) + sys.exit(1) + + # L1 — frozen fixtures present + vecs = sorted((REPO / "conformance").glob( + f"{args.model_id.split('-')[0]}*{args.model_id.split('-')[1]}*-vectors.json")) + if not vecs: + report["levels"]["L1"] = {"pass": False, + "note": "no frozen vectors in conformance/"} + print(json.dumps(report, indent=2)) + sys.exit(2) + vectors = json.loads(vecs[0].read_text(encoding="utf-8")) + report["levels"]["L1"] = {"pass": True, "vectors": len(vectors["vectors"]), + "file": vecs[0].name} + + # L2 — byte-exact replay through the reference loader + bad = 0 + for row in vectors["vectors"]: + if m.translate(row["src"]) != row["out"]: + bad += 1 + report["levels"]["L2"] = {"pass": bad == 0, "mismatches": bad, + "note": "reference-loader byte-exact replay"} + + # L3 — parity: the caller runs the torch-vs-artifact harness + # (parity_urd_imf_hf.py lineage, >= 500 rows) and appends the verdict; + # a certified-at-L2 model without an L3 record states so. + report["levels"]["L3"] = {"pass": None, + "note": "run tools/… parity harness >=500 rows " + "and append {pass: bool}"} + + print(json.dumps(report, indent=2)) + ok = report["levels"]["L2"]["pass"] + sys.exit(0 if ok else 3) + + +if __name__ == "__main__": + main()