feat: fetch_lexicon — canonical sha-verified lexicon loader (WO19) (#33)

Author: ronaldtse

src/interscript/ml/thai_hybrid.py

@@ -92,3 +92,28 @@ def build_lexicon_from_kaikki(entries, min_head_len: int = 1) -> Lexicon:
     for word, ipas in counts.items():
         lex[word] = max(ipas.items(), key=lambda kv: kv[1])[0]
     return lex
+
+
+LEXICON_URL = (
+    "https://github.com/interscript/interscript-models/releases/download/"
+    "tha-lexicon-kaikki-1.0/tha-lexicon-kaikki.json"
+)
+LEXICON_SHA256 = "6e44c7116aca3e570f6b8a12d9c0ca8f7631336147448d0818ad7e076b72b911"
+
+
+def fetch_lexicon(index_url=None, _base_url=None, _sha256=None) -> Lexicon:
+    """Fetch the canonical lexicon artifact, sha-verified (same trust
+    contract as model artifacts — no unverified data paths). Test hooks
+    take precedence over the release defaults."""
+    import hashlib
+    import json
+    import urllib.request
+
+    url = _base_url or LEXICON_URL
+    expected = _sha256 or LEXICON_SHA256
+    with urllib.request.urlopen(url) as r:
+        blob = r.read()
+    got = hashlib.sha256(blob).hexdigest()
+    if got != expected:
+        raise ValueError(f"lexicon sha256 mismatch: {got}")
+    return Lexicon(json.loads(blob.decode("utf-8")))

tests/test_thai_hybrid.py

@@ -54,3 +54,37 @@ def test_hybrid_all_oov_single_span():
     out = hybrid_translate("นอนๆ", make_lexicon(), model)
     assert out == "|นอนๆ|"
     assert model.calls == ["นอนๆ"]
+
+
+class TestFetchLexicon:
+    def test_fetches_sha_verified_and_builds_a_lexicon(self, tmp_path, monkeypatch):
+        import json as _json
+
+        lexicon = {"กิน": "kin", "ข้าว": "kʰâːw"}
+        blob = _json.dumps(lexicon, ensure_ascii=False).encode("utf-8")
+        import hashlib
+
+        sha = hashlib.sha256(blob).hexdigest()
+        (tmp_path / "lex.json").write_bytes(blob)
+        base = f"file://{tmp_path}/lex.json"
+
+        from interscript.ml import thai_hybrid as TH
+
+        got = TH.fetch_lexicon(index_url=None, _base_url=base, _sha256=sha)
+        assert got["กิน"] == "kin"
+        assert isinstance(got, TH.Lexicon)
+
+    def test_rejects_sha_mismatch(self, tmp_path):
+        (tmp_path / "lex.json").write_bytes(b'{"a":"b"}')
+        from interscript.ml import thai_hybrid as TH
+
+        try:
+            TH.fetch_lexicon(
+                index_url=None,
+                _base_url=f"file://{tmp_path}/lex.json",
+                _sha256="0" * 64,
+            )
+        except ValueError as e:
+            assert "sha256" in str(e)
+        else:
+            raise AssertionError("expected sha mismatch")