diff --git a/src/interscript/ml/thai_hybrid.py b/src/interscript/ml/thai_hybrid.py index 377329b..d8f4a78 100644 --- a/src/interscript/ml/thai_hybrid.py +++ b/src/interscript/ml/thai_hybrid.py @@ -92,3 +92,28 @@ def build_lexicon_from_kaikki(entries, min_head_len: int = 1) -> Lexicon: for word, ipas in counts.items(): lex[word] = max(ipas.items(), key=lambda kv: kv[1])[0] return lex + + +LEXICON_URL = ( + "https://github.com/interscript/interscript-models/releases/download/" + "tha-lexicon-kaikki-1.0/tha-lexicon-kaikki.json" +) +LEXICON_SHA256 = "6e44c7116aca3e570f6b8a12d9c0ca8f7631336147448d0818ad7e076b72b911" + + +def fetch_lexicon(index_url=None, _base_url=None, _sha256=None) -> Lexicon: + """Fetch the canonical lexicon artifact, sha-verified (same trust + contract as model artifacts — no unverified data paths). Test hooks + take precedence over the release defaults.""" + import hashlib + import json + import urllib.request + + url = _base_url or LEXICON_URL + expected = _sha256 or LEXICON_SHA256 + with urllib.request.urlopen(url) as r: + blob = r.read() + got = hashlib.sha256(blob).hexdigest() + if got != expected: + raise ValueError(f"lexicon sha256 mismatch: {got}") + return Lexicon(json.loads(blob.decode("utf-8"))) diff --git a/tests/test_thai_hybrid.py b/tests/test_thai_hybrid.py index bc685bb..2cb8349 100644 --- a/tests/test_thai_hybrid.py +++ b/tests/test_thai_hybrid.py @@ -54,3 +54,37 @@ def test_hybrid_all_oov_single_span(): out = hybrid_translate("นอนๆ", make_lexicon(), model) assert out == "|นอนๆ|" assert model.calls == ["นอนๆ"] + + +class TestFetchLexicon: + def test_fetches_sha_verified_and_builds_a_lexicon(self, tmp_path, monkeypatch): + import json as _json + + lexicon = {"กิน": "kin", "ข้าว": "kʰâːw"} + blob = _json.dumps(lexicon, ensure_ascii=False).encode("utf-8") + import hashlib + + sha = hashlib.sha256(blob).hexdigest() + (tmp_path / "lex.json").write_bytes(blob) + base = f"file://{tmp_path}/lex.json" + + from interscript.ml import thai_hybrid as TH + + got = TH.fetch_lexicon(index_url=None, _base_url=base, _sha256=sha) + assert got["กิน"] == "kin" + assert isinstance(got, TH.Lexicon) + + def test_rejects_sha_mismatch(self, tmp_path): + (tmp_path / "lex.json").write_bytes(b'{"a":"b"}') + from interscript.ml import thai_hybrid as TH + + try: + TH.fetch_lexicon( + index_url=None, + _base_url=f"file://{tmp_path}/lex.json", + _sha256="0" * 64, + ) + except ValueError as e: + assert "sha256" in str(e) + else: + raise AssertionError("expected sha mismatch")