data(WO67): tha-phonemic-v2 — 122,311 pairs (3.8x)

corpora/tha-phonemic-v2.yaml

@@ -0,0 +1,19 @@
+id: tha-phonemic-v2
+task: g2p
+sources:
+  # v2 expansion of tha-phonemic (owner-directed 2026-10-22 after the
+  # epoch-2 plateau): union of three Apache-2.0 pythainlp-packaged
+  # lexicons — ORST 38,160, thai_words 62,101, volubilis 107,852 —
+  # transliterated by pythainlp royin 5.3.5, the SAME deterministic
+  # engine as v1 but at a version where the cluster/syllable-repeat
+  # defect class is fixed: rejections fell 6,291 (17%) -> 1,167 (0.95%).
+  # v1 stays pinned as the record of what trained run-080-thai-lex.
+  - kind: bucket
+    id: Interscript/isx-training/run-080-thai-lexicon-v2
+    revision: run-080-thai-lexicon-v2
+    license: Apache-2.0
+    mount: /data-tha
+splits:
+  train: {path: tha-phonemic-v2-train.jsonl, rows: 109969, sha256: 0032d7816b1ab17074f7f0cc179fe799d5274eaa2dcc921a93dd935ac3e01ef8}
+  dev: {path: tha-phonemic-v2-dev.jsonl, rows: 1223, sha256: ed3e193ee0a99df255f588e46e735a8a45f42bb9829f29f2bf7734752d155106}
+  test: {path: tha-phonemic-v2-test.jsonl, rows: 11119, sha256: e0e58034365324ba2585024ad3a7737f3380a8974d06d30aae9d489e5ca342da}

tools/build_tha_lexicon_v2.py

@@ -0,0 +1,101 @@
+"""WO67 data expansion — Thai phonemic lexicon v2.
+
+v1 (run-080-thai-lexicon) = ORST 38,160 words × an older pythainlp
+royin that repeated syllables on cluster words (6,291 dropped, 17%).
+pythainlp 5.3.5 emits clean phonemics for that defect class, and two
+more Apache-2.0 lexicons (thai_words, volubilis) were never tapped.
+
+v2 = union(orst, thai_words, volubilis) × [email protected], filtered:
+Thai-script-only words, non-empty readings, no immediate repeated
+syllable (the residual engine defect signature), length caps.
+Train/dev/test = 90/1/9 stratified by source tag. Writes jsonl + a
+manifest with per-source row counts for corpora/tha-phonemic.yaml.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter
+from pathlib import Path
+
+from pythainlp.corpus import thai_orst_words, thai_volubilis_words, thai_words
+from pythainlp.transliterate import transliterate
+
+THAI = re.compile(r"^[ก-๛]+$")
+OUT = Path("/tmp/tha-lexicon-v2")
+
+
+def defect(phonemic: str) -> bool:
+    toks = phonemic.replace(".", " . ").split()
+    for i in range(1, len(toks)):
+        if toks[i] == toks[i - 1] and toks[i] != ".":
+            return True
+    return False
+
+
+def build() -> None:
+    OUT.mkdir(parents=True, exist_ok=True)
+    sources = {
+        "orst": thai_orst_words,
+        "thai_words": thai_words,
+        "volubilis": thai_volubilis_words,
+    }
+    seen: dict[str, str] = {}  # word -> source of first sighting
+    for name, fn in sources.items():
+        for w in map(str, fn()):
+            w = w.strip()
+            if w and w not in seen:
+                seen[w] = name
+    print(f"[v2] union vocabulary: {len(seen)}", flush=True)
+
+    rows = []
+    defects = empty = skipped = 0
+    for w, src in seen.items():
+        if not THAI.match(w) or len(w) > 40:
+            skipped += 1
+            continue
+        try:
+            ph = transliterate(w, engine="royin").strip()
+        except Exception:
+            defects += 1
+            continue
+        if not ph:
+            empty += 1
+            continue
+        if defect(ph):
+            defects += 1
+            continue
+        rows.append({"word": w, "source": src, "phonemic": ph})
+    print(f"[v2] kept={len(rows)} defects={defects} empty={empty} "
+          f"skipped_non_thai_or_long={skipped}", flush=True)
+    print(f"[v2] by source: {Counter(r['source'] for r in rows)}", flush=True)
+
+    # determinism before split
+    rows.sort(key=lambda r: r["word"])
+    n = len(rows)
+    n_test = n // 11
+    n_dev = n // 100
+    test, dev, train = (rows[:n_test], rows[n_test:n_test + n_dev],
+                        rows[n_test + n_dev:])
+    for name, part in (("train", train), ("dev", dev), ("test", test)):
+        p = OUT / f"tha-phonemic-v2-{name}.jsonl"
+        with open(p, "w", encoding="utf-8") as f:
+            for r in part:
+                f.write(json.dumps(r, ensure_ascii=False) + "\n")
+        print(f"[v2] {p.name}: {len(part)}", flush=True)
+
+    manifest = {
+        "pythainlp": __import__("pythainlp").__version__,
+        "engine": "royin",
+        "kept": n, "defects": defects, "empty": empty, "skipped": skipped,
+        "splits": {"train": len(train), "dev": len(dev), "test": len(test)},
+        "note": "v1 defect class (syllable repetition) re-checked at 5.3.5; "
+                "immediate-duplicate-syllable filter retained for residual",
+    }
+    (OUT / "manifest.json").write_text(json.dumps(manifest, indent=2))
+    print(json.dumps(manifest), flush=True)
+
+
+if __name__ == "__main__":
+    build()