diff --git a/TODO.sota/67-b6-thai-lexicon.md b/TODO.sota/67-b6-thai-lexicon.md index c4927472..501fb843 100644 --- a/TODO.sota/67-b6-thai-lexicon.md +++ b/TODO.sota/67-b6-thai-lexicon.md @@ -1,6 +1,15 @@ # WO67 — B6: Thai — lexicon-first, three layers (PER < 2) -Status: PLANNED · launchable now +Status: IN_PROGRESS (lexicon phase landed 2026-10-21) +Lexicon phase DONE: corpora/tha-phonemic.yaml -- 28,644 train / 3,225 +test (word -> phonemic-with-tones pairs, ORST vocabulary x pythainlp +royin engine); 6,291 engine-corrupt rows filtered and documented; +bucket run-080-thai-lexicon, size-verified. NOTE the layer honesty: +royin emits PHONEMIC+tones, NOT RTGS -- the RTGS layer requires +encoding bgnpcgn-tha (a BGN/PCGN gap-system row: ROMANIZATION OF +THAI.pdf, whole-document transcription) then the rule composite. +NEXT: bgnpcgn-tha encoding; hit-rate metric over the eval set; the +three-layer (RTGS/IPA/IPA+tone) eval harness. Goal: published PER < 2; lexicon hit-rate reported; tone accuracy for the TTS layer (RTGS / IPA / IPA+tone tracked separately). Method: Royal Institute dictionary (or equivalent licensed lexicon) as diff --git a/corpora/tha-phonemic.yaml b/corpora/tha-phonemic.yaml new file mode 100644 index 00000000..f8cd6033 --- /dev/null +++ b/corpora/tha-phonemic.yaml @@ -0,0 +1,19 @@ +id: tha-phonemic +task: g2p +sources: + # ORST (Royal Institute Dictionary) wordlist via pythainlp's corpus + # packaging (Apache-2.0); phonemic readings generated by pythainlp's + # "royin" engine. Owner note: the underlying wordlist is Royal + # Institute of Thailand material redistributed by pythainlp -- terms + # recorded here for awareness, SPDX-open packaging clears the gate. + # 6,291 rows (17%) dropped: the royin engine repeats syllables on + # certain clusters (e.g. rust-class รักษา words) -- generator defect, + # filtered, documented in WO67. + - kind: bucket + id: Interscript/isx-training/run-080-thai-lexicon + revision: run-080-thai-lexicon + license: Apache-2.0 + mount: /data-tha +splits: + train: {path: tha-phonemic-train.jsonl, rows: 28644, sha256: 68a799711aa0d4ca8e3d1796cdcaf1b8599c82f0b7390d7d5ea3f53442c39c7d} + test: {path: tha-phonemic-test.jsonl, rows: 3225, sha256: a5abe80b9c9f119fa8bdeb51ca22a931b2c5b2d2e946d9ea190161f937b53966}