diff --git a/TODO.sota/68-b7-khmer-lexicon.md b/TODO.sota/68-b7-khmer-lexicon.md index cde065a1..8d1a28f7 100644 --- a/TODO.sota/68-b7-khmer-lexicon.md +++ b/TODO.sota/68-b7-khmer-lexicon.md @@ -1,6 +1,13 @@ # WO68 — B7: Khmer — Chuon Nath lexicon conditioning (CER < 15) -Status: PLANNED · launchable now (data on disk: khmer-dict-spice) +Status: IN_PROGRESS (phase 1 landed 2026-10-21; phase 2 = training arm) +Phase 1 DONE: khm-dict corpus built + pinned (corpora/khm-dict.yaml) — +3,261 gold Chuon-Nath (word→IPA) pairs, 326-row holdout, 72,048 SBBIC +vocab rows; staged at bucket Interscript/isx-training/run-080-khm-lexicon +(size-verified). License LGPL-2.1 recorded; owner note on +LGPL-for-data semantics. NEXT: hit-rate vs the khm-latn eval set (in +the phase-2 job, which mounts it), lexicon-conditioned ByT5 fallback +training, three-way probe. Goal: khm-latn 27.4 → CER < 15 with hit-rate reported. Method: build the Chuon Nath-conditioned dataset (headword→reading pairs, syllabification-aware); lexicon-conditioned ByT5 (retrieval or diff --git a/corpora/khm-dict.yaml b/corpora/khm-dict.yaml new file mode 100644 index 00000000..8202f0e9 --- /dev/null +++ b/corpora/khm-dict.yaml @@ -0,0 +1,19 @@ +id: khm-dict +task: transliteration +sources: + # Chuon Nath dictionary data, digitalized by the Open Institute + # (SPICE / USAID-funded) — github.com/interscript/khmer-dict-spice. + # Repo LICENSE: LGPL-2.1 (SPDX-open; gate clears). Owner note: + # LGPL semantics for a data corpus are unusual; flagged for owner + # review without blocking (registry law: SPDX-open clears). + - kind: bucket + id: Interscript/isx-training/run-080-khm-lexicon + revision: run-080-khm-lexicon + license: LGPL-2.1 + mount: /data-khm +splits: + # Gold reading pairs (word -> IPA, Chuon Nath via SPICE ipa_dic): + # 3261 pairs, 326 held out as test. SBBIC vocabulary (72,048 words, + # no readings) carries the lexicon-coverage layer. + train: {path: khm-lexicon-train.jsonl, rows: 74983, sha256: 83eba4945044543c6a0af467ecdcb72a16b2f552fa86e041bb5eeca6e9b2e9be} + test: {path: khm-lexicon-test.jsonl, rows: 326, sha256: 23385241b606f3ca1f74d09df5bf34fe32b364545171edf157f0c90a58bd5b18}