From edeb0c15ac97be8c2b723487afaee95c5f38371c Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 21 Oct 2026 10:45:34 +0800 Subject: [PATCH] =?UTF-8?q?feat(WO68/B7):=20khm-dict=20corpus=20=E2=80=94?= =?UTF-8?q?=20Chuon=20Nath=20lexicon=20pinned=20and=20staged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3,261 gold (word→IPA) reading pairs + 72,048 SBBIC vocab rows, 326-row holdout; bucket-staged with size verification; LGPL-2.1 recorded with an owner note on LGPL-for-data semantics. Phase 2 (hit-rate + lexicon- conditioned fallback training) tracked in the WO. --- TODO.sota/68-b7-khmer-lexicon.md | 9 ++++++++- corpora/khm-dict.yaml | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 corpora/khm-dict.yaml diff --git a/TODO.sota/68-b7-khmer-lexicon.md b/TODO.sota/68-b7-khmer-lexicon.md index cde065a1..8d1a28f7 100644 --- a/TODO.sota/68-b7-khmer-lexicon.md +++ b/TODO.sota/68-b7-khmer-lexicon.md @@ -1,6 +1,13 @@ # WO68 — B7: Khmer — Chuon Nath lexicon conditioning (CER < 15) -Status: PLANNED · launchable now (data on disk: khmer-dict-spice) +Status: IN_PROGRESS (phase 1 landed 2026-10-21; phase 2 = training arm) +Phase 1 DONE: khm-dict corpus built + pinned (corpora/khm-dict.yaml) — +3,261 gold Chuon-Nath (word→IPA) pairs, 326-row holdout, 72,048 SBBIC +vocab rows; staged at bucket Interscript/isx-training/run-080-khm-lexicon +(size-verified). License LGPL-2.1 recorded; owner note on +LGPL-for-data semantics. NEXT: hit-rate vs the khm-latn eval set (in +the phase-2 job, which mounts it), lexicon-conditioned ByT5 fallback +training, three-way probe. Goal: khm-latn 27.4 → CER < 15 with hit-rate reported. Method: build the Chuon Nath-conditioned dataset (headword→reading pairs, syllabification-aware); lexicon-conditioned ByT5 (retrieval or diff --git a/corpora/khm-dict.yaml b/corpora/khm-dict.yaml new file mode 100644 index 00000000..8202f0e9 --- /dev/null +++ b/corpora/khm-dict.yaml @@ -0,0 +1,19 @@ +id: khm-dict +task: transliteration +sources: + # Chuon Nath dictionary data, digitalized by the Open Institute + # (SPICE / USAID-funded) — github.com/interscript/khmer-dict-spice. + # Repo LICENSE: LGPL-2.1 (SPDX-open; gate clears). Owner note: + # LGPL semantics for a data corpus are unusual; flagged for owner + # review without blocking (registry law: SPDX-open clears). + - kind: bucket + id: Interscript/isx-training/run-080-khm-lexicon + revision: run-080-khm-lexicon + license: LGPL-2.1 + mount: /data-khm +splits: + # Gold reading pairs (word -> IPA, Chuon Nath via SPICE ipa_dic): + # 3261 pairs, 326 held out as test. SBBIC vocabulary (72,048 words, + # no readings) carries the lexicon-coverage layer. + train: {path: khm-lexicon-train.jsonl, rows: 74983, sha256: 83eba4945044543c6a0af467ecdcb72a16b2f552fa86e041bb5eeca6e9b2e9be} + test: {path: khm-lexicon-test.jsonl, rows: 326, sha256: 23385241b606f3ca1f74d09df5bf34fe32b364545171edf157f0c90a58bd5b18}