diff --git a/src/sadeedbench/vrr.py b/src/sadeedbench/vrr.py new file mode 100644 index 00000000..b66868d4 --- /dev/null +++ b/src/sadeedbench/vrr.py @@ -0,0 +1,84 @@ +"""WO81 — valid-reading-rate (VRR), the ambiguity-aware metric. + +Undiacritized Arabic is an underspecified input: the skeleton is a +relation over readings, and one-gold DER penalizes every legitimate +alternative. VRR scores a prediction as valid when it matches ANY +dictionary-licensed reading of its skeleton. Reported alongside DER — +never instead of it. +""" + +from __future__ import annotations + +import re + +# Arabic diacritics: tanwin + harakat + shadda/sukun marks (0610-061A), +# honorific/annotation marks (064B-065F), superscript alef (0670). +_HARAKAT = frozenset( + chr(c) for c in list(range(0x0610, 0x061B)) + list(range(0x064B, 0x0660)) + + [0x0670] +) + + +def strip_harakat(text: str) -> str: + """The skeleton: the text with all diacritics removed.""" + return "".join(ch for ch in text if ch not in _HARAKAT) + + +def build_lexicon(vocalized_forms) -> dict[str, set[str]]: + """skeleton -> set of licensed fully-vocalized readings. + + A form with no diacritic is not a reading (it is its own skeleton) + and is excluded — otherwise VRR degenerates to 1.0 for any + undiacritized prediction. + """ + lex: dict[str, set[str]] = {} + for form in vocalized_forms: + skeleton = strip_harakat(form) + if form == skeleton: + continue + lex.setdefault(skeleton, set()).add(form) + return lex + + +def score_predictions_vrr(preds: list[str], + licensed: list[set[str]]) -> dict: + """VRR over rows: valid iff the exact prediction is licensed. + + A skeleton with no known readings can never validate — the row + still counts in the denominator (absence of evidence is not + validity). + """ + if len(preds) != len(licensed): + raise ValueError( + f"preds and licensed lengths differ: {len(preds)} vs " + f"{len(licensed)}") + valid = sum(1 for p, ls in zip(preds, licensed, strict=False) if p in ls) + return { + "vrr": valid / len(preds) if preds else 0.0, + "valid": valid, + "total": len(preds), + "convention": "row-level exact match against licensed readings; " + "readings built per build_lexicon (>=1 haraka)", + } + + +_WORD_RE = re.compile(r"[؀-ۿ]+") + + +def score_text_vrr(src: str, pred: str, lexicon: dict[str, set[str]]) -> tuple[int, int]: + """(valid_words, total_words) for one paragraph row. + + A word is valid iff its exact vocalized form is a licensed reading + of that word's skeleton. Non-Arabic tokens count as invalid-missing + evidence: they enter the denominator (the model must vocalize the + Arabic content words; Latin/numeric tokens carry no readings). + """ + src_words = _WORD_RE.findall(src) + pred_words = _WORD_RE.findall(pred) + if not src_words: + return 0, 0 + valid = 0 + for sw, pw in zip(src_words, pred_words, strict=False): + if pw in lexicon.get(strip_harakat(sw), ()): # exact licensed reading + valid += 1 + return valid, len(src_words) diff --git a/tests/test_sadeedbench_vrr.py b/tests/test_sadeedbench_vrr.py new file mode 100644 index 00000000..f9e0d97b --- /dev/null +++ b/tests/test_sadeedbench_vrr.py @@ -0,0 +1,70 @@ +"""WO81 — valid-reading-rate (VRR): a prediction counts when it matches +ANY dictionary-licensed reading of its skeleton. Single-gold DER +undercounts legitimate ambiguity; VRR is the ambiguity-aware complement. +""" + +from __future__ import annotations + +from sadeedbench.vrr import build_lexicon, score_predictions_vrr, strip_harakat + +# skeleton of "بِكَتاب" (kitab, "with a book") +VOCALIZED = "بِكِتَاب" +SKELETON = "بكتاب" + + +def test_strip_harakat_removes_only_diacritics() -> None: + assert strip_harakat(VOCALIZED) == SKELETON + assert strip_harakat(SKELETON) == SKELETON + # letters, spaces, punctuation survive + assert strip_harakat("مَدرسة، جامعة") == "مدرسة، جامعة" + + +def test_build_lexicon_groups_by_skeleton_and_needs_a_reading() -> None: + lex = build_lexicon([ + VOCALIZED, # بِكِتَاب — licensed reading of بكتاب + "بُكْتَاب", # another licensed reading + SKELETON, # bare form — not a reading, excluded + "مَدرسة", + ]) + assert lex[SKELETON] == {VOCALIZED, "بُكْتَاب"} + assert lex["مدرسة"] == {"مَدرسة"} + assert SKELETON not in lex["مدرسة"] + + +def test_vrr_scores_any_licensed_reading() -> None: + lex = build_lexicon([VOCALIZED, "بُكْتَاب"]) + rows = [ + (SKELETON, VOCALIZED), # exact licensed reading -> valid + (SKELETON, "بُكْتَاب"), # alternative reading -> valid + (SKELETON, "بَكَاتِيب"), # hallucinated -> invalid + ] + preds = [p for _, p in rows] + result = score_predictions_vrr(preds, [lex[_s] for _s, _ in rows]) + assert result["vrr"] == 2 / 3 + assert result["valid"] == 2 + assert result["total"] == 3 + + +def test_vrr_with_empty_licensed_set_is_never_valid() -> None: + result = score_predictions_vrr(["مَدرسة"], [set()]) + assert result["vrr"] == 0.0 + assert result["total"] == 1 + + +def test_vrr_mismatched_lengths_raise() -> None: + try: + score_predictions_vrr(["x"], [set(), set()]) + except ValueError as e: + assert "length" in str(e) + else: + raise AssertionError("expected ValueError") + + +def test_score_text_vrr_word_level() -> None: + from sadeedbench.vrr import score_text_vrr + + lex = build_lexicon([VOCALIZED, "مَدرسة"]) + # two Arabic words: one licensed reading, one hallucination; a Latin + # token in the source pairs with pred text but carries no readings + v, t = score_text_vrr("بكتاب مدرسة x", f"{VOCALIZED} بَكَاتِيب yy", lex) + assert (v, t) == (1, 2)