Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/interscript/ml/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@
from interscript.ml.loader import load_manifest, verify_and_read


def first_alternates(text: str) -> str:
"""Seq2seq output contract: models trained on multi-reference
corpora emit '/'-separated alternates per token; the runtime
returns the primary choice only."""
return " ".join(part.split("/", 1)[0] for part in text.split())


class Model:
"""A loaded, checksum-verified IMF v1 model.

Expand Down Expand Up @@ -61,7 +68,7 @@ def id(self) -> str:

def translate(self, text: str, max_len: int = 256) -> str:
token_ids = self.generate(text, max_len=max_len)
return decode(token_ids)
return first_alternates(decode(token_ids))

def generate(self, text: str, max_len: int = 256) -> list[int]:
ids = np.array([encode(text)], dtype=np.int64)
Expand Down
15 changes: 15 additions & 0 deletions tests/test_first_alternates.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
from interscript.ml.model import first_alternates


def test_first_alternates_keeps_primary_choice_per_token():
assert first_alternates("وِكَالَةُ/وِكَالَةُ اَلْأَرْصَادِ") == "وِكَالَةُ اَلْأَرْصَادِ"


def test_first_alternates_plain_text_untouched():
assert first_alternates("مرحبا بالعالم") == "مرحبا بالعالم"


def test_first_alternates_edges():
assert first_alternates("") == ""
assert first_alternates("a/b c/d") == "a c"
assert first_alternates("مَارْغِرِيتْ/مَارْغِرِيتْ/مَارْغِرِيتُ اَلثَّانِيَةُ") == "مَارْغِرِيتْ اَلثَّانِيَةُ"
Loading