diff --git a/conformance/fas-diac-base-vectors.json b/conformance/fas-diac-base-vectors.json new file mode 100644 index 00000000..f03d26ce --- /dev/null +++ b/conformance/fas-diac-base-vectors.json @@ -0,0 +1,58 @@ +{ + "model": "fas-diac-base-1.0", + "index_version": 12, + "decode": { + "mode": "single-row-greedy" + }, + "source": "first 12 rows of the 9288-row docs split (sha256 67513aa1)", + "vectors": [ + { + "src": "کلیدهای پرکینزمانند، به کاربران امکان نوشتن می‌دهند", + "out": "کِلِیدِهای پِرِکَینزمانند، بِه کَاربران امَکِان نِوشتن مِی‌دهند" + }, + { + "src": "مبلغ پرداختی برای تعمیرات خانه زیاد شد.", + "out": "مِبلغ پَرِداختی بَرِای تِعَمِیرات خَانه زِیاد شُد." + }, + { + "src": "کسی تو مهمونی ما نمیاد", + "out": "کِسی تُو مِهُمِونی مَا نِمِیاد" + }, + { + "src": "هر بار که به یاد عشق جوانی‌اش می‌افتد، می‌گریاند.", + "out": "هر بَار کِه بِه یاد عِشِق جَوانی‌اش مِی‌افتد، مِی‌گَریاند." + }, + { + "src": "دانشگاه نرفتم", + "out": "دَانَشِگاه نُرفتم" + }, + { + "src": "در این میان، بچه‌هایی که پدر و مادر آسیب‌دیده دارند، به مراتب، با چالش‌های بیشتری مواجه هستند. به نظر من، برای این‌که بتوانیم با مشکلات فرزند نوجوان‌مان", + "out": "در این مِیان، بِچَهِ‌هایی کِه پِدر و مَادر آسِیبِ‌دیده دَارند، بِه مَرِاتب، بَا چَالَشِ‌های بِیشتری مُواجه هستند. بِه نِظر من، بَرِای این‌�" + }, + { + "src": "تو مگه برده او هستی ک فرمانبرداری میکنی", + "out": "تُو مِگه بِرِده او هِستی کِ فَرُمَانبرداری مِیکِنی" + }, + { + "src": "پنجره اتاقم به سمت بر حیاط باز می‌شود.", + "out": "پِنِجره اتَاقم بِه سِمت بُر حَیاط بَاز مِی‌شود." + }, + { + "src": "نفس کشیدن در هوای تازه صبحگاهی لذت‌بخش است.", + "out": "نفس کِشِیدن در هَوای تَازِه صُبَحِگاهی لِذت‌بخش است." + }, + { + "src": "این قسم از مسیر همیشه ترافیک دارد.", + "out": "این قِسم از مِسیر هِمِیشه تِرَافیک دَارد." + }, + { + "src": "رسانهها میتوانند بازتاب افکار عمومی باشند.", + "out": "رَسِانهها مِیتوانند بَازِتاب افِکار عُمُومی بَاشند." + }, + { + "src": "او ایده‌های نوآورانه‌ای برای کسب و کار پرورد.", + "out": "او ایدَهِ‌های نُوآورِانه‌ای بَرِای کسب و کَار پرورد." + } + ] +} \ No newline at end of file diff --git a/conformance/urd-g2p-base-vectors.json b/conformance/urd-g2p-base-vectors.json new file mode 100644 index 00000000..a0fc591d --- /dev/null +++ b/conformance/urd-g2p-base-vectors.json @@ -0,0 +1,58 @@ +{ + "model": "urd-g2p-base-1.0", + "index_version": 12, + "decode": { + "mode": "single-row-greedy" + }, + "source": "first 12 rows of urd-g2p-dict", + "vectors": [ + { + "src": "چَشمگاہ", + "out": "t͡ʃəʃmˈɡaːh" + }, + { + "src": "سرد کرنے", + "out": "sərd̪.kərˈneː" + }, + { + "src": "المعدہ", + "out": "alˈmaːd̪a" + }, + { + "src": "غَیر فنکارہ", + "out": "ɣɛːr.fənˈkaːraː" + }, + { + "src": "دَندانوں کا سَیٹ", + "out": "d̪ənˈdaːnoːn.kaː.sɛːʈ" + }, + { + "src": "ہَستِیْاں", + "out": "həs.t̪iː.jãː" + }, + { + "src": "کفشوں", + "out": "kəfˈʃoː̃" + }, + { + "src": "پَہُنچنے والی", + "out": "pəˈɦʊnt͡ʃneː ʋaːˈliː" + }, + { + "src": "دَیر سے", + "out": "d̪ɛːr.seː" + }, + { + "src": "جشنِیَہ", + "out": "d͡ʒaʃ.niː.jaː" + }, + { + "src": "سنارنیں", + "out": "sɪˈnaːrnĩː" + }, + { + "src": "بلند ترين", + "out": "bʊˈlənd̪ t̪əˈriːn" + } + ] +} \ No newline at end of file diff --git a/runtime/src/interscript_ml/registry.py b/runtime/src/interscript_ml/registry.py index aede061f..7bc0a249 100644 --- a/runtime/src/interscript_ml/registry.py +++ b/runtime/src/interscript_ml/registry.py @@ -19,7 +19,7 @@ import yaml DEFAULT_INDEX_URL = ( - "https://github.com/interscript/interscript-ml/releases/download/index-v6/models-index.yaml" + "https://github.com/interscript/interscript-models/releases/download/index-v12/models-index.yaml" ) ENV_INDEX = "SECRYST_INDEX" ENV_CACHE = "SECRYST_CACHE" diff --git a/runtime/tests/test_registry.py b/runtime/tests/test_registry.py index e2c0c2e7..e41f2f04 100644 --- a/runtime/tests/test_registry.py +++ b/runtime/tests/test_registry.py @@ -174,7 +174,7 @@ def test_default_index_url_pins_current_release() -> None: from interscript_ml.registry import DEFAULT_INDEX_URL assert DEFAULT_INDEX_URL == ( - "https://github.com/interscript/interscript-ml" - "/releases/download/index-v6/models-index.yaml" + "https://github.com/interscript/interscript-models" + "/releases/download/index-v12/models-index.yaml" ) assert "raw.githubusercontent" not in DEFAULT_INDEX_URL diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index c5ffcf9e..19de739b 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -17,10 +17,18 @@ from __future__ import annotations +import warnings from pathlib import Path import modal +warnings.warn( + 'Modal is retired; this entrypoint is frozen for ' + 'provenance — launch via the pipeline runner (HF Jobs)', + DeprecationWarning, + stacklevel=2, +) + REPO_ROOT = Path(__file__).resolve().parent.parent.parent IMAGE = ( diff --git a/src/gpu/modal_export.py b/src/gpu/modal_export.py index a1037b0f..5d53d3b1 100644 --- a/src/gpu/modal_export.py +++ b/src/gpu/modal_export.py @@ -20,10 +20,18 @@ from __future__ import annotations import re +import warnings from pathlib import Path import modal +warnings.warn( + 'Modal is retired; this entrypoint is frozen for ' + 'provenance — launch via the pipeline runner (HF Jobs)', + DeprecationWarning, + stacklevel=2, +) + REPO_ROOT = Path(__file__).resolve().parent.parent.parent IMAGE = ( diff --git a/src/gpu/modal_golden_matrix.py b/src/gpu/modal_golden_matrix.py index 69353ae1..eac3eaeb 100644 --- a/src/gpu/modal_golden_matrix.py +++ b/src/gpu/modal_golden_matrix.py @@ -10,10 +10,18 @@ from __future__ import annotations +import warnings from pathlib import Path import modal +warnings.warn( + 'Modal is retired; this entrypoint is frozen for ' + 'provenance — launch via the pipeline runner (HF Jobs)', + DeprecationWarning, + stacklevel=2, +) + REPO_ROOT = Path(__file__).resolve().parent.parent.parent IMAGE = ( diff --git a/src/gpu/modal_score_artifact.py b/src/gpu/modal_score_artifact.py index ee076322..b037cf9a 100644 --- a/src/gpu/modal_score_artifact.py +++ b/src/gpu/modal_score_artifact.py @@ -10,10 +10,18 @@ from __future__ import annotations +import warnings from pathlib import Path import modal +warnings.warn( + 'Modal is retired; this entrypoint is frozen for ' + 'provenance — launch via the pipeline runner (HF Jobs)', + DeprecationWarning, + stacklevel=2, +) + REPO_ROOT = Path(__file__).resolve().parent.parent.parent IMAGE = ( diff --git a/src/gpu/modal_teacher_sadeed.py b/src/gpu/modal_teacher_sadeed.py index 627dd487..9e02fef6 100644 --- a/src/gpu/modal_teacher_sadeed.py +++ b/src/gpu/modal_teacher_sadeed.py @@ -13,10 +13,18 @@ from __future__ import annotations +import warnings from pathlib import Path import modal +warnings.warn( + 'Modal is retired; this entrypoint is frozen for ' + 'provenance — launch via the pipeline runner (HF Jobs)', + DeprecationWarning, + stacklevel=2, +) + REPO_ROOT = Path(__file__).resolve().parent.parent.parent IMAGE = ( diff --git a/src/gpu/modal_teacher_thai.py b/src/gpu/modal_teacher_thai.py index 54d558d8..0e65394e 100644 --- a/src/gpu/modal_teacher_thai.py +++ b/src/gpu/modal_teacher_thai.py @@ -16,10 +16,18 @@ from __future__ import annotations +import warnings from pathlib import Path import modal +warnings.warn( + 'Modal is retired; this entrypoint is frozen for ' + 'provenance — launch via the pipeline runner (HF Jobs)', + DeprecationWarning, + stacklevel=2, +) + REPO_ROOT = Path(__file__).resolve().parent.parent.parent IMAGE = ( diff --git a/src/gpu/modal_train.py b/src/gpu/modal_train.py index af8ea173..718c808a 100644 --- a/src/gpu/modal_train.py +++ b/src/gpu/modal_train.py @@ -20,8 +20,16 @@ from __future__ import annotations import sys +import warnings from pathlib import Path +warnings.warn( + 'Modal is retired; this entrypoint is frozen for ' + 'provenance — launch via the pipeline runner (HF Jobs)', + DeprecationWarning, + stacklevel=2, +) + try: import modal # type: ignore _MODAL_AVAILABLE = True diff --git a/src/pipeline/housekeeper.py b/src/pipeline/housekeeper.py new file mode 100644 index 00000000..90b55ec5 --- /dev/null +++ b/src/pipeline/housekeeper.py @@ -0,0 +1,80 @@ +"""WO56 — the housekeeper: bucket inventory + checkpoint hygiene. + +KLF's law, adopted: storage hygiene runs with a DRY-RUN DEFAULT and +never deletes without an explicit allowlist. This module only ever +CLASSIFIES; pruning is a separate, explicitly-authorized step that +takes the inventory as input. + +Classification (per run directory on the training bucket): +- terminal marker present (RELABEL_DONE / TRAIN_DONE / GATE_DONE / + VERIFY_DONE / verdict.json) → complete +- candidate epochs: epoch-N / oepoch-N dirs and best/ dirs +- stray: large files outside a recognized run shape +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +TERMINAL_MARKERS = { + "RELABEL_DONE", "TRAIN_DONE", "GATE_DONE", "VERIFY_DONE", "EXPORT_DONE", + "ANATOMY_DONE", "verdict.json", "relabel_stats.json", +} +EPOCH_DIR_PREFIXES = ("epoch-", "oepoch-") + + +@dataclass +class RunInventory: + name: str + complete: bool + markers: list[str] = field(default_factory=list) + epoch_dirs: list[str] = field(default_factory=list) + best_dirs: list[str] = field(default_factory=list) + files: int = 0 + bytes: int = 0 + notes: list[str] = field(default_factory=list) + + +def classify_run(name: str, listing: list[dict[str, Any]]) -> RunInventory: + """listing = HfFileSystem.ls() entries for one run dir (name/size).""" + inv = RunInventory(name=name, complete=False) + for entry in listing: + n = entry["name"].rsplit("/", 1)[-1] + size = entry.get("size", 0) or 0 + inv.files += 1 + inv.bytes += size + if n in TERMINAL_MARKERS: + inv.complete = True + inv.markers.append(n) + elif any(n.startswith(p) for p in EPOCH_DIR_PREFIXES): + inv.epoch_dirs.append(n) + elif n == "best": + inv.best_dirs.append(n) + if not inv.complete: + inv.notes.append("no terminal marker — possibly still running") + if inv.epoch_dirs and inv.complete: + inv.notes.append( + f"{len(inv.epoch_dirs)} epoch dirs are prune candidates " + "(keep best/ unless the recipe says otherwise)" + ) + return inv + + +def inventory(runs: dict[str, list[dict[str, Any]]]) -> list[RunInventory]: + return [classify_run(name, listing) for name, listing in sorted(runs.items())] + + +def prune_plan(inv: list[RunInventory], keep_best: bool = True) -> list[str]: + """The explicit, reviewable prune list. Dry-run output only — + executing it is a separate, owner-authorized step.""" + plan: list[str] = [] + for run in inv: + if run.complete and run.epoch_dirs: + for d in run.epoch_dirs: + plan.append(f"{run.name}/{d}") + if keep_best and not run.best_dirs: + plan.append( + f"{run.name} (WARNING: no best/ — keep epoch-final)" + ) + return plan diff --git a/tests/test_pipeline_housekeeper.py b/tests/test_pipeline_housekeeper.py new file mode 100644 index 00000000..fef5d713 --- /dev/null +++ b/tests/test_pipeline_housekeeper.py @@ -0,0 +1,51 @@ +"""WO56 — housekeeper tests: classify, never delete; dry-run default. + +The housekeeper's iron law: this module classifies and proposes. It +has no delete capability at all — pruning plans are explicit, +reviewable lists that a separate owner-authorized step consumes. +""" + +from __future__ import annotations + +from pipeline.housekeeper import classify_run, inventory, prune_plan + + +def _listing(*names: str) -> list[dict]: + return [{"name": f"run-x/{n}", "size": 100} for n in names] + + +def test_complete_run_with_epochs_flags_prune_candidates() -> None: + inv = classify_run("run-a", _listing( + "RELABEL_DONE", "best", "epoch-0", "epoch-1", "preds-p047.jsonl")) + assert inv.complete + assert inv.epoch_dirs == ["epoch-0", "epoch-1"] + assert any("prune candidates" in n for n in inv.notes) + + +def test_incomplete_run_never_flags_prune() -> None: + inv = classify_run("run-b", _listing("best", "epoch-0")) + assert not inv.complete + assert not prune_plan(inventory({"run-b": _listing("best", "epoch-0")})), ( + "an incomplete run must never appear in a prune plan" + ) + + +def test_prune_plan_keeps_best_and_warns_without_it() -> None: + inv = inventory({ + "run-a": _listing("TRAIN_DONE", "best", "epoch-0", "epoch-1"), + "run-c": _listing("GATE_DONE", "epoch-0"), # complete, no best/ + }) + plan = prune_plan(inv) + assert "run-a/epoch-0" in plan and "run-a/epoch-1" in plan + assert "run-c/epoch-0" in plan + assert any("no best/" in p for p in plan if p.startswith("run-c")), ( + "pruning the only checkpoint of a run must warn loudly" + ) + + +def test_prune_plan_is_dry_run_only() -> None: + """The plan is strings; nothing here can touch a filesystem.""" + inv = inventory({"run-a": _listing("TRAIN_DONE", "epoch-0")}) + assert all(isinstance(p, str) for p in prune_plan(inv)) + assert not hasattr(__import__("pipeline.housekeeper", + fromlist=["inventory"]), "delete")