diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1853a47..dc4e038 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,3 +27,39 @@ jobs: - run: pytest tests/ -v env: INTERSCRIPT_MAPS_PATH: ${{ github.workspace }}/maps/maps + + conformance-matrix: + name: Conformance matrix (frozen fixtures) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -e ".[dev]" + - name: Download fixtures (GitHub Releases channel) + run: | + mkdir -p fixtures && cd fixtures + BASE="https://github.com/interscript/interscript-models/releases/download" + curl -fsSL -o models-index.yaml "$BASE/index-v12/models-index.yaml" + curl -fsSL -o models-index.yaml.sha256 "$BASE/index-v12/models-index.yaml.sha256" + echo "$(cat models-index.yaml.sha256)" | sha256sum -c + for v in urd-diac-base fas-diac-base urd-g2p-base; do + curl -fsSL -O "$BASE/index-v12/$v-vectors.json" + done + # the two single-zip models; urd-g2p (split parts) resolves + # via the runtime resolver in a later matrix pass + for m in urd-diac-base-1.0 fas-diac-base-1.0; do + url=$(sed -n "/^ $m:/,/^ [a-z]/p" models-index.yaml | grep -m1 'url:' | awk '{print $2}') + curl -fsSL -o "$m.zip" "$url" + done + - name: Run the matrix + run: | + for pair in "urd-diac-base-1.0 urd-diac-base" "fas-diac-base-1.0 fas-diac-base"; do + set -- $pair + echo "::group::$1" + SECRYST_E2E_ZIP="$PWD/fixtures/$1.zip" \ + CONFORMANCE_VECTORS="$PWD/fixtures/$2-vectors.json" \ + pytest tests_ml/test_conformance.py -v + echo "::endgroup::" + done diff --git a/src/interscript/ml/model.py b/src/interscript/ml/model.py index fcc084e..9677a29 100644 --- a/src/interscript/ml/model.py +++ b/src/interscript/ml/model.py @@ -31,6 +31,10 @@ def __init__(self, zip_path: Path | str): options = ort.SessionOptions() options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + # single-thread decode: the conformance contract is byte-exact + # across environments (WO79); multi-thread fp16 ties flip rows + options.intra_op_num_threads = 1 + options.inter_op_num_threads = 1 graphs = verify_and_read(self.zip_path) self._encoder = ort.InferenceSession( graphs["encoder.onnx"], options, providers=_providers() diff --git a/tests_ml/test_conformance.py b/tests_ml/test_conformance.py new file mode 100644 index 0000000..ea48d50 --- /dev/null +++ b/tests_ml/test_conformance.py @@ -0,0 +1,36 @@ +"""The frozen-fixture contract (interscript-models/conformance/) — the +python runtime must reproduce a model's vectors byte-exactly under the +declared decode mode, or the change does not merge. + +Gated on env: SECRYST_E2E_ZIP (the artifact) + CONFORMANCE_VECTORS +(the fixture file), so CI runs without downloads.""" + +from __future__ import annotations + +import json +import os + +import pytest + +ort = pytest.importorskip("onnxruntime") + +from interscript.ml import Model # noqa: E402 + + +def test_conformance_vectors_byte_exact() -> None: + vectors_path = os.environ.get("CONFORMANCE_VECTORS") + zip_path = os.environ.get("SECRYST_E2E_ZIP") + if not vectors_path or not zip_path: + pytest.skip("set CONFORMANCE_VECTORS and SECRYST_E2E_ZIP") + vectors = json.loads(open(vectors_path, encoding="utf-8").read()) + assert vectors["decode"]["mode"] == "single-row-greedy" + model = Model.load(zip_path) + failures = [] + for v in vectors["vectors"]: + got = model.translate(v["src"]) + if got != v["out"]: + failures.append(f"{v['src']!r} -> {got!r} (expected {v['out']!r})") + assert not failures, ( + f"{vectors['model']}: {len(failures)}/{len(vectors['vectors'])} " + f"mismatched\n" + "\n".join(failures[:3]) + )