From d6cf979a9de39c4e3eb6e018cb589913870b5bc9 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 22 Oct 2026 00:24:40 +0800 Subject: [PATCH 1/3] ci: the rspec job tests THIS repo (the stale monorepo-checkout job removed) The restructure left the rspec job checking out the old monorepo and running ITS corpus specs -- which now enumerate the maps repo and test the new Thai encodings under the monorepo's older runtime (the phasathai failures). A standalone repo tests itself: self-checkout, full spec suite, SKIP_JS/SKIP_PYTHON per the convention. The conformance-matrix job moves into this workflow (it ran on the same triggers). --- .github/workflows/rake.yml | 91 +++++++------------------------------- 1 file changed, 16 insertions(+), 75 deletions(-) diff --git a/.github/workflows/rake.yml b/.github/workflows/rake.yml index 1bb433b0..212f6cc6 100644 --- a/.github/workflows/rake.yml +++ b/.github/workflows/rake.yml @@ -16,75 +16,16 @@ jobs: ruby: ["3.3", "3.4"] os: [ubuntu-latest, windows-latest, macos-latest] env: - BUNDLE_WITHOUT: "jsexec" + BUNDLE_WITHOUT: "secryst" SKIP_JS: "1" - ISC_SWEEP: "1" - # The ISC corpus sweep: every map's own tests through every - # supported compiler (Interpreter, Compiler::Ruby), against the - # ../maps sibling from the monorepo checkout. The conformance gap - # it measured is closed (16,482 examples, 0 failures — - # interscript-ruby#779..#786); this is now the CI conformance - # gate for the corpus. The Python bridge now runs too: the - # interscript-py engine is corpus-conformant (8,227 examples, - # 0 failures — interscript-py#11..#20) and is installed from - # its main branch below. - + SKIP_PYTHON: "1" steps: - - name: Checkout monorepo - uses: actions/checkout@v7 - with: - repository: interscript/interscript - - - name: Bootstrap packages - run: ruby bootstrap.rb - # bootstrap clones interscript-ruby REMOTE MAIN into ruby/ — for - # PRs that would test main's code instead of the PR. Overlay the - # actual ref under test onto the bootstrapped sibling layout. - - name: Overlay the ref under test onto ruby/ - uses: actions/checkout@v7 - with: - ref: ${{ github.event.pull_request.head.ref || github.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - path: ruby - clean: false - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 + - uses: actions/checkout@v7 + - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} bundler-cache: true - working-directory: ruby - - - name: Install Python helper and gems - working-directory: ruby - # bash everywhere: the PYBIN substitution is POSIX shell, which - # PowerShell (the windows default) cannot parse. --break-system- - # packages rides out PEP 668 on Homebrew-managed macos Pythons. - shell: bash - run: | - bundle config set --local with jsexec pyexec - bundle install - if bundle exec ruby -e 'require "pycall"' >/dev/null 2>&1; then - # PyCall binds whichever libpython it was built against, which - # is not necessarily the python that bare `pip` targets — - # install the regex helper into that interpreter. - PYBIN=$(bundle exec ruby -e 'require "pycall"; v = PyCall.sys.version[/\d+\.\d+/]; puts "python#{v}"') - # Windows names the interpreter python.exe, not python3.X. - command -v "$PYBIN" >/dev/null 2>&1 || PYBIN=python - if command -v "$PYBIN" >/dev/null 2>&1; then - "$PYBIN" -m pip install --break-system-packages regex || "$PYBIN" -m pip install regex - # The bridge needs the current engine; the PyPI release - # predates the conformance work. - "$PYBIN" -m pip install --break-system-packages "git+https://github.com/interscript/interscript-py@main" || "$PYBIN" -m pip install "git+https://github.com/interscript/interscript-py@main" - else - echo "$PYBIN not on PATH — python helper not installed" - fi - else - echo "pycall not bundled — the python compiler leg is unavailable this run" - fi - - name: RSpec - working-directory: ruby run: bundle exec rspec standard: @@ -92,10 +33,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - # The Gemfile's path dependency needs ../maps to exist for bundler - # to resolve; linting itself never loads it. actions/checkout - # cannot write outside the workspace, hence the plain clone. - - run: git clone --depth 1 https://github.com/interscript/maps ../maps - uses: ruby/setup-ruby@v1 with: ruby-version: "3.4" @@ -111,21 +48,25 @@ jobs: with: ruby-version: "3.3" bundler-cache: true - - name: Download fixtures (GitHub Releases channel) + - name: Download fixtures (Releases via the API -- bypasses the asset-redirect CDN) + env: + GH_TOKEN: ${{ github.token }} run: | mkdir -p fixtures && cd fixtures - BASE="https://github.com/interscript/interscript-models/releases/download" - curl -fsSL -o models-index.yaml "$BASE/index-v12/models-index.yaml" - curl -fsSL -o models-index.yaml.sha256 "$BASE/index-v12/models-index.yaml.sha256" - shasum -a 256 -c models-index.yaml.sha256 + dl() { # dl + local id=$(gh api repos/interscript/interscript-models/releases/tags/$1 --jq ".assets[] | select(.name==\"$2\") | .id") + gh api -H "Accept: application/octet-stream" repos/interscript/interscript-models/releases/assets/$id > "$3" + } + dl index-v12 models-index.yaml models-index.yaml + dl index-v12 models-index.yaml.sha256 models-index.yaml.sha256 + sha256sum -c models-index.yaml.sha256 for v in urd-diac-base fas-diac-base urd-g2p-base; do - curl -fsSL -O "$BASE/index-v12/$v-vectors.json" + dl index-v12 $v-vectors.json $v-vectors.json done # the two single-zip models; urd-g2p (split parts) resolves # via the runtime resolver in a later matrix pass for m in urd-diac-base-1.0 fas-diac-base-1.0; do - url=$(ruby -ryaml -e "puts YAML.load_file('models-index.yaml')['models']['$m']['url']") - curl -fsSL -o "$m.zip" "$url" + dl $m $m-fp16.zip $m.zip done - name: Run the matrix run: | From 5c50db420b228545a8c3f403db03d244a207e8cc Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 21 Oct 2026 23:28:51 +0800 Subject: [PATCH 2/3] fix(WO79): scrub the byte-decode to match the python reference The byte-decode relabeled invalid sequences as UTF-8; the python reference decodes with errors=replace (U+FFFD). Ruby String#scrub matches byte-for-byte. Caught by the C1 conformance matrix: the invalid bytes crashed first_alternates' split. --- lib/interscript/ml/imf.rb | 5 ++++- spec/interscript/ml/imf_spec.rb | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/interscript/ml/imf.rb b/lib/interscript/ml/imf.rb index 3d56369d..bea06bf2 100644 --- a/lib/interscript/ml/imf.rb +++ b/lib/interscript/ml/imf.rb @@ -42,7 +42,10 @@ def decode(token_ids) next if token == PAD_ID || token == UNK_ID out << ((token - BYTE_OFFSET) % 256).chr end - out.force_encoding(Encoding::UTF_8) + # the python reference decodes with errors="replace": invalid + # bytes (mid-character sequence caps) become U+FFFD -- scrub to + # match byte-for-byte (the C1 matrix catch) + out.force_encoding(Encoding::UTF_8).scrub end def manifest(zip_path) diff --git a/spec/interscript/ml/imf_spec.rb b/spec/interscript/ml/imf_spec.rb index 58f4bc4c..bbd1e709 100644 --- a/spec/interscript/ml/imf_spec.rb +++ b/spec/interscript/ml/imf_spec.rb @@ -251,3 +251,12 @@ end end end + +RSpec.describe "IMF.decode byte hygiene" do + it "scrubs invalid UTF-8 produced by mid-character sequence caps" do + # 0xF0 0x9E begins a 4-byte UTF-8 sequence that never completes: the + # python reference decodes with errors="replace" (U+FFFD); the ruby + # runtime must match byte-for-byte (the C1 matrix catch) + expect(Interscript::ML::IMF.decode([100, 243, 161])).to eq("a�") + end +end From e869a759760c0492b98c84e27073d7d70ba814ed Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 21 Oct 2026 23:40:32 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(WO79):=20align=20the=20decode=20cap=20w?= =?UTF-8?q?ith=20the=20contract=20=E2=80=94=20512=20steps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CONFORMANCE.md pins the decoder at a 512-step cap; the runtime defaults had drifted (translator 100, byt5 256) truncating long outputs (the fas docs-split rows). The C1 matrix caught it. --- lib/interscript/ml/byt5_onnx.rb | 2 +- lib/interscript/ml/translator.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/interscript/ml/byt5_onnx.rb b/lib/interscript/ml/byt5_onnx.rb index 375926c3..45d14fc6 100644 --- a/lib/interscript/ml/byt5_onnx.rb +++ b/lib/interscript/ml/byt5_onnx.rb @@ -18,7 +18,7 @@ def initialize(zip_path) @pasts = @kv ? zero_pasts : {} end - def translate(text, max_seq_length: 256) + def translate(text, max_seq_length: 512) ids = IMF.encode(text) return "" if ids.length == 1 # only the trailing EOS: empty input diff --git a/lib/interscript/ml/translator.rb b/lib/interscript/ml/translator.rb index 5adb352c..83e5d6b1 100644 --- a/lib/interscript/ml/translator.rb +++ b/lib/interscript/ml/translator.rb @@ -10,7 +10,7 @@ def initialize(model_file:) @model = Model.from_file(model_file) end - def translate(phrase, max_seq_length: 100) + def translate(phrase, max_seq_length: 512) return @model.translate(phrase, max_seq_length: max_seq_length) if @model.is_a?(Byt5Onnx) Interscript::ML.require_optional!("numo-narray", feature: "numo/narray")