diff --git a/lib/interscript/equivalence.rb b/lib/interscript/equivalence.rb new file mode 100644 index 00000000..efd18da9 --- /dev/null +++ b/lib/interscript/equivalence.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Interscript + # A3 (WO59): mechanical equivalence checking between two systems. + # + # Probes a sample set through both systems and reports the agreement + # ratio plus the minimal distinguishing samples. The sample set is + # caller-supplied or derived from the systems' own embedded tests. + class Equivalence + def self.check(first, second, samples:) + new(first, second).check(samples) + end + + # Embedded test inputs of the given systems (the natural probe set + # for near-equivalent authorities). + def self.samples_from_tests(*codes) + codes.flat_map do |code| + Array(Interscript.parse(code).tests&.data).map(&:first).compact + end.uniq + end + + def initialize(first, second) + @first = first + @second = second + end + + def check(samples) + distinguishing = [] + tested = 0 + matched = 0 + samples.each do |sample| + tested += 1 + a = Interscript.transliterate(@first, sample) + b = Interscript.transliterate(@second, sample) + if a == b + matched += 1 + else + distinguishing << {sample: sample, first: a, second: b} + end + end + ratio = tested.zero? ? 1.0 : matched.fdiv(tested) + {tested: tested, matched: matched, ratio: ratio, + distinguishing: distinguishing} + end + end +end diff --git a/lib/interscript/ml/byt5_onnx.rb b/lib/interscript/ml/byt5_onnx.rb index 375926c3..45d14fc6 100644 --- a/lib/interscript/ml/byt5_onnx.rb +++ b/lib/interscript/ml/byt5_onnx.rb @@ -18,7 +18,7 @@ def initialize(zip_path) @pasts = @kv ? zero_pasts : {} end - def translate(text, max_seq_length: 256) + def translate(text, max_seq_length: 512) ids = IMF.encode(text) return "" if ids.length == 1 # only the trailing EOS: empty input diff --git a/lib/interscript/ml/imf.rb b/lib/interscript/ml/imf.rb index 3d56369d..bea06bf2 100644 --- a/lib/interscript/ml/imf.rb +++ b/lib/interscript/ml/imf.rb @@ -42,7 +42,10 @@ def decode(token_ids) next if token == PAD_ID || token == UNK_ID out << ((token - BYTE_OFFSET) % 256).chr end - out.force_encoding(Encoding::UTF_8) + # the python reference decodes with errors="replace": invalid + # bytes (mid-character sequence caps) become U+FFFD -- scrub to + # match byte-for-byte (the C1 matrix catch) + out.force_encoding(Encoding::UTF_8).scrub end def manifest(zip_path) diff --git a/lib/interscript/ml/translator.rb b/lib/interscript/ml/translator.rb index 5adb352c..83e5d6b1 100644 --- a/lib/interscript/ml/translator.rb +++ b/lib/interscript/ml/translator.rb @@ -10,7 +10,7 @@ def initialize(model_file:) @model = Model.from_file(model_file) end - def translate(phrase, max_seq_length: 100) + def translate(phrase, max_seq_length: 512) return @model.translate(phrase, max_seq_length: max_seq_length) if @model.is_a?(Byt5Onnx) Interscript::ML.require_optional!("numo-narray", feature: "numo/narray") diff --git a/spec/interscript/equivalence_spec.rb b/spec/interscript/equivalence_spec.rb new file mode 100644 index 00000000..6b59e0da --- /dev/null +++ b/spec/interscript/equivalence_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require "json" +require "tmpdir" +require "interscript/equivalence" + +RSpec.describe Interscript::Equivalence do + def write_map(dir, name, body) + path = File.join(dir, "#{name}.isc") + File.write(path, body) + path + end + + def identical_pair(dir) + body = <<~ISC + system "TST:xx-Latn:Latn:1" { + metadata { + authority_id tst + id 1 + language iso-639-2:xx + source_script Latn + destination_script Latn + } + tests { "ab" -> "ab" } + stage main { parallel { sub "a" "a" } } + } + ISC + [write_map(dir, "tst-a-1", body), write_map(dir, "tst-b-1", body.gsub("tst-a-1", "tst-b-1"))] + end + + it "reports full agreement for identical systems" do + Dir.mktmpdir do |dir| + identical_pair(dir) + Interscript.load_path.unshift(dir) + begin + result = described_class.check("tst-a-1", "tst-b-1", samples: ["ab", "abc", "b"]) + expect(result[:ratio]).to eq(1.0) + expect(result[:distinguishing]).to be_empty + ensure + Interscript.load_path.delete(dir) + end + end + end + + it "finds the minimal distinguishing sample for divergent systems" do + Dir.mktmpdir do |dir| + body_a = <<~ISC + system "TST:xx-Latn:Latn:1" { + metadata { + authority_id tst + id 1 + language iso-639-2:xx + source_script Latn + destination_script Latn + } + stage main { parallel { sub "a" "x" } } + } + ISC + body_b = body_a.gsub('sub "a" "x"', 'sub "a" "y"').gsub("id 1", "id 2") + write_map(dir, "tst-c-1", body_a) + write_map(dir, "tst-c-2", body_b) + Interscript.load_path.unshift(dir) + begin + result = described_class.check("tst-c-1", "tst-c-2", samples: ["a", "b"]) + expect(result[:ratio]).to eq(0.5) + expect(result[:distinguishing]).to eq([{sample: "a", first: "x", second: "y"}]) + ensure + Interscript.load_path.delete(dir) + end + end + end +end diff --git a/spec/interscript/ml/conformance_spec.rb b/spec/interscript/ml/conformance_spec.rb index ff144fbe..1d9ef728 100644 --- a/spec/interscript/ml/conformance_spec.rb +++ b/spec/interscript/ml/conformance_spec.rb @@ -22,7 +22,8 @@ failures << "#{v["src"].inspect} -> #{got.inspect} (expected #{v["out"].inspect})" if got != v["out"] end expect(failures).to be_empty, - "#{vectors['model']}: #{failures.length}/#{vectors['vectors'].length} mismatched\n" + - failures.first(3).join("\n") + "#{vectors["model"]}: #{failures.length}/#{vectors["vectors"].length} mismatched +" + + failures.first(3).join("\n") end end diff --git a/spec/interscript/ml/imf_spec.rb b/spec/interscript/ml/imf_spec.rb index 58f4bc4c..bbd1e709 100644 --- a/spec/interscript/ml/imf_spec.rb +++ b/spec/interscript/ml/imf_spec.rb @@ -251,3 +251,12 @@ end end end + +RSpec.describe "IMF.decode byte hygiene" do + it "scrubs invalid UTF-8 produced by mid-character sequence caps" do + # 0xF0 0x9E begins a 4-byte UTF-8 sequence that never completes: the + # python reference decodes with errors="replace" (U+FFFD); the ruby + # runtime must match byte-for-byte (the C1 matrix catch) + expect(Interscript::ML::IMF.decode([100, 243, 161])).to eq("a�") + end +end