From 582d912f142478f8f96cd910832da04a6e7860fb Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 8 Nov 2026 20:10:12 +0800 Subject: [PATCH] =?UTF-8?q?fix(executor):=20compile=20constraints=20in=20c?= =?UTF-8?q?lass=20context=20=E2=80=94=20imported=20charset=20aliases=20res?= =?UTF-8?q?olve=20per-character?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the alalc-ell-Grek-Latn-2010 divergences (medial γκ rendered "gk" instead of "nk"): compileItem had a single alias expansion context, so an imported charset alias resolving to a plain string (unicode's greek, ~400 codepoints) compiled to regexpEscape() — a literal sequence. In a constraint position that sequence became a lookbehind/lookahead no input could ever satisfy, silently disabling every rule built on it: the map's before any(greek) ... after any("κΚ") + any(greek) γ→n rules never fired and the unconstrained parallel γ→g won. Ruby/RS resolve aliases in two contexts (RS builder.rs materialize_class vs materialize, convert_class_alias): constraints and any() arguments are CLASS context, where an imported Item::Str charset expands one entry per character; from/to stay sequence context; local string aliases stay sequences even in class context. This port mirrors that with a classCtx parameter on compileItem: constraint compilation sites (executeSubRule + parallel megaregexp) pass class context, the any() case class-resolves its alias arguments, capture innards and group parts stay sequence-resolved. Golden capture (interscript-rs tests/golden/isc-wf0.jsonl): pre-fix exactly 5 divergent rows, all on alalc-ell-Grek-Latn-2010 (γκέγκε, Λαγκαδάς, Παγκράτι, Κονγκό, Νταγκεστάν); post-fix all 11,048 rows byte-identical — no other map's output changes. New specs: - test/golden-isc-wf0.test.ts — the TS mirror of the RS golden_capture additivity gate (skips without sibling checkouts); - test/isc/constraints.test.ts — per-shape regression units (imported charset in class position, sum constraints, before+after combined, capital variants, local-alias sequence preservation, nth_string replacement) plus the five ell rows through the full path. --- src/runtime/compile-item.ts | 53 +++++++- src/runtime/executor.ts | 20 +-- test/golden-isc-wf0.test.ts | 91 ++++++++++++++ test/isc/constraints.test.ts | 234 +++++++++++++++++++++++++++++++++++ 4 files changed, 385 insertions(+), 13 deletions(-) create mode 100644 test/golden-isc-wf0.test.ts create mode 100644 test/isc/constraints.test.ts diff --git a/src/runtime/compile-item.ts b/src/runtime/compile-item.ts index 7b39230..1a7fc26 100644 --- a/src/runtime/compile-item.ts +++ b/src/runtime/compile-item.ts @@ -22,17 +22,33 @@ export interface CompiledItem { * Compile an Item into a RegExp source string + literal value. * Visitor-free dispatch — adding a new Item kind = adding a case here. * + * `classCtx` selects the compile context. Ruby/RS resolve aliases in two + * contexts (NodeAdapter#materialize_class vs #materialize, mirrored in + * interscript-rs builder.rs): + * + * - sequence context (default): an alias resolving to a String is a + * literal sequence; + * - class context (constraint positions — before/not_before/after/ + * not_after — and the arguments of `any()`): an IMPORTED charset alias + * (a dependency map's alias) resolving to a String is a per-character + * set (`convert_class_alias` expands Item::Str into one entry per + * char; Ruby `Any#to_regexp` builds a char class). A LOCAL string + * alias stays a sequence even in class context, matching RS. + * * Note: this is intentionally a single switch with exhaustive type * coverage. New Item kinds force this switch to be updated, which is * good — it surfaces the change rather than hiding it. */ -export function compileItem(item: Item, ctx: ExecutionContext): CompiledItem { +export function compileItem(item: Item, ctx: ExecutionContext, classCtx = false): CompiledItem { switch (item.kind) { case "string": return { re: regexpEscape(item.value), literal: item.value } case "capture_group": { - const inner = compileItem(item.data, ctx) + // RS converts capture innards in sequence context even inside a + // constraint (convert_class_fragment only class-resolves the + // top-level alias). + const inner = compileItem(item.data, ctx, false) return { re: `(${inner.re})`, literal: inner.literal } } @@ -47,15 +63,27 @@ export function compileItem(item: Item, ctx: ExecutionContext): CompiledItem { if (stdlib) return stdlib const resolved = ctx.resolveAlias(item.name) if (!resolved) return { re: "", literal: "" } - return compileItem(resolved, ctx) + if (classCtx && !ctx.map.aliases.has(item.name) && resolved.kind === "string") { + // Imported charset alias in class position: per-character set. + return stringCharset(resolved.value) + } + return compileItem(resolved, ctx, classCtx) } case "any": { - const parts = item.of.map((i) => compileItem(i, ctx).re) + // any() arguments are class-resolved regardless of the surrounding + // context (RS convert_any_arg routes alias entries through + // convert_set_entry -> convert_class_alias). Concatenation/capture + // entries inside a list stay sequence-resolved — only alias + // entries differ per context. + const parts = item.of.map((i) => compileItem(i, ctx, i.kind === "alias").re) // Mirror Ruby's Any#nth_string: the literal form picks the first // alternative. Sub rules with `to: any("ie")` use this for the // canonical replacement. - const firstLit = item.of.length > 0 ? compileItem(item.of[0]!, ctx).literal : "" + const firstLit = + item.of.length > 0 + ? compileItem(item.of[0]!, ctx, item.of[0]!.kind === "alias").literal + : "" return { re: `(?:${parts.join("|")})`, literal: firstLit } } @@ -97,6 +125,21 @@ export function compileItem(item: Item, ctx: ExecutionContext): CompiledItem { } } +/** + * Compile a String charset as a per-character SET (class context). + * Mirrors RS `convert_class_alias` expanding Item::Str into one entry + * per char, and Ruby `Any#to_regexp` building a char class from String + * data. Compiled as an alternation (not a `[...]` class) so astral + * code points and regexp-escaped chars stay correct under the `u` flag. + */ +function stringCharset(value: string): CompiledItem { + const chars = Array.from(value) + return { + re: `(?:${chars.map((c) => regexpEscape(c)).join("|")})`, + literal: chars[0] ?? "", + } +} + /** * Compile an Item to a literal string only (no regex semantics). * Used by the parallel-replace executor which needs exact strings. diff --git a/src/runtime/executor.ts b/src/runtime/executor.ts index d2e8332..73fc9a9 100644 --- a/src/runtime/executor.ts +++ b/src/runtime/executor.ts @@ -147,10 +147,12 @@ const executors: { [K in RuleKind]: RuleExecutorFor } = { const megaRules: MegaregexpRule[] = sorted.map(({ rule: r }) => { const from = compileItem(r.from!, ctx) - const before = r.before ? compileItem(r.before, ctx).re : "" - const after = r.after ? compileItem(r.after, ctx).re : "" - const notBefore = r.notBefore ? compileItem(r.notBefore, ctx).re : "" - const notAfter = r.notAfter ? compileItem(r.notAfter, ctx).re : "" + // Constraints compile in CLASS context (charset aliases resolve + // per-character — mirrors RS materialize_class). + const before = r.before ? compileItem(r.before, ctx, true).re : "" + const after = r.after ? compileItem(r.after, ctx, true).re : "" + const notBefore = r.notBefore ? compileItem(r.notBefore, ctx, true).re : "" + const notAfter = r.notAfter ? compileItem(r.notAfter, ctx, true).re : "" const pattern = [ before ? `(?<=${before})` : "", @@ -192,10 +194,12 @@ function executeSubRule(rule: SubRule, ctx: ExecutionContext): void { // Ruby's interpreter uses Ruby gsub with capture groups, which DOES // consume the surrounding context and re-inserts it via backreferences. // We use lookarounds for correctness with multi-byte scripts. - const before = rule.before ? compileItem(rule.before, ctx).re : "" - const after = rule.after ? compileItem(rule.after, ctx).re : "" - const notBefore = rule.notBefore ? compileItem(rule.notBefore, ctx).re : "" - const notAfter = rule.notAfter ? compileItem(rule.notAfter, ctx).re : "" + // Constraints compile in CLASS context (charset aliases resolve + // per-character — mirrors RS materialize_class / convert_class_alias). + const before = rule.before ? compileItem(rule.before, ctx, true).re : "" + const after = rule.after ? compileItem(rule.after, ctx, true).re : "" + const notBefore = rule.notBefore ? compileItem(rule.notBefore, ctx, true).re : "" + const notAfter = rule.notAfter ? compileItem(rule.notAfter, ctx, true).re : "" const patternParts: string[] = [] if (before) patternParts.push(`(?<=${before})`) diff --git a/test/golden-isc-wf0.test.ts b/test/golden-isc-wf0.test.ts new file mode 100644 index 0000000..8db3b48 --- /dev/null +++ b/test/golden-isc-wf0.test.ts @@ -0,0 +1,91 @@ +/** + * ISC-WF-0 golden capture additivity — the TypeScript mirror of + * interscript-rs tests/golden_capture.rs. + * + * Replays every row of the RS-captured golden file + * (interscript-rs/tests/golden/isc-wf0.jsonl) through THIS runtime and + * requires byte-identical output on every row. The golden is the + * cross-runtime contract: a delta here is a runtime regression (or a + * newly-fixed divergence being adopted into the golden). + * + * Skips when the sibling interscript-rs / maps checkouts are missing. + */ + +import { describe, it, expect } from "vitest" +import { readFileSync, readdirSync, existsSync } from "fs" +import { join } from "path" +import { configure, transliterate } from "../src/index.ts" +import { iscBundledStrategy } from "../src/isc/loader.ts" + +const REPO_ROOT = join(__dirname, "..", "..") +// GOLDEN_ROOT overrides the sibling-checkout root — used by scratch +// baselines that run this spec from a worktree outside the monorepo. +const SIBLING_ROOT = process.env.GOLDEN_ROOT ?? REPO_ROOT +const GOLDEN = join(SIBLING_ROOT, "interscript-rs", "tests", "golden", "isc-wf0.jsonl") +const MAPS_DIR = join(SIBLING_ROOT, "maps", "maps") +const LIBS_DIR = join(SIBLING_ROOT, "maps", "libs") + +interface GoldenRow { + readonly map: string + readonly src: string + readonly out: string +} + +function loadGolden(): GoldenRow[] { + if (!existsSync(GOLDEN) || !existsSync(MAPS_DIR)) return [] + return readFileSync(GOLDEN, "utf8") + .split("\n") + .filter((l) => l.trim().length > 0) + .map((l) => { + const v = JSON.parse(l) as GoldenRow + return { map: v.map, src: v.src, out: v.out } + }) +} + +function configureRuntime(): void { + const sources: Record = {} + for (const dir of [MAPS_DIR, LIBS_DIR]) { + if (!existsSync(dir)) continue + for (const f of readdirSync(dir).filter((f) => f.endsWith(".isc"))) { + sources[f.replace(/\.isc$/, "")] = readFileSync(join(dir, f), "utf8") + } + } + configure({ strategies: [iscBundledStrategy(sources)] }) +} + +describe("golden capture — interscript-rs isc-wf0 additivity", () => { + it("replays every golden row byte-identically", () => { + const rows = loadGolden() + // Sibling checkouts absent (standalone checkout) — nothing to verify. + if (rows.length === 0) { + console.warn("skip: golden capture or sibling maps checkout missing") + return + } + configureRuntime() + + const deltas: string[] = [] + const failedMaps = new Set() + let checked = 0 + for (const row of rows) { + let got: string + try { + got = transliterate(row.map, row.src) + } catch (e) { + got = `\0error:${(e as Error).message}` + } + checked++ + if (got === row.out) continue + failedMaps.add(row.map) + if (deltas.length < 10) { + deltas.push( + `${row.map}: ${JSON.stringify(row.src)} -> ${JSON.stringify(got)} (golden ${JSON.stringify(row.out)})`, + ) + } + } + expect( + [...failedMaps], + `golden divergence on ${failedMaps.size} map(s); first deltas:\n${deltas.join("\n")}`, + ).toEqual([]) + expect(checked).toBe(rows.length) + }, 120_000) +}) diff --git a/test/isc/constraints.test.ts b/test/isc/constraints.test.ts new file mode 100644 index 0000000..3083132 --- /dev/null +++ b/test/isc/constraints.test.ts @@ -0,0 +1,234 @@ +/** + * Contextual (before/after-constrained) rule regression specs. + * + * Root cause regression (2026-11): an IMPORTED charset alias (a + * dependency map's alias resolving to a plain string, e.g. unicode's + * `greek`) used in a class position — a constraint (before/not_before/ + * after/not_after) or an any() argument — compiled as a literal + * SEQUENCE of the whole charset instead of a per-character SET. The + * resulting lookbehind/lookahead could never match, silently disabling + * every constrained rule built on it (alalc-ell-Grek-Latn-2010's + * medial γκ → nk rules never fired). + * + * RS parity (interscript-rs builder.rs): + * - convert_class_alias: imported Item::Str charsets expand one entry + * per char in class context; LOCAL string aliases stay sequences; + * - convert_any_arg: alias entries class-resolve everywhere (from, + * constraints); concatenation entries stay sequence-resolved; + * - constraints materialize in class context (materialize_class); + * from/to materialize in sequence context. + * + * The five alalc-ell-Grek-Latn-2010 golden divergences this fixes: + * Λαγκαδάς → Lankadas, Παγκράτι → Pankrati, Νταγκεστάν → Ḏankestan, + * Κονγκό → Konnko, γκέγκε → gkenke (and Χονγκ Κονγκ → Chongk Kongk, + * which must keep its word-final γκ → "gk"). + */ + +import { describe, it, expect } from "vitest" +import { readFileSync, readdirSync, existsSync } from "fs" +import { join } from "path" +import { iscBundledStrategy } from "../../src/isc/loader.ts" +import { configure, transliterate } from "../../src/index.ts" + +// -- Shared fixture: a charset library map (mirrors maps/libs/unicode.isc) -- + +const LIB_CODE = "ts-constraint-lib" +const LIB_SRC = `system "${LIB_CODE}" { +aliases { + vowels = "aeiou" + upper = "KQV" +} +}` + +function configureWith(lib: boolean, sources: Record): void { + const all = lib ? { ...sources, [LIB_CODE]: LIB_SRC } : sources + configure({ strategies: [iscBundledStrategy(all)] }) +} + +describe("constraints — imported charset alias in class position", () => { + it("before any(charset) matches a single set char (root-cause shape)", () => { + configureWith(true, { + "ts-constraint-class-ctx": `system "ts-constraint-class-ctx" { +dependency "${LIB_CODE}" +stage main { + sub { + from "x" + to "X" + before any(vowels) + } +} +}`, + }) + // before = lookbehind: the set char must PRECEDE the match. + expect(transliterate("ts-constraint-class-ctx", "ax")).toBe("aX") + expect(transliterate("ts-constraint-class-ctx", "ux")).toBe("uX") + // "q" is not in the set — with the sequence bug this rule could + // NEVER fire at all; here it must fire on set membership only. + expect(transliterate("ts-constraint-class-ctx", "qx")).toBe("qx") + }) + + it("from any(charset) matches a single set char", () => { + configureWith(true, { + "ts-constraint-from-any": `system "ts-constraint-from-any" { +dependency "${LIB_CODE}" +stage main { + sub { + from any(vowels) + to "-" + } +} +}`, + }) + expect(transliterate("ts-constraint-from-any", "abc")).toBe("-bc") + }) + + it("to any(charset) replaces with the first set char (nth_string)", () => { + configureWith(true, { + "ts-constraint-to-any": `system "ts-constraint-to-any" { +dependency "${LIB_CODE}" +stage main { + sub { + from "q" + to any(upper) + } +} +}`, + }) + expect(transliterate("ts-constraint-to-any", "qq")).toBe("KK") + }) + + it("a sum constraint keeps per-char semantics for its charset part", () => { + // Mirrors alalc-ell-Grek-Latn-2010's after any("κΚ") + any(greek). + configureWith(true, { + "ts-constraint-sum": `system "ts-constraint-sum" { +dependency "${LIB_CODE}" +stage main { + sub { + from "g" + to "n" + after any("kK") + any(upper) + } +} +}`, + }) + expect(transliterate("ts-constraint-sum", "agKQ")).toBe("anKQ") + // a is not in the upper set — no fire + expect(transliterate("ts-constraint-sum", "agKa")).toBe("agKa") + // k (small) is in any("kK") but Q is needed after it + expect(transliterate("ts-constraint-sum", "agkQ")).toBe("ankQ") + }) + + it("a sum with boundary protects word-final positions (ell Χονγκ shape)", () => { + configureWith(true, { + "ts-constraint-boundary-sum": `system "ts-constraint-boundary-sum" { +dependency "${LIB_CODE}" +stage main { + sub { + from "g" + to "n" + before any(upper) + after any("kK") + boundary + } +} +}`, + }) + // g followed by word-final k — the after-sum blocks: g stays + expect(transliterate("ts-constraint-boundary-sum", "xgk x")).toBe("xgk x") + // g preceded by a capital and followed by word-final k — both + // constraints hold: fires + expect(transliterate("ts-constraint-boundary-sum", "Qgk x")).toBe("Qnk x") + // k not word-final — the after-sum fails: no fire + expect(transliterate("ts-constraint-boundary-sum", "Qgka")).toBe("Qgka") + // lowercase predecessor — before any(upper) fails: no fire + expect(transliterate("ts-constraint-boundary-sum", "qgk x")).toBe("qgk x") + }) +}) + +describe("constraints — before + after combined, capital variants", () => { + it("applies both lookarounds and discriminates case-variant rules", () => { + configureWith(false, { + "ts-constraint-combined": `system "ts-constraint-combined" { +stage main { + sub { + from "s" + to "1" + before "k" + after "a" + } + sub { + from "s" + to "2" + before "K" + after "a" + } +} +}`, + }) + expect(transliterate("ts-constraint-combined", "ksa")).toBe("k1a") + expect(transliterate("ts-constraint-combined", "Ksa")).toBe("K2a") + expect(transliterate("ts-constraint-combined", "ksb")).toBe("ksb") + expect(transliterate("ts-constraint-combined", "asa")).toBe("asa") + }) +}) + +describe("constraints — sequence context preserved (RS parity)", () => { + it("a LOCAL string alias in class position stays a sequence", () => { + // RS convert_class_alias: only IMPORTED charsets expand per-char; + // a local alias keeps its materialized shape (Str = sequence). + configureWith(false, { + "ts-constraint-local-alias": `system "ts-constraint-local-alias" { +aliases { loc = "ab" } +stage main { + sub { + from "s" + to "S" + before loc + } +} +}`, + }) + expect(transliterate("ts-constraint-local-alias", "abs")).toBe("abS") + expect(transliterate("ts-constraint-local-alias", "as")).toBe("as") + }) +}) + +// -- Full-path regression: the real alalc-ell-Grek-Latn-2010 map -- + +// MAPS_ROOT overrides the sibling-checkout root — used by scratch +// baselines that run this spec from a worktree outside the monorepo. +const MONOREPO_ROOT = process.env.MAPS_ROOT ?? join(__dirname, "..", "..", "..") +const MAPS_DIR = join(MONOREPO_ROOT, "maps", "maps") +const LIBS_DIR = join(MONOREPO_ROOT, "maps", "libs") + +describe("constraints — alalc-ell-Grek-Latn-2010 full path", () => { + const ELL = "alalc-ell-Grek-Latn-2010" + + function configureEll(): boolean { + if (!existsSync(MAPS_DIR) || !existsSync(LIBS_DIR)) return false + const sources: Record = {} + for (const dir of [MAPS_DIR, LIBS_DIR]) { + for (const f of readdirSync(dir).filter((f) => f.endsWith(".isc"))) { + sources[f.replace(/\.isc$/, "")] = readFileSync(join(dir, f), "utf8") + } + } + configure({ strategies: [iscBundledStrategy(sources)] }) + return true + } + + it("renders medial γκ as nk through the constrained rules", () => { + if (!configureEll()) { + console.warn("skip: no sibling maps checkout") + return + } + // The five golden divergences fixed by class-context compilation. + expect(transliterate(ELL, "Λαγκαδάς")).toBe("Lankadas") + expect(transliterate(ELL, "Παγκράτι")).toBe("Pankrati") + expect(transliterate(ELL, "Νταγκεστάν")).toBe("Ḏankestan") + expect(transliterate(ELL, "Κονγκό")).toBe("Konnko") + expect(transliterate(ELL, "γκέγκε")).toBe("gkenke") + // Word-final γκ keeps "gk" — the not_after-boundary sum blocks. + expect(transliterate(ELL, "Χονγκ Κονγκ")).toBe("Chongk Kongk") + // Capital Γ at word start has no lookbehind — unconstrained "G". + expect(transliterate(ELL, "Γκαμπόν")).toBe("Gkampon") + }) +})