fix: combined WASM hardening (stack guard, v18 errors, scan OOM, sync recovery, ESM, native parity)

.github/workflows/ci.yml

@@ -54,6 +54,9 @@ jobs:
         run: pnpm run build
         working-directory: ${{ matrix.package.path }}
 
+      - name: Native parity 🔍
+        run: node scripts/native-parity.js ${{ matrix.package.path }}
+
       - name: Upload WASM Artifacts 📦
         uses: actions/upload-artifact@v4
         with:

parser/package.json

@@ -4,43 +4,43 @@
   "author": "Constructive <[email protected]>",
   "description": "Multi-version PostgreSQL parser with dynamic version selection",
   "main": "./wasm/index.cjs",
-  "module": "./wasm/index.js",
+  "module": "./wasm/index.mjs",
   "types": "./wasm/index.d.ts",
   "exports": {
     ".": {
-      "import": "./wasm/index.js",
-      "require": "./wasm/index.cjs",
-      "types": "./wasm/index.d.ts"
+      "types": "./wasm/index.d.ts",
+      "import": "./wasm/index.mjs",
+      "require": "./wasm/index.cjs"
     },
     "./v13": {
-      "import": "./wasm/v13.js",
-      "require": "./wasm/v13.cjs",
-      "types": "./wasm/v13.d.ts"
+      "types": "./wasm/v13.d.ts",
+      "import": "./wasm/v13.mjs",
+      "require": "./wasm/v13.cjs"
     },
     "./v14": {
-      "import": "./wasm/v14.js",
-      "require": "./wasm/v14.cjs",
-      "types": "./wasm/v14.d.ts"
+      "types": "./wasm/v14.d.ts",
+      "import": "./wasm/v14.mjs",
+      "require": "./wasm/v14.cjs"
     },
     "./v15": {
-      "import": "./wasm/v15.js",
-      "require": "./wasm/v15.cjs",
-      "types": "./wasm/v15.d.ts"
+      "types": "./wasm/v15.d.ts",
+      "import": "./wasm/v15.mjs",
+      "require": "./wasm/v15.cjs"
     },
     "./v16": {
-      "import": "./wasm/v16.js",
-      "require": "./wasm/v16.cjs",
-      "types": "./wasm/v16.d.ts"
+      "types": "./wasm/v16.d.ts",
+      "import": "./wasm/v16.mjs",
+      "require": "./wasm/v16.cjs"
     },
     "./v17": {
-      "import": "./wasm/v17.js",
-      "require": "./wasm/v17.cjs",
-      "types": "./wasm/v17.d.ts"
+      "types": "./wasm/v17.d.ts",
+      "import": "./wasm/v17.mjs",
+      "require": "./wasm/v17.cjs"
     },
     "./v18": {
-      "import": "./wasm/v18.js",
-      "require": "./wasm/v18.cjs",
-      "types": "./wasm/v18.d.ts"
+      "types": "./wasm/v18.d.ts",
+      "import": "./wasm/v18.mjs",
+      "require": "./wasm/v18.cjs"
     }
   },
   "files": [

parser/scripts/prepare.js

@@ -45,8 +45,8 @@ function generateTemplateVars(versions) {
   // For TypeScript union: 13 | 14 | 15 | 16 | 17
   const versionUnion = versions.join(' | ');
 
-  // For ESM exports: export * as v13 from './v13/index.js';
-  const versionExports = versions.map(v => `export * as v${v} from './v${v}/index.js';`).join('\n');
+  // For ESM exports: export * as v13 from './v13/index.mjs';
+  const versionExports = versions.map(v => `export * as v${v} from './v${v}/index.mjs';`).join('\n');
 
   // For CommonJS requires: v13: require('./v13/index.cjs'),
   const versionRequires = versions.map(v => `  v${v}: require('./v${v}/index.cjs')`).join(',\n');
@@ -113,7 +113,9 @@ config.versions.forEach(version => {
   const files = fs.readdirSync(sourceWasmDir);
   files.forEach(file => {
     const sourcePath = path.join(sourceWasmDir, file);
-    const destPath = path.join(versionWasmDir, file);
+    // ESM files must be .mjs: the package has no "type": "module", so Node
+    // before 20.19 (and without syntax detection) would load a .js as CommonJS.
+    const destPath = path.join(versionWasmDir, file === 'index.js' ? 'index.mjs' : file);
 
     if (fs.statSync(sourcePath).isFile()) {
       console.log(`Copying ${file} for v${version}...`);
@@ -162,7 +164,7 @@ const templateVars = generateTemplateVars(config.versions);
 
 // Process each template
 const templates = [
-  { template: 'index.js.template', output: 'index.js' },
+  { template: 'index.js.template', output: 'index.mjs' },
   { template: 'index.cjs.template', output: 'index.cjs' },
   { template: 'index.d.ts.template', output: 'index.d.ts' }
 ];
@@ -182,9 +184,9 @@ templates.forEach(({ template, output }) => {
 config.versions.forEach(version => {
   // ESM version
   const esmContent = `// Re-export everything from the v${version} parser
-export * from './v${version}/index.js';
+export * from './v${version}/index.mjs';
 `;
-  fs.writeFileSync(path.join(wasmDir, `v${version}.js`), esmContent);
+  fs.writeFileSync(path.join(wasmDir, `v${version}.mjs`), esmContent);
 
   // CJS version
   const cjsContent = `// Re-export everything from the v${version} parser

parser/templates/index.js.template

@@ -33,7 +33,7 @@ export class Parser {
   }
 
   async _doLoad() {
-    const module = await import(`./v${this.version}/index.js`);
+    const module = await import(`./v${this.version}/index.mjs`);
     this.parser = module;
 
     if (this.parser.loadModule) {

parser/test/esm.test.mjs

@@ -1,6 +1,8 @@
 import { describe, it } from 'node:test';
 import assert from 'node:assert/strict';
 import { readFileSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
 
 const { versions } = JSON.parse(readFileSync(new URL('../wasm/build-info.json', import.meta.url), 'utf8'));
 
@@ -21,3 +23,27 @@ describe('ESM entry points', () => {
     });
   }
 });
+
+// Node 18 and Node 20 before 20.19 load a `.js` file in a package without
+// "type": "module" as CommonJS. Turning off syntax detection reproduces that
+// on current Node, so this fails if an ESM entry point is not `.mjs`.
+describe('ESM entry points without module syntax detection', () => {
+  const flags = process.allowedNodeEnvironmentFlags.has('--experimental-detect-module')
+    ? ['--no-experimental-detect-module']
+    : [];
+  const specifiers = ['@pgsql/parser', ...versions.map((v) => `@pgsql/parser/v${v}`)];
+  for (const specifier of specifiers) {
+    it(`imports ${specifier}`, () => {
+      const script = `const m = await import(${JSON.stringify(specifier)});
+        const lib = m.Parser ? new m.Parser() : m;
+        if (m.loadModule) await m.loadModule();
+        const result = m.Parser ? await lib.parse('SELECT 1') : lib.parseSync('SELECT 1');
+        if (result.stmts.length !== 1) throw new Error('bad parse');`;
+      const { status, stderr } = spawnSync(process.execPath, [...flags, '--input-type=module', '-e', script], {
+        cwd: fileURLToPath(new URL('..', import.meta.url)),
+        encoding: 'utf8',
+      });
+      assert.equal(status, 0, stderr);
+    });
+  }
+});

scripts/native-parity.js

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+/**
+ * Checks that a built versions/<n>/wasm parses exactly like libpg_query compiled natively
+ * from the same commit, over libpg_query's test SQL: test/sql/postgres_regress and
+ * plpgsql_regress (the PostgreSQL regression suite) plus its own deparse tests.
+ *
+ *   node scripts/native-parity.js versions/18     (after `pnpm build` in that directory)
+ *
+ * Reuses the libpg_query checkout the WASM build cloned into .cache, so it needs no network.
+ */
+const { execFileSync } = require('child_process');
+const fs = require('fs');
+const glob = require('glob');
+const os = require('os');
+const path = require('path');
+
+// FETCH_ALL is LONG_MAX, which is 32-bit on wasm32
+const FETCH_ALL_NATIVE = 9223372036854775807;
+const FETCH_ALL_WASM = 2147483647;
+
+async function main() {
+  const versionDir = path.resolve(process.argv[2] || '.');
+  const commit = require(path.join(versionDir, 'package.json'))['x-publish'].libpgQueryCommit;
+  const [source] = glob.sync(`.cache/*/libpg_query/${commit}`, { cwd: versionDir, absolute: true });
+  if (!source) throw new Error(`no libpg_query checkout for ${commit} under ${versionDir}/.cache; run pnpm build first`);
+
+  const work = fs.mkdtempSync(path.join(os.tmpdir(), 'libpg-query-native-'));
+  try {
+    await checkParity(versionDir, source, commit, work);
+  } finally {
+    fs.rmSync(work, { recursive: true, force: true });
+  }
+}
+
+async function checkParity(versionDir, source, commit, work) {
+  execFileSync('sh', ['-c', `git -C "${source}" archive ${commit} | tar -x -C "${work}"`]);
+  // glibc >= 2.38 declares strchrnul, which PG <= 14's vendored snprintf.c would otherwise redefine
+  const cppflags = process.platform === 'linux' ? 'CPPFLAGS=-DHAVE_STRCHRNUL=' : 'CPPFLAGS=';
+  execFileSync('make', ['-j', String(os.cpus().length), cppflags, 'build'], { cwd: work, stdio: ['ignore', 'ignore', 'inherit'] });
+  const driver = path.join(work, 'parity-driver');
+  execFileSync('cc', ['-O2', '-I', work, path.join(__dirname, 'native-parity/driver.c'), path.join(work, 'libpg_query.a'), '-lpthread', '-o', driver]);
+
+  const files = glob.sync('test/{sql/**/*.{sql,psql},plpgsql_samples.sql}', { cwd: work }).sort();
+  const statements = [];
+  for (const file of files) {
+    const split = JSON.parse(execFileSync(driver, ['split', path.join(work, file)], { maxBuffer: 1 << 28 }).toString());
+    if (split.error) continue; // e.g. strings.sql: intentionally invalid escapes the scanner rejects
+    split.stmts.forEach((sql, i) => statements.push({ key: `${file}#${i}`, sql }));
+  }
+
+  const input = path.join(work, 'statements.bin');
+  fs.writeFileSync(input, Buffer.concat(statements.map(({ sql }) => Buffer.from(sql + '\0'))));
+  const output = path.join(work, 'native.jsonl');
+  const fd = fs.openSync(output, 'w');
+  execFileSync(driver, ['parse', input], { stdio: ['ignore', fd, 'inherit'] });
+  fs.closeSync(fd);
+  const native = fs.readFileSync(output, 'utf8').trimEnd().split('\n');
+
+  const wasm = require(path.join(versionDir, 'wasm/index.cjs'));
+  await wasm.loadModule();
+
+  const diffs = [];
+  statements.forEach(({ key, sql }, i) => {
+    const expected = JSON.parse(native[i]);
+    let actual;
+    try {
+      actual = { parse: wasm.parseSync(sql) };
+    } catch (err) {
+      actual = { error: { message: err.message, cursorpos: err.sqlDetails ? err.sqlDetails.cursorPosition + 1 : undefined } };
+    }
+    if (expected.error && actual.error && expected.error.cursorpos === 0) expected.error.cursorpos = 1;
+    const normalize = (value) =>
+      JSON.stringify(value, (k, v) => ((k === 'howMany' || k === 'how_many') && v === FETCH_ALL_NATIVE ? FETCH_ALL_WASM : v));
+    const a = normalize(expected);
+    const b = JSON.stringify(actual);
+    if (a !== b) diffs.push({ key, native: a.slice(0, 300), wasm: b.slice(0, 300) });
+  });
+
+  console.log(`native parity: ${statements.length} statements from ${files.length} files @ ${commit.slice(0, 7)}, ${diffs.length} differences`);
+  for (const d of diffs.slice(0, 10)) console.log(JSON.stringify(d, null, 2));
+  if (diffs.length) process.exitCode = 1;
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

scripts/native-parity/driver.c

@@ -0,0 +1,95 @@
+/*
+ * Native side of scripts/native-parity.js.
+ *   driver split <file>   -> {"stmts":["...", ...]} or {"error":"..."}
+ *   driver parse <file>   -> one JSON line per NUL-separated statement:
+ *                            {"parse":<tree>} or {"error":{"message":"...","cursorpos":N}}
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "pg_query.h"
+
+static void print_json_string(const char *s) {
+  putchar('"');
+  for (const unsigned char *p = (const unsigned char *) s; p && *p; p++) {
+    if (*p == '"' || *p == '\\') {
+      putchar('\\');
+      putchar(*p);
+    } else if (*p < 0x20) {
+      printf("\\u%04x", *p);
+    } else {
+      putchar(*p);
+    }
+  }
+  putchar('"');
+}
+
+static char *read_file(const char *path, size_t *len) {
+  FILE *f = fopen(path, "rb");
+  if (!f) {
+    perror(path);
+    exit(2);
+  }
+  fseek(f, 0, SEEK_END);
+  long n = ftell(f);
+  fseek(f, 0, SEEK_SET);
+  char *buf = malloc(n + 1);
+  if (fread(buf, 1, n, f) != (size_t) n) {
+    perror(path);
+    exit(2);
+  }
+  buf[n] = 0;
+  fclose(f);
+  *len = n;
+  return buf;
+}
+
+int main(int argc, char **argv) {
+  size_t len;
+  if (argc != 3) {
+    fprintf(stderr, "usage: driver split|parse <file>\n");
+    return 1;
+  }
+  char *buf = read_file(argv[2], &len);
+
+  if (strcmp(argv[1], "split") == 0) {
+    PgQuerySplitResult r = pg_query_split_with_scanner(buf);
+    if (r.error) {
+      printf("{\"error\":");
+      print_json_string(r.error->message);
+      printf("}\n");
+    } else {
+      printf("{\"stmts\":[");
+      for (int i = 0; i < r.n_stmts; i++) {
+        int n = r.stmts[i]->stmt_len;
+        char *stmt = malloc(n + 1);
+        memcpy(stmt, buf + r.stmts[i]->stmt_location, n);
+        stmt[n] = 0;
+        if (i) putchar(',');
+        print_json_string(stmt);
+        free(stmt);
+      }
+      printf("]}\n");
+    }
+    pg_query_free_split_result(r);
+    return 0;
+  }
+
+  if (strcmp(argv[1], "parse") == 0) {
+    for (char *p = buf; p < buf + len; p += strlen(p) + 1) {
+      PgQueryParseResult r = pg_query_parse(p);
+      if (r.error) {
+        printf("{\"error\":{\"message\":");
+        print_json_string(r.error->message);
+        printf(",\"cursorpos\":%d}}\n", r.error->cursorpos);
+      } else {
+        printf("{\"parse\":%s}\n", r.parse_tree);
+      }
+      pg_query_free_parse_result(r);
+    }
+    return 0;
+  }
+
+  fprintf(stderr, "unknown mode %s\n", argv[1]);
+  return 1;
+}

templates/Makefile.template

@@ -78,7 +78,7 @@ ifdef EMSCRIPTEN
 		-I$(LIBPG_QUERY_DIR)/vendor \
 		-L$(LIBPG_QUERY_DIR) \
 		-sEXPORTED_FUNCTIONS="{{EXPORTED_FUNCTIONS}}" \
-		-sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \
+		-sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32','stackSave','stackRestore']" \
 		-sEXPORT_NAME="$(WASM_MODULE_NAME)" \
 		-sENVIRONMENT="web,node,worker" \
         -sASSERTIONS=0 \