From 548eac885718a9fd27ab20b9b061b25902a9b2e2 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 24 Oct 2026 01:46:17 +0000 Subject: [PATCH 01/15] fix: stack-depth guard and shadow-stack restore for deep recursion libpg_query's check_stack_depth() never had a base pointer, so recursion past the WASM stack trapped (memory access out of bounds). Call the fork's pg_query_set_stack_limit() at every export with the stack actually left (weak symbol: upstream pins skip it) and pin 17/18 to the fork branches. Restore the shadow stack pointer in callWasm after a throw so sync calls on the stale instance do not start half-way down it. Regression test: 10 parseSync of a 100k-term expression under --stack-size=8000. --- templates/Makefile.template | 2 +- templates/full/index.ts | 5 +++ templates/full/wasm_wrapper.c | 18 +++++++++++ templates/index.ts | 5 +++ templates/wasm_wrapper.c | 12 +++++++ versions/13/Makefile | 2 +- versions/13/src/index.ts | 5 +++ versions/13/src/wasm_wrapper.c | 12 +++++++ versions/14/Makefile | 2 +- versions/14/src/index.ts | 5 +++ versions/14/src/wasm_wrapper.c | 12 +++++++ versions/15/Makefile | 2 +- versions/15/src/index.ts | 5 +++ versions/15/src/wasm_wrapper.c | 12 +++++++ versions/16/Makefile | 2 +- versions/16/src/index.ts | 5 +++ versions/16/src/wasm_wrapper.c | 12 +++++++ versions/17/Makefile | 4 +-- versions/17/package.json | 4 +-- versions/17/src/index.ts | 5 +++ versions/17/src/wasm_wrapper.c | 12 +++++++ versions/17/test/wasm-stale-stack.test.js | 38 +++++++++++++++++++++++ versions/18/Makefile | 4 +-- versions/18/package.json | 4 +-- versions/18/src/index.ts | 5 +++ versions/18/src/wasm_wrapper.c | 18 +++++++++++ versions/18/test/wasm-stale-stack.test.js | 38 +++++++++++++++++++++++ 27 files changed, 237 insertions(+), 13 deletions(-) create mode 100644 versions/17/test/wasm-stale-stack.test.js create mode 100644 versions/18/test/wasm-stale-stack.test.js diff --git a/templates/Makefile.template b/templates/Makefile.template index 952520d5..5f82744c 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -76,7 +76,7 @@ ifdef EMSCRIPTEN -I$(LIBPG_QUERY_DIR)/vendor \ -L$(LIBPG_QUERY_DIR) \ -sEXPORTED_FUNCTIONS="{{EXPORTED_FUNCTIONS}}" \ - -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \ + -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32','stackSave','stackRestore']" \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ -sENVIRONMENT="web,node,worker" \ -sASSERTIONS=0 \ diff --git a/templates/full/index.ts b/templates/full/index.ts index 682f05b5..435df9a9 100644 --- a/templates/full/index.ts +++ b/templates/full/index.ts @@ -177,9 +177,14 @@ instantiate(); function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; + const module = wasmModule; + const stackPointer = module.stackSave(); try { return fn(); } catch (error) { + // The throw unwound the JS frames but not the C shadow stack; give it + // back so later sync calls on this instance do not start half-way down it. + module.stackRestore(stackPointer); failures++; stale = true; if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; diff --git a/templates/full/wasm_wrapper.c b/templates/full/wasm_wrapper.c index 268d2893..6e9bcd8f 100644 --- a/templates/full/wasm_wrapper.c +++ b/templates/full/wasm_wrapper.c @@ -1,10 +1,21 @@ #include "pg_query.h" #include "protobuf/pg_query.pb-c.h" #include +#include #include #include #include +/* libpg_query's check_stack_depth() needs a reference point and a limit per + * entry: the shadow stack left after the previous call, minus a 512 KiB + * margin for the frames between here and the recursion. */ +extern void pg_query_set_stack_limit(size_t bytes) __attribute__((weak)); +static void set_stack_guard(void) { + if (!pg_query_set_stack_limit) return; /* libpg_query without the guard (upstream pins) */ + size_t free_bytes = emscripten_stack_get_free(); + pg_query_set_stack_limit(free_bytes > 524288 ? free_bytes - 524288 : 0); +} + static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; } @@ -28,6 +39,7 @@ static void* safe_malloc(size_t size) { EMSCRIPTEN_KEEPALIVE char* wasm_parse_query(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } @@ -47,6 +59,7 @@ char* wasm_parse_query(const char* input) { EMSCRIPTEN_KEEPALIVE PgQueryParseResult* wasm_parse_query_raw(const char* input) { + set_stack_guard(); PgQueryParseResult* result = (PgQueryParseResult*)malloc(sizeof(PgQueryParseResult)); if (!result) { return NULL; @@ -66,6 +79,7 @@ void wasm_free_parse_result(PgQueryParseResult* result) { EMSCRIPTEN_KEEPALIVE char* wasm_parse_plpgsql(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } @@ -106,6 +120,7 @@ char* wasm_parse_plpgsql(const char* input) { EMSCRIPTEN_KEEPALIVE char* wasm_fingerprint(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } @@ -125,6 +140,7 @@ char* wasm_fingerprint(const char* input) { EMSCRIPTEN_KEEPALIVE char* wasm_normalize_query(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } @@ -165,6 +181,7 @@ typedef struct { EMSCRIPTEN_KEEPALIVE WasmDetailedResult* wasm_parse_query_detailed(const char* input) { + set_stack_guard(); WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); if (!result) { return NULL; @@ -365,6 +382,7 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi EMSCRIPTEN_KEEPALIVE char* wasm_scan(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } diff --git a/templates/index.ts b/templates/index.ts index 4a9a8929..6e487eb4 100644 --- a/templates/index.ts +++ b/templates/index.ts @@ -165,9 +165,14 @@ instantiate(); function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; + const module = wasmModule; + const stackPointer = module.stackSave(); try { return fn(); } catch (error) { + // The throw unwound the JS frames but not the C shadow stack; give it + // back so later sync calls on this instance do not start half-way down it. + module.stackRestore(stackPointer); failures++; stale = true; if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; diff --git a/templates/wasm_wrapper.c b/templates/wasm_wrapper.c index 59c5a979..f53bcecf 100644 --- a/templates/wasm_wrapper.c +++ b/templates/wasm_wrapper.c @@ -1,8 +1,19 @@ #include +#include #include #include #include +/* libpg_query's check_stack_depth() needs a reference point and a limit per + * entry: the shadow stack left after the previous call, minus a 512 KiB + * margin for the frames between here and the recursion. */ +extern void pg_query_set_stack_limit(size_t bytes) __attribute__((weak)); +static void set_stack_guard(void) { + if (!pg_query_set_stack_limit) return; /* libpg_query without the guard (upstream pins) */ + size_t free_bytes = emscripten_stack_get_free(); + pg_query_set_stack_limit(free_bytes > 524288 ? free_bytes - 524288 : 0); +} + static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; } @@ -18,6 +29,7 @@ static void* safe_malloc(size_t size) { // Raw struct access functions for parse EMSCRIPTEN_KEEPALIVE PgQueryParseResult* wasm_parse_query_raw(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return NULL; } diff --git a/versions/13/Makefile b/versions/13/Makefile index 47cdf4d7..fb5fc97c 100644 --- a/versions/13/Makefile +++ b/versions/13/Makefile @@ -80,7 +80,7 @@ ifdef EMSCRIPTEN -I$(LIBPG_QUERY_DIR)/vendor \ -L$(LIBPG_QUERY_DIR) \ -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query_raw','_wasm_free_parse_result']" \ - -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \ + -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32','stackSave','stackRestore']" \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ -sENVIRONMENT="web,node,worker" \ -sASSERTIONS=0 \ diff --git a/versions/13/src/index.ts b/versions/13/src/index.ts index ea70bb2a..7c1340a1 100644 --- a/versions/13/src/index.ts +++ b/versions/13/src/index.ts @@ -172,9 +172,14 @@ instantiate(); function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; + const module = wasmModule; + const stackPointer = module.stackSave(); try { return fn(); } catch (error) { + // The throw unwound the JS frames but not the C shadow stack; give it + // back so later sync calls on this instance do not start half-way down it. + module.stackRestore(stackPointer); failures++; stale = true; if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; diff --git a/versions/13/src/wasm_wrapper.c b/versions/13/src/wasm_wrapper.c index 38151683..2e13e261 100644 --- a/versions/13/src/wasm_wrapper.c +++ b/versions/13/src/wasm_wrapper.c @@ -6,10 +6,21 @@ */ #include +#include #include #include #include +/* libpg_query's check_stack_depth() needs a reference point and a limit per + * entry: the shadow stack left after the previous call, minus a 512 KiB + * margin for the frames between here and the recursion. */ +extern void pg_query_set_stack_limit(size_t bytes) __attribute__((weak)); +static void set_stack_guard(void) { + if (!pg_query_set_stack_limit) return; /* libpg_query without the guard (upstream pins) */ + size_t free_bytes = emscripten_stack_get_free(); + pg_query_set_stack_limit(free_bytes > 524288 ? free_bytes - 524288 : 0); +} + static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; } @@ -25,6 +36,7 @@ static void* safe_malloc(size_t size) { // Raw struct access functions for parse EMSCRIPTEN_KEEPALIVE PgQueryParseResult* wasm_parse_query_raw(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return NULL; } diff --git a/versions/14/Makefile b/versions/14/Makefile index 5fe746c8..57a74d36 100644 --- a/versions/14/Makefile +++ b/versions/14/Makefile @@ -75,7 +75,7 @@ ifdef EMSCRIPTEN -I$(LIBPG_QUERY_DIR)/vendor \ -L$(LIBPG_QUERY_DIR) \ -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query_raw','_wasm_free_parse_result']" \ - -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \ + -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32','stackSave','stackRestore']" \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ -sENVIRONMENT="web,node,worker" \ -sASSERTIONS=0 \ diff --git a/versions/14/src/index.ts b/versions/14/src/index.ts index ea70bb2a..7c1340a1 100644 --- a/versions/14/src/index.ts +++ b/versions/14/src/index.ts @@ -172,9 +172,14 @@ instantiate(); function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; + const module = wasmModule; + const stackPointer = module.stackSave(); try { return fn(); } catch (error) { + // The throw unwound the JS frames but not the C shadow stack; give it + // back so later sync calls on this instance do not start half-way down it. + module.stackRestore(stackPointer); failures++; stale = true; if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; diff --git a/versions/14/src/wasm_wrapper.c b/versions/14/src/wasm_wrapper.c index 38151683..2e13e261 100644 --- a/versions/14/src/wasm_wrapper.c +++ b/versions/14/src/wasm_wrapper.c @@ -6,10 +6,21 @@ */ #include +#include #include #include #include +/* libpg_query's check_stack_depth() needs a reference point and a limit per + * entry: the shadow stack left after the previous call, minus a 512 KiB + * margin for the frames between here and the recursion. */ +extern void pg_query_set_stack_limit(size_t bytes) __attribute__((weak)); +static void set_stack_guard(void) { + if (!pg_query_set_stack_limit) return; /* libpg_query without the guard (upstream pins) */ + size_t free_bytes = emscripten_stack_get_free(); + pg_query_set_stack_limit(free_bytes > 524288 ? free_bytes - 524288 : 0); +} + static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; } @@ -25,6 +36,7 @@ static void* safe_malloc(size_t size) { // Raw struct access functions for parse EMSCRIPTEN_KEEPALIVE PgQueryParseResult* wasm_parse_query_raw(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return NULL; } diff --git a/versions/15/Makefile b/versions/15/Makefile index a16a538c..af9a3f42 100644 --- a/versions/15/Makefile +++ b/versions/15/Makefile @@ -75,7 +75,7 @@ ifdef EMSCRIPTEN -I$(LIBPG_QUERY_DIR)/vendor \ -L$(LIBPG_QUERY_DIR) \ -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query_raw','_wasm_free_parse_result']" \ - -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \ + -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32','stackSave','stackRestore']" \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ -sENVIRONMENT="web,node,worker" \ -sASSERTIONS=0 \ diff --git a/versions/15/src/index.ts b/versions/15/src/index.ts index ea70bb2a..7c1340a1 100644 --- a/versions/15/src/index.ts +++ b/versions/15/src/index.ts @@ -172,9 +172,14 @@ instantiate(); function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; + const module = wasmModule; + const stackPointer = module.stackSave(); try { return fn(); } catch (error) { + // The throw unwound the JS frames but not the C shadow stack; give it + // back so later sync calls on this instance do not start half-way down it. + module.stackRestore(stackPointer); failures++; stale = true; if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; diff --git a/versions/15/src/wasm_wrapper.c b/versions/15/src/wasm_wrapper.c index 38151683..2e13e261 100644 --- a/versions/15/src/wasm_wrapper.c +++ b/versions/15/src/wasm_wrapper.c @@ -6,10 +6,21 @@ */ #include +#include #include #include #include +/* libpg_query's check_stack_depth() needs a reference point and a limit per + * entry: the shadow stack left after the previous call, minus a 512 KiB + * margin for the frames between here and the recursion. */ +extern void pg_query_set_stack_limit(size_t bytes) __attribute__((weak)); +static void set_stack_guard(void) { + if (!pg_query_set_stack_limit) return; /* libpg_query without the guard (upstream pins) */ + size_t free_bytes = emscripten_stack_get_free(); + pg_query_set_stack_limit(free_bytes > 524288 ? free_bytes - 524288 : 0); +} + static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; } @@ -25,6 +36,7 @@ static void* safe_malloc(size_t size) { // Raw struct access functions for parse EMSCRIPTEN_KEEPALIVE PgQueryParseResult* wasm_parse_query_raw(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return NULL; } diff --git a/versions/16/Makefile b/versions/16/Makefile index 05e7e6f3..f19ce690 100644 --- a/versions/16/Makefile +++ b/versions/16/Makefile @@ -75,7 +75,7 @@ ifdef EMSCRIPTEN -I$(LIBPG_QUERY_DIR)/vendor \ -L$(LIBPG_QUERY_DIR) \ -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query_raw','_wasm_free_parse_result']" \ - -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \ + -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32','stackSave','stackRestore']" \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ -sENVIRONMENT="web,node,worker" \ -sASSERTIONS=0 \ diff --git a/versions/16/src/index.ts b/versions/16/src/index.ts index ea70bb2a..7c1340a1 100644 --- a/versions/16/src/index.ts +++ b/versions/16/src/index.ts @@ -172,9 +172,14 @@ instantiate(); function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; + const module = wasmModule; + const stackPointer = module.stackSave(); try { return fn(); } catch (error) { + // The throw unwound the JS frames but not the C shadow stack; give it + // back so later sync calls on this instance do not start half-way down it. + module.stackRestore(stackPointer); failures++; stale = true; if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; diff --git a/versions/16/src/wasm_wrapper.c b/versions/16/src/wasm_wrapper.c index 38151683..2e13e261 100644 --- a/versions/16/src/wasm_wrapper.c +++ b/versions/16/src/wasm_wrapper.c @@ -6,10 +6,21 @@ */ #include +#include #include #include #include +/* libpg_query's check_stack_depth() needs a reference point and a limit per + * entry: the shadow stack left after the previous call, minus a 512 KiB + * margin for the frames between here and the recursion. */ +extern void pg_query_set_stack_limit(size_t bytes) __attribute__((weak)); +static void set_stack_guard(void) { + if (!pg_query_set_stack_limit) return; /* libpg_query without the guard (upstream pins) */ + size_t free_bytes = emscripten_stack_get_free(); + pg_query_set_stack_limit(free_bytes > 524288 ? free_bytes - 524288 : 0); +} + static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; } @@ -25,6 +36,7 @@ static void* safe_malloc(size_t size) { // Raw struct access functions for parse EMSCRIPTEN_KEEPALIVE PgQueryParseResult* wasm_parse_query_raw(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return NULL; } diff --git a/versions/17/Makefile b/versions/17/Makefile index 747eb3f0..3e8b3e28 100644 --- a/versions/17/Makefile +++ b/versions/17/Makefile @@ -17,7 +17,7 @@ WASM_STACK_SIZE ?= 33554432 # byte-for-byte. Set in package.json `x-publish` and applied by copy:templates. LIBPG_QUERY_REPO := https://github.com/constructive-io/libpg_query.git LIBPG_QUERY_TAG := 17-constructive -LIBPG_QUERY_COMMIT := 1ade880fd1bbc6e89c33ea2424ddc366c41c1408 +LIBPG_QUERY_COMMIT := 5691182b996645135febd16d6403668298bf0c8b CACHE_DIR := .cache @@ -75,7 +75,7 @@ ifdef EMSCRIPTEN -I$(LIBPG_QUERY_DIR)/vendor \ -L$(LIBPG_QUERY_DIR) \ -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query_raw','_wasm_free_parse_result']" \ - -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \ + -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32','stackSave','stackRestore']" \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ -sENVIRONMENT="web,node,worker" \ -sASSERTIONS=0 \ diff --git a/versions/17/package.json b/versions/17/package.json index 8f6e5e26..44c121a9 100644 --- a/versions/17/package.json +++ b/versions/17/package.json @@ -15,7 +15,7 @@ "distTag": "pg17", "libpgQueryRepo": "https://github.com/constructive-io/libpg_query.git", "libpgQueryTag": "17-constructive", - "libpgQueryCommit": "1ade880fd1bbc6e89c33ea2424ddc366c41c1408" + "libpgQueryCommit": "5691182b996645135febd16d6403668298bf0c8b" }, "files": [ "wasm/*" @@ -30,7 +30,7 @@ "wasm:rebuild": "pnpm wasm:make rebuild", "wasm:clean": "pnpm wasm:make clean", "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js" + "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/wasm-stale-stack.test.js" }, "author": "Constructive ", "license": "MIT", diff --git a/versions/17/src/index.ts b/versions/17/src/index.ts index ea70bb2a..7c1340a1 100644 --- a/versions/17/src/index.ts +++ b/versions/17/src/index.ts @@ -172,9 +172,14 @@ instantiate(); function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; + const module = wasmModule; + const stackPointer = module.stackSave(); try { return fn(); } catch (error) { + // The throw unwound the JS frames but not the C shadow stack; give it + // back so later sync calls on this instance do not start half-way down it. + module.stackRestore(stackPointer); failures++; stale = true; if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; diff --git a/versions/17/src/wasm_wrapper.c b/versions/17/src/wasm_wrapper.c index 38151683..2e13e261 100644 --- a/versions/17/src/wasm_wrapper.c +++ b/versions/17/src/wasm_wrapper.c @@ -6,10 +6,21 @@ */ #include +#include #include #include #include +/* libpg_query's check_stack_depth() needs a reference point and a limit per + * entry: the shadow stack left after the previous call, minus a 512 KiB + * margin for the frames between here and the recursion. */ +extern void pg_query_set_stack_limit(size_t bytes) __attribute__((weak)); +static void set_stack_guard(void) { + if (!pg_query_set_stack_limit) return; /* libpg_query without the guard (upstream pins) */ + size_t free_bytes = emscripten_stack_get_free(); + pg_query_set_stack_limit(free_bytes > 524288 ? free_bytes - 524288 : 0); +} + static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; } @@ -25,6 +36,7 @@ static void* safe_malloc(size_t size) { // Raw struct access functions for parse EMSCRIPTEN_KEEPALIVE PgQueryParseResult* wasm_parse_query_raw(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return NULL; } diff --git a/versions/17/test/wasm-stale-stack.test.js b/versions/17/test/wasm-stale-stack.test.js new file mode 100644 index 00000000..eec569f1 --- /dev/null +++ b/versions/17/test/wasm-stale-stack.test.js @@ -0,0 +1,38 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const path = require('node:path'); + +// A V8 stack overflow inside the parser unwinds the JS frames but not the C +// shadow stack. Sync calls in the same tick must keep using that instance until +// its replacement arrives, and used to start half-way down the stack: with a +// smaller native stack (--stack-size=8000) the third or fourth one overran the +// shadow stack and trapped instead of throwing RangeError. +const child = ` + const lib = require(process.argv[1]); + const deep = 'SELECT 1' + '+1'.repeat(100000); + (async () => { + await lib.loadModule(); + const seen = []; + for (let i = 0; i < 10; i++) { + try { lib.parseSync(deep); seen.push('ok'); } catch (error) { seen.push(error.constructor.name); } + } + await lib.loadModule(); + seen.push(lib.parseSync('SELECT 1').stmts.length); + console.log(JSON.stringify(seen)); + })(); +`; + +describe('Sync calls on a stale instance after a stack overflow', { timeout: 120000 }, () => { + it('throw RangeError (or an error from libpg_query), never a WebAssembly trap', () => { + const stdout = execFileSync( + process.execPath, + ['--stack-size=8000', '-e', child, path.join(__dirname, '..', 'wasm', 'index.cjs')], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] } + ); + const seen = JSON.parse(stdout.trim().split('\n').pop()); + assert.equal(seen.pop(), 1, 'the replacement instance parses'); + assert.ok(!seen.includes('ok'), 'the deep expression must not parse'); + for (const name of seen) assert.notEqual(name, 'RuntimeError', `trap: ${JSON.stringify(seen)}`); + }); +}); diff --git a/versions/18/Makefile b/versions/18/Makefile index c0c91c28..b1674817 100644 --- a/versions/18/Makefile +++ b/versions/18/Makefile @@ -17,7 +17,7 @@ WASM_STACK_SIZE ?= 33554432 # byte-for-byte. Set in package.json `x-publish` and applied by copy:templates. LIBPG_QUERY_REPO := https://github.com/constructive-io/libpg_query.git LIBPG_QUERY_TAG := 18-constructive -LIBPG_QUERY_COMMIT := b0f5f331dc50d69597ec610de7b21afa47a66808 +LIBPG_QUERY_COMMIT := 64e7edacbce8e376754d35240c55c62289783d44 CACHE_DIR := .cache @@ -75,7 +75,7 @@ ifdef EMSCRIPTEN -I$(LIBPG_QUERY_DIR)/vendor \ -L$(LIBPG_QUERY_DIR) \ -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_scan','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string','_wasm_parse_query_raw','_wasm_free_parse_result']" \ - -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \ + -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32','stackSave','stackRestore']" \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ -sENVIRONMENT="web,node,worker" \ -sASSERTIONS=0 \ diff --git a/versions/18/package.json b/versions/18/package.json index bcaaf273..779f2803 100644 --- a/versions/18/package.json +++ b/versions/18/package.json @@ -15,7 +15,7 @@ "distTag": "pg18", "libpgQueryRepo": "https://github.com/constructive-io/libpg_query.git", "libpgQueryTag": "18-constructive", - "libpgQueryCommit": "b0f5f331dc50d69597ec610de7b21afa47a66808", + "libpgQueryCommit": "64e7edacbce8e376754d35240c55c62289783d44", "fullApi": true }, "files": [ @@ -31,7 +31,7 @@ "wasm:rebuild": "pnpm wasm:make rebuild", "wasm:clean": "pnpm wasm:make clean", "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/pg18.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js" + "test": "node --test test/parsing.test.js test/pg18.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/wasm-stale-stack.test.js" }, "author": "Constructive ", "license": "MIT", diff --git a/versions/18/src/index.ts b/versions/18/src/index.ts index e905d6c6..3a997688 100644 --- a/versions/18/src/index.ts +++ b/versions/18/src/index.ts @@ -184,9 +184,14 @@ instantiate(); function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; + const module = wasmModule; + const stackPointer = module.stackSave(); try { return fn(); } catch (error) { + // The throw unwound the JS frames but not the C shadow stack; give it + // back so later sync calls on this instance do not start half-way down it. + module.stackRestore(stackPointer); failures++; stale = true; if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; diff --git a/versions/18/src/wasm_wrapper.c b/versions/18/src/wasm_wrapper.c index 63ae5dab..f76d6c75 100644 --- a/versions/18/src/wasm_wrapper.c +++ b/versions/18/src/wasm_wrapper.c @@ -8,10 +8,21 @@ #include "pg_query.h" #include "protobuf/pg_query.pb-c.h" #include +#include #include #include #include +/* libpg_query's check_stack_depth() needs a reference point and a limit per + * entry: the shadow stack left after the previous call, minus a 512 KiB + * margin for the frames between here and the recursion. */ +extern void pg_query_set_stack_limit(size_t bytes) __attribute__((weak)); +static void set_stack_guard(void) { + if (!pg_query_set_stack_limit) return; /* libpg_query without the guard (upstream pins) */ + size_t free_bytes = emscripten_stack_get_free(); + pg_query_set_stack_limit(free_bytes > 524288 ? free_bytes - 524288 : 0); +} + static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; } @@ -35,6 +46,7 @@ static void* safe_malloc(size_t size) { EMSCRIPTEN_KEEPALIVE char* wasm_parse_query(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } @@ -54,6 +66,7 @@ char* wasm_parse_query(const char* input) { EMSCRIPTEN_KEEPALIVE PgQueryParseResult* wasm_parse_query_raw(const char* input) { + set_stack_guard(); PgQueryParseResult* result = (PgQueryParseResult*)malloc(sizeof(PgQueryParseResult)); if (!result) { return NULL; @@ -73,6 +86,7 @@ void wasm_free_parse_result(PgQueryParseResult* result) { EMSCRIPTEN_KEEPALIVE char* wasm_parse_plpgsql(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } @@ -113,6 +127,7 @@ char* wasm_parse_plpgsql(const char* input) { EMSCRIPTEN_KEEPALIVE char* wasm_fingerprint(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } @@ -132,6 +147,7 @@ char* wasm_fingerprint(const char* input) { EMSCRIPTEN_KEEPALIVE char* wasm_normalize_query(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } @@ -172,6 +188,7 @@ typedef struct { EMSCRIPTEN_KEEPALIVE WasmDetailedResult* wasm_parse_query_detailed(const char* input) { + set_stack_guard(); WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); if (!result) { return NULL; @@ -372,6 +389,7 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi EMSCRIPTEN_KEEPALIVE char* wasm_scan(const char* input) { + set_stack_guard(); if (!validate_input(input)) { return safe_strdup("Invalid input: query cannot be null or empty"); } diff --git a/versions/18/test/wasm-stale-stack.test.js b/versions/18/test/wasm-stale-stack.test.js new file mode 100644 index 00000000..eec569f1 --- /dev/null +++ b/versions/18/test/wasm-stale-stack.test.js @@ -0,0 +1,38 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const path = require('node:path'); + +// A V8 stack overflow inside the parser unwinds the JS frames but not the C +// shadow stack. Sync calls in the same tick must keep using that instance until +// its replacement arrives, and used to start half-way down the stack: with a +// smaller native stack (--stack-size=8000) the third or fourth one overran the +// shadow stack and trapped instead of throwing RangeError. +const child = ` + const lib = require(process.argv[1]); + const deep = 'SELECT 1' + '+1'.repeat(100000); + (async () => { + await lib.loadModule(); + const seen = []; + for (let i = 0; i < 10; i++) { + try { lib.parseSync(deep); seen.push('ok'); } catch (error) { seen.push(error.constructor.name); } + } + await lib.loadModule(); + seen.push(lib.parseSync('SELECT 1').stmts.length); + console.log(JSON.stringify(seen)); + })(); +`; + +describe('Sync calls on a stale instance after a stack overflow', { timeout: 120000 }, () => { + it('throw RangeError (or an error from libpg_query), never a WebAssembly trap', () => { + const stdout = execFileSync( + process.execPath, + ['--stack-size=8000', '-e', child, path.join(__dirname, '..', 'wasm', 'index.cjs')], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] } + ); + const seen = JSON.parse(stdout.trim().split('\n').pop()); + assert.equal(seen.pop(), 1, 'the replacement instance parses'); + assert.ok(!seen.includes('ok'), 'the deep expression must not parse'); + for (const name of seen) assert.notEqual(name, 'RuntimeError', `trap: ${JSON.stringify(seen)}`); + }); +}); From 32656620063311ebc1560603b9b98cd8955dc215 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 24 Oct 2026 02:49:42 +0000 Subject: [PATCH 02/15] chore: pin v17/v18 to the fork tips that also free the PL/pgSQL alias list --- versions/17/Makefile | 2 +- versions/17/package.json | 2 +- versions/18/Makefile | 2 +- versions/18/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/versions/17/Makefile b/versions/17/Makefile index 3e8b3e28..d101e0c5 100644 --- a/versions/17/Makefile +++ b/versions/17/Makefile @@ -17,7 +17,7 @@ WASM_STACK_SIZE ?= 33554432 # byte-for-byte. Set in package.json `x-publish` and applied by copy:templates. LIBPG_QUERY_REPO := https://github.com/constructive-io/libpg_query.git LIBPG_QUERY_TAG := 17-constructive -LIBPG_QUERY_COMMIT := 5691182b996645135febd16d6403668298bf0c8b +LIBPG_QUERY_COMMIT := ab619b807feea69fdaaa4ea1516236b575bc6d1d CACHE_DIR := .cache diff --git a/versions/17/package.json b/versions/17/package.json index 44c121a9..f640640f 100644 --- a/versions/17/package.json +++ b/versions/17/package.json @@ -15,7 +15,7 @@ "distTag": "pg17", "libpgQueryRepo": "https://github.com/constructive-io/libpg_query.git", "libpgQueryTag": "17-constructive", - "libpgQueryCommit": "5691182b996645135febd16d6403668298bf0c8b" + "libpgQueryCommit": "ab619b807feea69fdaaa4ea1516236b575bc6d1d" }, "files": [ "wasm/*" diff --git a/versions/18/Makefile b/versions/18/Makefile index b1674817..a911156c 100644 --- a/versions/18/Makefile +++ b/versions/18/Makefile @@ -17,7 +17,7 @@ WASM_STACK_SIZE ?= 33554432 # byte-for-byte. Set in package.json `x-publish` and applied by copy:templates. LIBPG_QUERY_REPO := https://github.com/constructive-io/libpg_query.git LIBPG_QUERY_TAG := 18-constructive -LIBPG_QUERY_COMMIT := 64e7edacbce8e376754d35240c55c62289783d44 +LIBPG_QUERY_COMMIT := b66cc16ea07eeec340f317b91a93e0cbea7922c0 CACHE_DIR := .cache diff --git a/versions/18/package.json b/versions/18/package.json index 779f2803..f4d52b23 100644 --- a/versions/18/package.json +++ b/versions/18/package.json @@ -15,7 +15,7 @@ "distTag": "pg18", "libpgQueryRepo": "https://github.com/constructive-io/libpg_query.git", "libpgQueryTag": "18-constructive", - "libpgQueryCommit": "64e7edacbce8e376754d35240c55c62289783d44", + "libpgQueryCommit": "b66cc16ea07eeec340f317b91a93e0cbea7922c0", "fullApi": true }, "files": [ From eee3f00554c41c928cf46acbcb84d466c06a7d66 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 03:24:47 +0000 Subject: [PATCH 03/15] fix(parser): ship ESM entry points as .mjs so Node 18 can import them --- parser/package.json | 44 +++++++++++++++--------------- parser/scripts/prepare.js | 14 ++++++---- parser/templates/index.js.template | 2 +- parser/test/esm.test.mjs | 26 ++++++++++++++++++ 4 files changed, 57 insertions(+), 29 deletions(-) diff --git a/parser/package.json b/parser/package.json index 76f62ae4..c4ff7beb 100644 --- a/parser/package.json +++ b/parser/package.json @@ -4,43 +4,43 @@ "author": "Constructive ", "description": "Multi-version PostgreSQL parser with dynamic version selection", "main": "./wasm/index.cjs", - "module": "./wasm/index.js", + "module": "./wasm/index.mjs", "types": "./wasm/index.d.ts", "exports": { ".": { - "import": "./wasm/index.js", - "require": "./wasm/index.cjs", - "types": "./wasm/index.d.ts" + "types": "./wasm/index.d.ts", + "import": "./wasm/index.mjs", + "require": "./wasm/index.cjs" }, "./v13": { - "import": "./wasm/v13.js", - "require": "./wasm/v13.cjs", - "types": "./wasm/v13.d.ts" + "types": "./wasm/v13.d.ts", + "import": "./wasm/v13.mjs", + "require": "./wasm/v13.cjs" }, "./v14": { - "import": "./wasm/v14.js", - "require": "./wasm/v14.cjs", - "types": "./wasm/v14.d.ts" + "types": "./wasm/v14.d.ts", + "import": "./wasm/v14.mjs", + "require": "./wasm/v14.cjs" }, "./v15": { - "import": "./wasm/v15.js", - "require": "./wasm/v15.cjs", - "types": "./wasm/v15.d.ts" + "types": "./wasm/v15.d.ts", + "import": "./wasm/v15.mjs", + "require": "./wasm/v15.cjs" }, "./v16": { - "import": "./wasm/v16.js", - "require": "./wasm/v16.cjs", - "types": "./wasm/v16.d.ts" + "types": "./wasm/v16.d.ts", + "import": "./wasm/v16.mjs", + "require": "./wasm/v16.cjs" }, "./v17": { - "import": "./wasm/v17.js", - "require": "./wasm/v17.cjs", - "types": "./wasm/v17.d.ts" + "types": "./wasm/v17.d.ts", + "import": "./wasm/v17.mjs", + "require": "./wasm/v17.cjs" }, "./v18": { - "import": "./wasm/v18.js", - "require": "./wasm/v18.cjs", - "types": "./wasm/v18.d.ts" + "types": "./wasm/v18.d.ts", + "import": "./wasm/v18.mjs", + "require": "./wasm/v18.cjs" } }, "files": [ diff --git a/parser/scripts/prepare.js b/parser/scripts/prepare.js index 9c1ac240..e767e3fe 100644 --- a/parser/scripts/prepare.js +++ b/parser/scripts/prepare.js @@ -45,8 +45,8 @@ function generateTemplateVars(versions) { // For TypeScript union: 13 | 14 | 15 | 16 | 17 const versionUnion = versions.join(' | '); - // For ESM exports: export * as v13 from './v13/index.js'; - const versionExports = versions.map(v => `export * as v${v} from './v${v}/index.js';`).join('\n'); + // For ESM exports: export * as v13 from './v13/index.mjs'; + const versionExports = versions.map(v => `export * as v${v} from './v${v}/index.mjs';`).join('\n'); // For CommonJS requires: v13: require('./v13/index.cjs'), const versionRequires = versions.map(v => ` v${v}: require('./v${v}/index.cjs')`).join(',\n'); @@ -113,7 +113,9 @@ config.versions.forEach(version => { const files = fs.readdirSync(sourceWasmDir); files.forEach(file => { const sourcePath = path.join(sourceWasmDir, file); - const destPath = path.join(versionWasmDir, file); + // ESM files must be .mjs: the package has no "type": "module", so Node + // before 20.19 (and without syntax detection) would load a .js as CommonJS. + const destPath = path.join(versionWasmDir, file === 'index.js' ? 'index.mjs' : file); if (fs.statSync(sourcePath).isFile()) { console.log(`Copying ${file} for v${version}...`); @@ -162,7 +164,7 @@ const templateVars = generateTemplateVars(config.versions); // Process each template const templates = [ - { template: 'index.js.template', output: 'index.js' }, + { template: 'index.js.template', output: 'index.mjs' }, { template: 'index.cjs.template', output: 'index.cjs' }, { template: 'index.d.ts.template', output: 'index.d.ts' } ]; @@ -182,9 +184,9 @@ templates.forEach(({ template, output }) => { config.versions.forEach(version => { // ESM version const esmContent = `// Re-export everything from the v${version} parser -export * from './v${version}/index.js'; +export * from './v${version}/index.mjs'; `; - fs.writeFileSync(path.join(wasmDir, `v${version}.js`), esmContent); + fs.writeFileSync(path.join(wasmDir, `v${version}.mjs`), esmContent); // CJS version const cjsContent = `// Re-export everything from the v${version} parser diff --git a/parser/templates/index.js.template b/parser/templates/index.js.template index 6716d1eb..96276df1 100644 --- a/parser/templates/index.js.template +++ b/parser/templates/index.js.template @@ -33,7 +33,7 @@ export class Parser { } async _doLoad() { - const module = await import(`./v${this.version}/index.js`); + const module = await import(`./v${this.version}/index.mjs`); this.parser = module; if (this.parser.loadModule) { diff --git a/parser/test/esm.test.mjs b/parser/test/esm.test.mjs index c5241dfe..67b953c9 100644 --- a/parser/test/esm.test.mjs +++ b/parser/test/esm.test.mjs @@ -1,6 +1,8 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; const { versions } = JSON.parse(readFileSync(new URL('../wasm/build-info.json', import.meta.url), 'utf8')); @@ -21,3 +23,27 @@ describe('ESM entry points', () => { }); } }); + +// Node 18 and Node 20 before 20.19 load a `.js` file in a package without +// "type": "module" as CommonJS. Turning off syntax detection reproduces that +// on current Node, so this fails if an ESM entry point is not `.mjs`. +describe('ESM entry points without module syntax detection', () => { + const flags = process.allowedNodeEnvironmentFlags.has('--experimental-detect-module') + ? ['--no-experimental-detect-module'] + : []; + const specifiers = ['@pgsql/parser', ...versions.map((v) => `@pgsql/parser/v${v}`)]; + for (const specifier of specifiers) { + it(`imports ${specifier}`, () => { + const script = `const m = await import(${JSON.stringify(specifier)}); + const lib = m.Parser ? new m.Parser() : m; + if (m.loadModule) await m.loadModule(); + const result = m.Parser ? await lib.parse('SELECT 1') : lib.parseSync('SELECT 1'); + if (result.stmts.length !== 1) throw new Error('bad parse');`; + const { status, stderr } = spawnSync(process.execPath, [...flags, '--input-type=module', '-e', script], { + cwd: fileURLToPath(new URL('..', import.meta.url)), + encoding: 'utf8', + }); + assert.equal(status, 0, stderr); + }); + } +}); From 229f31505160ef10fac7094a3e0f0019455e0be0 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 03:35:02 +0000 Subject: [PATCH 04/15] fix(v18): throw SqlError from fingerprint/normalize/scan on every libpg_query error --- templates/full/index.ts | 71 ++++++----- templates/full/wasm_wrapper.c | 180 +++++++++++++++------------ versions/18/src/index.ts | 71 ++++++----- versions/18/src/wasm_wrapper.c | 180 +++++++++++++++------------ versions/18/test/fingerprint.test.js | 32 +++++ versions/18/test/normalize.test.js | 32 +++++ versions/18/test/scan.test.js | 44 +++++++ 7 files changed, 384 insertions(+), 226 deletions(-) diff --git a/templates/full/index.ts b/templates/full/index.ts index e97b6096..767089ae 100644 --- a/templates/full/index.ts +++ b/templates/full/index.ts @@ -127,6 +127,7 @@ interface WasmModule { _wasm_fingerprint: (queryPtr: number) => number; _wasm_normalize_query: (queryPtr: number) => number; _wasm_scan: (queryPtr: number) => number; + _wasm_free_detailed_result: (ptr: number) => void; lengthBytesUTF8: (str: string) => number; stringToUTF8: (str: string, ptr: number, len: number) => void; UTF8ToString: (ptr: number) => string; @@ -244,6 +245,28 @@ function ptrToString(ptr: number): string { return wasmModule.UTF8ToString(ptr); } +function readDetailedResult(resultPtr: number): string { + if (!resultPtr) { + throw new Error('libpg_query returned no result: out of WASM memory (heap limit is WASM_MAXIMUM_MEMORY, 1 GiB); parse large inputs in smaller pieces'); + } + const ptrAt = (offset: number) => wasmModule.getValue(resultPtr + offset, 'i32'); + const str = (ptr: number) => (ptr ? wasmModule.UTF8ToString(ptr) : undefined); + if (ptrAt(0)) { + const message = str(ptrAt(4)) ?? 'Unknown error'; + const lineno = ptrAt(16); + const cursorpos = ptrAt(20); + throw new SqlError(message, { + message, + cursorPosition: cursorpos > 0 ? cursorpos - 1 : 0, + fileName: str(ptrAt(12)), + functionName: str(ptrAt(8)), + lineNumber: lineno > 0 ? lineno : undefined, + context: str(ptrAt(24)) + }); + } + return str(ptrAt(28)) ?? ''; +} + export const parse = awaitInit(async (query: string): Promise => { // Input validation if (query === null || query === undefined) { @@ -338,18 +361,14 @@ export const fingerprint = awaitInit(async (query: string): Promise => { try { resultPtr = callWasm(() => wasmModule._wasm_fingerprint(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return resultStr; } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -362,18 +381,14 @@ export const normalize = awaitInit(async (query: string): Promise => { try { resultPtr = callWasm(() => wasmModule._wasm_normalize_query(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return resultStr; } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -484,18 +499,14 @@ export function fingerprintSync(query: string): string { try { resultPtr = callWasm(() => wasmModule._wasm_fingerprint(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return resultStr; } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -511,18 +522,14 @@ export function normalizeSync(query: string): string { try { resultPtr = callWasm(() => wasmModule._wasm_normalize_query(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return resultStr; } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -535,18 +542,14 @@ export const scan = awaitInit(async (query: string): Promise => { try { resultPtr = callWasm(() => wasmModule._wasm_scan(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return JSON.parse(resultStr); } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -562,18 +565,14 @@ export function scanSync(query: string): ScanResult { try { resultPtr = callWasm(() => wasmModule._wasm_scan(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return JSON.parse(resultStr); } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } diff --git a/templates/full/wasm_wrapper.c b/templates/full/wasm_wrapper.c index 268d2893..81911fab 100644 --- a/templates/full/wasm_wrapper.c +++ b/templates/full/wasm_wrapper.c @@ -104,53 +104,6 @@ char* wasm_parse_plpgsql(const char* input) { return wrapped_result; } -EMSCRIPTEN_KEEPALIVE -char* wasm_fingerprint(const char* input) { - if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); - } - - PgQueryFingerprintResult result = pg_query_fingerprint(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_fingerprint_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); - } - - char* fingerprint_str = safe_strdup(result.fingerprint_str); - pg_query_free_fingerprint_result(result); - return fingerprint_str; -} - -EMSCRIPTEN_KEEPALIVE -char* wasm_normalize_query(const char* input) { - if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); - } - - PgQueryNormalizeResult result = pg_query_normalize(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_normalize_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); - } - - char* normalized = safe_strdup(result.normalized_query); - pg_query_free_normalize_result(result); - - if (!normalized) { - return safe_strdup("Memory allocation failed"); - } - - return normalized; -} - - - - - typedef struct { int has_error; char* message; @@ -230,6 +183,80 @@ void wasm_free_detailed_result(WasmDetailedResult* result) { } } +static WasmDetailedResult* detailed_result_new(void) { + WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); + if (result) { + memset(result, 0, sizeof(WasmDetailedResult)); + } + return result; +} + +static WasmDetailedResult* detailed_set_error(WasmDetailedResult* result, PgQueryError* error) { + result->has_error = 1; + result->message = safe_strdup(error->message); + result->funcname = error->funcname ? safe_strdup(error->funcname) : NULL; + result->filename = error->filename ? safe_strdup(error->filename) : NULL; + result->context = error->context ? safe_strdup(error->context) : NULL; + result->lineno = error->lineno; + result->cursorpos = error->cursorpos; + return result; +} + +static WasmDetailedResult* detailed_set_message(WasmDetailedResult* result, const char* message) { + result->has_error = 1; + result->message = safe_strdup(message); + return result; +} + +static WasmDetailedResult* detailed_set_data(WasmDetailedResult* result, char* data) { + if (!data) { + return detailed_set_message(result, "Memory allocation failed"); + } + result->data = data; + result->data_len = strlen(data); + return result; +} + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_fingerprint(const char* input) { + WasmDetailedResult* result = detailed_result_new(); + if (!result) { + return NULL; + } + if (!validate_input(input)) { + return detailed_set_message(result, "Invalid input: query cannot be null or empty"); + } + + PgQueryFingerprintResult fingerprint_result = pg_query_fingerprint(input); + if (fingerprint_result.error) { + detailed_set_error(result, fingerprint_result.error); + } else { + detailed_set_data(result, safe_strdup(fingerprint_result.fingerprint_str)); + } + pg_query_free_fingerprint_result(fingerprint_result); + return result; +} + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_normalize_query(const char* input) { + WasmDetailedResult* result = detailed_result_new(); + if (!result) { + return NULL; + } + if (!validate_input(input)) { + return detailed_set_message(result, "Invalid input: query cannot be null or empty"); + } + + PgQueryNormalizeResult normalize_result = pg_query_normalize(input); + if (normalize_result.error) { + detailed_set_error(result, normalize_result.error); + } else { + detailed_set_data(result, safe_strdup(normalize_result.normalized_query)); + } + pg_query_free_normalize_result(normalize_result); + return result; +} + static const char* get_token_name(PgQuery__Token token_type) { // Map some common token types to readable names // Note: This is a simplified mapping - full enum lookup would require more complexity @@ -272,14 +299,14 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi } // Size the buffer from token text length, not token count: each token - // contributes at most ~170 bytes of fixed JSON plus up to 2 bytes per input - // byte once escaped. + // contributes at most ~170 bytes of fixed JSON plus up to 6 bytes per input + // byte once escaped (\\u00XX for control characters). size_t estimated_size = 1024; for (size_t i = 0; i < scan_result->n_tokens; i++) { PgQuery__ScanToken *token = scan_result->tokens[i]; int token_length = token->end - token->start; if (token_length < 0) token_length = 0; - estimated_size += (size_t) token_length * 2 + 200; + estimated_size += (size_t) token_length * 6 + 200; } char* json = safe_malloc(estimated_size); if (!json) { @@ -305,7 +332,7 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi token_text[token_length] = '\0'; // Escape token text for JSON - char* escaped_text = safe_malloc(token_length * 2 + 1); + char* escaped_text = safe_malloc(token_length * 6 + 1); if (!escaped_text) { free(token_text); continue; @@ -326,6 +353,8 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi } else if (c == '\t') { escaped_text[escaped_pos++] = '\\'; escaped_text[escaped_pos++] = 't'; + } else if ((unsigned char) c < 0x20) { + escaped_pos += snprintf(escaped_text + escaped_pos, 7, "\\u%04x", (unsigned char) c); } else { escaped_text[escaped_pos++] = c; } @@ -364,36 +393,33 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi } EMSCRIPTEN_KEEPALIVE -char* wasm_scan(const char* input) { +WasmDetailedResult* wasm_scan(const char* input) { + WasmDetailedResult* result = detailed_result_new(); + if (!result) { + return NULL; + } if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); + return detailed_set_message(result, "Invalid input: query cannot be null or empty"); } - - PgQueryScanResult result = pg_query_scan(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_scan_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + + PgQueryScanResult scan_result = pg_query_scan(input); + if (scan_result.error) { + detailed_set_error(result, scan_result.error); + pg_query_free_scan_result(scan_result); + return result; } - - // Unpack protobuf data - PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( - NULL, result.pbuf.len, (void *) result.pbuf.data); - - if (!scan_result) { - pg_query_free_scan_result(result); - return safe_strdup("Failed to unpack scan result"); + + PgQuery__ScanResult *unpacked = pg_query__scan_result__unpack( + NULL, scan_result.pbuf.len, (void *) scan_result.pbuf.data); + if (!unpacked) { + pg_query_free_scan_result(scan_result); + return detailed_set_message(result, "Failed to unpack scan result"); } - - // Convert to JSON - char* json_result = build_scan_json(scan_result, input); - - // Clean up - pg_query__scan_result__free_unpacked(scan_result, NULL); - pg_query_free_scan_result(result); - - return json_result ? json_result : safe_strdup("{\"version\":0,\"tokens\":[]}"); + + detailed_set_data(result, build_scan_json(unpacked, input)); + pg_query__scan_result__free_unpacked(unpacked, NULL); + pg_query_free_scan_result(scan_result); + return result; } EMSCRIPTEN_KEEPALIVE diff --git a/versions/18/src/index.ts b/versions/18/src/index.ts index 56c93d61..31ce9660 100644 --- a/versions/18/src/index.ts +++ b/versions/18/src/index.ts @@ -134,6 +134,7 @@ interface WasmModule { _wasm_fingerprint: (queryPtr: number) => number; _wasm_normalize_query: (queryPtr: number) => number; _wasm_scan: (queryPtr: number) => number; + _wasm_free_detailed_result: (ptr: number) => void; lengthBytesUTF8: (str: string) => number; stringToUTF8: (str: string, ptr: number, len: number) => void; UTF8ToString: (ptr: number) => string; @@ -251,6 +252,28 @@ function ptrToString(ptr: number): string { return wasmModule.UTF8ToString(ptr); } +function readDetailedResult(resultPtr: number): string { + if (!resultPtr) { + throw new Error('libpg_query returned no result: out of WASM memory (heap limit is WASM_MAXIMUM_MEMORY, 1 GiB); parse large inputs in smaller pieces'); + } + const ptrAt = (offset: number) => wasmModule.getValue(resultPtr + offset, 'i32'); + const str = (ptr: number) => (ptr ? wasmModule.UTF8ToString(ptr) : undefined); + if (ptrAt(0)) { + const message = str(ptrAt(4)) ?? 'Unknown error'; + const lineno = ptrAt(16); + const cursorpos = ptrAt(20); + throw new SqlError(message, { + message, + cursorPosition: cursorpos > 0 ? cursorpos - 1 : 0, + fileName: str(ptrAt(12)), + functionName: str(ptrAt(8)), + lineNumber: lineno > 0 ? lineno : undefined, + context: str(ptrAt(24)) + }); + } + return str(ptrAt(28)) ?? ''; +} + export const parse = awaitInit(async (query: string): Promise => { // Input validation if (query === null || query === undefined) { @@ -345,18 +368,14 @@ export const fingerprint = awaitInit(async (query: string): Promise => { try { resultPtr = callWasm(() => wasmModule._wasm_fingerprint(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return resultStr; } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -369,18 +388,14 @@ export const normalize = awaitInit(async (query: string): Promise => { try { resultPtr = callWasm(() => wasmModule._wasm_normalize_query(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return resultStr; } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -491,18 +506,14 @@ export function fingerprintSync(query: string): string { try { resultPtr = callWasm(() => wasmModule._wasm_fingerprint(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return resultStr; } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -518,18 +529,14 @@ export function normalizeSync(query: string): string { try { resultPtr = callWasm(() => wasmModule._wasm_normalize_query(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return resultStr; } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -542,18 +549,14 @@ export const scan = awaitInit(async (query: string): Promise => { try { resultPtr = callWasm(() => wasmModule._wasm_scan(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return JSON.parse(resultStr); } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } @@ -569,18 +572,14 @@ export function scanSync(query: string): ScanResult { try { resultPtr = callWasm(() => wasmModule._wasm_scan(queryPtr)); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } + const resultStr = readDetailedResult(resultPtr); return JSON.parse(resultStr); } finally { if (failures === failuresBefore) { wasmModule._free(queryPtr); if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); + wasmModule._wasm_free_detailed_result(resultPtr); } } } diff --git a/versions/18/src/wasm_wrapper.c b/versions/18/src/wasm_wrapper.c index 63ae5dab..2ee54101 100644 --- a/versions/18/src/wasm_wrapper.c +++ b/versions/18/src/wasm_wrapper.c @@ -111,53 +111,6 @@ char* wasm_parse_plpgsql(const char* input) { return wrapped_result; } -EMSCRIPTEN_KEEPALIVE -char* wasm_fingerprint(const char* input) { - if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); - } - - PgQueryFingerprintResult result = pg_query_fingerprint(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_fingerprint_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); - } - - char* fingerprint_str = safe_strdup(result.fingerprint_str); - pg_query_free_fingerprint_result(result); - return fingerprint_str; -} - -EMSCRIPTEN_KEEPALIVE -char* wasm_normalize_query(const char* input) { - if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); - } - - PgQueryNormalizeResult result = pg_query_normalize(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_normalize_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); - } - - char* normalized = safe_strdup(result.normalized_query); - pg_query_free_normalize_result(result); - - if (!normalized) { - return safe_strdup("Memory allocation failed"); - } - - return normalized; -} - - - - - typedef struct { int has_error; char* message; @@ -237,6 +190,80 @@ void wasm_free_detailed_result(WasmDetailedResult* result) { } } +static WasmDetailedResult* detailed_result_new(void) { + WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); + if (result) { + memset(result, 0, sizeof(WasmDetailedResult)); + } + return result; +} + +static WasmDetailedResult* detailed_set_error(WasmDetailedResult* result, PgQueryError* error) { + result->has_error = 1; + result->message = safe_strdup(error->message); + result->funcname = error->funcname ? safe_strdup(error->funcname) : NULL; + result->filename = error->filename ? safe_strdup(error->filename) : NULL; + result->context = error->context ? safe_strdup(error->context) : NULL; + result->lineno = error->lineno; + result->cursorpos = error->cursorpos; + return result; +} + +static WasmDetailedResult* detailed_set_message(WasmDetailedResult* result, const char* message) { + result->has_error = 1; + result->message = safe_strdup(message); + return result; +} + +static WasmDetailedResult* detailed_set_data(WasmDetailedResult* result, char* data) { + if (!data) { + return detailed_set_message(result, "Memory allocation failed"); + } + result->data = data; + result->data_len = strlen(data); + return result; +} + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_fingerprint(const char* input) { + WasmDetailedResult* result = detailed_result_new(); + if (!result) { + return NULL; + } + if (!validate_input(input)) { + return detailed_set_message(result, "Invalid input: query cannot be null or empty"); + } + + PgQueryFingerprintResult fingerprint_result = pg_query_fingerprint(input); + if (fingerprint_result.error) { + detailed_set_error(result, fingerprint_result.error); + } else { + detailed_set_data(result, safe_strdup(fingerprint_result.fingerprint_str)); + } + pg_query_free_fingerprint_result(fingerprint_result); + return result; +} + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_normalize_query(const char* input) { + WasmDetailedResult* result = detailed_result_new(); + if (!result) { + return NULL; + } + if (!validate_input(input)) { + return detailed_set_message(result, "Invalid input: query cannot be null or empty"); + } + + PgQueryNormalizeResult normalize_result = pg_query_normalize(input); + if (normalize_result.error) { + detailed_set_error(result, normalize_result.error); + } else { + detailed_set_data(result, safe_strdup(normalize_result.normalized_query)); + } + pg_query_free_normalize_result(normalize_result); + return result; +} + static const char* get_token_name(PgQuery__Token token_type) { // Map some common token types to readable names // Note: This is a simplified mapping - full enum lookup would require more complexity @@ -279,14 +306,14 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi } // Size the buffer from token text length, not token count: each token - // contributes at most ~170 bytes of fixed JSON plus up to 2 bytes per input - // byte once escaped. + // contributes at most ~170 bytes of fixed JSON plus up to 6 bytes per input + // byte once escaped (\\u00XX for control characters). size_t estimated_size = 1024; for (size_t i = 0; i < scan_result->n_tokens; i++) { PgQuery__ScanToken *token = scan_result->tokens[i]; int token_length = token->end - token->start; if (token_length < 0) token_length = 0; - estimated_size += (size_t) token_length * 2 + 200; + estimated_size += (size_t) token_length * 6 + 200; } char* json = safe_malloc(estimated_size); if (!json) { @@ -312,7 +339,7 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi token_text[token_length] = '\0'; // Escape token text for JSON - char* escaped_text = safe_malloc(token_length * 2 + 1); + char* escaped_text = safe_malloc(token_length * 6 + 1); if (!escaped_text) { free(token_text); continue; @@ -333,6 +360,8 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi } else if (c == '\t') { escaped_text[escaped_pos++] = '\\'; escaped_text[escaped_pos++] = 't'; + } else if ((unsigned char) c < 0x20) { + escaped_pos += snprintf(escaped_text + escaped_pos, 7, "\\u%04x", (unsigned char) c); } else { escaped_text[escaped_pos++] = c; } @@ -371,36 +400,33 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi } EMSCRIPTEN_KEEPALIVE -char* wasm_scan(const char* input) { +WasmDetailedResult* wasm_scan(const char* input) { + WasmDetailedResult* result = detailed_result_new(); + if (!result) { + return NULL; + } if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); + return detailed_set_message(result, "Invalid input: query cannot be null or empty"); } - - PgQueryScanResult result = pg_query_scan(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_scan_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + + PgQueryScanResult scan_result = pg_query_scan(input); + if (scan_result.error) { + detailed_set_error(result, scan_result.error); + pg_query_free_scan_result(scan_result); + return result; } - - // Unpack protobuf data - PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( - NULL, result.pbuf.len, (void *) result.pbuf.data); - - if (!scan_result) { - pg_query_free_scan_result(result); - return safe_strdup("Failed to unpack scan result"); + + PgQuery__ScanResult *unpacked = pg_query__scan_result__unpack( + NULL, scan_result.pbuf.len, (void *) scan_result.pbuf.data); + if (!unpacked) { + pg_query_free_scan_result(scan_result); + return detailed_set_message(result, "Failed to unpack scan result"); } - - // Convert to JSON - char* json_result = build_scan_json(scan_result, input); - - // Clean up - pg_query__scan_result__free_unpacked(scan_result, NULL); - pg_query_free_scan_result(result); - - return json_result ? json_result : safe_strdup("{\"version\":0,\"tokens\":[]}"); + + detailed_set_data(result, build_scan_json(unpacked, input)); + pg_query__scan_result__free_unpacked(unpacked, NULL); + pg_query_free_scan_result(scan_result); + return result; } EMSCRIPTEN_KEEPALIVE diff --git a/versions/18/test/fingerprint.test.js b/versions/18/test/fingerprint.test.js index 07171c82..e38a0df5 100644 --- a/versions/18/test/fingerprint.test.js +++ b/versions/18/test/fingerprint.test.js @@ -65,3 +65,35 @@ describe("Query Fingerprinting", () => { }); }); }); + +describe("fingerprint errors", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + // Errors raised outside the grammar's "syntax error" path used to come back + // as the fingerprint result instead of being thrown. + const failing = [ + ["select percentile_cont(p order by p) within group (order by x) from t", "cannot use multiple ORDER BY clauses with WITHIN GROUP"], + ["CREATE TABLE p (a int) PARTITION BY MAGIC (a)", 'unrecognized partitioning strategy "magic"'], + ]; + + for (const [sql, message] of failing) { + it(`fingerprintSync throws SqlError for ${sql}`, () => { + assert.throws(() => query.fingerprintSync(sql), (err) => { + assert.equal(err.name, "SqlError"); + assert.equal(err.message, message); + assert.equal(typeof err.sqlDetails.cursorPosition, "number"); + return true; + }); + }); + + it(`fingerprint rejects with SqlError for ${sql}`, async () => { + await assert.rejects(query.fingerprint(sql), (err) => { + assert.equal(err.name, "SqlError"); + assert.equal(err.message, message); + return true; + }); + }); + } +}); diff --git a/versions/18/test/normalize.test.js b/versions/18/test/normalize.test.js index 14f0a431..68932a68 100644 --- a/versions/18/test/normalize.test.js +++ b/versions/18/test/normalize.test.js @@ -67,3 +67,35 @@ describe("Query Normalization", () => { }); }); }); + +describe("normalize errors", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + // Errors raised outside the grammar's "syntax error" path used to come back + // as the normalize result instead of being thrown. + const failing = [ + ["select percentile_cont(p order by p) within group (order by x) from t", "cannot use multiple ORDER BY clauses with WITHIN GROUP"], + ["CREATE TABLE p (a int) PARTITION BY MAGIC (a)", 'unrecognized partitioning strategy "magic"'], + ]; + + for (const [sql, message] of failing) { + it(`normalizeSync throws SqlError for ${sql}`, () => { + assert.throws(() => query.normalizeSync(sql), (err) => { + assert.equal(err.name, "SqlError"); + assert.equal(err.message, message); + assert.equal(typeof err.sqlDetails.cursorPosition, "number"); + return true; + }); + }); + + it(`normalize rejects with SqlError for ${sql}`, async () => { + await assert.rejects(query.normalize(sql), (err) => { + assert.equal(err.name, "SqlError"); + assert.equal(err.message, message); + return true; + }); + }); + } +}); diff --git a/versions/18/test/scan.test.js b/versions/18/test/scan.test.js index bd927874..5d1a3424 100644 --- a/versions/18/test/scan.test.js +++ b/versions/18/test/scan.test.js @@ -305,3 +305,47 @@ comment */ SELECT 2`; }); }); }); + +describe("scan errors", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + // Scanner errors used to surface as a JSON.parse SyntaxError on the message. + const failing = [ + ["SELECT 'abc", "unterminated quoted string at or near \"'abc\""], + ["SELECT $$x", "unterminated dollar-quoted string at or near \"$$x\""], + ]; + + for (const [sql, message] of failing) { + it(`scanSync throws SqlError for ${sql}`, () => { + assert.throws(() => query.scanSync(sql), (err) => { + assert.equal(err.name, "SqlError"); + assert.equal(err.message, message); + assert.equal(typeof err.sqlDetails.cursorPosition, "number"); + return true; + }); + }); + + it(`scan rejects with SqlError for ${sql}`, async () => { + await assert.rejects(query.scan(sql), (err) => { + assert.equal(err.name, "SqlError"); + assert.equal(err.message, message); + return true; + }); + }); + } +}); + +describe("scan control characters", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + it("escapes control characters in token text", () => { + const sql = "SELECT E'a\u0002b'"; + const result = query.scanSync(sql); + const literal = result.tokens.find((t) => t.tokenName === "SCONST"); + assert.equal(literal.text, "E'a\u0002b'"); + }); +}); From ed28cca9fe98fb613480228d56c382b84afb422d Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 03:41:22 +0000 Subject: [PATCH 05/15] test: assert SqlError source fields; declare new WASM exports in libpg-query.d.ts --- templates/full/libpg-query.d.ts | 3 +++ versions/18/src/libpg-query.d.ts | 3 +++ versions/18/test/fingerprint.test.js | 3 +++ versions/18/test/normalize.test.js | 3 +++ versions/18/test/scan.test.js | 3 +++ 5 files changed, 15 insertions(+) diff --git a/templates/full/libpg-query.d.ts b/templates/full/libpg-query.d.ts index 731e1a99..dd0cd1c4 100644 --- a/templates/full/libpg-query.d.ts +++ b/templates/full/libpg-query.d.ts @@ -7,9 +7,12 @@ declare module './libpg-query.js' { _wasm_parse_plpgsql: (queryPtr: number) => number; _wasm_fingerprint: (queryPtr: number) => number; _wasm_normalize_query: (queryPtr: number) => number; + _wasm_scan: (queryPtr: number) => number; + _wasm_free_detailed_result: (ptr: number) => void; lengthBytesUTF8: (str: string) => number; stringToUTF8: (str: string, ptr: number, len: number) => void; UTF8ToString: (ptr: number) => string; + getValue: (ptr: number, type: string) => number; HEAPU8: Uint8Array; } diff --git a/versions/18/src/libpg-query.d.ts b/versions/18/src/libpg-query.d.ts index 152d8414..e19be14e 100644 --- a/versions/18/src/libpg-query.d.ts +++ b/versions/18/src/libpg-query.d.ts @@ -14,9 +14,12 @@ declare module './libpg-query.js' { _wasm_parse_plpgsql: (queryPtr: number) => number; _wasm_fingerprint: (queryPtr: number) => number; _wasm_normalize_query: (queryPtr: number) => number; + _wasm_scan: (queryPtr: number) => number; + _wasm_free_detailed_result: (ptr: number) => void; lengthBytesUTF8: (str: string) => number; stringToUTF8: (str: string, ptr: number, len: number) => void; UTF8ToString: (ptr: number) => string; + getValue: (ptr: number, type: string) => number; HEAPU8: Uint8Array; } diff --git a/versions/18/test/fingerprint.test.js b/versions/18/test/fingerprint.test.js index e38a0df5..e3132948 100644 --- a/versions/18/test/fingerprint.test.js +++ b/versions/18/test/fingerprint.test.js @@ -84,6 +84,9 @@ describe("fingerprint errors", () => { assert.equal(err.name, "SqlError"); assert.equal(err.message, message); assert.equal(typeof err.sqlDetails.cursorPosition, "number"); + assert.equal(typeof err.sqlDetails.fileName, "string"); + assert.equal(typeof err.sqlDetails.functionName, "string"); + assert.ok(err.sqlDetails.lineNumber > 0); return true; }); }); diff --git a/versions/18/test/normalize.test.js b/versions/18/test/normalize.test.js index 68932a68..404ee341 100644 --- a/versions/18/test/normalize.test.js +++ b/versions/18/test/normalize.test.js @@ -86,6 +86,9 @@ describe("normalize errors", () => { assert.equal(err.name, "SqlError"); assert.equal(err.message, message); assert.equal(typeof err.sqlDetails.cursorPosition, "number"); + assert.equal(typeof err.sqlDetails.fileName, "string"); + assert.equal(typeof err.sqlDetails.functionName, "string"); + assert.ok(err.sqlDetails.lineNumber > 0); return true; }); }); diff --git a/versions/18/test/scan.test.js b/versions/18/test/scan.test.js index 5d1a3424..f008c9cd 100644 --- a/versions/18/test/scan.test.js +++ b/versions/18/test/scan.test.js @@ -323,6 +323,9 @@ describe("scan errors", () => { assert.equal(err.name, "SqlError"); assert.equal(err.message, message); assert.equal(typeof err.sqlDetails.cursorPosition, "number"); + assert.equal(typeof err.sqlDetails.fileName, "string"); + assert.equal(typeof err.sqlDetails.functionName, "string"); + assert.ok(err.sqlDetails.lineNumber > 0); return true; }); }); From 4761bd1098019444463915fb48b6c44817911bf6 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 03:47:27 +0000 Subject: [PATCH 06/15] ci: check WASM parse output against native libpg_query on its test SQL --- .github/workflows/ci.yml | 3 ++ scripts/native-parity.js | 78 ++++++++++++++++++++++++++++ scripts/native-parity/driver.c | 95 ++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 scripts/native-parity.js create mode 100644 scripts/native-parity/driver.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c90099d6..590b3df3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,9 @@ jobs: run: pnpm run build working-directory: ${{ matrix.package.path }} + - name: Native parity 🔍 + run: node scripts/native-parity.js ${{ matrix.package.path }} + - name: Upload WASM Artifacts 📦 uses: actions/upload-artifact@v4 with: diff --git a/scripts/native-parity.js b/scripts/native-parity.js new file mode 100644 index 00000000..f8c7ee09 --- /dev/null +++ b/scripts/native-parity.js @@ -0,0 +1,78 @@ +#!/usr/bin/env node +/** + * Checks that a built versions//wasm parses exactly like libpg_query compiled natively + * from the same commit, over libpg_query's test SQL: test/sql/postgres_regress and + * plpgsql_regress (the PostgreSQL regression suite) plus its own deparse tests. + * + * node scripts/native-parity.js versions/18 (after `pnpm build` in that directory) + * + * Reuses the libpg_query checkout the WASM build cloned into .cache, so it needs no network. + */ +const { execFileSync } = require('child_process'); +const fs = require('fs'); +const glob = require('glob'); +const os = require('os'); +const path = require('path'); + +// FETCH_ALL is LONG_MAX, which is 32-bit on wasm32 +const FETCH_ALL_NATIVE = 9223372036854775807; +const FETCH_ALL_WASM = 2147483647; + +async function main() { + const versionDir = path.resolve(process.argv[2] || '.'); + const commit = require(path.join(versionDir, 'package.json'))['x-publish'].libpgQueryCommit; + const [source] = glob.sync(`.cache/*/libpg_query/${commit}`, { cwd: versionDir, absolute: true }); + if (!source) throw new Error(`no libpg_query checkout for ${commit} under ${versionDir}/.cache; run pnpm build first`); + + const work = fs.mkdtempSync(path.join(os.tmpdir(), 'libpg-query-native-')); + execFileSync('sh', ['-c', `git -C "${source}" archive ${commit} | tar -x -C "${work}"`]); + execFileSync('make', ['-j', String(os.cpus().length), 'build'], { cwd: work, stdio: ['ignore', 'ignore', 'inherit'] }); + const driver = path.join(work, 'parity-driver'); + execFileSync('cc', ['-O2', '-I', work, path.join(__dirname, 'native-parity/driver.c'), path.join(work, 'libpg_query.a'), '-lpthread', '-o', driver]); + + const files = glob.sync('test/{sql/**/*.{sql,psql},plpgsql_samples.sql}', { cwd: work }).sort(); + const statements = []; + for (const file of files) { + const split = JSON.parse(execFileSync(driver, ['split', path.join(work, file)], { maxBuffer: 1 << 28 }).toString()); + if (split.error) continue; // e.g. strings.sql: intentionally invalid escapes the scanner rejects + split.stmts.forEach((sql, i) => statements.push({ key: `${file}#${i}`, sql })); + } + + const input = path.join(work, 'statements.bin'); + fs.writeFileSync(input, Buffer.concat(statements.map(({ sql }) => Buffer.from(sql + '\0')))); + const output = path.join(work, 'native.jsonl'); + const fd = fs.openSync(output, 'w'); + execFileSync(driver, ['parse', input], { stdio: ['ignore', fd, 'inherit'] }); + fs.closeSync(fd); + const native = fs.readFileSync(output, 'utf8').trimEnd().split('\n'); + + const wasm = require(path.join(versionDir, 'wasm/index.cjs')); + await wasm.loadModule(); + + const diffs = []; + statements.forEach(({ key, sql }, i) => { + const expected = JSON.parse(native[i]); + let actual; + try { + actual = { parse: wasm.parseSync(sql) }; + } catch (err) { + actual = { error: { message: err.message, cursorpos: err.sqlDetails ? err.sqlDetails.cursorPosition + 1 : undefined } }; + } + if (expected.error && actual.error && expected.error.cursorpos === 0) expected.error.cursorpos = 1; + const normalize = (value) => + JSON.stringify(value, (k, v) => ((k === 'howMany' || k === 'how_many') && v === FETCH_ALL_NATIVE ? FETCH_ALL_WASM : v)); + const a = normalize(expected); + const b = JSON.stringify(actual); + if (a !== b) diffs.push({ key, native: a.slice(0, 300), wasm: b.slice(0, 300) }); + }); + + fs.rmSync(work, { recursive: true, force: true }); + console.log(`native parity: ${statements.length} statements from ${files.length} files @ ${commit.slice(0, 7)}, ${diffs.length} differences`); + for (const d of diffs.slice(0, 10)) console.log(JSON.stringify(d, null, 2)); + if (diffs.length) process.exit(1); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/native-parity/driver.c b/scripts/native-parity/driver.c new file mode 100644 index 00000000..95f95838 --- /dev/null +++ b/scripts/native-parity/driver.c @@ -0,0 +1,95 @@ +/* + * Native side of scripts/native-parity.js. + * driver split -> {"stmts":["...", ...]} or {"error":"..."} + * driver parse -> one JSON line per NUL-separated statement: + * {"parse":} or {"error":{"message":"...","cursorpos":N}} + */ +#include +#include +#include +#include "pg_query.h" + +static void print_json_string(const char *s) { + putchar('"'); + for (const unsigned char *p = (const unsigned char *) s; p && *p; p++) { + if (*p == '"' || *p == '\\') { + putchar('\\'); + putchar(*p); + } else if (*p < 0x20) { + printf("\\u%04x", *p); + } else { + putchar(*p); + } + } + putchar('"'); +} + +static char *read_file(const char *path, size_t *len) { + FILE *f = fopen(path, "rb"); + if (!f) { + perror(path); + exit(2); + } + fseek(f, 0, SEEK_END); + long n = ftell(f); + fseek(f, 0, SEEK_SET); + char *buf = malloc(n + 1); + if (fread(buf, 1, n, f) != (size_t) n) { + perror(path); + exit(2); + } + buf[n] = 0; + fclose(f); + *len = n; + return buf; +} + +int main(int argc, char **argv) { + size_t len; + if (argc != 3) { + fprintf(stderr, "usage: driver split|parse \n"); + return 1; + } + char *buf = read_file(argv[2], &len); + + if (strcmp(argv[1], "split") == 0) { + PgQuerySplitResult r = pg_query_split_with_scanner(buf); + if (r.error) { + printf("{\"error\":"); + print_json_string(r.error->message); + printf("}\n"); + } else { + printf("{\"stmts\":["); + for (int i = 0; i < r.n_stmts; i++) { + int n = r.stmts[i]->stmt_len; + char *stmt = malloc(n + 1); + memcpy(stmt, buf + r.stmts[i]->stmt_location, n); + stmt[n] = 0; + if (i) putchar(','); + print_json_string(stmt); + free(stmt); + } + printf("]}\n"); + } + pg_query_free_split_result(r); + return 0; + } + + if (strcmp(argv[1], "parse") == 0) { + for (char *p = buf; p < buf + len; p += strlen(p) + 1) { + PgQueryParseResult r = pg_query_parse(p); + if (r.error) { + printf("{\"error\":{\"message\":"); + print_json_string(r.error->message); + printf(",\"cursorpos\":%d}}\n", r.error->cursorpos); + } else { + printf("{\"parse\":%s}\n", r.parse_tree); + } + pg_query_free_parse_result(r); + } + return 0; + } + + fprintf(stderr, "unknown mode %s\n", argv[1]); + return 1; +} From 6610a93a656bd417c4cdd19356d5bf90921a0a79 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 03:54:16 +0000 Subject: [PATCH 07/15] fix(v18): throw out of WASM memory from scan instead of returning partial or empty tokens --- templates/full/wasm_wrapper.c | 36 ++++++++++++++++++---------- versions/18/src/wasm_wrapper.c | 36 ++++++++++++++++++---------- versions/18/test/fingerprint.test.js | 6 +++++ versions/18/test/normalize.test.js | 6 +++++ versions/18/test/scan.test.js | 20 ++++++++++++++++ 5 files changed, 80 insertions(+), 24 deletions(-) diff --git a/templates/full/wasm_wrapper.c b/templates/full/wasm_wrapper.c index 81911fab..ec87a0e7 100644 --- a/templates/full/wasm_wrapper.c +++ b/templates/full/wasm_wrapper.c @@ -4,6 +4,7 @@ #include #include #include +#include static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; @@ -208,9 +209,11 @@ static WasmDetailedResult* detailed_set_message(WasmDetailedResult* result, cons return result; } +// Out of memory returns NULL, which the JS side reports as out of WASM memory. static WasmDetailedResult* detailed_set_data(WasmDetailedResult* result, char* data) { if (!data) { - return detailed_set_message(result, "Memory allocation failed"); + wasm_free_detailed_result(result); + return NULL; } result->data = data; result->data_len = strlen(data); @@ -231,7 +234,7 @@ WasmDetailedResult* wasm_fingerprint(const char* input) { if (fingerprint_result.error) { detailed_set_error(result, fingerprint_result.error); } else { - detailed_set_data(result, safe_strdup(fingerprint_result.fingerprint_str)); + result = detailed_set_data(result, safe_strdup(fingerprint_result.fingerprint_str)); } pg_query_free_fingerprint_result(fingerprint_result); return result; @@ -251,7 +254,7 @@ WasmDetailedResult* wasm_normalize_query(const char* input) { if (normalize_result.error) { detailed_set_error(result, normalize_result.error); } else { - detailed_set_data(result, safe_strdup(normalize_result.normalized_query)); + result = detailed_set_data(result, safe_strdup(normalize_result.normalized_query)); } pg_query_free_normalize_result(normalize_result); return result; @@ -293,9 +296,10 @@ static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) { } } +// Returns NULL when out of memory: a partial token list must never pass as a result. static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* original_sql) { if (!scan_result || !original_sql) { - return safe_strdup("{\"version\":0,\"tokens\":[]}"); + return NULL; } // Size the buffer from token text length, not token count: each token @@ -306,11 +310,14 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi PgQuery__ScanToken *token = scan_result->tokens[i]; int token_length = token->end - token->start; if (token_length < 0) token_length = 0; + if ((size_t) token_length > (SIZE_MAX - 200 - estimated_size) / 6) { + return NULL; + } estimated_size += (size_t) token_length * 6 + 200; } char* json = safe_malloc(estimated_size); if (!json) { - return safe_strdup("{\"version\":0,\"tokens\":[]}"); + return NULL; } // Start building JSON @@ -323,8 +330,11 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi int token_length = token->end - token->start; if (token_length < 0) token_length = 0; // Safety check - char* token_text = safe_malloc(token_length + 1); - if (!token_text) continue; + char* token_text = safe_malloc((size_t) token_length + 1); + if (!token_text) { + free(json); + return NULL; + } if (token_length > 0) { strncpy(token_text, &original_sql[token->start], token_length); @@ -332,10 +342,11 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi token_text[token_length] = '\0'; // Escape token text for JSON - char* escaped_text = safe_malloc(token_length * 6 + 1); + char* escaped_text = safe_malloc((size_t) token_length * 6 + 1); if (!escaped_text) { free(token_text); - continue; + free(json); + return NULL; } int escaped_pos = 0; @@ -413,13 +424,14 @@ WasmDetailedResult* wasm_scan(const char* input) { NULL, scan_result.pbuf.len, (void *) scan_result.pbuf.data); if (!unpacked) { pg_query_free_scan_result(scan_result); - return detailed_set_message(result, "Failed to unpack scan result"); + wasm_free_detailed_result(result); + return NULL; } - detailed_set_data(result, build_scan_json(unpacked, input)); + char* json = build_scan_json(unpacked, input); pg_query__scan_result__free_unpacked(unpacked, NULL); pg_query_free_scan_result(scan_result); - return result; + return detailed_set_data(result, json); } EMSCRIPTEN_KEEPALIVE diff --git a/versions/18/src/wasm_wrapper.c b/versions/18/src/wasm_wrapper.c index 2ee54101..e3ea6a92 100644 --- a/versions/18/src/wasm_wrapper.c +++ b/versions/18/src/wasm_wrapper.c @@ -11,6 +11,7 @@ #include #include #include +#include static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; @@ -215,9 +216,11 @@ static WasmDetailedResult* detailed_set_message(WasmDetailedResult* result, cons return result; } +// Out of memory returns NULL, which the JS side reports as out of WASM memory. static WasmDetailedResult* detailed_set_data(WasmDetailedResult* result, char* data) { if (!data) { - return detailed_set_message(result, "Memory allocation failed"); + wasm_free_detailed_result(result); + return NULL; } result->data = data; result->data_len = strlen(data); @@ -238,7 +241,7 @@ WasmDetailedResult* wasm_fingerprint(const char* input) { if (fingerprint_result.error) { detailed_set_error(result, fingerprint_result.error); } else { - detailed_set_data(result, safe_strdup(fingerprint_result.fingerprint_str)); + result = detailed_set_data(result, safe_strdup(fingerprint_result.fingerprint_str)); } pg_query_free_fingerprint_result(fingerprint_result); return result; @@ -258,7 +261,7 @@ WasmDetailedResult* wasm_normalize_query(const char* input) { if (normalize_result.error) { detailed_set_error(result, normalize_result.error); } else { - detailed_set_data(result, safe_strdup(normalize_result.normalized_query)); + result = detailed_set_data(result, safe_strdup(normalize_result.normalized_query)); } pg_query_free_normalize_result(normalize_result); return result; @@ -300,9 +303,10 @@ static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) { } } +// Returns NULL when out of memory: a partial token list must never pass as a result. static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* original_sql) { if (!scan_result || !original_sql) { - return safe_strdup("{\"version\":0,\"tokens\":[]}"); + return NULL; } // Size the buffer from token text length, not token count: each token @@ -313,11 +317,14 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi PgQuery__ScanToken *token = scan_result->tokens[i]; int token_length = token->end - token->start; if (token_length < 0) token_length = 0; + if ((size_t) token_length > (SIZE_MAX - 200 - estimated_size) / 6) { + return NULL; + } estimated_size += (size_t) token_length * 6 + 200; } char* json = safe_malloc(estimated_size); if (!json) { - return safe_strdup("{\"version\":0,\"tokens\":[]}"); + return NULL; } // Start building JSON @@ -330,8 +337,11 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi int token_length = token->end - token->start; if (token_length < 0) token_length = 0; // Safety check - char* token_text = safe_malloc(token_length + 1); - if (!token_text) continue; + char* token_text = safe_malloc((size_t) token_length + 1); + if (!token_text) { + free(json); + return NULL; + } if (token_length > 0) { strncpy(token_text, &original_sql[token->start], token_length); @@ -339,10 +349,11 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi token_text[token_length] = '\0'; // Escape token text for JSON - char* escaped_text = safe_malloc(token_length * 6 + 1); + char* escaped_text = safe_malloc((size_t) token_length * 6 + 1); if (!escaped_text) { free(token_text); - continue; + free(json); + return NULL; } int escaped_pos = 0; @@ -420,13 +431,14 @@ WasmDetailedResult* wasm_scan(const char* input) { NULL, scan_result.pbuf.len, (void *) scan_result.pbuf.data); if (!unpacked) { pg_query_free_scan_result(scan_result); - return detailed_set_message(result, "Failed to unpack scan result"); + wasm_free_detailed_result(result); + return NULL; } - detailed_set_data(result, build_scan_json(unpacked, input)); + char* json = build_scan_json(unpacked, input); pg_query__scan_result__free_unpacked(unpacked, NULL); pg_query_free_scan_result(scan_result); - return result; + return detailed_set_data(result, json); } EMSCRIPTEN_KEEPALIVE diff --git a/versions/18/test/fingerprint.test.js b/versions/18/test/fingerprint.test.js index e3132948..e6ce519a 100644 --- a/versions/18/test/fingerprint.test.js +++ b/versions/18/test/fingerprint.test.js @@ -76,8 +76,14 @@ describe("fingerprint errors", () => { const failing = [ ["select percentile_cont(p order by p) within group (order by x) from t", "cannot use multiple ORDER BY clauses with WITHIN GROUP"], ["CREATE TABLE p (a int) PARTITION BY MAGIC (a)", 'unrecognized partitioning strategy "magic"'], + ["SELECT 'abc", "unterminated quoted string at or near \"'abc\""], ]; + it("throws SqlError for empty input", async () => { + assert.throws(() => query.fingerprintSync(""), { name: "SqlError", message: /^Invalid input/ }); + await assert.rejects(query.fingerprint(""), { name: "SqlError", message: /^Invalid input/ }); + }); + for (const [sql, message] of failing) { it(`fingerprintSync throws SqlError for ${sql}`, () => { assert.throws(() => query.fingerprintSync(sql), (err) => { diff --git a/versions/18/test/normalize.test.js b/versions/18/test/normalize.test.js index 404ee341..a7d996d9 100644 --- a/versions/18/test/normalize.test.js +++ b/versions/18/test/normalize.test.js @@ -78,8 +78,14 @@ describe("normalize errors", () => { const failing = [ ["select percentile_cont(p order by p) within group (order by x) from t", "cannot use multiple ORDER BY clauses with WITHIN GROUP"], ["CREATE TABLE p (a int) PARTITION BY MAGIC (a)", 'unrecognized partitioning strategy "magic"'], + ['SELECT "x', "unterminated quoted identifier at or near \"\"x\""], ]; + it("throws SqlError for empty input", async () => { + assert.throws(() => query.normalizeSync(""), { name: "SqlError", message: /^Invalid input/ }); + await assert.rejects(query.normalize(""), { name: "SqlError", message: /^Invalid input/ }); + }); + for (const [sql, message] of failing) { it(`normalizeSync throws SqlError for ${sql}`, () => { assert.throws(() => query.normalizeSync(sql), (err) => { diff --git a/versions/18/test/scan.test.js b/versions/18/test/scan.test.js index f008c9cd..9f121d21 100644 --- a/versions/18/test/scan.test.js +++ b/versions/18/test/scan.test.js @@ -340,6 +340,26 @@ describe("scan errors", () => { } }); +describe("scan input limits", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + it("throws SqlError for empty input", async () => { + assert.throws(() => query.scanSync(""), { name: "SqlError", message: /^Invalid input/ }); + await assert.rejects(query.scan(""), { name: "SqlError", message: /^Invalid input/ }); + }); + + // About 11 MB: the token JSON no longer fits in the 1 GiB heap. This used to + // return { version: 0, tokens: [] } instead of throwing. + it("throws when the token JSON does not fit in WASM memory, then recovers", async () => { + const sql = "SELECT a, b FROM t WHERE c = 1;\n".repeat(360000); + assert.throws(() => query.scanSync(sql), /out of WASM memory/); + await assert.rejects(query.scan(sql), /out of WASM memory/); + assert.equal(query.scanSync("SELECT 1").tokens.length, 2); + }); +}); + describe("scan control characters", () => { before(async () => { await query.parse("SELECT 1"); From 5d4b70f51089c3a65a00cc2fe60af70fe514941a Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 03:55:23 +0000 Subject: [PATCH 08/15] native-parity: build PG <= 14 sources on glibc >= 2.38; always clean up the temp dir --- scripts/native-parity.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/native-parity.js b/scripts/native-parity.js index f8c7ee09..da4b31e3 100644 --- a/scripts/native-parity.js +++ b/scripts/native-parity.js @@ -25,8 +25,18 @@ async function main() { if (!source) throw new Error(`no libpg_query checkout for ${commit} under ${versionDir}/.cache; run pnpm build first`); const work = fs.mkdtempSync(path.join(os.tmpdir(), 'libpg-query-native-')); + try { + await checkParity(versionDir, source, commit, work); + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +} + +async function checkParity(versionDir, source, commit, work) { execFileSync('sh', ['-c', `git -C "${source}" archive ${commit} | tar -x -C "${work}"`]); - execFileSync('make', ['-j', String(os.cpus().length), 'build'], { cwd: work, stdio: ['ignore', 'ignore', 'inherit'] }); + // glibc >= 2.38 declares strchrnul, which PG <= 14's vendored snprintf.c would otherwise redefine + const cppflags = process.platform === 'linux' ? 'CPPFLAGS=-DHAVE_STRCHRNUL=' : 'CPPFLAGS='; + execFileSync('make', ['-j', String(os.cpus().length), cppflags, 'build'], { cwd: work, stdio: ['ignore', 'ignore', 'inherit'] }); const driver = path.join(work, 'parity-driver'); execFileSync('cc', ['-O2', '-I', work, path.join(__dirname, 'native-parity/driver.c'), path.join(work, 'libpg_query.a'), '-lpthread', '-o', driver]); @@ -66,10 +76,9 @@ async function main() { if (a !== b) diffs.push({ key, native: a.slice(0, 300), wasm: b.slice(0, 300) }); }); - fs.rmSync(work, { recursive: true, force: true }); console.log(`native parity: ${statements.length} statements from ${files.length} files @ ${commit.slice(0, 7)}, ${diffs.length} differences`); for (const d of diffs.slice(0, 10)) console.log(JSON.stringify(d, null, 2)); - if (diffs.length) process.exit(1); + if (diffs.length) process.exitCode = 1; } main().catch((err) => { From ab4aedb60d22f665eb60cf0dc59d5035fb747393 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 03:58:12 +0000 Subject: [PATCH 09/15] test(v18): scan of 6M short tokens throws out of WASM memory --- versions/18/test/scan.test.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/versions/18/test/scan.test.js b/versions/18/test/scan.test.js index 9f121d21..c96ef4eb 100644 --- a/versions/18/test/scan.test.js +++ b/versions/18/test/scan.test.js @@ -358,6 +358,12 @@ describe("scan input limits", () => { await assert.rejects(query.scan(sql), /out of WASM memory/); assert.equal(query.scanSync("SELECT 1").tokens.length, 2); }); + + // 6M one-character tokens (20 MB) used to return tokens: [] as well. + it("throws for many short tokens instead of returning none", () => { + assert.equal(query.scanSync("SELECT 1;\n".repeat(1000000)).tokens.length, 3000000); + assert.throws(() => query.scanSync("SELECT 1;\n".repeat(2000000)), /out of WASM memory/); + }); }); describe("scan control characters", () => { From b30efdb076bc3bb98c8f665d042f302f28472531 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 04:12:06 +0000 Subject: [PATCH 10/15] fix(v18): pin libpg_query with checked scan allocations; report its out of memory as out of WASM memory --- templates/full/index.ts | 8 +++++++- versions/18/Makefile | 2 +- versions/18/package.json | 2 +- versions/18/src/index.ts | 8 +++++++- versions/18/test/scan.test.js | 29 +++++++++++++++++++++++++++++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/templates/full/index.ts b/templates/full/index.ts index 767089ae..a47644a4 100644 --- a/templates/full/index.ts +++ b/templates/full/index.ts @@ -245,14 +245,20 @@ function ptrToString(ptr: number): string { return wasmModule.UTF8ToString(ptr); } +const NO_RESULT_OUT_OF_MEMORY = 'libpg_query returned no result: out of WASM memory (heap limit is WASM_MAXIMUM_MEMORY, 1 GiB); parse large inputs in smaller pieces'; + function readDetailedResult(resultPtr: number): string { if (!resultPtr) { - throw new Error('libpg_query returned no result: out of WASM memory (heap limit is WASM_MAXIMUM_MEMORY, 1 GiB); parse large inputs in smaller pieces'); + throw new Error(NO_RESULT_OUT_OF_MEMORY); } const ptrAt = (offset: number) => wasmModule.getValue(resultPtr + offset, 'i32'); const str = (ptr: number) => (ptr ? wasmModule.UTF8ToString(ptr) : undefined); if (ptrAt(0)) { const message = str(ptrAt(4)) ?? 'Unknown error'; + // libpg_query reports a failed malloc as a plain "out of memory" error. + if (message === 'out of memory') { + throw new Error(NO_RESULT_OUT_OF_MEMORY); + } const lineno = ptrAt(16); const cursorpos = ptrAt(20); throw new SqlError(message, { diff --git a/versions/18/Makefile b/versions/18/Makefile index 268e2f15..ebe0b546 100644 --- a/versions/18/Makefile +++ b/versions/18/Makefile @@ -19,7 +19,7 @@ WASM_STACK_SIZE ?= 33554432 # byte-for-byte. Set in package.json `x-publish` and applied by copy:templates. LIBPG_QUERY_REPO := https://github.com/constructive-io/libpg_query.git LIBPG_QUERY_TAG := 18-constructive -LIBPG_QUERY_COMMIT := b0f5f331dc50d69597ec610de7b21afa47a66808 +LIBPG_QUERY_COMMIT := 8076f09aecc655a717e4a038e182a9dd41cc3eb8 CACHE_DIR := .cache diff --git a/versions/18/package.json b/versions/18/package.json index 1ee183f3..be6400d3 100644 --- a/versions/18/package.json +++ b/versions/18/package.json @@ -15,7 +15,7 @@ "distTag": "pg18", "libpgQueryRepo": "https://github.com/constructive-io/libpg_query.git", "libpgQueryTag": "18-constructive", - "libpgQueryCommit": "b0f5f331dc50d69597ec610de7b21afa47a66808", + "libpgQueryCommit": "8076f09aecc655a717e4a038e182a9dd41cc3eb8", "fullApi": true }, "files": [ diff --git a/versions/18/src/index.ts b/versions/18/src/index.ts index 31ce9660..badaa198 100644 --- a/versions/18/src/index.ts +++ b/versions/18/src/index.ts @@ -252,14 +252,20 @@ function ptrToString(ptr: number): string { return wasmModule.UTF8ToString(ptr); } +const NO_RESULT_OUT_OF_MEMORY = 'libpg_query returned no result: out of WASM memory (heap limit is WASM_MAXIMUM_MEMORY, 1 GiB); parse large inputs in smaller pieces'; + function readDetailedResult(resultPtr: number): string { if (!resultPtr) { - throw new Error('libpg_query returned no result: out of WASM memory (heap limit is WASM_MAXIMUM_MEMORY, 1 GiB); parse large inputs in smaller pieces'); + throw new Error(NO_RESULT_OUT_OF_MEMORY); } const ptrAt = (offset: number) => wasmModule.getValue(resultPtr + offset, 'i32'); const str = (ptr: number) => (ptr ? wasmModule.UTF8ToString(ptr) : undefined); if (ptrAt(0)) { const message = str(ptrAt(4)) ?? 'Unknown error'; + // libpg_query reports a failed malloc as a plain "out of memory" error. + if (message === 'out of memory') { + throw new Error(NO_RESULT_OUT_OF_MEMORY); + } const lineno = ptrAt(16); const cursorpos = ptrAt(20); throw new SqlError(message, { diff --git a/versions/18/test/scan.test.js b/versions/18/test/scan.test.js index c96ef4eb..c23fd65c 100644 --- a/versions/18/test/scan.test.js +++ b/versions/18/test/scan.test.js @@ -1,6 +1,7 @@ const query = require("../"); const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); describe("Query Scanning", () => { before(async () => { @@ -364,6 +365,34 @@ describe("scan input limits", () => { assert.equal(query.scanSync("SELECT 1;\n".repeat(1000000)).tokens.length, 3000000); assert.throws(() => query.scanSync("SELECT 1;\n".repeat(2000000)), /out of WASM memory/); }); + + // About 50 MB (18.6M tokens) used to make libpg_query write the token protobuf + // through a NULL pointer, abort, and break the next sync call. Runs in a child + // process because the abort corrupts the instance. + it("throws out of WASM memory at 50 MB, then sync and async calls work", () => { + const script = `const query = require(${JSON.stringify(require.resolve("../"))}); + (async () => { + await query.parse("SELECT 1"); + let message = "no error"; + try { + query.scanSync("SELECT a, b FROM t WHERE c = 1;\\n".repeat(1690000)); + } catch (error) { + message = error.message; + } + const sync = query.scanSync("SELECT 1").tokens.length; + const async = (await query.scan("SELECT 1")).tokens.length; + process.stdout.write("\\nresult=" + JSON.stringify({ message, sync, async })); + })();`; + const output = execFileSync(process.execPath, ["-e", script], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + maxBuffer: 256 * 1024 * 1024, + }); + const result = JSON.parse(output.match(/^result=(.*)$/m)[1]); + assert.match(result.message, /out of WASM memory/); + assert.equal(result.sync, 2); + assert.equal(result.async, 2); + }); }); describe("scan control characters", () => { From 564154f2174eb19fec8da38518fe90fec124721a Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 04:18:19 +0000 Subject: [PATCH 11/15] fix(wasm): replace a failed instance synchronously so the next call never runs on a broken one --- templates/full/index.ts | 120 +++++++++++++--------- templates/index.ts | 100 +++++++++++------- versions/13/src/index.ts | 100 +++++++++++------- versions/13/test/wasm-recovery.test.js | 136 +++++++++++-------------- versions/14/src/index.ts | 100 +++++++++++------- versions/14/test/wasm-recovery.test.js | 136 +++++++++++-------------- versions/15/src/index.ts | 100 +++++++++++------- versions/15/test/wasm-recovery.test.js | 136 +++++++++++-------------- versions/16/src/index.ts | 100 +++++++++++------- versions/16/test/wasm-recovery.test.js | 136 +++++++++++-------------- versions/17/src/index.ts | 100 +++++++++++------- versions/17/test/wasm-recovery.test.js | 136 +++++++++++-------------- versions/18/src/index.ts | 120 +++++++++++++--------- versions/18/test/wasm-recovery.test.js | 136 +++++++++++-------------- 14 files changed, 896 insertions(+), 760 deletions(-) diff --git a/templates/full/index.ts b/templates/full/index.ts index e97b6096..8f1b971d 100644 --- a/templates/full/index.ts +++ b/templates/full/index.ts @@ -136,44 +136,71 @@ interface WasmModule { let wasmModule: WasmModule; -// After a WASM call throws, the instance is stale: async calls wait for its -// replacement, while sync calls, which cannot wait, keep using it until the -// replacement is ready. One replacement loads at a time; the next call retries -// a failed load. -let stale = false; +// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a +// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and +// can leave the instance corrupt. callWasm() therefore swaps in a fresh instance +// before rethrowing. It builds that instance synchronously from the compiled +// module, so the very next call, sync or async, already runs on a clean one. +let compiled: WebAssembly.Module | undefined; let loading: Promise | undefined; -// Counts WASM calls that threw, so cleanup skips an instance that threw mid-call. +// Counts WASM calls that threw, so cleanup skips memory of the replaced instance. let failures = 0; -// Sync calls stop using a stale instance after this many failures on it: past -// about 45, a stack overflow can make the next one spin forever. -const MAX_STALE_FAILURES = 10; -let staleFailures = 0; +// Why the last synchronous replacement failed; the next loadModule() retries. +let replaceError: unknown; + +// Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. +async function compileWasm(url: string): Promise { + const proc = (globalThis as any).process; + if (proc?.versions?.node && proc.type !== 'renderer') { + return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + } + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); + return WebAssembly.compile(await response.arrayBuffer()); +} function instantiate(): Promise { if (loading) return loading; - const promise = PgQueryModule().then( - (module: WasmModule) => { - wasmModule = module; - stale = false; - staleFailures = 0; - loading = undefined; - }, - (error: unknown) => { - loading = undefined; - throw error; - } - ); + const promise = (async () => { + let wasmUrl: string | undefined; + const module: WasmModule = await PgQueryModule({ + locateFile: (path: string, dir: string) => (wasmUrl = dir + path), + }); + compiled ??= await compileWasm(wasmUrl); + wasmModule = module; + replaceError = undefined; + })(); loading = promise; - // Rejects only for the callers awaiting it. - promise.catch(() => {}); + promise.then( + () => { loading = undefined; }, + () => { loading = undefined; } + ); return promise; } -instantiate(); +// Rejects only for the callers awaiting it. +instantiate().catch(() => {}); + +function replaceInstance(): void { + let error: unknown = new Error('the WASM loader did not instantiate synchronously'); + const module: any = { + instantiateWasm(imports: WebAssembly.Imports, receive: (instance: WebAssembly.Instance) => void) { + try { + receive(new WebAssembly.Instance(compiled, imports)); + error = undefined; + } catch (e) { + error = e; + } + return {}; + }, + }; + // receive() sets the exports and memory views, so the instance is usable right + // away; the loader runs the rest of its start-up (stdio streams) on a later microtask. + PgQueryModule(module).catch(() => {}); + wasmModule = error === undefined ? module : undefined; + replaceError = error; +} -// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a -// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and -// leaves the instance corrupt, so replace it with a fresh one. function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; @@ -181,9 +208,7 @@ function callWasm(fn: () => number): number { return fn(); } catch (error) { failures++; - stale = true; - if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; - instantiate(); + replaceInstance(); if (proc) proc.exitCode = exitCode; if (error instanceof Error) throw error; const reason = typeof error === 'object' && error !== null @@ -197,17 +222,22 @@ function callWasm(fn: () => number): number { } function ensureLoaded() { - if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); + // A replacement that failed to start is retried here, for callers that never await. + if (!wasmModule && compiled) replaceInstance(); + if (wasmModule) return; + if (replaceError !== undefined) { + throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); + } + throw new Error("WASM module not initialized. Call `loadModule()` first."); } export async function loadModule(): Promise { - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); } function awaitInit Promise>(fn: T): T { return (async (...args: Parameters) => { - // Re-checked after every await: another call may have failed in between. - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); return fn(...args); }) as T; } @@ -381,9 +411,7 @@ export const normalize = awaitInit(async (query: string): Promise => { // Sync versions export function parseSync(query: string): ParseResult { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); // Input validation if (query === null || query === undefined) { @@ -447,9 +475,7 @@ export function parseSync(query: string): ParseResult { } export function parsePlPgSQLSync(query: string): ParseResult { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); const queryPtr = stringToPtr(query); const failuresBefore = failures; let resultPtr = 0; @@ -475,9 +501,7 @@ export function parsePlPgSQLSync(query: string): ParseResult { } export function fingerprintSync(query: string): string { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); const queryPtr = stringToPtr(query); const failuresBefore = failures; let resultPtr = 0; @@ -502,9 +526,7 @@ export function fingerprintSync(query: string): string { } export function normalizeSync(query: string): string { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); const queryPtr = stringToPtr(query); const failuresBefore = failures; let resultPtr = 0; @@ -553,9 +575,7 @@ export const scan = awaitInit(async (query: string): Promise => { }); export function scanSync(query: string): ScanResult { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); const queryPtr = stringToPtr(query); const failuresBefore = failures; let resultPtr = 0; diff --git a/templates/index.ts b/templates/index.ts index 4eb97848..4a86e126 100644 --- a/templates/index.ts +++ b/templates/index.ts @@ -124,44 +124,71 @@ export function hasSqlDetails(error: any): error is Error & { sqlDetails: SqlErr 'cursorPosition' in (error as any).sqlDetails; } -// After a WASM call throws, the instance is stale: async calls wait for its -// replacement, while sync calls, which cannot wait, keep using it until the -// replacement is ready. One replacement loads at a time; the next call retries -// a failed load. -let stale = false; +// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a +// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and +// can leave the instance corrupt. callWasm() therefore swaps in a fresh instance +// before rethrowing. It builds that instance synchronously from the compiled +// module, so the very next call, sync or async, already runs on a clean one. +let compiled: WebAssembly.Module | undefined; let loading: Promise | undefined; -// Counts WASM calls that threw, so cleanup skips an instance that threw mid-call. +// Counts WASM calls that threw, so cleanup skips memory of the replaced instance. let failures = 0; -// Sync calls stop using a stale instance after this many failures on it: past -// about 45, a stack overflow can make the next one spin forever. -const MAX_STALE_FAILURES = 10; -let staleFailures = 0; +// Why the last synchronous replacement failed; the next loadModule() retries. +let replaceError: unknown; + +// Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. +async function compileWasm(url: string): Promise { + const proc = (globalThis as any).process; + if (proc?.versions?.node && proc.type !== 'renderer') { + return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + } + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); + return WebAssembly.compile(await response.arrayBuffer()); +} function instantiate(): Promise { if (loading) return loading; - const promise = PgQueryModule().then( - (module: any) => { - wasmModule = module; - stale = false; - staleFailures = 0; - loading = undefined; - }, - (error: unknown) => { - loading = undefined; - throw error; - } - ); + const promise = (async () => { + let wasmUrl: string | undefined; + const module: any = await PgQueryModule({ + locateFile: (path: string, dir: string) => (wasmUrl = dir + path), + }); + compiled ??= await compileWasm(wasmUrl); + wasmModule = module; + replaceError = undefined; + })(); loading = promise; - // Rejects only for the callers awaiting it. - promise.catch(() => {}); + promise.then( + () => { loading = undefined; }, + () => { loading = undefined; } + ); return promise; } -instantiate(); +// Rejects only for the callers awaiting it. +instantiate().catch(() => {}); + +function replaceInstance(): void { + let error: unknown = new Error('the WASM loader did not instantiate synchronously'); + const module: any = { + instantiateWasm(imports: WebAssembly.Imports, receive: (instance: WebAssembly.Instance) => void) { + try { + receive(new WebAssembly.Instance(compiled, imports)); + error = undefined; + } catch (e) { + error = e; + } + return {}; + }, + }; + // receive() sets the exports and memory views, so the instance is usable right + // away; the loader runs the rest of its start-up (stdio streams) on a later microtask. + PgQueryModule(module).catch(() => {}); + wasmModule = error === undefined ? module : undefined; + replaceError = error; +} -// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a -// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and -// leaves the instance corrupt, so replace it with a fresh one. function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; @@ -169,9 +196,7 @@ function callWasm(fn: () => number): number { return fn(); } catch (error) { failures++; - stale = true; - if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; - instantiate(); + replaceInstance(); if (proc) proc.exitCode = exitCode; if (error instanceof Error) throw error; const reason = typeof error === 'object' && error !== null @@ -185,17 +210,22 @@ function callWasm(fn: () => number): number { } function ensureLoaded() { - if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); + // A replacement that failed to start is retried here, for callers that never await. + if (!wasmModule && compiled) replaceInstance(); + if (wasmModule) return; + if (replaceError !== undefined) { + throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); + } + throw new Error("WASM module not initialized. Call `loadModule()` first."); } export async function loadModule() { - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); } function awaitInit any>(fn: T): T { return (async (...args: Parameters) => { - // Re-checked after every await: another call may have failed in between. - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); return fn(...args); }) as T; } diff --git a/versions/13/src/index.ts b/versions/13/src/index.ts index f39ac71c..40babf7c 100644 --- a/versions/13/src/index.ts +++ b/versions/13/src/index.ts @@ -131,44 +131,71 @@ export function hasSqlDetails(error: any): error is Error & { sqlDetails: SqlErr 'cursorPosition' in (error as any).sqlDetails; } -// After a WASM call throws, the instance is stale: async calls wait for its -// replacement, while sync calls, which cannot wait, keep using it until the -// replacement is ready. One replacement loads at a time; the next call retries -// a failed load. -let stale = false; +// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a +// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and +// can leave the instance corrupt. callWasm() therefore swaps in a fresh instance +// before rethrowing. It builds that instance synchronously from the compiled +// module, so the very next call, sync or async, already runs on a clean one. +let compiled: WebAssembly.Module | undefined; let loading: Promise | undefined; -// Counts WASM calls that threw, so cleanup skips an instance that threw mid-call. +// Counts WASM calls that threw, so cleanup skips memory of the replaced instance. let failures = 0; -// Sync calls stop using a stale instance after this many failures on it: past -// about 45, a stack overflow can make the next one spin forever. -const MAX_STALE_FAILURES = 10; -let staleFailures = 0; +// Why the last synchronous replacement failed; the next loadModule() retries. +let replaceError: unknown; + +// Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. +async function compileWasm(url: string): Promise { + const proc = (globalThis as any).process; + if (proc?.versions?.node && proc.type !== 'renderer') { + return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + } + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); + return WebAssembly.compile(await response.arrayBuffer()); +} function instantiate(): Promise { if (loading) return loading; - const promise = PgQueryModule().then( - (module: any) => { - wasmModule = module; - stale = false; - staleFailures = 0; - loading = undefined; - }, - (error: unknown) => { - loading = undefined; - throw error; - } - ); + const promise = (async () => { + let wasmUrl: string | undefined; + const module: any = await PgQueryModule({ + locateFile: (path: string, dir: string) => (wasmUrl = dir + path), + }); + compiled ??= await compileWasm(wasmUrl); + wasmModule = module; + replaceError = undefined; + })(); loading = promise; - // Rejects only for the callers awaiting it. - promise.catch(() => {}); + promise.then( + () => { loading = undefined; }, + () => { loading = undefined; } + ); return promise; } -instantiate(); +// Rejects only for the callers awaiting it. +instantiate().catch(() => {}); + +function replaceInstance(): void { + let error: unknown = new Error('the WASM loader did not instantiate synchronously'); + const module: any = { + instantiateWasm(imports: WebAssembly.Imports, receive: (instance: WebAssembly.Instance) => void) { + try { + receive(new WebAssembly.Instance(compiled, imports)); + error = undefined; + } catch (e) { + error = e; + } + return {}; + }, + }; + // receive() sets the exports and memory views, so the instance is usable right + // away; the loader runs the rest of its start-up (stdio streams) on a later microtask. + PgQueryModule(module).catch(() => {}); + wasmModule = error === undefined ? module : undefined; + replaceError = error; +} -// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a -// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and -// leaves the instance corrupt, so replace it with a fresh one. function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; @@ -176,9 +203,7 @@ function callWasm(fn: () => number): number { return fn(); } catch (error) { failures++; - stale = true; - if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; - instantiate(); + replaceInstance(); if (proc) proc.exitCode = exitCode; if (error instanceof Error) throw error; const reason = typeof error === 'object' && error !== null @@ -192,17 +217,22 @@ function callWasm(fn: () => number): number { } function ensureLoaded() { - if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); + // A replacement that failed to start is retried here, for callers that never await. + if (!wasmModule && compiled) replaceInstance(); + if (wasmModule) return; + if (replaceError !== undefined) { + throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); + } + throw new Error("WASM module not initialized. Call `loadModule()` first."); } export async function loadModule() { - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); } function awaitInit any>(fn: T): T { return (async (...args: Parameters) => { - // Re-checked after every await: another call may have failed in between. - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); return fn(...args); }) as T; } diff --git a/versions/13/test/wasm-recovery.test.js b/versions/13/test/wasm-recovery.test.js index 6d16d379..98f6dc99 100644 --- a/versions/13/test/wasm-recovery.test.js +++ b/versions/13/test/wasm-recovery.test.js @@ -22,21 +22,24 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1').stmts.length, 1); }); - // Without the 10-failure cap, about 45 overflows on one instance made the next one spin forever. - it('stops sync calls on the stale instance after 10 real overflows in one synchronous loop', async () => { + it('keeps sync calls working through 60 overflows in one synchronous loop', async () => { await loadModule(); - const errors = []; + const results = []; for (let i = 0; i < 60; i++) { - try { - parseSync(DEEP); - } catch (error) { - errors.push(error instanceof RangeError ? 'overflow' : error.message); + for (const query of [DEEP, 'SELECT 1']) { + try { + results.push(parseSync(query).stmts.length); + } catch (error) { + results.push(error instanceof RangeError ? 'overflow' : error.message); + } } } - assert.deepEqual(errors.slice(0, 10), Array(10).fill('overflow')); - for (const message of errors.slice(10)) assert.match(message, /Call `?loadModule\(\)`? first/); - await loadModule(); - assert.equal(parseSync('SELECT 1').stmts.length, 1); + assert.deepEqual(results, Array(60).fill(['overflow', 1]).flat()); + }); + + it('gives the next sync call a fresh instance after the memory limit is hit', () => { + assert.throws(() => parseSync('SELECT 1;\n'.repeat(5000000)), /out of WASM memory/); + assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); it('frees each replaced instance, so WASM memory stays at one instance', () => { @@ -118,16 +121,22 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(loads, 3); }); - it('lets loadModule() reject on a failed reload and retry on the next call', async () => { + it('names the cause when a fresh instance cannot start, and loadModule() retries', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); + const isolated = loadIsolated((real, options) => { + if (++loads === 2 || loads === 3) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await assert.rejects(isolated.loadModule(), /simulated load failure/); + assert.throws(() => isolated.parseSync(DEEP), RangeError); + assert.throws(() => isolated.parseSync('SELECT 1'), (error) => { + assert.match(error.message, /Could not start a fresh WASM instance/); + assert.match(error.cause.message, /simulated instantiate failure/); + return true; + }); await isolated.loadModule(); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); - assert.equal(loads, 3); + assert.equal(loads, 4); }); it('keeps sync calls in the same synchronous loop working after a failure', () => { @@ -145,18 +154,16 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.deepEqual(results, expected); }); - it('retries a failed fresh load on the next failure, for callers that never await', async () => { + it('retries a fresh instance that failed to start on the next sync call', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); - await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(loads, 2); - assert.throws(() => isolated.parseSync(DEEP)); - assert.equal(loads, 3); + const isolated = loadIsolated((real, options) => { + if (++loads === 2) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); + assert.throws(() => isolated.parseSync(DEEP), RangeError); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); + assert.equal(loads, 3); }); it('makes loadModule() wait again when a queued call breaks the fresh instance', async () => { @@ -201,9 +208,15 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => let used; let frees = 0; let throwNext; - const isolated = loadIsolated(async (real, ...args) => { + const isolated = loadIsolated((real, options) => { const id = ++loads; - const module = await real(...args); + const ready = real(options); + // A replacement instance is built synchronously; the first load is async. + if (typeof options._malloc === 'function') return instrument(options, id) && ready; + return ready.then((module) => instrument(module, id)); + }); + + function instrument(module, id) { const free = module._free; module._free = (ptr) => { frees++; @@ -221,7 +234,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => }; } return module; - }); + } async function failOnce(name, query, thrown) { await isolated.loadModule(); @@ -242,73 +255,38 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => assert.equal(exitCodeAfter, 0, `${name} changed process.exitCode`); assert.equal(frees, freesBefore, `${name} freed memory in the instance that threw`); assert.equal(loads, loadsBefore + 1, `${name} did not start a fresh instance`); - const stale = used; const syncName = name.endsWith('Sync') ? name : 'parseSync'; - assert.ok(isolated[syncName](query), `${syncName} failed while the fresh instance loads`); - assert.equal(used, stale, 'a sync call did not run on the stale instance'); + assert.ok(isolated[syncName](query), `${syncName} failed right after the failure`); + assert.equal(used, loads, `${syncName} ran on the instance that threw`); assert.ok(frees > freesBefore, 'a successful call did not free its memory'); - // A sync caller gets the fresh instance once it awaits loadModule(); async calls wait by themselves. - if (name.endsWith('Sync')) await isolated.loadModule(); assert.ok(await isolated[name](query), `${name} failed after recovery`); - assert.equal(used, loads, `${name} ran on the stale instance, not the fresh one`); + assert.equal(used, loads, `${name} ran on the instance that threw`); return caught; } - it('starts one fresh instance for many failures in one synchronous loop', async () => { + it('starts a fresh instance for every failure in one synchronous loop', async () => { await isolated.loadModule(); const loadsBefore = loads; - for (let i = 0; i < 5; i++) { - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - } - assert.equal(loads, loadsBefore + 1); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); - }); - - it('stops sync calls on a stale instance after 10 failures until the fresh one loads', async () => { - await isolated.loadModule(); - for (let i = 0; i < 9; i++) { + for (let i = 0; i < 30; i++) { throwNext = new RangeError('Maximum call stack size exceeded'); assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); + assert.ok(isolated.parseSync('SELECT 1')); + assert.equal(used, loads, 'a sync call ran on the instance that threw'); } - assert.ok(isolated.parseSync('SELECT 1')); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - assert.throws(() => isolated.parseSync('SELECT 1'), /Call `?loadModule\(\)`? first/); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); + assert.equal(loads, loadsBefore + 30); }); - it('makes loadModule() and queued calls wait again when a queued call breaks the fresh instance', async () => { + it('never runs a queued call on the instance that threw', async () => { await isolated.loadModule(); throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); const after = isolated.parse('SELECT 1'); await assert.rejects(failing, RangeError); assert.ok(await after); - assert.equal(used, loads, 'a queued call ran on the instance that just threw'); - await ready; - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); - }); - - it('makes loadModule() wait again when a call queued ahead of it breaks the fresh instance', async () => { + assert.equal(used, loads, 'a queued call ran on the instance that threw'); await isolated.loadModule(); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); - await assert.rejects(failing, RangeError); - await ready; assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); + assert.equal(used, loads); }); it('names a thrown object that has no message', async () => { @@ -335,6 +313,14 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => } }); +// Makes the loader's synchronous instantiation fail, as if the new instance's memory could not be allocated. +function failInstantiate(options) { + const instantiateWasm = options.instantiateWasm; + if (!instantiateWasm) return; + options.instantiateWasm = (imports) => + instantiateWasm(imports, () => { throw new Error('simulated instantiate failure'); }); +} + // A copy of the wrapper whose WASM factory is wrapped, leaving the shared copy alone. function loadIsolated(wrapFactory) { const indexPath = require.resolve('../wasm/index.cjs'); diff --git a/versions/14/src/index.ts b/versions/14/src/index.ts index f39ac71c..40babf7c 100644 --- a/versions/14/src/index.ts +++ b/versions/14/src/index.ts @@ -131,44 +131,71 @@ export function hasSqlDetails(error: any): error is Error & { sqlDetails: SqlErr 'cursorPosition' in (error as any).sqlDetails; } -// After a WASM call throws, the instance is stale: async calls wait for its -// replacement, while sync calls, which cannot wait, keep using it until the -// replacement is ready. One replacement loads at a time; the next call retries -// a failed load. -let stale = false; +// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a +// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and +// can leave the instance corrupt. callWasm() therefore swaps in a fresh instance +// before rethrowing. It builds that instance synchronously from the compiled +// module, so the very next call, sync or async, already runs on a clean one. +let compiled: WebAssembly.Module | undefined; let loading: Promise | undefined; -// Counts WASM calls that threw, so cleanup skips an instance that threw mid-call. +// Counts WASM calls that threw, so cleanup skips memory of the replaced instance. let failures = 0; -// Sync calls stop using a stale instance after this many failures on it: past -// about 45, a stack overflow can make the next one spin forever. -const MAX_STALE_FAILURES = 10; -let staleFailures = 0; +// Why the last synchronous replacement failed; the next loadModule() retries. +let replaceError: unknown; + +// Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. +async function compileWasm(url: string): Promise { + const proc = (globalThis as any).process; + if (proc?.versions?.node && proc.type !== 'renderer') { + return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + } + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); + return WebAssembly.compile(await response.arrayBuffer()); +} function instantiate(): Promise { if (loading) return loading; - const promise = PgQueryModule().then( - (module: any) => { - wasmModule = module; - stale = false; - staleFailures = 0; - loading = undefined; - }, - (error: unknown) => { - loading = undefined; - throw error; - } - ); + const promise = (async () => { + let wasmUrl: string | undefined; + const module: any = await PgQueryModule({ + locateFile: (path: string, dir: string) => (wasmUrl = dir + path), + }); + compiled ??= await compileWasm(wasmUrl); + wasmModule = module; + replaceError = undefined; + })(); loading = promise; - // Rejects only for the callers awaiting it. - promise.catch(() => {}); + promise.then( + () => { loading = undefined; }, + () => { loading = undefined; } + ); return promise; } -instantiate(); +// Rejects only for the callers awaiting it. +instantiate().catch(() => {}); + +function replaceInstance(): void { + let error: unknown = new Error('the WASM loader did not instantiate synchronously'); + const module: any = { + instantiateWasm(imports: WebAssembly.Imports, receive: (instance: WebAssembly.Instance) => void) { + try { + receive(new WebAssembly.Instance(compiled, imports)); + error = undefined; + } catch (e) { + error = e; + } + return {}; + }, + }; + // receive() sets the exports and memory views, so the instance is usable right + // away; the loader runs the rest of its start-up (stdio streams) on a later microtask. + PgQueryModule(module).catch(() => {}); + wasmModule = error === undefined ? module : undefined; + replaceError = error; +} -// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a -// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and -// leaves the instance corrupt, so replace it with a fresh one. function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; @@ -176,9 +203,7 @@ function callWasm(fn: () => number): number { return fn(); } catch (error) { failures++; - stale = true; - if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; - instantiate(); + replaceInstance(); if (proc) proc.exitCode = exitCode; if (error instanceof Error) throw error; const reason = typeof error === 'object' && error !== null @@ -192,17 +217,22 @@ function callWasm(fn: () => number): number { } function ensureLoaded() { - if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); + // A replacement that failed to start is retried here, for callers that never await. + if (!wasmModule && compiled) replaceInstance(); + if (wasmModule) return; + if (replaceError !== undefined) { + throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); + } + throw new Error("WASM module not initialized. Call `loadModule()` first."); } export async function loadModule() { - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); } function awaitInit any>(fn: T): T { return (async (...args: Parameters) => { - // Re-checked after every await: another call may have failed in between. - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); return fn(...args); }) as T; } diff --git a/versions/14/test/wasm-recovery.test.js b/versions/14/test/wasm-recovery.test.js index 6d16d379..98f6dc99 100644 --- a/versions/14/test/wasm-recovery.test.js +++ b/versions/14/test/wasm-recovery.test.js @@ -22,21 +22,24 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1').stmts.length, 1); }); - // Without the 10-failure cap, about 45 overflows on one instance made the next one spin forever. - it('stops sync calls on the stale instance after 10 real overflows in one synchronous loop', async () => { + it('keeps sync calls working through 60 overflows in one synchronous loop', async () => { await loadModule(); - const errors = []; + const results = []; for (let i = 0; i < 60; i++) { - try { - parseSync(DEEP); - } catch (error) { - errors.push(error instanceof RangeError ? 'overflow' : error.message); + for (const query of [DEEP, 'SELECT 1']) { + try { + results.push(parseSync(query).stmts.length); + } catch (error) { + results.push(error instanceof RangeError ? 'overflow' : error.message); + } } } - assert.deepEqual(errors.slice(0, 10), Array(10).fill('overflow')); - for (const message of errors.slice(10)) assert.match(message, /Call `?loadModule\(\)`? first/); - await loadModule(); - assert.equal(parseSync('SELECT 1').stmts.length, 1); + assert.deepEqual(results, Array(60).fill(['overflow', 1]).flat()); + }); + + it('gives the next sync call a fresh instance after the memory limit is hit', () => { + assert.throws(() => parseSync('SELECT 1;\n'.repeat(5000000)), /out of WASM memory/); + assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); it('frees each replaced instance, so WASM memory stays at one instance', () => { @@ -118,16 +121,22 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(loads, 3); }); - it('lets loadModule() reject on a failed reload and retry on the next call', async () => { + it('names the cause when a fresh instance cannot start, and loadModule() retries', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); + const isolated = loadIsolated((real, options) => { + if (++loads === 2 || loads === 3) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await assert.rejects(isolated.loadModule(), /simulated load failure/); + assert.throws(() => isolated.parseSync(DEEP), RangeError); + assert.throws(() => isolated.parseSync('SELECT 1'), (error) => { + assert.match(error.message, /Could not start a fresh WASM instance/); + assert.match(error.cause.message, /simulated instantiate failure/); + return true; + }); await isolated.loadModule(); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); - assert.equal(loads, 3); + assert.equal(loads, 4); }); it('keeps sync calls in the same synchronous loop working after a failure', () => { @@ -145,18 +154,16 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.deepEqual(results, expected); }); - it('retries a failed fresh load on the next failure, for callers that never await', async () => { + it('retries a fresh instance that failed to start on the next sync call', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); - await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(loads, 2); - assert.throws(() => isolated.parseSync(DEEP)); - assert.equal(loads, 3); + const isolated = loadIsolated((real, options) => { + if (++loads === 2) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); + assert.throws(() => isolated.parseSync(DEEP), RangeError); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); + assert.equal(loads, 3); }); it('makes loadModule() wait again when a queued call breaks the fresh instance', async () => { @@ -201,9 +208,15 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => let used; let frees = 0; let throwNext; - const isolated = loadIsolated(async (real, ...args) => { + const isolated = loadIsolated((real, options) => { const id = ++loads; - const module = await real(...args); + const ready = real(options); + // A replacement instance is built synchronously; the first load is async. + if (typeof options._malloc === 'function') return instrument(options, id) && ready; + return ready.then((module) => instrument(module, id)); + }); + + function instrument(module, id) { const free = module._free; module._free = (ptr) => { frees++; @@ -221,7 +234,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => }; } return module; - }); + } async function failOnce(name, query, thrown) { await isolated.loadModule(); @@ -242,73 +255,38 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => assert.equal(exitCodeAfter, 0, `${name} changed process.exitCode`); assert.equal(frees, freesBefore, `${name} freed memory in the instance that threw`); assert.equal(loads, loadsBefore + 1, `${name} did not start a fresh instance`); - const stale = used; const syncName = name.endsWith('Sync') ? name : 'parseSync'; - assert.ok(isolated[syncName](query), `${syncName} failed while the fresh instance loads`); - assert.equal(used, stale, 'a sync call did not run on the stale instance'); + assert.ok(isolated[syncName](query), `${syncName} failed right after the failure`); + assert.equal(used, loads, `${syncName} ran on the instance that threw`); assert.ok(frees > freesBefore, 'a successful call did not free its memory'); - // A sync caller gets the fresh instance once it awaits loadModule(); async calls wait by themselves. - if (name.endsWith('Sync')) await isolated.loadModule(); assert.ok(await isolated[name](query), `${name} failed after recovery`); - assert.equal(used, loads, `${name} ran on the stale instance, not the fresh one`); + assert.equal(used, loads, `${name} ran on the instance that threw`); return caught; } - it('starts one fresh instance for many failures in one synchronous loop', async () => { + it('starts a fresh instance for every failure in one synchronous loop', async () => { await isolated.loadModule(); const loadsBefore = loads; - for (let i = 0; i < 5; i++) { - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - } - assert.equal(loads, loadsBefore + 1); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); - }); - - it('stops sync calls on a stale instance after 10 failures until the fresh one loads', async () => { - await isolated.loadModule(); - for (let i = 0; i < 9; i++) { + for (let i = 0; i < 30; i++) { throwNext = new RangeError('Maximum call stack size exceeded'); assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); + assert.ok(isolated.parseSync('SELECT 1')); + assert.equal(used, loads, 'a sync call ran on the instance that threw'); } - assert.ok(isolated.parseSync('SELECT 1')); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - assert.throws(() => isolated.parseSync('SELECT 1'), /Call `?loadModule\(\)`? first/); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); + assert.equal(loads, loadsBefore + 30); }); - it('makes loadModule() and queued calls wait again when a queued call breaks the fresh instance', async () => { + it('never runs a queued call on the instance that threw', async () => { await isolated.loadModule(); throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); const after = isolated.parse('SELECT 1'); await assert.rejects(failing, RangeError); assert.ok(await after); - assert.equal(used, loads, 'a queued call ran on the instance that just threw'); - await ready; - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); - }); - - it('makes loadModule() wait again when a call queued ahead of it breaks the fresh instance', async () => { + assert.equal(used, loads, 'a queued call ran on the instance that threw'); await isolated.loadModule(); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); - await assert.rejects(failing, RangeError); - await ready; assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); + assert.equal(used, loads); }); it('names a thrown object that has no message', async () => { @@ -335,6 +313,14 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => } }); +// Makes the loader's synchronous instantiation fail, as if the new instance's memory could not be allocated. +function failInstantiate(options) { + const instantiateWasm = options.instantiateWasm; + if (!instantiateWasm) return; + options.instantiateWasm = (imports) => + instantiateWasm(imports, () => { throw new Error('simulated instantiate failure'); }); +} + // A copy of the wrapper whose WASM factory is wrapped, leaving the shared copy alone. function loadIsolated(wrapFactory) { const indexPath = require.resolve('../wasm/index.cjs'); diff --git a/versions/15/src/index.ts b/versions/15/src/index.ts index f39ac71c..40babf7c 100644 --- a/versions/15/src/index.ts +++ b/versions/15/src/index.ts @@ -131,44 +131,71 @@ export function hasSqlDetails(error: any): error is Error & { sqlDetails: SqlErr 'cursorPosition' in (error as any).sqlDetails; } -// After a WASM call throws, the instance is stale: async calls wait for its -// replacement, while sync calls, which cannot wait, keep using it until the -// replacement is ready. One replacement loads at a time; the next call retries -// a failed load. -let stale = false; +// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a +// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and +// can leave the instance corrupt. callWasm() therefore swaps in a fresh instance +// before rethrowing. It builds that instance synchronously from the compiled +// module, so the very next call, sync or async, already runs on a clean one. +let compiled: WebAssembly.Module | undefined; let loading: Promise | undefined; -// Counts WASM calls that threw, so cleanup skips an instance that threw mid-call. +// Counts WASM calls that threw, so cleanup skips memory of the replaced instance. let failures = 0; -// Sync calls stop using a stale instance after this many failures on it: past -// about 45, a stack overflow can make the next one spin forever. -const MAX_STALE_FAILURES = 10; -let staleFailures = 0; +// Why the last synchronous replacement failed; the next loadModule() retries. +let replaceError: unknown; + +// Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. +async function compileWasm(url: string): Promise { + const proc = (globalThis as any).process; + if (proc?.versions?.node && proc.type !== 'renderer') { + return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + } + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); + return WebAssembly.compile(await response.arrayBuffer()); +} function instantiate(): Promise { if (loading) return loading; - const promise = PgQueryModule().then( - (module: any) => { - wasmModule = module; - stale = false; - staleFailures = 0; - loading = undefined; - }, - (error: unknown) => { - loading = undefined; - throw error; - } - ); + const promise = (async () => { + let wasmUrl: string | undefined; + const module: any = await PgQueryModule({ + locateFile: (path: string, dir: string) => (wasmUrl = dir + path), + }); + compiled ??= await compileWasm(wasmUrl); + wasmModule = module; + replaceError = undefined; + })(); loading = promise; - // Rejects only for the callers awaiting it. - promise.catch(() => {}); + promise.then( + () => { loading = undefined; }, + () => { loading = undefined; } + ); return promise; } -instantiate(); +// Rejects only for the callers awaiting it. +instantiate().catch(() => {}); + +function replaceInstance(): void { + let error: unknown = new Error('the WASM loader did not instantiate synchronously'); + const module: any = { + instantiateWasm(imports: WebAssembly.Imports, receive: (instance: WebAssembly.Instance) => void) { + try { + receive(new WebAssembly.Instance(compiled, imports)); + error = undefined; + } catch (e) { + error = e; + } + return {}; + }, + }; + // receive() sets the exports and memory views, so the instance is usable right + // away; the loader runs the rest of its start-up (stdio streams) on a later microtask. + PgQueryModule(module).catch(() => {}); + wasmModule = error === undefined ? module : undefined; + replaceError = error; +} -// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a -// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and -// leaves the instance corrupt, so replace it with a fresh one. function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; @@ -176,9 +203,7 @@ function callWasm(fn: () => number): number { return fn(); } catch (error) { failures++; - stale = true; - if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; - instantiate(); + replaceInstance(); if (proc) proc.exitCode = exitCode; if (error instanceof Error) throw error; const reason = typeof error === 'object' && error !== null @@ -192,17 +217,22 @@ function callWasm(fn: () => number): number { } function ensureLoaded() { - if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); + // A replacement that failed to start is retried here, for callers that never await. + if (!wasmModule && compiled) replaceInstance(); + if (wasmModule) return; + if (replaceError !== undefined) { + throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); + } + throw new Error("WASM module not initialized. Call `loadModule()` first."); } export async function loadModule() { - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); } function awaitInit any>(fn: T): T { return (async (...args: Parameters) => { - // Re-checked after every await: another call may have failed in between. - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); return fn(...args); }) as T; } diff --git a/versions/15/test/wasm-recovery.test.js b/versions/15/test/wasm-recovery.test.js index 6d16d379..98f6dc99 100644 --- a/versions/15/test/wasm-recovery.test.js +++ b/versions/15/test/wasm-recovery.test.js @@ -22,21 +22,24 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1').stmts.length, 1); }); - // Without the 10-failure cap, about 45 overflows on one instance made the next one spin forever. - it('stops sync calls on the stale instance after 10 real overflows in one synchronous loop', async () => { + it('keeps sync calls working through 60 overflows in one synchronous loop', async () => { await loadModule(); - const errors = []; + const results = []; for (let i = 0; i < 60; i++) { - try { - parseSync(DEEP); - } catch (error) { - errors.push(error instanceof RangeError ? 'overflow' : error.message); + for (const query of [DEEP, 'SELECT 1']) { + try { + results.push(parseSync(query).stmts.length); + } catch (error) { + results.push(error instanceof RangeError ? 'overflow' : error.message); + } } } - assert.deepEqual(errors.slice(0, 10), Array(10).fill('overflow')); - for (const message of errors.slice(10)) assert.match(message, /Call `?loadModule\(\)`? first/); - await loadModule(); - assert.equal(parseSync('SELECT 1').stmts.length, 1); + assert.deepEqual(results, Array(60).fill(['overflow', 1]).flat()); + }); + + it('gives the next sync call a fresh instance after the memory limit is hit', () => { + assert.throws(() => parseSync('SELECT 1;\n'.repeat(5000000)), /out of WASM memory/); + assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); it('frees each replaced instance, so WASM memory stays at one instance', () => { @@ -118,16 +121,22 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(loads, 3); }); - it('lets loadModule() reject on a failed reload and retry on the next call', async () => { + it('names the cause when a fresh instance cannot start, and loadModule() retries', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); + const isolated = loadIsolated((real, options) => { + if (++loads === 2 || loads === 3) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await assert.rejects(isolated.loadModule(), /simulated load failure/); + assert.throws(() => isolated.parseSync(DEEP), RangeError); + assert.throws(() => isolated.parseSync('SELECT 1'), (error) => { + assert.match(error.message, /Could not start a fresh WASM instance/); + assert.match(error.cause.message, /simulated instantiate failure/); + return true; + }); await isolated.loadModule(); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); - assert.equal(loads, 3); + assert.equal(loads, 4); }); it('keeps sync calls in the same synchronous loop working after a failure', () => { @@ -145,18 +154,16 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.deepEqual(results, expected); }); - it('retries a failed fresh load on the next failure, for callers that never await', async () => { + it('retries a fresh instance that failed to start on the next sync call', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); - await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(loads, 2); - assert.throws(() => isolated.parseSync(DEEP)); - assert.equal(loads, 3); + const isolated = loadIsolated((real, options) => { + if (++loads === 2) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); + assert.throws(() => isolated.parseSync(DEEP), RangeError); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); + assert.equal(loads, 3); }); it('makes loadModule() wait again when a queued call breaks the fresh instance', async () => { @@ -201,9 +208,15 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => let used; let frees = 0; let throwNext; - const isolated = loadIsolated(async (real, ...args) => { + const isolated = loadIsolated((real, options) => { const id = ++loads; - const module = await real(...args); + const ready = real(options); + // A replacement instance is built synchronously; the first load is async. + if (typeof options._malloc === 'function') return instrument(options, id) && ready; + return ready.then((module) => instrument(module, id)); + }); + + function instrument(module, id) { const free = module._free; module._free = (ptr) => { frees++; @@ -221,7 +234,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => }; } return module; - }); + } async function failOnce(name, query, thrown) { await isolated.loadModule(); @@ -242,73 +255,38 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => assert.equal(exitCodeAfter, 0, `${name} changed process.exitCode`); assert.equal(frees, freesBefore, `${name} freed memory in the instance that threw`); assert.equal(loads, loadsBefore + 1, `${name} did not start a fresh instance`); - const stale = used; const syncName = name.endsWith('Sync') ? name : 'parseSync'; - assert.ok(isolated[syncName](query), `${syncName} failed while the fresh instance loads`); - assert.equal(used, stale, 'a sync call did not run on the stale instance'); + assert.ok(isolated[syncName](query), `${syncName} failed right after the failure`); + assert.equal(used, loads, `${syncName} ran on the instance that threw`); assert.ok(frees > freesBefore, 'a successful call did not free its memory'); - // A sync caller gets the fresh instance once it awaits loadModule(); async calls wait by themselves. - if (name.endsWith('Sync')) await isolated.loadModule(); assert.ok(await isolated[name](query), `${name} failed after recovery`); - assert.equal(used, loads, `${name} ran on the stale instance, not the fresh one`); + assert.equal(used, loads, `${name} ran on the instance that threw`); return caught; } - it('starts one fresh instance for many failures in one synchronous loop', async () => { + it('starts a fresh instance for every failure in one synchronous loop', async () => { await isolated.loadModule(); const loadsBefore = loads; - for (let i = 0; i < 5; i++) { - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - } - assert.equal(loads, loadsBefore + 1); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); - }); - - it('stops sync calls on a stale instance after 10 failures until the fresh one loads', async () => { - await isolated.loadModule(); - for (let i = 0; i < 9; i++) { + for (let i = 0; i < 30; i++) { throwNext = new RangeError('Maximum call stack size exceeded'); assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); + assert.ok(isolated.parseSync('SELECT 1')); + assert.equal(used, loads, 'a sync call ran on the instance that threw'); } - assert.ok(isolated.parseSync('SELECT 1')); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - assert.throws(() => isolated.parseSync('SELECT 1'), /Call `?loadModule\(\)`? first/); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); + assert.equal(loads, loadsBefore + 30); }); - it('makes loadModule() and queued calls wait again when a queued call breaks the fresh instance', async () => { + it('never runs a queued call on the instance that threw', async () => { await isolated.loadModule(); throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); const after = isolated.parse('SELECT 1'); await assert.rejects(failing, RangeError); assert.ok(await after); - assert.equal(used, loads, 'a queued call ran on the instance that just threw'); - await ready; - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); - }); - - it('makes loadModule() wait again when a call queued ahead of it breaks the fresh instance', async () => { + assert.equal(used, loads, 'a queued call ran on the instance that threw'); await isolated.loadModule(); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); - await assert.rejects(failing, RangeError); - await ready; assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); + assert.equal(used, loads); }); it('names a thrown object that has no message', async () => { @@ -335,6 +313,14 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => } }); +// Makes the loader's synchronous instantiation fail, as if the new instance's memory could not be allocated. +function failInstantiate(options) { + const instantiateWasm = options.instantiateWasm; + if (!instantiateWasm) return; + options.instantiateWasm = (imports) => + instantiateWasm(imports, () => { throw new Error('simulated instantiate failure'); }); +} + // A copy of the wrapper whose WASM factory is wrapped, leaving the shared copy alone. function loadIsolated(wrapFactory) { const indexPath = require.resolve('../wasm/index.cjs'); diff --git a/versions/16/src/index.ts b/versions/16/src/index.ts index f39ac71c..40babf7c 100644 --- a/versions/16/src/index.ts +++ b/versions/16/src/index.ts @@ -131,44 +131,71 @@ export function hasSqlDetails(error: any): error is Error & { sqlDetails: SqlErr 'cursorPosition' in (error as any).sqlDetails; } -// After a WASM call throws, the instance is stale: async calls wait for its -// replacement, while sync calls, which cannot wait, keep using it until the -// replacement is ready. One replacement loads at a time; the next call retries -// a failed load. -let stale = false; +// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a +// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and +// can leave the instance corrupt. callWasm() therefore swaps in a fresh instance +// before rethrowing. It builds that instance synchronously from the compiled +// module, so the very next call, sync or async, already runs on a clean one. +let compiled: WebAssembly.Module | undefined; let loading: Promise | undefined; -// Counts WASM calls that threw, so cleanup skips an instance that threw mid-call. +// Counts WASM calls that threw, so cleanup skips memory of the replaced instance. let failures = 0; -// Sync calls stop using a stale instance after this many failures on it: past -// about 45, a stack overflow can make the next one spin forever. -const MAX_STALE_FAILURES = 10; -let staleFailures = 0; +// Why the last synchronous replacement failed; the next loadModule() retries. +let replaceError: unknown; + +// Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. +async function compileWasm(url: string): Promise { + const proc = (globalThis as any).process; + if (proc?.versions?.node && proc.type !== 'renderer') { + return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + } + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); + return WebAssembly.compile(await response.arrayBuffer()); +} function instantiate(): Promise { if (loading) return loading; - const promise = PgQueryModule().then( - (module: any) => { - wasmModule = module; - stale = false; - staleFailures = 0; - loading = undefined; - }, - (error: unknown) => { - loading = undefined; - throw error; - } - ); + const promise = (async () => { + let wasmUrl: string | undefined; + const module: any = await PgQueryModule({ + locateFile: (path: string, dir: string) => (wasmUrl = dir + path), + }); + compiled ??= await compileWasm(wasmUrl); + wasmModule = module; + replaceError = undefined; + })(); loading = promise; - // Rejects only for the callers awaiting it. - promise.catch(() => {}); + promise.then( + () => { loading = undefined; }, + () => { loading = undefined; } + ); return promise; } -instantiate(); +// Rejects only for the callers awaiting it. +instantiate().catch(() => {}); + +function replaceInstance(): void { + let error: unknown = new Error('the WASM loader did not instantiate synchronously'); + const module: any = { + instantiateWasm(imports: WebAssembly.Imports, receive: (instance: WebAssembly.Instance) => void) { + try { + receive(new WebAssembly.Instance(compiled, imports)); + error = undefined; + } catch (e) { + error = e; + } + return {}; + }, + }; + // receive() sets the exports and memory views, so the instance is usable right + // away; the loader runs the rest of its start-up (stdio streams) on a later microtask. + PgQueryModule(module).catch(() => {}); + wasmModule = error === undefined ? module : undefined; + replaceError = error; +} -// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a -// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and -// leaves the instance corrupt, so replace it with a fresh one. function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; @@ -176,9 +203,7 @@ function callWasm(fn: () => number): number { return fn(); } catch (error) { failures++; - stale = true; - if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; - instantiate(); + replaceInstance(); if (proc) proc.exitCode = exitCode; if (error instanceof Error) throw error; const reason = typeof error === 'object' && error !== null @@ -192,17 +217,22 @@ function callWasm(fn: () => number): number { } function ensureLoaded() { - if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); + // A replacement that failed to start is retried here, for callers that never await. + if (!wasmModule && compiled) replaceInstance(); + if (wasmModule) return; + if (replaceError !== undefined) { + throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); + } + throw new Error("WASM module not initialized. Call `loadModule()` first."); } export async function loadModule() { - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); } function awaitInit any>(fn: T): T { return (async (...args: Parameters) => { - // Re-checked after every await: another call may have failed in between. - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); return fn(...args); }) as T; } diff --git a/versions/16/test/wasm-recovery.test.js b/versions/16/test/wasm-recovery.test.js index 6d16d379..98f6dc99 100644 --- a/versions/16/test/wasm-recovery.test.js +++ b/versions/16/test/wasm-recovery.test.js @@ -22,21 +22,24 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1').stmts.length, 1); }); - // Without the 10-failure cap, about 45 overflows on one instance made the next one spin forever. - it('stops sync calls on the stale instance after 10 real overflows in one synchronous loop', async () => { + it('keeps sync calls working through 60 overflows in one synchronous loop', async () => { await loadModule(); - const errors = []; + const results = []; for (let i = 0; i < 60; i++) { - try { - parseSync(DEEP); - } catch (error) { - errors.push(error instanceof RangeError ? 'overflow' : error.message); + for (const query of [DEEP, 'SELECT 1']) { + try { + results.push(parseSync(query).stmts.length); + } catch (error) { + results.push(error instanceof RangeError ? 'overflow' : error.message); + } } } - assert.deepEqual(errors.slice(0, 10), Array(10).fill('overflow')); - for (const message of errors.slice(10)) assert.match(message, /Call `?loadModule\(\)`? first/); - await loadModule(); - assert.equal(parseSync('SELECT 1').stmts.length, 1); + assert.deepEqual(results, Array(60).fill(['overflow', 1]).flat()); + }); + + it('gives the next sync call a fresh instance after the memory limit is hit', () => { + assert.throws(() => parseSync('SELECT 1;\n'.repeat(5000000)), /out of WASM memory/); + assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); it('frees each replaced instance, so WASM memory stays at one instance', () => { @@ -118,16 +121,22 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(loads, 3); }); - it('lets loadModule() reject on a failed reload and retry on the next call', async () => { + it('names the cause when a fresh instance cannot start, and loadModule() retries', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); + const isolated = loadIsolated((real, options) => { + if (++loads === 2 || loads === 3) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await assert.rejects(isolated.loadModule(), /simulated load failure/); + assert.throws(() => isolated.parseSync(DEEP), RangeError); + assert.throws(() => isolated.parseSync('SELECT 1'), (error) => { + assert.match(error.message, /Could not start a fresh WASM instance/); + assert.match(error.cause.message, /simulated instantiate failure/); + return true; + }); await isolated.loadModule(); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); - assert.equal(loads, 3); + assert.equal(loads, 4); }); it('keeps sync calls in the same synchronous loop working after a failure', () => { @@ -145,18 +154,16 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.deepEqual(results, expected); }); - it('retries a failed fresh load on the next failure, for callers that never await', async () => { + it('retries a fresh instance that failed to start on the next sync call', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); - await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(loads, 2); - assert.throws(() => isolated.parseSync(DEEP)); - assert.equal(loads, 3); + const isolated = loadIsolated((real, options) => { + if (++loads === 2) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); + assert.throws(() => isolated.parseSync(DEEP), RangeError); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); + assert.equal(loads, 3); }); it('makes loadModule() wait again when a queued call breaks the fresh instance', async () => { @@ -201,9 +208,15 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => let used; let frees = 0; let throwNext; - const isolated = loadIsolated(async (real, ...args) => { + const isolated = loadIsolated((real, options) => { const id = ++loads; - const module = await real(...args); + const ready = real(options); + // A replacement instance is built synchronously; the first load is async. + if (typeof options._malloc === 'function') return instrument(options, id) && ready; + return ready.then((module) => instrument(module, id)); + }); + + function instrument(module, id) { const free = module._free; module._free = (ptr) => { frees++; @@ -221,7 +234,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => }; } return module; - }); + } async function failOnce(name, query, thrown) { await isolated.loadModule(); @@ -242,73 +255,38 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => assert.equal(exitCodeAfter, 0, `${name} changed process.exitCode`); assert.equal(frees, freesBefore, `${name} freed memory in the instance that threw`); assert.equal(loads, loadsBefore + 1, `${name} did not start a fresh instance`); - const stale = used; const syncName = name.endsWith('Sync') ? name : 'parseSync'; - assert.ok(isolated[syncName](query), `${syncName} failed while the fresh instance loads`); - assert.equal(used, stale, 'a sync call did not run on the stale instance'); + assert.ok(isolated[syncName](query), `${syncName} failed right after the failure`); + assert.equal(used, loads, `${syncName} ran on the instance that threw`); assert.ok(frees > freesBefore, 'a successful call did not free its memory'); - // A sync caller gets the fresh instance once it awaits loadModule(); async calls wait by themselves. - if (name.endsWith('Sync')) await isolated.loadModule(); assert.ok(await isolated[name](query), `${name} failed after recovery`); - assert.equal(used, loads, `${name} ran on the stale instance, not the fresh one`); + assert.equal(used, loads, `${name} ran on the instance that threw`); return caught; } - it('starts one fresh instance for many failures in one synchronous loop', async () => { + it('starts a fresh instance for every failure in one synchronous loop', async () => { await isolated.loadModule(); const loadsBefore = loads; - for (let i = 0; i < 5; i++) { - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - } - assert.equal(loads, loadsBefore + 1); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); - }); - - it('stops sync calls on a stale instance after 10 failures until the fresh one loads', async () => { - await isolated.loadModule(); - for (let i = 0; i < 9; i++) { + for (let i = 0; i < 30; i++) { throwNext = new RangeError('Maximum call stack size exceeded'); assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); + assert.ok(isolated.parseSync('SELECT 1')); + assert.equal(used, loads, 'a sync call ran on the instance that threw'); } - assert.ok(isolated.parseSync('SELECT 1')); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - assert.throws(() => isolated.parseSync('SELECT 1'), /Call `?loadModule\(\)`? first/); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); + assert.equal(loads, loadsBefore + 30); }); - it('makes loadModule() and queued calls wait again when a queued call breaks the fresh instance', async () => { + it('never runs a queued call on the instance that threw', async () => { await isolated.loadModule(); throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); const after = isolated.parse('SELECT 1'); await assert.rejects(failing, RangeError); assert.ok(await after); - assert.equal(used, loads, 'a queued call ran on the instance that just threw'); - await ready; - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); - }); - - it('makes loadModule() wait again when a call queued ahead of it breaks the fresh instance', async () => { + assert.equal(used, loads, 'a queued call ran on the instance that threw'); await isolated.loadModule(); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); - await assert.rejects(failing, RangeError); - await ready; assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); + assert.equal(used, loads); }); it('names a thrown object that has no message', async () => { @@ -335,6 +313,14 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => } }); +// Makes the loader's synchronous instantiation fail, as if the new instance's memory could not be allocated. +function failInstantiate(options) { + const instantiateWasm = options.instantiateWasm; + if (!instantiateWasm) return; + options.instantiateWasm = (imports) => + instantiateWasm(imports, () => { throw new Error('simulated instantiate failure'); }); +} + // A copy of the wrapper whose WASM factory is wrapped, leaving the shared copy alone. function loadIsolated(wrapFactory) { const indexPath = require.resolve('../wasm/index.cjs'); diff --git a/versions/17/src/index.ts b/versions/17/src/index.ts index f39ac71c..40babf7c 100644 --- a/versions/17/src/index.ts +++ b/versions/17/src/index.ts @@ -131,44 +131,71 @@ export function hasSqlDetails(error: any): error is Error & { sqlDetails: SqlErr 'cursorPosition' in (error as any).sqlDetails; } -// After a WASM call throws, the instance is stale: async calls wait for its -// replacement, while sync calls, which cannot wait, keep using it until the -// replacement is ready. One replacement loads at a time; the next call retries -// a failed load. -let stale = false; +// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a +// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and +// can leave the instance corrupt. callWasm() therefore swaps in a fresh instance +// before rethrowing. It builds that instance synchronously from the compiled +// module, so the very next call, sync or async, already runs on a clean one. +let compiled: WebAssembly.Module | undefined; let loading: Promise | undefined; -// Counts WASM calls that threw, so cleanup skips an instance that threw mid-call. +// Counts WASM calls that threw, so cleanup skips memory of the replaced instance. let failures = 0; -// Sync calls stop using a stale instance after this many failures on it: past -// about 45, a stack overflow can make the next one spin forever. -const MAX_STALE_FAILURES = 10; -let staleFailures = 0; +// Why the last synchronous replacement failed; the next loadModule() retries. +let replaceError: unknown; + +// Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. +async function compileWasm(url: string): Promise { + const proc = (globalThis as any).process; + if (proc?.versions?.node && proc.type !== 'renderer') { + return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + } + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); + return WebAssembly.compile(await response.arrayBuffer()); +} function instantiate(): Promise { if (loading) return loading; - const promise = PgQueryModule().then( - (module: any) => { - wasmModule = module; - stale = false; - staleFailures = 0; - loading = undefined; - }, - (error: unknown) => { - loading = undefined; - throw error; - } - ); + const promise = (async () => { + let wasmUrl: string | undefined; + const module: any = await PgQueryModule({ + locateFile: (path: string, dir: string) => (wasmUrl = dir + path), + }); + compiled ??= await compileWasm(wasmUrl); + wasmModule = module; + replaceError = undefined; + })(); loading = promise; - // Rejects only for the callers awaiting it. - promise.catch(() => {}); + promise.then( + () => { loading = undefined; }, + () => { loading = undefined; } + ); return promise; } -instantiate(); +// Rejects only for the callers awaiting it. +instantiate().catch(() => {}); + +function replaceInstance(): void { + let error: unknown = new Error('the WASM loader did not instantiate synchronously'); + const module: any = { + instantiateWasm(imports: WebAssembly.Imports, receive: (instance: WebAssembly.Instance) => void) { + try { + receive(new WebAssembly.Instance(compiled, imports)); + error = undefined; + } catch (e) { + error = e; + } + return {}; + }, + }; + // receive() sets the exports and memory views, so the instance is usable right + // away; the loader runs the rest of its start-up (stdio streams) on a later microtask. + PgQueryModule(module).catch(() => {}); + wasmModule = error === undefined ? module : undefined; + replaceError = error; +} -// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a -// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and -// leaves the instance corrupt, so replace it with a fresh one. function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; @@ -176,9 +203,7 @@ function callWasm(fn: () => number): number { return fn(); } catch (error) { failures++; - stale = true; - if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; - instantiate(); + replaceInstance(); if (proc) proc.exitCode = exitCode; if (error instanceof Error) throw error; const reason = typeof error === 'object' && error !== null @@ -192,17 +217,22 @@ function callWasm(fn: () => number): number { } function ensureLoaded() { - if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); + // A replacement that failed to start is retried here, for callers that never await. + if (!wasmModule && compiled) replaceInstance(); + if (wasmModule) return; + if (replaceError !== undefined) { + throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); + } + throw new Error("WASM module not initialized. Call `loadModule()` first."); } export async function loadModule() { - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); } function awaitInit any>(fn: T): T { return (async (...args: Parameters) => { - // Re-checked after every await: another call may have failed in between. - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); return fn(...args); }) as T; } diff --git a/versions/17/test/wasm-recovery.test.js b/versions/17/test/wasm-recovery.test.js index 6d16d379..98f6dc99 100644 --- a/versions/17/test/wasm-recovery.test.js +++ b/versions/17/test/wasm-recovery.test.js @@ -22,21 +22,24 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1').stmts.length, 1); }); - // Without the 10-failure cap, about 45 overflows on one instance made the next one spin forever. - it('stops sync calls on the stale instance after 10 real overflows in one synchronous loop', async () => { + it('keeps sync calls working through 60 overflows in one synchronous loop', async () => { await loadModule(); - const errors = []; + const results = []; for (let i = 0; i < 60; i++) { - try { - parseSync(DEEP); - } catch (error) { - errors.push(error instanceof RangeError ? 'overflow' : error.message); + for (const query of [DEEP, 'SELECT 1']) { + try { + results.push(parseSync(query).stmts.length); + } catch (error) { + results.push(error instanceof RangeError ? 'overflow' : error.message); + } } } - assert.deepEqual(errors.slice(0, 10), Array(10).fill('overflow')); - for (const message of errors.slice(10)) assert.match(message, /Call `?loadModule\(\)`? first/); - await loadModule(); - assert.equal(parseSync('SELECT 1').stmts.length, 1); + assert.deepEqual(results, Array(60).fill(['overflow', 1]).flat()); + }); + + it('gives the next sync call a fresh instance after the memory limit is hit', () => { + assert.throws(() => parseSync('SELECT 1;\n'.repeat(5000000)), /out of WASM memory/); + assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); it('frees each replaced instance, so WASM memory stays at one instance', () => { @@ -118,16 +121,22 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(loads, 3); }); - it('lets loadModule() reject on a failed reload and retry on the next call', async () => { + it('names the cause when a fresh instance cannot start, and loadModule() retries', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); + const isolated = loadIsolated((real, options) => { + if (++loads === 2 || loads === 3) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await assert.rejects(isolated.loadModule(), /simulated load failure/); + assert.throws(() => isolated.parseSync(DEEP), RangeError); + assert.throws(() => isolated.parseSync('SELECT 1'), (error) => { + assert.match(error.message, /Could not start a fresh WASM instance/); + assert.match(error.cause.message, /simulated instantiate failure/); + return true; + }); await isolated.loadModule(); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); - assert.equal(loads, 3); + assert.equal(loads, 4); }); it('keeps sync calls in the same synchronous loop working after a failure', () => { @@ -145,18 +154,16 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.deepEqual(results, expected); }); - it('retries a failed fresh load on the next failure, for callers that never await', async () => { + it('retries a fresh instance that failed to start on the next sync call', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); - await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(loads, 2); - assert.throws(() => isolated.parseSync(DEEP)); - assert.equal(loads, 3); + const isolated = loadIsolated((real, options) => { + if (++loads === 2) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); + assert.throws(() => isolated.parseSync(DEEP), RangeError); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); + assert.equal(loads, 3); }); it('makes loadModule() wait again when a queued call breaks the fresh instance', async () => { @@ -201,9 +208,15 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => let used; let frees = 0; let throwNext; - const isolated = loadIsolated(async (real, ...args) => { + const isolated = loadIsolated((real, options) => { const id = ++loads; - const module = await real(...args); + const ready = real(options); + // A replacement instance is built synchronously; the first load is async. + if (typeof options._malloc === 'function') return instrument(options, id) && ready; + return ready.then((module) => instrument(module, id)); + }); + + function instrument(module, id) { const free = module._free; module._free = (ptr) => { frees++; @@ -221,7 +234,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => }; } return module; - }); + } async function failOnce(name, query, thrown) { await isolated.loadModule(); @@ -242,73 +255,38 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => assert.equal(exitCodeAfter, 0, `${name} changed process.exitCode`); assert.equal(frees, freesBefore, `${name} freed memory in the instance that threw`); assert.equal(loads, loadsBefore + 1, `${name} did not start a fresh instance`); - const stale = used; const syncName = name.endsWith('Sync') ? name : 'parseSync'; - assert.ok(isolated[syncName](query), `${syncName} failed while the fresh instance loads`); - assert.equal(used, stale, 'a sync call did not run on the stale instance'); + assert.ok(isolated[syncName](query), `${syncName} failed right after the failure`); + assert.equal(used, loads, `${syncName} ran on the instance that threw`); assert.ok(frees > freesBefore, 'a successful call did not free its memory'); - // A sync caller gets the fresh instance once it awaits loadModule(); async calls wait by themselves. - if (name.endsWith('Sync')) await isolated.loadModule(); assert.ok(await isolated[name](query), `${name} failed after recovery`); - assert.equal(used, loads, `${name} ran on the stale instance, not the fresh one`); + assert.equal(used, loads, `${name} ran on the instance that threw`); return caught; } - it('starts one fresh instance for many failures in one synchronous loop', async () => { + it('starts a fresh instance for every failure in one synchronous loop', async () => { await isolated.loadModule(); const loadsBefore = loads; - for (let i = 0; i < 5; i++) { - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - } - assert.equal(loads, loadsBefore + 1); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); - }); - - it('stops sync calls on a stale instance after 10 failures until the fresh one loads', async () => { - await isolated.loadModule(); - for (let i = 0; i < 9; i++) { + for (let i = 0; i < 30; i++) { throwNext = new RangeError('Maximum call stack size exceeded'); assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); + assert.ok(isolated.parseSync('SELECT 1')); + assert.equal(used, loads, 'a sync call ran on the instance that threw'); } - assert.ok(isolated.parseSync('SELECT 1')); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - assert.throws(() => isolated.parseSync('SELECT 1'), /Call `?loadModule\(\)`? first/); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); + assert.equal(loads, loadsBefore + 30); }); - it('makes loadModule() and queued calls wait again when a queued call breaks the fresh instance', async () => { + it('never runs a queued call on the instance that threw', async () => { await isolated.loadModule(); throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); const after = isolated.parse('SELECT 1'); await assert.rejects(failing, RangeError); assert.ok(await after); - assert.equal(used, loads, 'a queued call ran on the instance that just threw'); - await ready; - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); - }); - - it('makes loadModule() wait again when a call queued ahead of it breaks the fresh instance', async () => { + assert.equal(used, loads, 'a queued call ran on the instance that threw'); await isolated.loadModule(); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); - await assert.rejects(failing, RangeError); - await ready; assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); + assert.equal(used, loads); }); it('names a thrown object that has no message', async () => { @@ -335,6 +313,14 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => } }); +// Makes the loader's synchronous instantiation fail, as if the new instance's memory could not be allocated. +function failInstantiate(options) { + const instantiateWasm = options.instantiateWasm; + if (!instantiateWasm) return; + options.instantiateWasm = (imports) => + instantiateWasm(imports, () => { throw new Error('simulated instantiate failure'); }); +} + // A copy of the wrapper whose WASM factory is wrapped, leaving the shared copy alone. function loadIsolated(wrapFactory) { const indexPath = require.resolve('../wasm/index.cjs'); diff --git a/versions/18/src/index.ts b/versions/18/src/index.ts index 56c93d61..097078df 100644 --- a/versions/18/src/index.ts +++ b/versions/18/src/index.ts @@ -143,44 +143,71 @@ interface WasmModule { let wasmModule: WasmModule; -// After a WASM call throws, the instance is stale: async calls wait for its -// replacement, while sync calls, which cannot wait, keep using it until the -// replacement is ready. One replacement loads at a time; the next call retries -// a failed load. -let stale = false; +// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a +// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and +// can leave the instance corrupt. callWasm() therefore swaps in a fresh instance +// before rethrowing. It builds that instance synchronously from the compiled +// module, so the very next call, sync or async, already runs on a clean one. +let compiled: WebAssembly.Module | undefined; let loading: Promise | undefined; -// Counts WASM calls that threw, so cleanup skips an instance that threw mid-call. +// Counts WASM calls that threw, so cleanup skips memory of the replaced instance. let failures = 0; -// Sync calls stop using a stale instance after this many failures on it: past -// about 45, a stack overflow can make the next one spin forever. -const MAX_STALE_FAILURES = 10; -let staleFailures = 0; +// Why the last synchronous replacement failed; the next loadModule() retries. +let replaceError: unknown; + +// Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. +async function compileWasm(url: string): Promise { + const proc = (globalThis as any).process; + if (proc?.versions?.node && proc.type !== 'renderer') { + return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + } + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); + return WebAssembly.compile(await response.arrayBuffer()); +} function instantiate(): Promise { if (loading) return loading; - const promise = PgQueryModule().then( - (module: WasmModule) => { - wasmModule = module; - stale = false; - staleFailures = 0; - loading = undefined; - }, - (error: unknown) => { - loading = undefined; - throw error; - } - ); + const promise = (async () => { + let wasmUrl: string | undefined; + const module: WasmModule = await PgQueryModule({ + locateFile: (path: string, dir: string) => (wasmUrl = dir + path), + }); + compiled ??= await compileWasm(wasmUrl); + wasmModule = module; + replaceError = undefined; + })(); loading = promise; - // Rejects only for the callers awaiting it. - promise.catch(() => {}); + promise.then( + () => { loading = undefined; }, + () => { loading = undefined; } + ); return promise; } -instantiate(); +// Rejects only for the callers awaiting it. +instantiate().catch(() => {}); + +function replaceInstance(): void { + let error: unknown = new Error('the WASM loader did not instantiate synchronously'); + const module: any = { + instantiateWasm(imports: WebAssembly.Imports, receive: (instance: WebAssembly.Instance) => void) { + try { + receive(new WebAssembly.Instance(compiled, imports)); + error = undefined; + } catch (e) { + error = e; + } + return {}; + }, + }; + // receive() sets the exports and memory views, so the instance is usable right + // away; the loader runs the rest of its start-up (stdio streams) on a later microtask. + PgQueryModule(module).catch(() => {}); + wasmModule = error === undefined ? module : undefined; + replaceError = error; +} -// An exception out of a WASM call (V8 stack overflow on deeply nested SQL, a -// trap, or exit() once the heap limit is hit) skips libpg_query's cleanup and -// leaves the instance corrupt, so replace it with a fresh one. function callWasm(fn: () => number): number { const proc = (globalThis as any).process; const exitCode = proc?.exitCode; @@ -188,9 +215,7 @@ function callWasm(fn: () => number): number { return fn(); } catch (error) { failures++; - stale = true; - if (++staleFailures >= MAX_STALE_FAILURES) wasmModule = undefined; - instantiate(); + replaceInstance(); if (proc) proc.exitCode = exitCode; if (error instanceof Error) throw error; const reason = typeof error === 'object' && error !== null @@ -204,17 +229,22 @@ function callWasm(fn: () => number): number { } function ensureLoaded() { - if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); + // A replacement that failed to start is retried here, for callers that never await. + if (!wasmModule && compiled) replaceInstance(); + if (wasmModule) return; + if (replaceError !== undefined) { + throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); + } + throw new Error("WASM module not initialized. Call `loadModule()` first."); } export async function loadModule(): Promise { - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); } function awaitInit Promise>(fn: T): T { return (async (...args: Parameters) => { - // Re-checked after every await: another call may have failed in between. - while (!wasmModule || stale) await instantiate(); + while (!wasmModule) await instantiate(); return fn(...args); }) as T; } @@ -388,9 +418,7 @@ export const normalize = awaitInit(async (query: string): Promise => { // Sync versions export function parseSync(query: string): ParseResult { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); // Input validation if (query === null || query === undefined) { @@ -454,9 +482,7 @@ export function parseSync(query: string): ParseResult { } export function parsePlPgSQLSync(query: string): ParseResult { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); const queryPtr = stringToPtr(query); const failuresBefore = failures; let resultPtr = 0; @@ -482,9 +508,7 @@ export function parsePlPgSQLSync(query: string): ParseResult { } export function fingerprintSync(query: string): string { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); const queryPtr = stringToPtr(query); const failuresBefore = failures; let resultPtr = 0; @@ -509,9 +533,7 @@ export function fingerprintSync(query: string): string { } export function normalizeSync(query: string): string { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); const queryPtr = stringToPtr(query); const failuresBefore = failures; let resultPtr = 0; @@ -560,9 +582,7 @@ export const scan = awaitInit(async (query: string): Promise => { }); export function scanSync(query: string): ScanResult { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } + ensureLoaded(); const queryPtr = stringToPtr(query); const failuresBefore = failures; let resultPtr = 0; diff --git a/versions/18/test/wasm-recovery.test.js b/versions/18/test/wasm-recovery.test.js index 6d16d379..98f6dc99 100644 --- a/versions/18/test/wasm-recovery.test.js +++ b/versions/18/test/wasm-recovery.test.js @@ -22,21 +22,24 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1').stmts.length, 1); }); - // Without the 10-failure cap, about 45 overflows on one instance made the next one spin forever. - it('stops sync calls on the stale instance after 10 real overflows in one synchronous loop', async () => { + it('keeps sync calls working through 60 overflows in one synchronous loop', async () => { await loadModule(); - const errors = []; + const results = []; for (let i = 0; i < 60; i++) { - try { - parseSync(DEEP); - } catch (error) { - errors.push(error instanceof RangeError ? 'overflow' : error.message); + for (const query of [DEEP, 'SELECT 1']) { + try { + results.push(parseSync(query).stmts.length); + } catch (error) { + results.push(error instanceof RangeError ? 'overflow' : error.message); + } } } - assert.deepEqual(errors.slice(0, 10), Array(10).fill('overflow')); - for (const message of errors.slice(10)) assert.match(message, /Call `?loadModule\(\)`? first/); - await loadModule(); - assert.equal(parseSync('SELECT 1').stmts.length, 1); + assert.deepEqual(results, Array(60).fill(['overflow', 1]).flat()); + }); + + it('gives the next sync call a fresh instance after the memory limit is hit', () => { + assert.throws(() => parseSync('SELECT 1;\n'.repeat(5000000)), /out of WASM memory/); + assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); it('frees each replaced instance, so WASM memory stays at one instance', () => { @@ -118,16 +121,22 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(loads, 3); }); - it('lets loadModule() reject on a failed reload and retry on the next call', async () => { + it('names the cause when a fresh instance cannot start, and loadModule() retries', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); + const isolated = loadIsolated((real, options) => { + if (++loads === 2 || loads === 3) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await assert.rejects(isolated.loadModule(), /simulated load failure/); + assert.throws(() => isolated.parseSync(DEEP), RangeError); + assert.throws(() => isolated.parseSync('SELECT 1'), (error) => { + assert.match(error.message, /Could not start a fresh WASM instance/); + assert.match(error.cause.message, /simulated instantiate failure/); + return true; + }); await isolated.loadModule(); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); - assert.equal(loads, 3); + assert.equal(loads, 4); }); it('keeps sync calls in the same synchronous loop working after a failure', () => { @@ -145,18 +154,16 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.deepEqual(results, expected); }); - it('retries a failed fresh load on the next failure, for callers that never await', async () => { + it('retries a fresh instance that failed to start on the next sync call', async () => { let loads = 0; - const isolated = loadIsolated((real, ...args) => - ++loads === 2 ? Promise.reject(new Error('simulated load failure')) : real(...args)); - await isolated.loadModule(); - assert.throws(() => isolated.parseSync(DEEP)); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(loads, 2); - assert.throws(() => isolated.parseSync(DEEP)); - assert.equal(loads, 3); + const isolated = loadIsolated((real, options) => { + if (++loads === 2) failInstantiate(options); + return real(options); + }); await isolated.loadModule(); + assert.throws(() => isolated.parseSync(DEEP), RangeError); assert.equal(isolated.parseSync('SELECT 1').stmts.length, 1); + assert.equal(loads, 3); }); it('makes loadModule() wait again when a queued call breaks the fresh instance', async () => { @@ -201,9 +208,15 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => let used; let frees = 0; let throwNext; - const isolated = loadIsolated(async (real, ...args) => { + const isolated = loadIsolated((real, options) => { const id = ++loads; - const module = await real(...args); + const ready = real(options); + // A replacement instance is built synchronously; the first load is async. + if (typeof options._malloc === 'function') return instrument(options, id) && ready; + return ready.then((module) => instrument(module, id)); + }); + + function instrument(module, id) { const free = module._free; module._free = (ptr) => { frees++; @@ -221,7 +234,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => }; } return module; - }); + } async function failOnce(name, query, thrown) { await isolated.loadModule(); @@ -242,73 +255,38 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => assert.equal(exitCodeAfter, 0, `${name} changed process.exitCode`); assert.equal(frees, freesBefore, `${name} freed memory in the instance that threw`); assert.equal(loads, loadsBefore + 1, `${name} did not start a fresh instance`); - const stale = used; const syncName = name.endsWith('Sync') ? name : 'parseSync'; - assert.ok(isolated[syncName](query), `${syncName} failed while the fresh instance loads`); - assert.equal(used, stale, 'a sync call did not run on the stale instance'); + assert.ok(isolated[syncName](query), `${syncName} failed right after the failure`); + assert.equal(used, loads, `${syncName} ran on the instance that threw`); assert.ok(frees > freesBefore, 'a successful call did not free its memory'); - // A sync caller gets the fresh instance once it awaits loadModule(); async calls wait by themselves. - if (name.endsWith('Sync')) await isolated.loadModule(); assert.ok(await isolated[name](query), `${name} failed after recovery`); - assert.equal(used, loads, `${name} ran on the stale instance, not the fresh one`); + assert.equal(used, loads, `${name} ran on the instance that threw`); return caught; } - it('starts one fresh instance for many failures in one synchronous loop', async () => { + it('starts a fresh instance for every failure in one synchronous loop', async () => { await isolated.loadModule(); const loadsBefore = loads; - for (let i = 0; i < 5; i++) { - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - } - assert.equal(loads, loadsBefore + 1); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); - }); - - it('stops sync calls on a stale instance after 10 failures until the fresh one loads', async () => { - await isolated.loadModule(); - for (let i = 0; i < 9; i++) { + for (let i = 0; i < 30; i++) { throwNext = new RangeError('Maximum call stack size exceeded'); assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); + assert.ok(isolated.parseSync('SELECT 1')); + assert.equal(used, loads, 'a sync call ran on the instance that threw'); } - assert.ok(isolated.parseSync('SELECT 1')); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - assert.throws(() => isolated.parseSync('SELECT 1'), /Call `?loadModule\(\)`? first/); - await isolated.loadModule(); - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads); + assert.equal(loads, loadsBefore + 30); }); - it('makes loadModule() and queued calls wait again when a queued call breaks the fresh instance', async () => { + it('never runs a queued call on the instance that threw', async () => { await isolated.loadModule(); throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); const after = isolated.parse('SELECT 1'); await assert.rejects(failing, RangeError); assert.ok(await after); - assert.equal(used, loads, 'a queued call ran on the instance that just threw'); - await ready; - assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); - }); - - it('makes loadModule() wait again when a call queued ahead of it breaks the fresh instance', async () => { + assert.equal(used, loads, 'a queued call ran on the instance that threw'); await isolated.loadModule(); - throwNext = new RangeError('Maximum call stack size exceeded'); - assert.throws(() => isolated.parseSync('SELECT 1'), RangeError); - const failing = isolated.parse('SELECT 1'); - throwNext = new RangeError('Maximum call stack size exceeded'); - const ready = isolated.loadModule(); - await assert.rejects(failing, RangeError); - await ready; assert.ok(isolated.parseSync('SELECT 1')); - assert.equal(used, loads, 'loadModule() resolved before the fresh instance loaded'); + assert.equal(used, loads); }); it('names a thrown object that has no message', async () => { @@ -335,6 +313,14 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => } }); +// Makes the loader's synchronous instantiation fail, as if the new instance's memory could not be allocated. +function failInstantiate(options) { + const instantiateWasm = options.instantiateWasm; + if (!instantiateWasm) return; + options.instantiateWasm = (imports) => + instantiateWasm(imports, () => { throw new Error('simulated instantiate failure'); }); +} + // A copy of the wrapper whose WASM factory is wrapped, leaving the shared copy alone. function loadIsolated(wrapFactory) { const indexPath = require.resolve('../wasm/index.cjs'); From 7a4d982dcb43d1362f1cc0db563a30a25ac689e6 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 04:27:20 +0000 Subject: [PATCH 12/15] fix(wasm): read the wasm file with a dynamic import so ESM builds work in Node --- templates/full/index.ts | 3 ++- templates/index.ts | 3 ++- versions/13/src/index.ts | 3 ++- versions/14/src/index.ts | 3 ++- versions/15/src/index.ts | 3 ++- versions/16/src/index.ts | 3 ++- versions/17/src/index.ts | 3 ++- versions/18/src/index.ts | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/templates/full/index.ts b/templates/full/index.ts index 8f1b971d..7b460d86 100644 --- a/templates/full/index.ts +++ b/templates/full/index.ts @@ -152,7 +152,8 @@ let replaceError: unknown; async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; if (proc?.versions?.node && proc.type !== 'renderer') { - return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + const fs = await import('node:fs'); + return WebAssembly.compile(fs.readFileSync(url.startsWith('file://') ? new URL(url) : url)); } const response = await fetch(url, { credentials: 'same-origin' }); if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); diff --git a/templates/index.ts b/templates/index.ts index 4a86e126..83a8389b 100644 --- a/templates/index.ts +++ b/templates/index.ts @@ -140,7 +140,8 @@ let replaceError: unknown; async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; if (proc?.versions?.node && proc.type !== 'renderer') { - return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + const fs = await import('node:fs'); + return WebAssembly.compile(fs.readFileSync(url.startsWith('file://') ? new URL(url) : url)); } const response = await fetch(url, { credentials: 'same-origin' }); if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); diff --git a/versions/13/src/index.ts b/versions/13/src/index.ts index 40babf7c..b307ae31 100644 --- a/versions/13/src/index.ts +++ b/versions/13/src/index.ts @@ -147,7 +147,8 @@ let replaceError: unknown; async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; if (proc?.versions?.node && proc.type !== 'renderer') { - return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + const fs = await import('node:fs'); + return WebAssembly.compile(fs.readFileSync(url.startsWith('file://') ? new URL(url) : url)); } const response = await fetch(url, { credentials: 'same-origin' }); if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); diff --git a/versions/14/src/index.ts b/versions/14/src/index.ts index 40babf7c..b307ae31 100644 --- a/versions/14/src/index.ts +++ b/versions/14/src/index.ts @@ -147,7 +147,8 @@ let replaceError: unknown; async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; if (proc?.versions?.node && proc.type !== 'renderer') { - return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + const fs = await import('node:fs'); + return WebAssembly.compile(fs.readFileSync(url.startsWith('file://') ? new URL(url) : url)); } const response = await fetch(url, { credentials: 'same-origin' }); if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); diff --git a/versions/15/src/index.ts b/versions/15/src/index.ts index 40babf7c..b307ae31 100644 --- a/versions/15/src/index.ts +++ b/versions/15/src/index.ts @@ -147,7 +147,8 @@ let replaceError: unknown; async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; if (proc?.versions?.node && proc.type !== 'renderer') { - return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + const fs = await import('node:fs'); + return WebAssembly.compile(fs.readFileSync(url.startsWith('file://') ? new URL(url) : url)); } const response = await fetch(url, { credentials: 'same-origin' }); if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); diff --git a/versions/16/src/index.ts b/versions/16/src/index.ts index 40babf7c..b307ae31 100644 --- a/versions/16/src/index.ts +++ b/versions/16/src/index.ts @@ -147,7 +147,8 @@ let replaceError: unknown; async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; if (proc?.versions?.node && proc.type !== 'renderer') { - return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + const fs = await import('node:fs'); + return WebAssembly.compile(fs.readFileSync(url.startsWith('file://') ? new URL(url) : url)); } const response = await fetch(url, { credentials: 'same-origin' }); if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); diff --git a/versions/17/src/index.ts b/versions/17/src/index.ts index 40babf7c..b307ae31 100644 --- a/versions/17/src/index.ts +++ b/versions/17/src/index.ts @@ -147,7 +147,8 @@ let replaceError: unknown; async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; if (proc?.versions?.node && proc.type !== 'renderer') { - return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + const fs = await import('node:fs'); + return WebAssembly.compile(fs.readFileSync(url.startsWith('file://') ? new URL(url) : url)); } const response = await fetch(url, { credentials: 'same-origin' }); if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); diff --git a/versions/18/src/index.ts b/versions/18/src/index.ts index 097078df..e39f80d6 100644 --- a/versions/18/src/index.ts +++ b/versions/18/src/index.ts @@ -159,7 +159,8 @@ let replaceError: unknown; async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; if (proc?.versions?.node && proc.type !== 'renderer') { - return WebAssembly.compile(require('node:fs').readFileSync(url.startsWith('file://') ? new URL(url) : url)); + const fs = await import('node:fs'); + return WebAssembly.compile(fs.readFileSync(url.startsWith('file://') ? new URL(url) : url)); } const response = await fetch(url, { credentials: 'same-origin' }); if (!response.ok) throw new Error(`Failed to load ${url}: HTTP ${response.status}`); From abc325b801903bee008690dedae699ff3b8e5e83 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 04:39:42 +0000 Subject: [PATCH 13/15] fix(v18): type stackSave/stackRestore on the full-API WasmModule --- templates/full/index.ts | 2 ++ versions/18/src/index.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/templates/full/index.ts b/templates/full/index.ts index ab4220c5..72987348 100644 --- a/templates/full/index.ts +++ b/templates/full/index.ts @@ -133,6 +133,8 @@ interface WasmModule { UTF8ToString: (ptr: number) => string; getValue: (ptr: number, type: string) => number; HEAPU8: Uint8Array; + stackSave: () => number; + stackRestore: (ptr: number) => void; } let wasmModule: WasmModule; diff --git a/versions/18/src/index.ts b/versions/18/src/index.ts index e91912c5..dba473db 100644 --- a/versions/18/src/index.ts +++ b/versions/18/src/index.ts @@ -140,6 +140,8 @@ interface WasmModule { UTF8ToString: (ptr: number) => string; getValue: (ptr: number, type: string) => number; HEAPU8: Uint8Array; + stackSave: () => number; + stackRestore: (ptr: number) => void; } let wasmModule: WasmModule; From e87978ccb8333c15174cf3fd4975cbb231b451ad Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 04:43:46 +0000 Subject: [PATCH 14/15] fix(wasm): cap out-of-memory instances held in one synchronous run --- templates/full/index.ts | 49 +++++++++++++++++++------- templates/index.ts | 49 +++++++++++++++++++------- versions/13/src/index.ts | 49 +++++++++++++++++++------- versions/13/test/wasm-recovery.test.js | 33 ++++++++++++++++- versions/14/src/index.ts | 49 +++++++++++++++++++------- versions/14/test/wasm-recovery.test.js | 33 ++++++++++++++++- versions/15/src/index.ts | 49 +++++++++++++++++++------- versions/15/test/wasm-recovery.test.js | 33 ++++++++++++++++- versions/16/src/index.ts | 49 +++++++++++++++++++------- versions/16/test/wasm-recovery.test.js | 33 ++++++++++++++++- versions/17/src/index.ts | 49 +++++++++++++++++++------- versions/17/test/wasm-recovery.test.js | 33 ++++++++++++++++- versions/18/src/index.ts | 49 +++++++++++++++++++------- versions/18/test/wasm-recovery.test.js | 33 ++++++++++++++++- 14 files changed, 480 insertions(+), 110 deletions(-) diff --git a/templates/full/index.ts b/templates/full/index.ts index 7b460d86..36db44a3 100644 --- a/templates/full/index.ts +++ b/templates/full/index.ts @@ -147,6 +147,12 @@ let loading: Promise | undefined; let failures = 0; // Why the last synchronous replacement failed; the next loadModule() retries. let replaceError: unknown; +// A replaced instance is freed only after the current synchronous run yields. +// At most one that grew past half the 1 GiB heap limit may wait for that, so +// repeated out-of-memory failures can't pile up gigabytes; small ones (stack +// overflows on deep nesting) are not limited. +const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; +let largeUnfreed = 0; // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { @@ -207,28 +213,45 @@ function callWasm(fn: () => number): number { const exitCode = proc?.exitCode; try { return fn(); - } catch (error) { + } catch (thrown) { failures++; - replaceInstance(); - if (proc) proc.exitCode = exitCode; - if (error instanceof Error) throw error; - const reason = typeof error === 'object' && error !== null - ? ((error as any).message ?? (error as any).constructor?.name) - : error; - throw new Error( - `libpg_query terminated (${reason}), most likely out of WASM memory; parse large inputs in smaller pieces`, - { cause: error } + const error = thrown instanceof Error ? thrown : new Error( + `libpg_query terminated (${describeThrown(thrown)}), most likely out of WASM memory; parse large inputs in smaller pieces`, + // A copy: Emscripten's own exit object keeps the whole instance alive. + { cause: typeof thrown === 'object' && thrown !== null ? { ...thrown } : thrown } ); + if (wasmModule.HEAPU8.length > LARGE_INSTANCE_BYTES && largeUnfreed++ === 0) { + queueMicrotask(() => { largeUnfreed = 0; }); + } + if (largeUnfreed <= 1) { + replaceInstance(); + } else { + wasmModule = undefined; + replaceError = new Error( + 'WASM memory limit hit repeatedly in one synchronous run; yield (await) to free memory', + { cause: error } + ); + } + if (proc) proc.exitCode = exitCode; + throw error; } } +function describeThrown(thrown: unknown) { + return typeof thrown === 'object' && thrown !== null + ? ((thrown as any).message ?? (thrown as any).constructor?.name) + : thrown; +} + function ensureLoaded() { - // A replacement that failed to start is retried here, for callers that never await. - if (!wasmModule && compiled) replaceInstance(); if (wasmModule) return; - if (replaceError !== undefined) { + // A replacement that was skipped or failed to start is retried here, for callers that never await. + if (compiled && largeUnfreed <= 1) { + replaceInstance(); + if (wasmModule) return; throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); } + if (replaceError !== undefined) throw replaceError; throw new Error("WASM module not initialized. Call `loadModule()` first."); } diff --git a/templates/index.ts b/templates/index.ts index 83a8389b..dbcffed1 100644 --- a/templates/index.ts +++ b/templates/index.ts @@ -135,6 +135,12 @@ let loading: Promise | undefined; let failures = 0; // Why the last synchronous replacement failed; the next loadModule() retries. let replaceError: unknown; +// A replaced instance is freed only after the current synchronous run yields. +// At most one that grew past half the 1 GiB heap limit may wait for that, so +// repeated out-of-memory failures can't pile up gigabytes; small ones (stack +// overflows on deep nesting) are not limited. +const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; +let largeUnfreed = 0; // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { @@ -195,28 +201,45 @@ function callWasm(fn: () => number): number { const exitCode = proc?.exitCode; try { return fn(); - } catch (error) { + } catch (thrown) { failures++; - replaceInstance(); - if (proc) proc.exitCode = exitCode; - if (error instanceof Error) throw error; - const reason = typeof error === 'object' && error !== null - ? ((error as any).message ?? (error as any).constructor?.name) - : error; - throw new Error( - `libpg_query terminated (${reason}), most likely out of WASM memory; parse large inputs in smaller pieces`, - { cause: error } + const error = thrown instanceof Error ? thrown : new Error( + `libpg_query terminated (${describeThrown(thrown)}), most likely out of WASM memory; parse large inputs in smaller pieces`, + // A copy: Emscripten's own exit object keeps the whole instance alive. + { cause: typeof thrown === 'object' && thrown !== null ? { ...thrown } : thrown } ); + if (wasmModule.HEAPU8.length > LARGE_INSTANCE_BYTES && largeUnfreed++ === 0) { + queueMicrotask(() => { largeUnfreed = 0; }); + } + if (largeUnfreed <= 1) { + replaceInstance(); + } else { + wasmModule = undefined; + replaceError = new Error( + 'WASM memory limit hit repeatedly in one synchronous run; yield (await) to free memory', + { cause: error } + ); + } + if (proc) proc.exitCode = exitCode; + throw error; } } +function describeThrown(thrown: unknown) { + return typeof thrown === 'object' && thrown !== null + ? ((thrown as any).message ?? (thrown as any).constructor?.name) + : thrown; +} + function ensureLoaded() { - // A replacement that failed to start is retried here, for callers that never await. - if (!wasmModule && compiled) replaceInstance(); if (wasmModule) return; - if (replaceError !== undefined) { + // A replacement that was skipped or failed to start is retried here, for callers that never await. + if (compiled && largeUnfreed <= 1) { + replaceInstance(); + if (wasmModule) return; throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); } + if (replaceError !== undefined) throw replaceError; throw new Error("WASM module not initialized. Call `loadModule()` first."); } diff --git a/versions/13/src/index.ts b/versions/13/src/index.ts index b307ae31..8a82ff32 100644 --- a/versions/13/src/index.ts +++ b/versions/13/src/index.ts @@ -142,6 +142,12 @@ let loading: Promise | undefined; let failures = 0; // Why the last synchronous replacement failed; the next loadModule() retries. let replaceError: unknown; +// A replaced instance is freed only after the current synchronous run yields. +// At most one that grew past half the 1 GiB heap limit may wait for that, so +// repeated out-of-memory failures can't pile up gigabytes; small ones (stack +// overflows on deep nesting) are not limited. +const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; +let largeUnfreed = 0; // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { @@ -202,28 +208,45 @@ function callWasm(fn: () => number): number { const exitCode = proc?.exitCode; try { return fn(); - } catch (error) { + } catch (thrown) { failures++; - replaceInstance(); - if (proc) proc.exitCode = exitCode; - if (error instanceof Error) throw error; - const reason = typeof error === 'object' && error !== null - ? ((error as any).message ?? (error as any).constructor?.name) - : error; - throw new Error( - `libpg_query terminated (${reason}), most likely out of WASM memory; parse large inputs in smaller pieces`, - { cause: error } + const error = thrown instanceof Error ? thrown : new Error( + `libpg_query terminated (${describeThrown(thrown)}), most likely out of WASM memory; parse large inputs in smaller pieces`, + // A copy: Emscripten's own exit object keeps the whole instance alive. + { cause: typeof thrown === 'object' && thrown !== null ? { ...thrown } : thrown } ); + if (wasmModule.HEAPU8.length > LARGE_INSTANCE_BYTES && largeUnfreed++ === 0) { + queueMicrotask(() => { largeUnfreed = 0; }); + } + if (largeUnfreed <= 1) { + replaceInstance(); + } else { + wasmModule = undefined; + replaceError = new Error( + 'WASM memory limit hit repeatedly in one synchronous run; yield (await) to free memory', + { cause: error } + ); + } + if (proc) proc.exitCode = exitCode; + throw error; } } +function describeThrown(thrown: unknown) { + return typeof thrown === 'object' && thrown !== null + ? ((thrown as any).message ?? (thrown as any).constructor?.name) + : thrown; +} + function ensureLoaded() { - // A replacement that failed to start is retried here, for callers that never await. - if (!wasmModule && compiled) replaceInstance(); if (wasmModule) return; - if (replaceError !== undefined) { + // A replacement that was skipped or failed to start is retried here, for callers that never await. + if (compiled && largeUnfreed <= 1) { + replaceInstance(); + if (wasmModule) return; throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); } + if (replaceError !== undefined) throw replaceError; throw new Error("WASM module not initialized. Call `loadModule()` first."); } diff --git a/versions/13/test/wasm-recovery.test.js b/versions/13/test/wasm-recovery.test.js index 98f6dc99..88947786 100644 --- a/versions/13/test/wasm-recovery.test.js +++ b/versions/13/test/wasm-recovery.test.js @@ -42,6 +42,37 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); + // Each replaced instance stays allocated until the synchronous run yields. + it('stops replacing after repeated out-of-memory failures in one synchronous run, and recovers after an await', () => { + const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); + (async () => { + await lib.loadModule(); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + let peak = 0; + for (let i = 0; i < 10; i++) { + try { lib.parseSync(huge); } catch (error) { messages.push(error.message + ' / ' + error.cause?.message); } + peak = Math.max(peak, process.memoryUsage().rss); + } + await new Promise((resolve) => setTimeout(resolve, 0)); + const after = lib.parseSync('SELECT 1').stmts.length; + process.stdout.write('\\nresult=' + JSON.stringify({ messages, peak, after })); + })();`; + const output = execFileSync(process.execPath, ['-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const { messages, peak, after } = JSON.parse(output.match(/^result=(.*)$/m)[1]); + assert.equal(messages.length, 10); + assert.match(messages[0], /out of WASM memory/); + assert.match(messages[1], /out of WASM memory/); + for (const message of messages.slice(2)) { + assert.match(message, /^WASM memory limit hit repeatedly in one synchronous run; yield \(await\) to free memory \/ .*out of WASM memory/); + } + assert.ok(peak < 2.5 * 1024 ** 3, `peak RSS ${Math.round(peak / 1024 ** 2)} MiB`); + assert.equal(after, 1); + }); + it('frees each replaced instance, so WASM memory stays at one instance', () => { const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); (async () => { @@ -308,7 +339,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => const caught = await failOnce(name, query, thrown); assert.ok(caught instanceof Error); assert.match(caught.message, /out of WASM memory/); - assert.equal(caught.cause, thrown); + assert.deepEqual(caught.cause, thrown); }); } }); diff --git a/versions/14/src/index.ts b/versions/14/src/index.ts index b307ae31..8a82ff32 100644 --- a/versions/14/src/index.ts +++ b/versions/14/src/index.ts @@ -142,6 +142,12 @@ let loading: Promise | undefined; let failures = 0; // Why the last synchronous replacement failed; the next loadModule() retries. let replaceError: unknown; +// A replaced instance is freed only after the current synchronous run yields. +// At most one that grew past half the 1 GiB heap limit may wait for that, so +// repeated out-of-memory failures can't pile up gigabytes; small ones (stack +// overflows on deep nesting) are not limited. +const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; +let largeUnfreed = 0; // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { @@ -202,28 +208,45 @@ function callWasm(fn: () => number): number { const exitCode = proc?.exitCode; try { return fn(); - } catch (error) { + } catch (thrown) { failures++; - replaceInstance(); - if (proc) proc.exitCode = exitCode; - if (error instanceof Error) throw error; - const reason = typeof error === 'object' && error !== null - ? ((error as any).message ?? (error as any).constructor?.name) - : error; - throw new Error( - `libpg_query terminated (${reason}), most likely out of WASM memory; parse large inputs in smaller pieces`, - { cause: error } + const error = thrown instanceof Error ? thrown : new Error( + `libpg_query terminated (${describeThrown(thrown)}), most likely out of WASM memory; parse large inputs in smaller pieces`, + // A copy: Emscripten's own exit object keeps the whole instance alive. + { cause: typeof thrown === 'object' && thrown !== null ? { ...thrown } : thrown } ); + if (wasmModule.HEAPU8.length > LARGE_INSTANCE_BYTES && largeUnfreed++ === 0) { + queueMicrotask(() => { largeUnfreed = 0; }); + } + if (largeUnfreed <= 1) { + replaceInstance(); + } else { + wasmModule = undefined; + replaceError = new Error( + 'WASM memory limit hit repeatedly in one synchronous run; yield (await) to free memory', + { cause: error } + ); + } + if (proc) proc.exitCode = exitCode; + throw error; } } +function describeThrown(thrown: unknown) { + return typeof thrown === 'object' && thrown !== null + ? ((thrown as any).message ?? (thrown as any).constructor?.name) + : thrown; +} + function ensureLoaded() { - // A replacement that failed to start is retried here, for callers that never await. - if (!wasmModule && compiled) replaceInstance(); if (wasmModule) return; - if (replaceError !== undefined) { + // A replacement that was skipped or failed to start is retried here, for callers that never await. + if (compiled && largeUnfreed <= 1) { + replaceInstance(); + if (wasmModule) return; throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); } + if (replaceError !== undefined) throw replaceError; throw new Error("WASM module not initialized. Call `loadModule()` first."); } diff --git a/versions/14/test/wasm-recovery.test.js b/versions/14/test/wasm-recovery.test.js index 98f6dc99..88947786 100644 --- a/versions/14/test/wasm-recovery.test.js +++ b/versions/14/test/wasm-recovery.test.js @@ -42,6 +42,37 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); + // Each replaced instance stays allocated until the synchronous run yields. + it('stops replacing after repeated out-of-memory failures in one synchronous run, and recovers after an await', () => { + const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); + (async () => { + await lib.loadModule(); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + let peak = 0; + for (let i = 0; i < 10; i++) { + try { lib.parseSync(huge); } catch (error) { messages.push(error.message + ' / ' + error.cause?.message); } + peak = Math.max(peak, process.memoryUsage().rss); + } + await new Promise((resolve) => setTimeout(resolve, 0)); + const after = lib.parseSync('SELECT 1').stmts.length; + process.stdout.write('\\nresult=' + JSON.stringify({ messages, peak, after })); + })();`; + const output = execFileSync(process.execPath, ['-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const { messages, peak, after } = JSON.parse(output.match(/^result=(.*)$/m)[1]); + assert.equal(messages.length, 10); + assert.match(messages[0], /out of WASM memory/); + assert.match(messages[1], /out of WASM memory/); + for (const message of messages.slice(2)) { + assert.match(message, /^WASM memory limit hit repeatedly in one synchronous run; yield \(await\) to free memory \/ .*out of WASM memory/); + } + assert.ok(peak < 2.5 * 1024 ** 3, `peak RSS ${Math.round(peak / 1024 ** 2)} MiB`); + assert.equal(after, 1); + }); + it('frees each replaced instance, so WASM memory stays at one instance', () => { const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); (async () => { @@ -308,7 +339,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => const caught = await failOnce(name, query, thrown); assert.ok(caught instanceof Error); assert.match(caught.message, /out of WASM memory/); - assert.equal(caught.cause, thrown); + assert.deepEqual(caught.cause, thrown); }); } }); diff --git a/versions/15/src/index.ts b/versions/15/src/index.ts index b307ae31..8a82ff32 100644 --- a/versions/15/src/index.ts +++ b/versions/15/src/index.ts @@ -142,6 +142,12 @@ let loading: Promise | undefined; let failures = 0; // Why the last synchronous replacement failed; the next loadModule() retries. let replaceError: unknown; +// A replaced instance is freed only after the current synchronous run yields. +// At most one that grew past half the 1 GiB heap limit may wait for that, so +// repeated out-of-memory failures can't pile up gigabytes; small ones (stack +// overflows on deep nesting) are not limited. +const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; +let largeUnfreed = 0; // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { @@ -202,28 +208,45 @@ function callWasm(fn: () => number): number { const exitCode = proc?.exitCode; try { return fn(); - } catch (error) { + } catch (thrown) { failures++; - replaceInstance(); - if (proc) proc.exitCode = exitCode; - if (error instanceof Error) throw error; - const reason = typeof error === 'object' && error !== null - ? ((error as any).message ?? (error as any).constructor?.name) - : error; - throw new Error( - `libpg_query terminated (${reason}), most likely out of WASM memory; parse large inputs in smaller pieces`, - { cause: error } + const error = thrown instanceof Error ? thrown : new Error( + `libpg_query terminated (${describeThrown(thrown)}), most likely out of WASM memory; parse large inputs in smaller pieces`, + // A copy: Emscripten's own exit object keeps the whole instance alive. + { cause: typeof thrown === 'object' && thrown !== null ? { ...thrown } : thrown } ); + if (wasmModule.HEAPU8.length > LARGE_INSTANCE_BYTES && largeUnfreed++ === 0) { + queueMicrotask(() => { largeUnfreed = 0; }); + } + if (largeUnfreed <= 1) { + replaceInstance(); + } else { + wasmModule = undefined; + replaceError = new Error( + 'WASM memory limit hit repeatedly in one synchronous run; yield (await) to free memory', + { cause: error } + ); + } + if (proc) proc.exitCode = exitCode; + throw error; } } +function describeThrown(thrown: unknown) { + return typeof thrown === 'object' && thrown !== null + ? ((thrown as any).message ?? (thrown as any).constructor?.name) + : thrown; +} + function ensureLoaded() { - // A replacement that failed to start is retried here, for callers that never await. - if (!wasmModule && compiled) replaceInstance(); if (wasmModule) return; - if (replaceError !== undefined) { + // A replacement that was skipped or failed to start is retried here, for callers that never await. + if (compiled && largeUnfreed <= 1) { + replaceInstance(); + if (wasmModule) return; throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); } + if (replaceError !== undefined) throw replaceError; throw new Error("WASM module not initialized. Call `loadModule()` first."); } diff --git a/versions/15/test/wasm-recovery.test.js b/versions/15/test/wasm-recovery.test.js index 98f6dc99..88947786 100644 --- a/versions/15/test/wasm-recovery.test.js +++ b/versions/15/test/wasm-recovery.test.js @@ -42,6 +42,37 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); + // Each replaced instance stays allocated until the synchronous run yields. + it('stops replacing after repeated out-of-memory failures in one synchronous run, and recovers after an await', () => { + const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); + (async () => { + await lib.loadModule(); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + let peak = 0; + for (let i = 0; i < 10; i++) { + try { lib.parseSync(huge); } catch (error) { messages.push(error.message + ' / ' + error.cause?.message); } + peak = Math.max(peak, process.memoryUsage().rss); + } + await new Promise((resolve) => setTimeout(resolve, 0)); + const after = lib.parseSync('SELECT 1').stmts.length; + process.stdout.write('\\nresult=' + JSON.stringify({ messages, peak, after })); + })();`; + const output = execFileSync(process.execPath, ['-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const { messages, peak, after } = JSON.parse(output.match(/^result=(.*)$/m)[1]); + assert.equal(messages.length, 10); + assert.match(messages[0], /out of WASM memory/); + assert.match(messages[1], /out of WASM memory/); + for (const message of messages.slice(2)) { + assert.match(message, /^WASM memory limit hit repeatedly in one synchronous run; yield \(await\) to free memory \/ .*out of WASM memory/); + } + assert.ok(peak < 2.5 * 1024 ** 3, `peak RSS ${Math.round(peak / 1024 ** 2)} MiB`); + assert.equal(after, 1); + }); + it('frees each replaced instance, so WASM memory stays at one instance', () => { const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); (async () => { @@ -308,7 +339,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => const caught = await failOnce(name, query, thrown); assert.ok(caught instanceof Error); assert.match(caught.message, /out of WASM memory/); - assert.equal(caught.cause, thrown); + assert.deepEqual(caught.cause, thrown); }); } }); diff --git a/versions/16/src/index.ts b/versions/16/src/index.ts index b307ae31..8a82ff32 100644 --- a/versions/16/src/index.ts +++ b/versions/16/src/index.ts @@ -142,6 +142,12 @@ let loading: Promise | undefined; let failures = 0; // Why the last synchronous replacement failed; the next loadModule() retries. let replaceError: unknown; +// A replaced instance is freed only after the current synchronous run yields. +// At most one that grew past half the 1 GiB heap limit may wait for that, so +// repeated out-of-memory failures can't pile up gigabytes; small ones (stack +// overflows on deep nesting) are not limited. +const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; +let largeUnfreed = 0; // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { @@ -202,28 +208,45 @@ function callWasm(fn: () => number): number { const exitCode = proc?.exitCode; try { return fn(); - } catch (error) { + } catch (thrown) { failures++; - replaceInstance(); - if (proc) proc.exitCode = exitCode; - if (error instanceof Error) throw error; - const reason = typeof error === 'object' && error !== null - ? ((error as any).message ?? (error as any).constructor?.name) - : error; - throw new Error( - `libpg_query terminated (${reason}), most likely out of WASM memory; parse large inputs in smaller pieces`, - { cause: error } + const error = thrown instanceof Error ? thrown : new Error( + `libpg_query terminated (${describeThrown(thrown)}), most likely out of WASM memory; parse large inputs in smaller pieces`, + // A copy: Emscripten's own exit object keeps the whole instance alive. + { cause: typeof thrown === 'object' && thrown !== null ? { ...thrown } : thrown } ); + if (wasmModule.HEAPU8.length > LARGE_INSTANCE_BYTES && largeUnfreed++ === 0) { + queueMicrotask(() => { largeUnfreed = 0; }); + } + if (largeUnfreed <= 1) { + replaceInstance(); + } else { + wasmModule = undefined; + replaceError = new Error( + 'WASM memory limit hit repeatedly in one synchronous run; yield (await) to free memory', + { cause: error } + ); + } + if (proc) proc.exitCode = exitCode; + throw error; } } +function describeThrown(thrown: unknown) { + return typeof thrown === 'object' && thrown !== null + ? ((thrown as any).message ?? (thrown as any).constructor?.name) + : thrown; +} + function ensureLoaded() { - // A replacement that failed to start is retried here, for callers that never await. - if (!wasmModule && compiled) replaceInstance(); if (wasmModule) return; - if (replaceError !== undefined) { + // A replacement that was skipped or failed to start is retried here, for callers that never await. + if (compiled && largeUnfreed <= 1) { + replaceInstance(); + if (wasmModule) return; throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); } + if (replaceError !== undefined) throw replaceError; throw new Error("WASM module not initialized. Call `loadModule()` first."); } diff --git a/versions/16/test/wasm-recovery.test.js b/versions/16/test/wasm-recovery.test.js index 98f6dc99..88947786 100644 --- a/versions/16/test/wasm-recovery.test.js +++ b/versions/16/test/wasm-recovery.test.js @@ -42,6 +42,37 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); + // Each replaced instance stays allocated until the synchronous run yields. + it('stops replacing after repeated out-of-memory failures in one synchronous run, and recovers after an await', () => { + const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); + (async () => { + await lib.loadModule(); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + let peak = 0; + for (let i = 0; i < 10; i++) { + try { lib.parseSync(huge); } catch (error) { messages.push(error.message + ' / ' + error.cause?.message); } + peak = Math.max(peak, process.memoryUsage().rss); + } + await new Promise((resolve) => setTimeout(resolve, 0)); + const after = lib.parseSync('SELECT 1').stmts.length; + process.stdout.write('\\nresult=' + JSON.stringify({ messages, peak, after })); + })();`; + const output = execFileSync(process.execPath, ['-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const { messages, peak, after } = JSON.parse(output.match(/^result=(.*)$/m)[1]); + assert.equal(messages.length, 10); + assert.match(messages[0], /out of WASM memory/); + assert.match(messages[1], /out of WASM memory/); + for (const message of messages.slice(2)) { + assert.match(message, /^WASM memory limit hit repeatedly in one synchronous run; yield \(await\) to free memory \/ .*out of WASM memory/); + } + assert.ok(peak < 2.5 * 1024 ** 3, `peak RSS ${Math.round(peak / 1024 ** 2)} MiB`); + assert.equal(after, 1); + }); + it('frees each replaced instance, so WASM memory stays at one instance', () => { const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); (async () => { @@ -308,7 +339,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => const caught = await failOnce(name, query, thrown); assert.ok(caught instanceof Error); assert.match(caught.message, /out of WASM memory/); - assert.equal(caught.cause, thrown); + assert.deepEqual(caught.cause, thrown); }); } }); diff --git a/versions/17/src/index.ts b/versions/17/src/index.ts index b307ae31..8a82ff32 100644 --- a/versions/17/src/index.ts +++ b/versions/17/src/index.ts @@ -142,6 +142,12 @@ let loading: Promise | undefined; let failures = 0; // Why the last synchronous replacement failed; the next loadModule() retries. let replaceError: unknown; +// A replaced instance is freed only after the current synchronous run yields. +// At most one that grew past half the 1 GiB heap limit may wait for that, so +// repeated out-of-memory failures can't pile up gigabytes; small ones (stack +// overflows on deep nesting) are not limited. +const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; +let largeUnfreed = 0; // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { @@ -202,28 +208,45 @@ function callWasm(fn: () => number): number { const exitCode = proc?.exitCode; try { return fn(); - } catch (error) { + } catch (thrown) { failures++; - replaceInstance(); - if (proc) proc.exitCode = exitCode; - if (error instanceof Error) throw error; - const reason = typeof error === 'object' && error !== null - ? ((error as any).message ?? (error as any).constructor?.name) - : error; - throw new Error( - `libpg_query terminated (${reason}), most likely out of WASM memory; parse large inputs in smaller pieces`, - { cause: error } + const error = thrown instanceof Error ? thrown : new Error( + `libpg_query terminated (${describeThrown(thrown)}), most likely out of WASM memory; parse large inputs in smaller pieces`, + // A copy: Emscripten's own exit object keeps the whole instance alive. + { cause: typeof thrown === 'object' && thrown !== null ? { ...thrown } : thrown } ); + if (wasmModule.HEAPU8.length > LARGE_INSTANCE_BYTES && largeUnfreed++ === 0) { + queueMicrotask(() => { largeUnfreed = 0; }); + } + if (largeUnfreed <= 1) { + replaceInstance(); + } else { + wasmModule = undefined; + replaceError = new Error( + 'WASM memory limit hit repeatedly in one synchronous run; yield (await) to free memory', + { cause: error } + ); + } + if (proc) proc.exitCode = exitCode; + throw error; } } +function describeThrown(thrown: unknown) { + return typeof thrown === 'object' && thrown !== null + ? ((thrown as any).message ?? (thrown as any).constructor?.name) + : thrown; +} + function ensureLoaded() { - // A replacement that failed to start is retried here, for callers that never await. - if (!wasmModule && compiled) replaceInstance(); if (wasmModule) return; - if (replaceError !== undefined) { + // A replacement that was skipped or failed to start is retried here, for callers that never await. + if (compiled && largeUnfreed <= 1) { + replaceInstance(); + if (wasmModule) return; throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); } + if (replaceError !== undefined) throw replaceError; throw new Error("WASM module not initialized. Call `loadModule()` first."); } diff --git a/versions/17/test/wasm-recovery.test.js b/versions/17/test/wasm-recovery.test.js index 98f6dc99..88947786 100644 --- a/versions/17/test/wasm-recovery.test.js +++ b/versions/17/test/wasm-recovery.test.js @@ -42,6 +42,37 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); + // Each replaced instance stays allocated until the synchronous run yields. + it('stops replacing after repeated out-of-memory failures in one synchronous run, and recovers after an await', () => { + const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); + (async () => { + await lib.loadModule(); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + let peak = 0; + for (let i = 0; i < 10; i++) { + try { lib.parseSync(huge); } catch (error) { messages.push(error.message + ' / ' + error.cause?.message); } + peak = Math.max(peak, process.memoryUsage().rss); + } + await new Promise((resolve) => setTimeout(resolve, 0)); + const after = lib.parseSync('SELECT 1').stmts.length; + process.stdout.write('\\nresult=' + JSON.stringify({ messages, peak, after })); + })();`; + const output = execFileSync(process.execPath, ['-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const { messages, peak, after } = JSON.parse(output.match(/^result=(.*)$/m)[1]); + assert.equal(messages.length, 10); + assert.match(messages[0], /out of WASM memory/); + assert.match(messages[1], /out of WASM memory/); + for (const message of messages.slice(2)) { + assert.match(message, /^WASM memory limit hit repeatedly in one synchronous run; yield \(await\) to free memory \/ .*out of WASM memory/); + } + assert.ok(peak < 2.5 * 1024 ** 3, `peak RSS ${Math.round(peak / 1024 ** 2)} MiB`); + assert.equal(after, 1); + }); + it('frees each replaced instance, so WASM memory stays at one instance', () => { const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); (async () => { @@ -308,7 +339,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => const caught = await failOnce(name, query, thrown); assert.ok(caught instanceof Error); assert.match(caught.message, /out of WASM memory/); - assert.equal(caught.cause, thrown); + assert.deepEqual(caught.cause, thrown); }); } }); diff --git a/versions/18/src/index.ts b/versions/18/src/index.ts index e39f80d6..3a33f843 100644 --- a/versions/18/src/index.ts +++ b/versions/18/src/index.ts @@ -154,6 +154,12 @@ let loading: Promise | undefined; let failures = 0; // Why the last synchronous replacement failed; the next loadModule() retries. let replaceError: unknown; +// A replaced instance is freed only after the current synchronous run yields. +// At most one that grew past half the 1 GiB heap limit may wait for that, so +// repeated out-of-memory failures can't pile up gigabytes; small ones (stack +// overflows on deep nesting) are not limited. +const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; +let largeUnfreed = 0; // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { @@ -214,28 +220,45 @@ function callWasm(fn: () => number): number { const exitCode = proc?.exitCode; try { return fn(); - } catch (error) { + } catch (thrown) { failures++; - replaceInstance(); - if (proc) proc.exitCode = exitCode; - if (error instanceof Error) throw error; - const reason = typeof error === 'object' && error !== null - ? ((error as any).message ?? (error as any).constructor?.name) - : error; - throw new Error( - `libpg_query terminated (${reason}), most likely out of WASM memory; parse large inputs in smaller pieces`, - { cause: error } + const error = thrown instanceof Error ? thrown : new Error( + `libpg_query terminated (${describeThrown(thrown)}), most likely out of WASM memory; parse large inputs in smaller pieces`, + // A copy: Emscripten's own exit object keeps the whole instance alive. + { cause: typeof thrown === 'object' && thrown !== null ? { ...thrown } : thrown } ); + if (wasmModule.HEAPU8.length > LARGE_INSTANCE_BYTES && largeUnfreed++ === 0) { + queueMicrotask(() => { largeUnfreed = 0; }); + } + if (largeUnfreed <= 1) { + replaceInstance(); + } else { + wasmModule = undefined; + replaceError = new Error( + 'WASM memory limit hit repeatedly in one synchronous run; yield (await) to free memory', + { cause: error } + ); + } + if (proc) proc.exitCode = exitCode; + throw error; } } +function describeThrown(thrown: unknown) { + return typeof thrown === 'object' && thrown !== null + ? ((thrown as any).message ?? (thrown as any).constructor?.name) + : thrown; +} + function ensureLoaded() { - // A replacement that failed to start is retried here, for callers that never await. - if (!wasmModule && compiled) replaceInstance(); if (wasmModule) return; - if (replaceError !== undefined) { + // A replacement that was skipped or failed to start is retried here, for callers that never await. + if (compiled && largeUnfreed <= 1) { + replaceInstance(); + if (wasmModule) return; throw new Error('Could not start a fresh WASM instance after a failed call; await `loadModule()` to retry', { cause: replaceError }); } + if (replaceError !== undefined) throw replaceError; throw new Error("WASM module not initialized. Call `loadModule()` first."); } diff --git a/versions/18/test/wasm-recovery.test.js b/versions/18/test/wasm-recovery.test.js index 98f6dc99..88947786 100644 --- a/versions/18/test/wasm-recovery.test.js +++ b/versions/18/test/wasm-recovery.test.js @@ -42,6 +42,37 @@ describe('Recovery after the WASM parser throws', { timeout: 120000 }, () => { assert.equal(parseSync('SELECT 1;\n'.repeat(1000000)).stmts.length, 1000000); }); + // Each replaced instance stays allocated until the synchronous run yields. + it('stops replacing after repeated out-of-memory failures in one synchronous run, and recovers after an await', () => { + const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); + (async () => { + await lib.loadModule(); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + let peak = 0; + for (let i = 0; i < 10; i++) { + try { lib.parseSync(huge); } catch (error) { messages.push(error.message + ' / ' + error.cause?.message); } + peak = Math.max(peak, process.memoryUsage().rss); + } + await new Promise((resolve) => setTimeout(resolve, 0)); + const after = lib.parseSync('SELECT 1').stmts.length; + process.stdout.write('\\nresult=' + JSON.stringify({ messages, peak, after })); + })();`; + const output = execFileSync(process.execPath, ['-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const { messages, peak, after } = JSON.parse(output.match(/^result=(.*)$/m)[1]); + assert.equal(messages.length, 10); + assert.match(messages[0], /out of WASM memory/); + assert.match(messages[1], /out of WASM memory/); + for (const message of messages.slice(2)) { + assert.match(message, /^WASM memory limit hit repeatedly in one synchronous run; yield \(await\) to free memory \/ .*out of WASM memory/); + } + assert.ok(peak < 2.5 * 1024 ** 3, `peak RSS ${Math.round(peak / 1024 ** 2)} MiB`); + assert.equal(after, 1); + }); + it('frees each replaced instance, so WASM memory stays at one instance', () => { const script = `const lib = require(${JSON.stringify(require.resolve('../wasm/index.cjs'))}); (async () => { @@ -308,7 +339,7 @@ describe('Recovery from a throw in each entry point', { timeout: 120000 }, () => const caught = await failOnce(name, query, thrown); assert.ok(caught instanceof Error); assert.match(caught.message, /out of WASM memory/); - assert.equal(caught.cause, thrown); + assert.deepEqual(caught.cause, thrown); }); } }); From 7abd626fbb8f4c41620d284009ed68e03ea94051 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sun, 25 Oct 2026 05:07:19 +0000 Subject: [PATCH 15/15] test(v18): BEGIN ATOMIC SQL function parses without aborting --- versions/18/test/plpgsql.test.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/versions/18/test/plpgsql.test.js b/versions/18/test/plpgsql.test.js index e3235c83..62b2cbff 100644 --- a/versions/18/test/plpgsql.test.js +++ b/versions/18/test/plpgsql.test.js @@ -37,6 +37,14 @@ describe("PL/pgSQL Parsing", () => { assert.ok(result.plpgsql_funcs.length > 0); }); + it("parses a BEGIN ATOMIC SQL function, which has no body string, without aborting", () => { + const result = query.parsePlPgSQLSync( + "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT 1; END" + ); + assert.ok(Array.isArray(result.plpgsql_funcs)); + assert.equal(query.parseSync("SELECT 1").stmts.length, 1); + }); + it("should fail on invalid PL/pgSQL", () => { assert.throws(() => query.parsePlPgSQLSync("NOT A FUNCTION"), Error); });