From e47a9f82117a5107349a9400a05d57a92fde12cf Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 22:55:34 +0000 Subject: [PATCH 1/8] fix(objectql): a write answers every stored declared column the driver did not echo as null, as the next read does (#29538) Claude-Session: https://claude.ai/code/session_01ADHkmQjPiXgNoSaEpqeGvY Co-authored-by: Claude --- packages/objectql/src/declared-read-columns.ts | 5 ++++- packages/objectql/src/engine.ts | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/objectql/src/declared-read-columns.ts b/packages/objectql/src/declared-read-columns.ts index 7cb95364703..2b6000e47a2 100644 --- a/packages/objectql/src/declared-read-columns.ts +++ b/packages/objectql/src/declared-read-columns.ts @@ -159,7 +159,10 @@ export function rowsWithDeclaredColumnsOnly(rows: T[], declared: ReadonlySet< * engine's own widening (formula inputs, a file field's owner key, each cut * back off the answer before the verb returns) are what count; * - no projection (a whole-record GET, a list read) selects every declared - * column. + * column; + * - [#29538] a write's answer (the insert result, single or batch, and the + * by-id update readback) names no projection either, so it is answered on + * the whole stored record: the write answer equals the next read of the row. * * ## What stays absent, and why * diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 366ab8abff6..619b5e9a5a0 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -16686,6 +16686,17 @@ export class ObjectQL implements IObjectQLEngine { result = isBatch ? rowsWithDeclaredColumnsOnly(result as unknown[], declaredInsertColumns) : withDeclaredColumnsOnly(result, declaredInsertColumns); + // [#29538] …and answered on every stored declared column, as the next + // read of the row answers it (#29408): a column the driver did not + // echo (driver-mongodb and driver-memory store only written keys) is + // `null`. A write names no projection, so it selects the whole stored + // record. Before the formulas, so a formula over an unwritten column + // evaluates against `null` as the read's does; inside the verb, so the + // security middleware's field mask still deletes an FLS-refused key. + const insertAnswerColumns = selectedStoredColumns(schemaForValidation, undefined); + result = isBatch + ? rowsWithSelectedColumnsAnswered(result as unknown[], insertAnswerColumns) + : withSelectedColumnsAnswered(result, insertAnswerColumns); // Coerce `boolean` fields (SQLite/libsql return 0/1) to real booleans on // the after-hook view so flow trigger conditions (`record.is_escalated @@ -18355,6 +18366,9 @@ export class ObjectQL implements IObjectQLEngine { // `data.record.updated` event and the door's ingress strip. A // `null` readback (the row left the caller's scope) stays `null`. result = withDeclaredColumnsOnly(result, declaredColumnSet(updateSchema)); + // [#29538] Same answer on the stored declared columns as the + // insert result and the next read (#29408). + result = withSelectedColumnsAnswered(result, selectedStoredColumns(updateSchema, undefined)); // [#27495] The handle the stored row named before, where this // statement cleared or replaced it. retiredSecrets = retiredSecretSlots(updateSchema, [priorRecord], hookContext.input.data); From 1cb3f6a3d3c6169565a34029e5aa4282a0f44493 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 22:59:46 +0000 Subject: [PATCH 2/8] test(objectql, runtime, driver-mongodb): pin the write answer to the next read per path and driver; changeset (#29538) Claude-Session: https://claude.ai/code/session_01ADHkmQjPiXgNoSaEpqeGvY Co-authored-by: Claude --- ...29538-write-answer-declared-column-null.md | 13 ++ ...godb-declared-column-null-on-write.test.ts | 140 +++++++++++ .../src/write-answer-declared-columns.test.ts | 194 +++++++++++++++ .../src/declared-column-null-on-write.test.ts | 220 ++++++++++++++++++ 4 files changed, 567 insertions(+) create mode 100644 .changeset/29538-write-answer-declared-column-null.md create mode 100644 packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-write.test.ts create mode 100644 packages/objectql/src/write-answer-declared-columns.test.ts create mode 100644 packages/runtime/src/declared-column-null-on-write.test.ts diff --git a/.changeset/29538-write-answer-declared-column-null.md b/.changeset/29538-write-answer-declared-column-null.md new file mode 100644 index 00000000000..7f110e25962 --- /dev/null +++ b/.changeset/29538-write-answer-declared-column-null.md @@ -0,0 +1,13 @@ +--- +'@objectstack/objectql': patch +--- + +fix(objectql): a write answers every stored declared column the driver did not echo as `null`, so the insert and update answer is the record the next read returns, on every driver (#29538) + +Clause-②: no — nothing a request may send is refused or newly admitted, and nothing a write stores changes; measured on base and head through the engine and the protocol doors on SQLite, driver-memory and driver-mongodb (fake `Db`), every write was accepted on both. An insert or by-id update answer gains a `null` key for a stored declared column the driver did not echo, and on driver-mongodb and driver-memory a `formula` over such a column in that answer now evaluates against that `null`, as the next read and driver-sql already did. +Metadata-protocol: none + +- **Before.** After #29408, `find` and `findOne` answer an unwritten declared column `null` on every driver, but the write answer still came straight from the driver. driver-sql echoed `null` (its table has the column); driver-mongodb and driver-memory echoed only the keys the write stored. So a create on mongo or memory answered `{ title: 'x' }` and the very next `GET` of the same row answered `{ title: 'x', note: null }`, and a formula reading `note` answered `null` (a fault) in the create response and its value on the `GET`. +- **Now.** The engine answers the insert result (single, batch, and each `insertMany` outcome) and the by-id update readback on every stored declared column, `null` where the driver echoed no key, before the formulas and the `afterInsert` / `afterUpdate` hooks. Every door on those two verbs answers alike: `POST /data/:object`, `PATCH /data/:object/:id`, `createManyData`, `updateManyData`, and `batchData` create / update / both upsert arms. +- **Unchanged.** What a write stores. A predicate update still answers its affected-row count. A column field-level security refuses the writer stays absent (the security mask still deletes it), and so do a name outside the declared set, an injected column a federated (`external`) object has no storage for, and a `created_at` / `updated_at` a federated object neither declares nor has injected — the read's own exclusions. +- **Who sees it.** A client of an object served by driver-mongodb or driver-memory that diffs a create or update response against its own state, or renders it: a field the save never mentioned now arrives in that response as `null`, as it does on the next read, instead of appearing only on the next read. In-process readers of the write result see the same: an `afterInsert` / `afterUpdate` hook, a `data.record.created` / `data.record.updated` event payload and the webhooks that deliver it. A reader that told a missing key from `null` (`'x' in row`, `has(record.x)`) now reads what driver-sql already answered; test emptiness with `record.x != null`. A `formula` over an unwritten field in a write response on those two drivers changes value: it used to answer `null` with a fault, and now answers what the next read answers (`record.qty != null ? record.qty * 2 : -1` was `null`, now `-1`). diff --git a/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-write.test.ts b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-write.test.ts new file mode 100644 index 00000000000..44c3def7f5d --- /dev/null +++ b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-write.test.ts @@ -0,0 +1,140 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#29538] driver-mongodb through the engine: a write answers the record the + * next read answers — a declared column the write never touched is `null` in + * the insert and update answer, as on the following `findOne` (#29408). + * + * This driver stores no key for a field given no value (`withoutUndefinedOwnKeys` + * at the insert doors), its `create` / `bulkCreate` echo the document they + * inserted, and its `update` re-reads the document. So the row each write + * hands the engine is sparse. Measured on `origin/main` before the fix with + * this same harness: every write answer below carried no `note` / `qty` key + * and answered the formula `doubled` as `null` (a fault on the missing + * `qty`), while the next `findOne` answered `note: null`, `qty: null` and + * `doubled: -1`. + * + * Runs without a server, on the fake `Db` of + * `mongodb-declared-column-null-on-read.test.ts`, with the write calls the + * engine's insert and by-id update doors reach. The fake keeps each document + * as inserted and answers a projection as a real mongod does — the named keys + * the document HAS. The real-server suites are gated on `mongodb-memory-server` + * (#5517). + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { MongoDBDriver } from './mongodb-driver.js'; + +type Doc = Record; + +/** A fake `Db`: one collection, stored documents kept as inserted. */ +function fakeDb() { + const docs: Doc[] = []; + const project = (doc: Doc, projection: Record | undefined): Doc => { + const keep = Object.keys(projection ?? {}).filter((k) => projection![k] === 1); + const out: Doc = {}; + for (const [k, v] of Object.entries(doc)) { + if (k === '_id' && projection?._id === 0) continue; + if (keep.length > 0 && !keep.includes(k)) continue; + out[k] = v; + } + return out; + }; + // Equality on plain keys only — the writes and reads below address a row + // by `id` alone. A combinator or any other operator is REFUSED, never read + // as a field name. + const matches = (doc: Doc, filter: Doc | undefined): boolean => + Object.entries(filter ?? {}).every(([k, v]) => { + if (k.startsWith('$')) throw new Error(`fake Db: filter operator ${k} is not implemented`); + if (v !== null && typeof v === 'object') { + const ops = Object.keys(v as Doc); + if (ops.length !== 1 || ops[0] !== '$eq') throw new Error(`fake Db: operator ${ops.join(',')} is not implemented`); + return doc[k] === (v as Doc).$eq; + } + return doc[k] === v; + }); + const collection = { + insertOne: async (doc: Doc) => { docs.push({ _id: `oid_${docs.length}`, ...doc }); return {}; }, + insertMany: async (many: Doc[]) => { for (const doc of many) docs.push({ _id: `oid_${docs.length}`, ...doc }); return {}; }, + updateOne: async (filter: Doc, update: { $set?: Doc }) => { + const updateOps = Object.keys(update); + if (updateOps.length !== 1 || updateOps[0] !== '$set') throw new Error(`fake Db: update ${updateOps.join(',')} is not implemented`); + const d = docs.find((x) => matches(x, filter)); + if (d) Object.assign(d, update.$set); + return { matchedCount: d ? 1 : 0 }; + }, + findOne: async (filter: Doc, options: { projection?: Record }) => { + const d = docs.find((x) => matches(x, filter)); + return d ? project(d, options?.projection) : null; + }, + find: (filter: Doc, options: { projection?: Record }) => ({ + toArray: async () => docs.filter((x) => matches(x, filter)).map((d) => project(d, options?.projection)), + }), + }; + return { db: { collection: () => collection }, docs }; +} + +const OBJECT = 'nk_mongo_write_item'; + +const own = (row: unknown, key: string): boolean => + row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); + +describe('[#29538] driver-mongodb: a write answers the record the next read answers', () => { + let engine: ObjectQL; + let docs: Doc[]; + + beforeEach(async () => { + const driver = new MongoDBDriver({ url: 'mongodb://127.0.0.1:1/unused' }); + const fake = fakeDb(); + docs = fake.docs; + (driver as any).db = fake.db; + (driver as any).connect = async () => {}; + (driver as any).disconnect = async () => {}; + engine = new ObjectQL(); + engine.registerDriver(driver as never, true); + await engine.init(); + engine.registry.registerObject({ + name: OBJECT, + label: 'Item', + fields: { + name: { type: 'text' }, + note: { type: 'text' }, + qty: { type: 'number' }, + doubled: { type: 'formula', expression: { dialect: 'cel', source: 'record.qty != null ? record.qty * 2 : -1' } }, + }, + } as never); + }); + + const expectAnswerIsNextRead = async (answer: Doc) => { + expect(answer).toStrictEqual(await engine.findOne(OBJECT, { where: { id: answer.id } })); + expect(own(answer, 'note')).toBe(true); + expect(answer.note).toBeNull(); + expect(own(answer, 'qty')).toBe(true); + expect(answer.qty).toBeNull(); + // FORMULA CONTROL: evaluated against the `null`, as the read evaluates it. + expect(answer.doubled).toBe(-1); + }; + + it('insert: the answer deep-equals the next findOne', async () => { + const answer = await engine.insert(OBJECT, { name: 'Ada' }) as Doc; + expect(own(docs[0], 'note')).toBe(false); + await expectAnswerIsNextRead(answer); + }); + + it('batch insert (bulkCreate): every answered row deep-equals its next findOne', async () => { + const answers = await engine.insert(OBJECT, [{ name: 'Ada' }, { name: 'Bo' }]) as Doc[]; + expect(answers).toHaveLength(2); + for (const answer of answers) await expectAnswerIsNextRead(answer); + }); + + it('update by id: the answer deep-equals the next findOne', async () => { + const created = await engine.insert(OBJECT, { name: 'Ada' }) as Doc; + const answer = await engine.update(OBJECT, { id: created.id, name: 'Ada L.' }) as Doc; + expect(answer.name).toBe('Ada L.'); + await expectAnswerIsNextRead(answer); + // The stored document still carries no key for the column: only the + // answer is shaped, never what the write stores. + expect(own(docs[0], 'note')).toBe(false); + }); +}); diff --git a/packages/objectql/src/write-answer-declared-columns.test.ts b/packages/objectql/src/write-answer-declared-columns.test.ts new file mode 100644 index 00000000000..4a4c1dce169 --- /dev/null +++ b/packages/objectql/src/write-answer-declared-columns.test.ts @@ -0,0 +1,194 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#29538] A write answers the record the next read of the row answers: every + * stored declared column the driver did not echo is `null`, as on driver-sql. + * + * The write-side twin of #29408 (`selected-declared-columns-answered.test.ts`, + * the read half). The driver here stores only the keys a write carried and + * echoes exactly the stored row, the driver-mongodb / driver-memory shape. + * Measured on `origin/main` before the fix, through the engine and the + * protocol doors: on that shape every write answer below left out each + * unwritten declared column (`note`, `qty` and the registry-injected system + * columns) while the next `findOne` answered them `null`, and the formula + * `doubled` read `null` in the write answer (a fault on the missing `qty`) + * but `-1` on the read. driver-sql answered both the same. + * + * driver-mongodb is pinned in its own package + * (`mongodb-declared-column-null-on-write.test.ts`) and driver-sql with the + * security plugin composed in `@objectstack/runtime` + * (`declared-column-null-on-write.test.ts`), where a field-level-refused + * column is pinned absent; this file is the driver-memory shape's pin. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; + +import { ObjectQL } from './engine.js'; + +type Row = Record; + +interface DriverAst { + where?: Record; + fields?: string[]; + limit?: number; +} + +function makeSparseDriver(name: string) { + const tables = new Map>(); + const tableFor = (o: string): Map => { + let t = tables.get(o); + if (!t) { t = new Map(); tables.set(o, t); } + return t; + }; + // Equality on plain keys only. A combinator or any other operator is + // REFUSED, never read as a field name. + const matches = (row: Row, where: Record | undefined): boolean => { + if (!where) return true; + for (const [k, v] of Object.entries(where)) { + if (k.startsWith('$')) throw new Error(`sparse driver: filter operator ${k} is not implemented`); + if (v !== null && typeof v === 'object') throw new Error(`sparse driver: operator on ${k} is not implemented`); + if ((row[k] ?? null) !== (v ?? null)) return false; + } + return true; + }; + const run = (object: string, ast: DriverAst | undefined): Row[] => { + let out = Array.from(tableFor(object).values()).filter((r) => matches(r, ast?.where)); + if (typeof ast?.limit === 'number') out = out.slice(0, ast.limit); + const fields = Array.isArray(ast?.fields) && ast.fields.length > 0 ? ast.fields : undefined; + return out.map((r) => (fields + ? Object.fromEntries(Object.entries(r).filter(([k]) => k === 'id' || fields.includes(k))) + : { ...r })); + }; + let seq = 0; + // A key given `undefined` is not stored, as driver-mongodb's insert doors + // drop it; the echo is the stored row and nothing else. + const stored = (data: Row): Row => Object.fromEntries(Object.entries(data).filter(([, v]) => v !== undefined)); + const driver = { + name, version: '0.0.0', supports: {}, + async connect(): Promise {}, + async disconnect(): Promise {}, + async checkHealth(): Promise { return true; }, + async execute(): Promise { return null; }, + async find(object: string, ast?: DriverAst): Promise { return run(object, ast); }, + async findOne(object: string, ast?: DriverAst): Promise { return run(object, ast)[0] ?? null; }, + async create(object: string, data: Row): Promise { + seq += 1; + const row: Row = { ...stored(data), id: (data.id as string | undefined) ?? `new_${seq}` }; + tableFor(object).set(String(row.id), row); + return { ...row }; + }, + async update(object: string, id: string, data: Row): Promise { + const prior = tableFor(object).get(id); + if (!prior) return null; + const next: Row = { ...prior, ...stored(data), id }; + tableFor(object).set(id, next); + return { ...next }; + }, + async updateMany(object: string, ast: DriverAst, data: Row): Promise { + const hits = Array.from(tableFor(object).values()).filter((r) => matches(r, ast?.where)); + for (const r of hits) tableFor(object).set(String(r.id), { ...r, ...stored(data) }); + return hits.length; + }, + async delete(object: string, id: string): Promise { return tableFor(object).delete(id); }, + async count(object: string, ast?: DriverAst): Promise { return run(object, ast).length; }, + registerExternalObject() {}, + async syncSchema() {}, + }; + return { + driver, + stored: (object: string, id: string) => tableFor(object).get(id), + }; +} + +const ITEM = 'nk_write_item'; +const PACKAGE_ID = 'com.example.nk_write'; + +/** `note` and `qty` are never written; `doubled` reads the unwritten `qty`. */ +const ITEM_FIELDS = { + title: { type: 'text' }, + note: { type: 'text' }, + qty: { type: 'number' }, + doubled: { type: 'formula', expression: { dialect: 'cel', source: 'record.qty != null ? record.qty * 2 : -1' } }, +}; + +async function boot() { + const engine = new ObjectQL(); + const local = makeSparseDriver('memory'); + engine.registerDriver(local.driver as never, true); + await engine.init(); + engine.registry.registerObject({ name: ITEM, label: 'Item', fields: ITEM_FIELDS } as never, PACKAGE_ID); + return { engine, local, protocol: new ObjectStackProtocolImplementation(engine as never) }; +} + +const own = (row: unknown, key: string): boolean => + row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); + +type Booted = Awaited>; +type Answer = Row | null | undefined; + +/** One write door: performs the write and hands back the record body it answered. */ +type WritePath = [string, (h: Booted, existingId: string) => Promise]; + +const WRITE_PATHS: WritePath[] = [ + ['engine.insert', async (h) => await h.engine.insert(ITEM, { title: 'a' }) as Row], + ['engine.insert (batch)', async (h) => ((await h.engine.insert(ITEM, [{ title: 'b1' }, { title: 'b2' }])) as Row[])[1]], + ['engine.insertMany (partial outcomes)', async (h) => ((await h.engine.insertMany(ITEM, [{ title: 'c' }])) as Array<{ record?: Row }>)[0]!.record], + ['engine.update by id', async (h, id) => await h.engine.update(ITEM, { id, title: 'u' }) as Row], + ['createData (POST /data/:object)', async (h) => ((await h.protocol.createData({ object: ITEM, data: { title: 'd' } })) as { record: Row }).record], + ['updateData (PATCH /data/:object/:id)', async (h, id) => ((await h.protocol.updateData({ object: ITEM, id, data: { title: 'p' } })) as { record: Row }).record], + ['createManyData', async (h) => ((await h.protocol.createManyData({ object: ITEM, records: [{ title: 'm' }] })) as { records: Row[] }).records[0]], + ['updateManyData', async (h, id) => ((await h.protocol.updateManyData({ object: ITEM, records: [{ id, data: { title: 'n' } }] } as never)) as { results: Array<{ data: Row }> }).results[0]!.data], + ['batch create', async (h) => ((await h.protocol.batchData({ object: ITEM, request: { operation: 'create', records: [{ data: { title: 'bc' } }] } as never })) as { results: Array<{ data: Row }> }).results[0]!.data], + ['batch update', async (h, id) => ((await h.protocol.batchData({ object: ITEM, request: { operation: 'update', records: [{ id, data: { title: 'bu' } }] } as never })) as { results: Array<{ data: Row }> }).results[0]!.data], + ['batch upsert, update arm', async (h, id) => ((await h.protocol.batchData({ object: ITEM, request: { operation: 'upsert', records: [{ id, data: { title: 'xu' } }] } as never })) as { results: Array<{ data: Row }> }).results[0]!.data], + ['batch upsert, create arm', async (h) => ((await h.protocol.batchData({ object: ITEM, request: { operation: 'upsert', records: [{ id: 'nk_up_new', data: { title: 'xc' } }] } as never })) as { results: Array<{ data: Row }> }).results[0]!.data], +]; + +describe('[#29538] a write answers the record the next read answers', () => { + let h: Booted; + let existingId: string; + beforeEach(async () => { + h = await boot(); + existingId = String(((await h.engine.insert(ITEM, { title: 'seed' })) as Row).id); + }); + + it('the fixture is real: the stored row carries no key for the unwritten columns', () => { + const stored = h.local.stored(ITEM, existingId)!; + expect(own(stored, 'note')).toBe(false); + expect(own(stored, 'qty')).toBe(false); + }); + + it.each(WRITE_PATHS)('%s: the write answer deep-equals the next findOne', async (_label, write) => { + const answer = await write(h, existingId); + expect(answer).toBeTruthy(); + const next = await h.engine.findOne(ITEM, { where: { id: answer!.id } }); + expect(answer).toStrictEqual(next); + // The unwritten stored columns are present as `null`, not missing keys. + expect(own(answer, 'note')).toBe(true); + expect(answer!.note).toBeNull(); + expect(own(answer, 'qty')).toBe(true); + expect(answer!.qty).toBeNull(); + }); + + it.each(WRITE_PATHS)('%s: FORMULA CONTROL — the formula over the unwritten column evaluates against null, as the read does', async (_label, write) => { + const answer = await write(h, existingId); + expect(answer!.doubled).toBe(-1); + }); + + it('a stored value and a stored null are answered as stored', async () => { + const answer = await h.engine.update(ITEM, { id: existingId, note: 'kept', qty: null }) as Row; + expect(answer).toMatchObject({ note: 'kept', qty: null, doubled: -1 }); + expect(answer).toStrictEqual(await h.engine.findOne(ITEM, { where: { id: existingId } })); + }); + + it('the driver\'s stored row is never mutated by the answer', async () => { + await h.engine.update(ITEM, { id: existingId, title: 'again' }); + expect(own(h.local.stored(ITEM, existingId), 'note')).toBe(false); + }); + + it('a predicate update still answers its affected-row count, not a record body', async () => { + const count = await h.engine.update(ITEM, { title: 'many' }, { where: { title: 'seed' }, multi: true } as never); + expect(count).toBe(1); + }); +}); diff --git a/packages/runtime/src/declared-column-null-on-write.test.ts b/packages/runtime/src/declared-column-null-on-write.test.ts new file mode 100644 index 00000000000..39f7a06e83e --- /dev/null +++ b/packages/runtime/src/declared-column-null-on-write.test.ts @@ -0,0 +1,220 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#29538] A write answers the record the next read answers, on every driver: + * a declared column the write never touched is `null` in the insert and + * update answer, as it is on the following `findOne` (#29408) — and a column + * field-level security refuses the writer stays ABSENT from both. + * + * Measured on `origin/main` before the fix, through the engine and the + * protocol doors (insert, batch insert, `insertMany`, update by id, + * `createData`, `updateData`, `createManyData`, `updateManyData`, and + * `batchData` create / update / both upsert arms): driver-sql answered every + * write exactly as its next `findOne`; driver-memory and driver-mongodb + * answered each unwritten declared column as a MISSING key that the next read + * answered `null`, and a formula over it as `null` (a fault) that the next + * read answered with its value. + * + * Two drivers here. driver-sql (SQLite), where the write and the read were + * already the same — the parity control. And a stub that stores and echoes + * only the keys a write carried — the driver-mongodb / driver-memory shape + * (driver-memory itself is not bound: a new test consumer of it is a + * maintainer ruling, `check:driver-memory-census`; driver-mongodb is pinned + * in its own package). On that stub the engine's fill is what puts `note` in + * the answer, so it is where the field-level pin bites: the security plugin + * is composed with a member who may not read `secret`, and the fill must run + * inside the verb, before the security middleware's field mask deletes the + * refused key — a refused column reappearing as `null` would tell the member + * the field exists and is empty. A system write of the same shape is the + * control: there `secret` is a readable unwritten column, answered `null`. + * + * Reads `@objectstack/objectql` from its built `dist/`, as every runtime suite + * does: rebuild objectql before trusting a run of this file. + */ + +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { PermissionSetSchema } from '@objectstack/spec/security'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { SecurityPlugin } from '@objectstack/plugin-security'; +import { installDefaultOrganization } from './default-organization.testkit.js'; + +type Row = Record; + +const LEDGER = 'nk_null_write_ledger'; + +const SYS_CTX = { isSystem: true, userId: 'usr_system' }; + +const MEMBER_SET = PermissionSetSchema.parse({ + name: 'member_default', + label: 'Member', + objects: { '*': { allowRead: true, allowCreate: true, allowEdit: true, allowDelete: true } }, + fields: { [`${LEDGER}.secret`]: { readable: false, editable: false } }, +}); + +const MEMBER_CTX = { userId: 'usr_member', positions: [] as string[], permissions: [MEMBER_SET.name], posture: 'MEMBER' as const }; + +const own = (row: unknown, key: string): boolean => + row != null && typeof row === 'object' && Object.prototype.hasOwnProperty.call(row, key); + +/** + * Stores only the keys a write carried and echoes the stored row — no key for + * a column the write did not touch. Equality on plain keys only; any operator + * is REFUSED, never read as a field name. + */ +function makeSparseDriver() { + const tables = new Map>(); + const tableFor = (o: string): Map => { + let t = tables.get(o); + if (!t) { t = new Map(); tables.set(o, t); } + return t; + }; + const matches = (row: Row, where: Row | undefined): boolean => { + if (!where) return true; + for (const [k, v] of Object.entries(where)) { + if (k.startsWith('$')) throw new Error(`sparse driver: filter operator ${k} is not implemented`); + if (v !== null && typeof v === 'object') throw new Error(`sparse driver: operator on ${k} is not implemented`); + if ((row[k] ?? null) !== (v ?? null)) return false; + } + return true; + }; + const run = (object: string, ast: { where?: Row; fields?: string[]; limit?: number } | undefined): Row[] => { + let out = Array.from(tableFor(object).values()).filter((r) => matches(r, ast?.where)); + if (typeof ast?.limit === 'number') out = out.slice(0, ast.limit); + const fields = Array.isArray(ast?.fields) && ast.fields.length > 0 ? ast.fields : undefined; + return out.map((r) => (fields + ? Object.fromEntries(Object.entries(r).filter(([k]) => k === 'id' || fields.includes(k))) + : { ...r })); + }; + const stored = (data: Row): Row => Object.fromEntries(Object.entries(data).filter(([, v]) => v !== undefined)); + let seq = 0; + return { + name: 'memory', version: '0.0.0', supports: {}, + async connect(): Promise {}, + async disconnect(): Promise {}, + async checkHealth(): Promise { return true; }, + async execute(): Promise { return null; }, + async find(object: string, ast?: { where?: Row; fields?: string[]; limit?: number }): Promise { return run(object, ast); }, + async findOne(object: string, ast?: { where?: Row; fields?: string[]; limit?: number }): Promise { return run(object, ast)[0] ?? null; }, + async create(object: string, data: Row): Promise { + seq += 1; + const row: Row = { ...stored(data), id: (data.id as string | undefined) ?? `sp_${seq}` }; + tableFor(object).set(String(row.id), row); + return { ...row }; + }, + async update(object: string, id: string, data: Row): Promise { + const prior = tableFor(object).get(id); + if (!prior) return null; + const next: Row = { ...prior, ...stored(data), id }; + tableFor(object).set(id, next); + return { ...next }; + }, + async delete(object: string, id: string): Promise { return tableFor(object).delete(id); }, + async count(object: string, ast?: { where?: Row }): Promise { return run(object, ast).length; }, + async syncSchema(): Promise {}, + }; +} + +const DRIVERS: Array<[string, () => unknown]> = [ + ['driver-sql (SQLite)', () => new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true } as never)], + ['a driver that echoes only written keys (driver-mongodb / driver-memory shape)', makeSparseDriver], +]; + +describe.each(DRIVERS)('[#29538] the write answer is the next read — %s', (_label, makeDriver) => { + let engine: ObjectQL; + let seq = 0; + const nextId = () => `nkw_${++seq}`; + + beforeAll(async () => { + engine = new ObjectQL(); + engine.registerDriver(makeDriver() as never, true); + await engine.init(); + engine.registerApp({ + id: 'com.objectstack.qa.declared-column-null-on-write', + name: 'Declared column null on write', + version: '1.0.0', + type: 'plugin', + scope: 'system', + objects: [ + { + name: LEDGER, + label: 'Ledger', + sharingModel: 'public_read_write', + fields: { + title: { name: 'title', type: 'text' }, + note: { name: 'note', type: 'text' }, + secret: { name: 'secret', type: 'text' }, + qty: { name: 'qty', type: 'number' }, + doubled: { name: 'doubled', type: 'formula', expression: { dialect: 'cel', source: 'record.qty != null ? record.qty * 2 : -1' } }, + }, + }, + ], + } as never); + await engine.syncSchemas(); + await installDefaultOrganization(engine); + + const services: Record = { + manifest: { register: vi.fn() }, + objectql: engine, + metadata: { + get: async (_type: string, name: string) => engine.getSchema(name) ?? null, + list: async () => [MEMBER_SET], + }, + }; + const ctx = { + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + registerService: vi.fn(), + getService: (name: string) => { + if (!(name in services)) throw new Error(`service not registered: ${name}`); + return services[name]; + }, + }; + const plugin = new SecurityPlugin({ fallbackPermissionSet: 'member_default' }); + await plugin.init(ctx as never); + await plugin.start(ctx as never); + vi.spyOn((engine as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => undefined); + }, 60_000); + + afterAll(async () => { + try { await engine?.destroy(); } catch { /* noop */ } + }); + + const readBack = (id: unknown, context: unknown) => + engine.findOne(LEDGER, { where: { id }, context } as never); + + it('member insert: the answer deep-equals the next findOne, the unwritten column null', async () => { + const answer = await engine.insert(LEDGER, { id: nextId(), title: 'written' }, { context: MEMBER_CTX } as never) as Row; + expect(answer).toStrictEqual(await readBack(answer.id, MEMBER_CTX)); + expect(own(answer, 'note')).toBe(true); + expect(answer.note).toBeNull(); + expect(answer.doubled).toBe(-1); + }); + + it('member update by id: the answer deep-equals the next findOne, the unwritten column null', async () => { + const id = nextId(); + await engine.insert(LEDGER, { id, title: 'written' }, { context: SYS_CTX } as never); + const answer = await engine.update(LEDGER, { id, title: 'edited' }, { context: MEMBER_CTX } as never) as Row; + expect(answer).toStrictEqual(await readBack(id, MEMBER_CTX)); + expect(own(answer, 'note')).toBe(true); + expect(answer.note).toBeNull(); + expect(answer.doubled).toBe(-1); + }); + + it('FLS-REFUSED stays absent: a column the member may not read never appears in the member\'s write answer', async () => { + const created = await engine.insert(LEDGER, { id: nextId(), title: 'written' }, { context: MEMBER_CTX } as never) as Row; + expect(own(created, 'secret')).toBe(false); + const batch = await engine.insert(LEDGER, [{ id: nextId(), title: 'b' }], { context: MEMBER_CTX } as never) as Row[]; + expect(own(batch[0], 'secret')).toBe(false); + const updated = await engine.update(LEDGER, { id: created.id, title: 'edited' }, { context: MEMBER_CTX } as never) as Row; + expect(own(updated, 'secret')).toBe(false); + // …while the readable unwritten column beside it is answered. + expect(own(updated, 'note')).toBe(true); + }); + + it('control: the same column, written and answered to the system, is a readable unwritten column and answers null', async () => { + const created = await engine.insert(LEDGER, { id: nextId(), title: 'written' }, { context: SYS_CTX } as never) as Row; + expect(own(created, 'secret')).toBe(true); + expect(created.secret).toBeNull(); + expect(created).toStrictEqual(await readBack(created.id, SYS_CTX)); + }); +}); From 2fc0141e63d4518281b5ff1677927245696fa51f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 23:49:07 +0000 Subject: [PATCH 3/8] test(objectql): read "left unset" off the stored row, not the write answer, which now answers the unstored column null (#29538) Claude-Session: https://claude.ai/code/session_01ADHkmQjPiXgNoSaEpqeGvY Co-authored-by: Claude --- .../src/engine-default-value-tokens.test.ts | 7 +++++-- ...ngine-lookup-referential-integrity.test.ts | 6 +++++- .../src/engine-select-option-default.test.ts | 20 +++++++++++++++---- .../engine-write-formula-hydration.test.ts | 8 ++++++-- .../src/protocol-save-notice-answer.test.ts | 11 +++++++--- 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/packages/objectql/src/engine-default-value-tokens.test.ts b/packages/objectql/src/engine-default-value-tokens.test.ts index 80185c2f696..b72cc81d102 100644 --- a/packages/objectql/src/engine-default-value-tokens.test.ts +++ b/packages/objectql/src/engine-default-value-tokens.test.ts @@ -103,8 +103,11 @@ describe('[#4560] the `current_user` defaultValue token is engine-owned', () => // decision a column DEFAULT used to override, writing the literal string // `current_user` into a lookup('sys_user') column (#4560). const row: any = await engine.insert('tok_doc', { title: 'B' }, { context: { isSystem: true } } as any); - expect(row.owner).toBeUndefined(); - expect(row.owner).not.toBe('current_user'); + // [#29538] The answer is the next read's record: a column nothing was + // stored in reads `null`. "Unset" is the STORED row's own: no key at all. + expect(row.owner).toBeNull(); + const stored = await engine.getDriverForObject('tok_doc')!.findOne('tok_doc', { where: { id: row.id } } as any); + expect(stored).not.toHaveProperty('owner'); }); it('an explicit null is treated as "not supplied" and still resolves the token (#2706)', async () => { diff --git a/packages/objectql/src/engine-lookup-referential-integrity.test.ts b/packages/objectql/src/engine-lookup-referential-integrity.test.ts index 4f79484f7b3..3f7341c6b8f 100644 --- a/packages/objectql/src/engine-lookup-referential-integrity.test.ts +++ b/packages/objectql/src/engine-lookup-referential-integrity.test.ts @@ -370,7 +370,11 @@ describe('[#4441] a lookup id that resolves to nothing is refused', () => { const row: any = await engine.insert( 'ref_audit_note', { note: 'n', recorded_by: 'ps_does_not_exist' }, { context: userCtx } as any, ); - expect(row.recorded_by, 'stripped before the reference check, not refused by it').toBeUndefined(); + // [#29538] The answer reads the unstored column `null`, as the next read + // does; the strip is the STORED row's own: no key at all. + expect(row.recorded_by, 'stripped before the reference check, not refused by it').toBeNull(); + const stored = await engine.getDriverForObject('ref_audit_note')!.findOne('ref_audit_note', { where: { id: row.id } } as any); + expect(stored, 'stripped before the reference check, not refused by it').not.toHaveProperty('recorded_by'); }); it('…and the issue\'s own fields are NOT readonly, so they stay enforced', () => { diff --git a/packages/objectql/src/engine-select-option-default.test.ts b/packages/objectql/src/engine-select-option-default.test.ts index 55c693f71ab..f4b4a057f3f 100644 --- a/packages/objectql/src/engine-select-option-default.test.ts +++ b/packages/objectql/src/engine-select-option-default.test.ts @@ -72,6 +72,10 @@ function makeStubDriver() { const sys = { context: { isSystem: true } } as any; +/** The row as the stub driver stored it — what the write persisted, before any engine shaping. */ +const storedRow = (engine: ObjectQL, object: string, id: unknown) => + engine.getDriverForObject(object)!.findOne(object, { where: { id } } as any); + describe('[#7246] the option marked `default: true` is the field default', () => { let engine: ObjectQL; @@ -132,7 +136,10 @@ describe('[#7246] the option marked `default: true` is the field default', () => }, } as any, 'test.issue7246'); const row: any = await engine.insert('opt_unmarked', { title: 'D' }, sys); - expect(row.status).toBeUndefined(); + // [#29538] The answer reads the unstored column `null`, as the next read + // does; "left alone" is the STORED row's own: no key at all. + expect(row.status).toBeNull(); + expect(await storedRow(engine, 'opt_unmarked', row.id)).not.toHaveProperty('status'); }); }); @@ -348,8 +355,11 @@ describe('[#7246] the default follows the FIELD shape, not the option count', () { label: 'Quick', value: 'quick' }, ]) as any, 'test.issue7246'); const row: any = await engine.insert('opt_valueless', {}, sys); - expect(row.tags).toBeUndefined(); - expect(row.tags).not.toBeNull(); + // [#29538] The answer reads the unstored column `null`, as the next read + // does; "left unset" (no value, not even a stored `null`) is the STORED + // row's own: no key at all. + expect(row.tags).toBeNull(); + expect(await storedRow(engine, 'opt_valueless', row.id)).not.toHaveProperty('tags'); }); }); @@ -400,6 +410,8 @@ describe('[#7246] the lint heuristic and the engine agree BY CONSTRUCTION', () = }, } as any, 'test.issue7246'); const row: any = await engine.insert('opt_alias', {}, sys); - expect(row.status).toBeUndefined(); + // [#29538] The answer reads the unstored column `null`; the STORED row has no key. + expect(row.status).toBeNull(); + expect(await storedRow(engine, 'opt_alias', row.id)).not.toHaveProperty('status'); }); }); diff --git a/packages/objectql/src/engine-write-formula-hydration.test.ts b/packages/objectql/src/engine-write-formula-hydration.test.ts index 5e32a5df9bd..a7697cfe820 100644 --- a/packages/objectql/src/engine-write-formula-hydration.test.ts +++ b/packages/objectql/src/engine-write-formula-hydration.test.ts @@ -557,8 +557,12 @@ describe('#5504 — cost threshold: same gate the read path uses', () => { it('an object declaring NO formula evaluates nothing on write', async () => { const created = await rig.protocol.createData({ object: 'wf_plain', data: { name: 'nothing derived' } }); expect(evaluate).not.toHaveBeenCalled(); - // …and the response carries exactly the stored keys — no phantom columns. - expect(Object.keys(created.record as Rec).sort()).toEqual(['id', 'name']); + // …and the response carries exactly the stored values — no phantom columns. + // [#29538] Every other stored declared column is answered `null`, as the + // next read answers it: the response IS that read's record. + const record = created.record as Rec; + expect(Object.keys(record).filter((k) => record[k] !== null).sort()).toEqual(['id', 'name']); + expect(record).toStrictEqual(await rig.engine.findOne('wf_plain', { where: { id: record.id } })); }); it('an object declaring formulas evaluates once per formula field per row', async () => { diff --git a/packages/objectql/src/protocol-save-notice-answer.test.ts b/packages/objectql/src/protocol-save-notice-answer.test.ts index b0e15dcad9f..49af9f3c31a 100644 --- a/packages/objectql/src/protocol-save-notice-answer.test.ts +++ b/packages/objectql/src/protocol-save-notice-answer.test.ts @@ -109,7 +109,9 @@ const beforeSay = async (ctx: any) => { }; const afterSay = async (ctx: any) => { const say = ctx.result?.say_after; - if (say !== undefined) listenerOf(ctx)?.(say); + // [#29538] `!= null`: the write result answers an unwritten declared column + // `null`, as the next read does. + if (say != null) listenerOf(ctx)?.(say); if (nestedSay !== undefined && ctx.event === 'afterInsert') { const nested = nestedSay; nestedSay = undefined; @@ -136,7 +138,9 @@ async function boot(surface: 'envelope' | 'declarative' | 'member') { for (const e of ['beforeInsert', 'afterInsert', 'beforeDelete', 'afterDelete', 'beforeFind'] as const) { engine.registerHook(e, async (ctx: any) => { memberSeen.push({ event: e, type: typeof ctx.notice }); - const say = e === 'beforeInsert' ? ctx.input?.data?.say : e === 'afterInsert' ? ctx.result?.say_after : undefined; + // [#29538] The write result answers an unwritten declared column + // `null`, as the next read does: on the RESULT, `null` is "not said". + const say = e === 'beforeInsert' ? ctx.input?.data?.say : e === 'afterInsert' ? (ctx.result?.say_after ?? undefined) : undefined; if (say !== undefined) ctx.notice(say); }, { object: DEAL.name }); } @@ -144,7 +148,8 @@ async function boot(surface: 'envelope' | 'declarative' | 'member') { { name: 'sn_deal_member', object: DEAL.name, events: ['beforeUpdate', 'afterUpdate'], handler: async (ctx: any) => { - const say = ctx.event === 'beforeUpdate' ? ctx.input.say : ctx.result?.say_after; + // [#29538] On the RESULT, `null` is "not said", as above. + const say = ctx.event === 'beforeUpdate' ? ctx.input.say : (ctx.result?.say_after ?? undefined); if (say !== undefined) ctx.notice(say); }, }, From 8bf733db387a64af868ca0b8c8749a8a20f4ff16 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 01:59:48 +0000 Subject: [PATCH 4/8] fix(objectql): a write never answers null for a column it carried a value for, so a driver that stores it without echoing it leaves it absent (#29538) The fill now skips a column the insert payload, or the by-id PATCH and then the stored row before it, holds a non-null value for. On driver-sql, driver-memory and driver-mongodb the echo is the stored row, so nothing changes there; on a driver that stores a column without echoing it, the answer no longer claims a null the row does not hold, which a reader that layers the payload under the answer (the record-change flow record) read over the written value. Claude-Session: https://claude.ai/code/session_01JQwxbXRug3R6WZAjtFuAoS Co-Authored-By: Claude --- .../objectql/src/declared-read-columns.ts | 35 +++++++++++++ packages/objectql/src/engine.ts | 17 +++++-- .../src/write-answer-declared-columns.test.ts | 49 +++++++++++++++++-- 3 files changed, 92 insertions(+), 9 deletions(-) diff --git a/packages/objectql/src/declared-read-columns.ts b/packages/objectql/src/declared-read-columns.ts index 2b6000e47a2..e6e2b7095f1 100644 --- a/packages/objectql/src/declared-read-columns.ts +++ b/packages/objectql/src/declared-read-columns.ts @@ -163,6 +163,9 @@ export function rowsWithDeclaredColumnsOnly(rows: T[], declared: ReadonlySet< * - [#29538] a write's answer (the insert result, single or batch, and the * by-id update readback) names no projection either, so it is answered on * the whole stored record: the write answer equals the next read of the row. + * Less the columns the write itself carried a value for + * ({@link storedColumnsLeftUnwritten}): those are stored, so a driver that + * did not echo one leaves it absent rather than answered `null`. * * ## What stays absent, and why * @@ -236,6 +239,38 @@ export function withSelectedColumnsAnswered(row: T, selected: ReadonlySet | undefined, + ...writeImages: unknown[] +): ReadonlySet | undefined { + if (!selected || selected.size === 0) return selected; + const images = writeImages.filter( + (image): image is Record => !!image && typeof image === 'object' && !Array.isArray(image), + ); + let left: Set | undefined; + for (const name of selected) { + const decider = images.find((image) => image[name] !== undefined); + if (decider === undefined || decider[name] === null) continue; + left ??= new Set(selected); + left.delete(name); + } + return left ?? selected; +} + /** {@link withSelectedColumnsAnswered} over a driver's result page. */ export function rowsWithSelectedColumnsAnswered(rows: T[], selected: ReadonlySet | undefined): T[] { if (!selected || selected.size === 0) return rows; diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 619b5e9a5a0..80f86b0dee0 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -392,6 +392,7 @@ import { rowsWithDeclaredColumnsOnly, rowsWithSelectedColumnsAnswered, selectedStoredColumns, + storedColumnsLeftUnwritten, withDeclaredColumnsOnly, withSelectedColumnsAnswered, } from './declared-read-columns.js'; @@ -16693,10 +16694,15 @@ export class ObjectQL implements IObjectQLEngine { // record. Before the formulas, so a formula over an unwritten column // evaluates against `null` as the read's does; inside the verb, so the // security middleware's field mask still deletes an FLS-refused key. + // A column the row's payload carried a value for is stored, so it is + // never answered `null`: a driver that did not echo it leaves it + // absent (`storedColumnsLeftUnwritten`). Rows pair with `liveRows` by + // index, which the one-row-per-input guard above holds. const insertAnswerColumns = selectedStoredColumns(schemaForValidation, undefined); result = isBatch - ? rowsWithSelectedColumnsAnswered(result as unknown[], insertAnswerColumns) - : withSelectedColumnsAnswered(result, insertAnswerColumns); + ? (result as unknown[]).map((row, i) => + withSelectedColumnsAnswered(row, storedColumnsLeftUnwritten(insertAnswerColumns, liveRows[i]))) + : withSelectedColumnsAnswered(result, storedColumnsLeftUnwritten(insertAnswerColumns, liveRows[0])); // Coerce `boolean` fields (SQLite/libsql return 0/1) to real booleans on // the after-hook view so flow trigger conditions (`record.is_escalated @@ -18367,8 +18373,11 @@ export class ObjectQL implements IObjectQLEngine { // `null` readback (the row left the caller's scope) stays `null`. result = withDeclaredColumnsOnly(result, declaredColumnSet(updateSchema)); // [#29538] Same answer on the stored declared columns as the - // insert result and the next read (#29408). - result = withSelectedColumnsAnswered(result, selectedStoredColumns(updateSchema, undefined)); + // insert result and the next read (#29408), less those the + // PATCH or the stored row before it carried a value for. + result = withSelectedColumnsAnswered(result, storedColumnsLeftUnwritten( + selectedStoredColumns(updateSchema, undefined), hookContext.input.data, priorRecord, + )); // [#27495] The handle the stored row named before, where this // statement cleared or replaced it. retiredSecrets = retiredSecretSlots(updateSchema, [priorRecord], hookContext.input.data); diff --git a/packages/objectql/src/write-answer-declared-columns.test.ts b/packages/objectql/src/write-answer-declared-columns.test.ts index 4a4c1dce169..1669cc826fe 100644 --- a/packages/objectql/src/write-answer-declared-columns.test.ts +++ b/packages/objectql/src/write-answer-declared-columns.test.ts @@ -34,7 +34,7 @@ interface DriverAst { limit?: number; } -function makeSparseDriver(name: string) { +function makeSparseDriver(name: string, unechoed: readonly string[] = []) { const tables = new Map>(); const tableFor = (o: string): Map => { let t = tables.get(o); @@ -64,6 +64,8 @@ function makeSparseDriver(name: string) { // A key given `undefined` is not stored, as driver-mongodb's insert doors // drop it; the echo is the stored row and nothing else. const stored = (data: Row): Row => Object.fromEntries(Object.entries(data).filter(([, v]) => v !== undefined)); + // A driver that stores a column but does not echo it (#1872's shape). + const echo = (row: Row): Row => Object.fromEntries(Object.entries(row).filter(([k]) => !unechoed.includes(k))); const driver = { name, version: '0.0.0', supports: {}, async connect(): Promise {}, @@ -76,14 +78,14 @@ function makeSparseDriver(name: string) { seq += 1; const row: Row = { ...stored(data), id: (data.id as string | undefined) ?? `new_${seq}` }; tableFor(object).set(String(row.id), row); - return { ...row }; + return echo(row); }, async update(object: string, id: string, data: Row): Promise { const prior = tableFor(object).get(id); if (!prior) return null; const next: Row = { ...prior, ...stored(data), id }; tableFor(object).set(id, next); - return { ...next }; + return echo(next); }, async updateMany(object: string, ast: DriverAst, data: Row): Promise { const hits = Array.from(tableFor(object).values()).filter((r) => matches(r, ast?.where)); @@ -112,9 +114,9 @@ const ITEM_FIELDS = { doubled: { type: 'formula', expression: { dialect: 'cel', source: 'record.qty != null ? record.qty * 2 : -1' } }, }; -async function boot() { +async function boot(unechoed: readonly string[] = []) { const engine = new ObjectQL(); - const local = makeSparseDriver('memory'); + const local = makeSparseDriver('memory', unechoed); engine.registerDriver(local.driver as never, true); await engine.init(); engine.registry.registerObject({ name: ITEM, label: 'Item', fields: ITEM_FIELDS } as never, PACKAGE_ID); @@ -192,3 +194,40 @@ describe('[#29538] a write answers the record the next read answers', () => { expect(count).toBe(1); }); }); + +describe('[#29538] a column the write carried a value for is never answered null', () => { + // The driver stores `note` but leaves it out of its echo. The answer must not + // claim a `null` the row does not hold; a reader layering the payload under + // the answer (trigger-record-change's flow record) keeps the written value. + let h: Booted; + beforeEach(async () => { h = await boot(['note']); }); + + it('insert: a written, unechoed column stays absent; an unwritten one is null', async () => { + const answer = await h.engine.insert(ITEM, { title: 'w', note: 'written' }) as Row; + expect(h.local.stored(ITEM, String(answer.id))!.note).toBe('written'); + expect(own(answer, 'note')).toBe(false); + expect(answer.qty).toBeNull(); + }); + + it('insert (batch): each row is judged by its own payload', async () => { + const [carried, bare] = await h.engine.insert(ITEM, [{ title: 'x', note: 'n1' }, { title: 'y' }]) as Row[]; + expect(own(carried, 'note')).toBe(false); + expect(own(bare, 'note')).toBe(true); + expect(bare!.note).toBeNull(); + }); + + it('update by id: a value the prior row stored and the echo dropped stays absent', async () => { + const seeded = await h.engine.insert(ITEM, { title: 's', note: 'kept' }) as Row; + const answer = await h.engine.update(ITEM, { id: seeded.id, title: 's2' }) as Row; + expect(h.local.stored(ITEM, String(seeded.id))!.note).toBe('kept'); + expect(own(answer, 'note')).toBe(false); + expect(answer.qty).toBeNull(); + }); + + it('update by id: a PATCH that clears the prior value is answered null', async () => { + const seeded = await h.engine.insert(ITEM, { title: 's', note: 'kept' }) as Row; + const answer = await h.engine.update(ITEM, { id: seeded.id, note: null }) as Row; + expect(own(answer, 'note')).toBe(true); + expect(answer.note).toBeNull(); + }); +}); From f847848c60b563fb234031e3333fcc27d85e3dfb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 02:02:21 +0000 Subject: [PATCH 5/8] chore(changeset): name the carried-value rule of the write-answer fill (#29538) Claude-Session: https://claude.ai/code/session_01JQwxbXRug3R6WZAjtFuAoS Co-Authored-By: Claude --- .changeset/29538-write-answer-declared-column-null.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/29538-write-answer-declared-column-null.md b/.changeset/29538-write-answer-declared-column-null.md index 7f110e25962..4a2a3254ad4 100644 --- a/.changeset/29538-write-answer-declared-column-null.md +++ b/.changeset/29538-write-answer-declared-column-null.md @@ -9,5 +9,6 @@ Metadata-protocol: none - **Before.** After #29408, `find` and `findOne` answer an unwritten declared column `null` on every driver, but the write answer still came straight from the driver. driver-sql echoed `null` (its table has the column); driver-mongodb and driver-memory echoed only the keys the write stored. So a create on mongo or memory answered `{ title: 'x' }` and the very next `GET` of the same row answered `{ title: 'x', note: null }`, and a formula reading `note` answered `null` (a fault) in the create response and its value on the `GET`. - **Now.** The engine answers the insert result (single, batch, and each `insertMany` outcome) and the by-id update readback on every stored declared column, `null` where the driver echoed no key, before the formulas and the `afterInsert` / `afterUpdate` hooks. Every door on those two verbs answers alike: `POST /data/:object`, `PATCH /data/:object/:id`, `createManyData`, `updateManyData`, and `batchData` create / update / both upsert arms. +- **Never a `null` the row does not hold.** A column the insert payload, or the PATCH and then the stored row before it, carries a value for is stored, so a driver that stores it without echoing it leaves it absent from the answer, as before, rather than answered `null`. A reader that layers the payload under the answer (the record-change flow's `record`, which overlays the write result on the input) keeps the value it wrote. - **Unchanged.** What a write stores. A predicate update still answers its affected-row count. A column field-level security refuses the writer stays absent (the security mask still deletes it), and so do a name outside the declared set, an injected column a federated (`external`) object has no storage for, and a `created_at` / `updated_at` a federated object neither declares nor has injected — the read's own exclusions. - **Who sees it.** A client of an object served by driver-mongodb or driver-memory that diffs a create or update response against its own state, or renders it: a field the save never mentioned now arrives in that response as `null`, as it does on the next read, instead of appearing only on the next read. In-process readers of the write result see the same: an `afterInsert` / `afterUpdate` hook, a `data.record.created` / `data.record.updated` event payload and the webhooks that deliver it. A reader that told a missing key from `null` (`'x' in row`, `has(record.x)`) now reads what driver-sql already answered; test emptiness with `record.x != null`. A `formula` over an unwritten field in a write response on those two drivers changes value: it used to answer `null` with a fault, and now answers what the next read answers (`record.qty != null ? record.qty * 2 : -1` was `null`, now `-1`). From 94b04574b557635bbb4bcb5abcc207183b550bb9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 04:20:13 +0000 Subject: [PATCH 6/8] test(objectql, driver-mongodb): give each mongo write arm a fake of only the native members its path calls; type the stored-row reads (#29538) The single-row arms' Collection fake declares insertOne / updateOne / findOne, the batch arm's declares insertMany / find (its next read is a find). The three stored-row reads in the forced objectql test edits name their object in the AST instead of erasing the options with `as any`. Claude-Session: https://claude.ai/code/session_01JQwxbXRug3R6WZAjtFuAoS Co-Authored-By: Claude --- ...godb-declared-column-null-on-write.test.ts | 62 ++++++++++++++----- .../src/engine-default-value-tokens.test.ts | 2 +- ...ngine-lookup-referential-integrity.test.ts | 2 +- .../src/engine-select-option-default.test.ts | 2 +- 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-write.test.ts b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-write.test.ts index 44c3def7f5d..4be8afc5cff 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-write.test.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-declared-column-null-on-write.test.ts @@ -28,8 +28,14 @@ import { MongoDBDriver } from './mongodb-driver.js'; type Doc = Record; -/** A fake `Db`: one collection, stored documents kept as inserted. */ -function fakeDb() { +/** + * A fake `Db`: one store of documents kept as inserted, served through a + * collection that declares ONLY the native `Collection` members the path under + * test calls. Two shapes, one per path: the single-row doors (`create`, + * `update`, `findOne`) reach `insertOne` / `updateOne` / `findOne`; the batch + * door (`bulkCreate`) reaches `insertMany`, and its next read is a `find`. + */ +function fakeStore() { const docs: Doc[] = []; const project = (doc: Doc, projection: Record | undefined): Doc => { const keep = Object.keys(projection ?? {}).filter((k) => projection![k] === 1); @@ -54,25 +60,43 @@ function fakeDb() { } return doc[k] === v; }); + const push = (doc: Doc) => { docs.push({ _id: `oid_${docs.length}`, ...doc }); }; + const pick = (filter: Doc) => docs.find((x) => matches(x, filter)); + const pickAll = (filter: Doc, options?: { projection?: Record }) => + docs.filter((x) => matches(x, filter)).map((d) => project(d, options?.projection)); + return { docs, push, pick, pickAll, project }; +} + +/** The single-row doors: `create`, `update` and `findOne`. */ +function singleRowDb() { + const store = fakeStore(); const collection = { - insertOne: async (doc: Doc) => { docs.push({ _id: `oid_${docs.length}`, ...doc }); return {}; }, - insertMany: async (many: Doc[]) => { for (const doc of many) docs.push({ _id: `oid_${docs.length}`, ...doc }); return {}; }, + insertOne: async (doc: Doc) => { store.push(doc); return {}; }, updateOne: async (filter: Doc, update: { $set?: Doc }) => { const updateOps = Object.keys(update); if (updateOps.length !== 1 || updateOps[0] !== '$set') throw new Error(`fake Db: update ${updateOps.join(',')} is not implemented`); - const d = docs.find((x) => matches(x, filter)); + const d = store.pick(filter); if (d) Object.assign(d, update.$set); return { matchedCount: d ? 1 : 0 }; }, findOne: async (filter: Doc, options: { projection?: Record }) => { - const d = docs.find((x) => matches(x, filter)); - return d ? project(d, options?.projection) : null; + const d = store.pick(filter); + return d ? store.project(d, options?.projection) : null; }, + }; + return { db: { collection: () => collection }, docs: store.docs }; +} + +/** The batch door: `bulkCreate`, and `find` for the next read. */ +function batchDb() { + const store = fakeStore(); + const collection = { + insertMany: async (many: Doc[]) => { for (const doc of many) store.push(doc); return {}; }, find: (filter: Doc, options: { projection?: Record }) => ({ - toArray: async () => docs.filter((x) => matches(x, filter)).map((d) => project(d, options?.projection)), + toArray: async () => store.pickAll(filter, options), }), }; - return { db: { collection: () => collection }, docs }; + return { db: { collection: () => collection }, docs: store.docs }; } const OBJECT = 'nk_mongo_write_item'; @@ -84,9 +108,8 @@ describe('[#29538] driver-mongodb: a write answers the record the next read answ let engine: ObjectQL; let docs: Doc[]; - beforeEach(async () => { + const boot = async (fake: { db: unknown; docs: Doc[] }) => { const driver = new MongoDBDriver({ url: 'mongodb://127.0.0.1:1/unused' }); - const fake = fakeDb(); docs = fake.docs; (driver as any).db = fake.db; (driver as any).connect = async () => {}; @@ -104,10 +127,12 @@ describe('[#29538] driver-mongodb: a write answers the record the next read answ doubled: { type: 'formula', expression: { dialect: 'cel', source: 'record.qty != null ? record.qty * 2 : -1' } }, }, } as never); - }); + }; - const expectAnswerIsNextRead = async (answer: Doc) => { - expect(answer).toStrictEqual(await engine.findOne(OBJECT, { where: { id: answer.id } })); + beforeEach(async () => { await boot(singleRowDb()); }); + + const expectAnswerIsNextRead = async (answer: Doc, next?: Doc) => { + expect(answer).toStrictEqual(next ?? await engine.findOne(OBJECT, { where: { id: answer.id } })); expect(own(answer, 'note')).toBe(true); expect(answer.note).toBeNull(); expect(own(answer, 'qty')).toBe(true); @@ -122,10 +147,15 @@ describe('[#29538] driver-mongodb: a write answers the record the next read answ await expectAnswerIsNextRead(answer); }); - it('batch insert (bulkCreate): every answered row deep-equals its next findOne', async () => { + it('batch insert (bulkCreate): every answered row deep-equals its next read', async () => { + await boot(batchDb()); const answers = await engine.insert(OBJECT, [{ name: 'Ada' }, { name: 'Bo' }]) as Doc[]; expect(answers).toHaveLength(2); - for (const answer of answers) await expectAnswerIsNextRead(answer); + expect(own(docs[0], 'note')).toBe(false); + for (const answer of answers) { + const [next] = await engine.find(OBJECT, { where: { id: answer.id } }) as Doc[]; + await expectAnswerIsNextRead(answer, next); + } }); it('update by id: the answer deep-equals the next findOne', async () => { diff --git a/packages/objectql/src/engine-default-value-tokens.test.ts b/packages/objectql/src/engine-default-value-tokens.test.ts index b72cc81d102..49ced8bdd0d 100644 --- a/packages/objectql/src/engine-default-value-tokens.test.ts +++ b/packages/objectql/src/engine-default-value-tokens.test.ts @@ -106,7 +106,7 @@ describe('[#4560] the `current_user` defaultValue token is engine-owned', () => // [#29538] The answer is the next read's record: a column nothing was // stored in reads `null`. "Unset" is the STORED row's own: no key at all. expect(row.owner).toBeNull(); - const stored = await engine.getDriverForObject('tok_doc')!.findOne('tok_doc', { where: { id: row.id } } as any); + const stored = await engine.getDriverForObject('tok_doc')!.findOne('tok_doc', { object: 'tok_doc', where: { id: row.id } }); expect(stored).not.toHaveProperty('owner'); }); diff --git a/packages/objectql/src/engine-lookup-referential-integrity.test.ts b/packages/objectql/src/engine-lookup-referential-integrity.test.ts index 3f7341c6b8f..00e920a8749 100644 --- a/packages/objectql/src/engine-lookup-referential-integrity.test.ts +++ b/packages/objectql/src/engine-lookup-referential-integrity.test.ts @@ -373,7 +373,7 @@ describe('[#4441] a lookup id that resolves to nothing is refused', () => { // [#29538] The answer reads the unstored column `null`, as the next read // does; the strip is the STORED row's own: no key at all. expect(row.recorded_by, 'stripped before the reference check, not refused by it').toBeNull(); - const stored = await engine.getDriverForObject('ref_audit_note')!.findOne('ref_audit_note', { where: { id: row.id } } as any); + const stored = await engine.getDriverForObject('ref_audit_note')!.findOne('ref_audit_note', { object: 'ref_audit_note', where: { id: row.id } }); expect(stored, 'stripped before the reference check, not refused by it').not.toHaveProperty('recorded_by'); }); diff --git a/packages/objectql/src/engine-select-option-default.test.ts b/packages/objectql/src/engine-select-option-default.test.ts index f4b4a057f3f..d1fc1aefa6f 100644 --- a/packages/objectql/src/engine-select-option-default.test.ts +++ b/packages/objectql/src/engine-select-option-default.test.ts @@ -74,7 +74,7 @@ const sys = { context: { isSystem: true } } as any; /** The row as the stub driver stored it — what the write persisted, before any engine shaping. */ const storedRow = (engine: ObjectQL, object: string, id: unknown) => - engine.getDriverForObject(object)!.findOne(object, { where: { id } } as any); + engine.getDriverForObject(object)!.findOne(object, { object, where: { id } }); describe('[#7246] the option marked `default: true` is the field default', () => { let engine: ObjectQL; From cc071d1ae4b99b9069cb5b75fee148f73c59bf4c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 05:14:19 +0000 Subject: [PATCH 7/8] test(objectql): the sparse fake's refusals no longer name the driver literal, so the where-matcher lift judges it (#29538) check:where-matcher reads identifiers out of the template text too, lifted the `driver` literal and its `name` shorthand with the matcher, and filed it unjudged. The matcher refuses every operator it does not implement; it now reads CONFORMING. Claude-Session: https://claude.ai/code/session_01JQwxbXRug3R6WZAjtFuAoS Co-Authored-By: Claude --- packages/objectql/src/write-answer-declared-columns.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/objectql/src/write-answer-declared-columns.test.ts b/packages/objectql/src/write-answer-declared-columns.test.ts index 1669cc826fe..b1ed83cd5b8 100644 --- a/packages/objectql/src/write-answer-declared-columns.test.ts +++ b/packages/objectql/src/write-answer-declared-columns.test.ts @@ -46,8 +46,8 @@ function makeSparseDriver(name: string, unechoed: readonly string[] = []) { const matches = (row: Row, where: Record | undefined): boolean => { if (!where) return true; for (const [k, v] of Object.entries(where)) { - if (k.startsWith('$')) throw new Error(`sparse driver: filter operator ${k} is not implemented`); - if (v !== null && typeof v === 'object') throw new Error(`sparse driver: operator on ${k} is not implemented`); + if (k.startsWith('$')) throw new Error(`sparse fake: filter operator ${k} is not implemented`); + if (v !== null && typeof v === 'object') throw new Error(`sparse fake: operator on ${k} is not implemented`); if ((row[k] ?? null) !== (v ?? null)) return false; } return true; From 77b6b65dd2a9a51b058f8daba8f18e42766fbe2f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 06:44:14 +0000 Subject: [PATCH 8/8] test(objectql): read the stored row with a plain driver query, which needs neither an object key nor an `as any` (#29538) Claude-Session: https://claude.ai/code/session_01JQwxbXRug3R6WZAjtFuAoS Co-Authored-By: Claude --- packages/objectql/src/engine-default-value-tokens.test.ts | 2 +- .../objectql/src/engine-lookup-referential-integrity.test.ts | 2 +- packages/objectql/src/engine-select-option-default.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/objectql/src/engine-default-value-tokens.test.ts b/packages/objectql/src/engine-default-value-tokens.test.ts index 49ced8bdd0d..87b4e0983e9 100644 --- a/packages/objectql/src/engine-default-value-tokens.test.ts +++ b/packages/objectql/src/engine-default-value-tokens.test.ts @@ -106,7 +106,7 @@ describe('[#4560] the `current_user` defaultValue token is engine-owned', () => // [#29538] The answer is the next read's record: a column nothing was // stored in reads `null`. "Unset" is the STORED row's own: no key at all. expect(row.owner).toBeNull(); - const stored = await engine.getDriverForObject('tok_doc')!.findOne('tok_doc', { object: 'tok_doc', where: { id: row.id } }); + const stored = await engine.getDriverForObject('tok_doc')!.findOne('tok_doc', { where: { id: row.id } }); expect(stored).not.toHaveProperty('owner'); }); diff --git a/packages/objectql/src/engine-lookup-referential-integrity.test.ts b/packages/objectql/src/engine-lookup-referential-integrity.test.ts index 00e920a8749..db80392de74 100644 --- a/packages/objectql/src/engine-lookup-referential-integrity.test.ts +++ b/packages/objectql/src/engine-lookup-referential-integrity.test.ts @@ -373,7 +373,7 @@ describe('[#4441] a lookup id that resolves to nothing is refused', () => { // [#29538] The answer reads the unstored column `null`, as the next read // does; the strip is the STORED row's own: no key at all. expect(row.recorded_by, 'stripped before the reference check, not refused by it').toBeNull(); - const stored = await engine.getDriverForObject('ref_audit_note')!.findOne('ref_audit_note', { object: 'ref_audit_note', where: { id: row.id } }); + const stored = await engine.getDriverForObject('ref_audit_note')!.findOne('ref_audit_note', { where: { id: row.id } }); expect(stored, 'stripped before the reference check, not refused by it').not.toHaveProperty('recorded_by'); }); diff --git a/packages/objectql/src/engine-select-option-default.test.ts b/packages/objectql/src/engine-select-option-default.test.ts index d1fc1aefa6f..ff030662ff8 100644 --- a/packages/objectql/src/engine-select-option-default.test.ts +++ b/packages/objectql/src/engine-select-option-default.test.ts @@ -74,7 +74,7 @@ const sys = { context: { isSystem: true } } as any; /** The row as the stub driver stored it — what the write persisted, before any engine shaping. */ const storedRow = (engine: ObjectQL, object: string, id: unknown) => - engine.getDriverForObject(object)!.findOne(object, { object, where: { id } }); + engine.getDriverForObject(object)!.findOne(object, { where: { id } }); describe('[#7246] the option marked `default: true` is the field default', () => { let engine: ObjectQL;