From ad69ef14b4f399d5e16a5bbc3a51b9ef63290343 Mon Sep 17 00:00:00 2001 From: IGNGserver <176017966+IGNGserver@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:28:16 +0800 Subject: [PATCH 1/9] =?UTF-8?q?test(db):=20=E5=A2=9E=E5=8A=A0=20MySQL=208.?= =?UTF-8?q?4=20=E6=96=B9=E8=A8=80=E5=85=BC=E5=AE=B9=E6=8E=A2=E9=92=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 用真实 MySQL 8.4 逐条验证移植依赖的 SQL 形态,结论作为后续方言层与 Store 移植的依据: - RETURNING 不可用,需 insertId 或同事务回查 - ON DUPLICATE KEY UPDATE 支持行别名 - 生成列唯一索引可模拟部分唯一索引(墓碑并存) - 多行 VALUES 批量写、JSON_TABLE、JSON_ARRAYAGG 均可用 - JSON_ARRAYAGG 不保序,顺序需应用层补齐 - DELETE 同表子查询被 1093/1235 拒绝,改用 DELETE ORDER BY LIMIT - DATETIME 拒绝带 Z 的 ISO 串,参数需归一 - rank/cursor 是保留字,必须反引号引用 --- package.json | 4 +- packages/database/package.json | 1 + pnpm-lock.yaml | 92 ++++- scripts/mysql-compat.ts | 652 +++++++++++++++++++++++++++++++++ 4 files changed, 742 insertions(+), 7 deletions(-) create mode 100644 scripts/mysql-compat.ts diff --git a/package.json b/package.json index 4874748..e0710ed 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,8 @@ "android:assembleRelease": "pnpm brand:sync && tsx scripts/android-assemble.ts", "db:migrate": "tsx scripts/migrate.ts", "openapi:check": "tsx scripts/openapi-check.ts", - "verify:m3e:web": "tsx scripts/verify-m3e-web.ts" + "verify:m3e:web": "tsx scripts/verify-m3e-web.ts", + "test:mysql-compat": "tsx scripts/mysql-compat.ts" }, "devDependencies": { "@axe-core/playwright": "^4.13.0", @@ -44,6 +45,7 @@ "eslint-import-resolver-typescript": "^4.4.4", "eslint-plugin-import": "^2.32.0", "eslint-plugin-react-hooks": "^5.2.0", + "mysql2": "^3.24.6", "prettier": "^3.6.2", "tsx": "^4.20.5", "typescript": "^5.9.2", diff --git a/packages/database/package.json b/packages/database/package.json index ede67ea..f1ef067 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -19,6 +19,7 @@ }, "dependencies": { "drizzle-orm": "^0.45.4", + "mysql2": "^3.24.6", "pg": "^8.16.3" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 72d3d96..09f2906 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: eslint-plugin-react-hooks: specifier: ^5.2.0 version: 5.2.0(eslint@9.39.5(jiti@2.7.0)(supports-color@7.2.0)) + mysql2: + specifier: ^3.24.6 + version: 3.24.6(@types/node@24.13.3) prettier: specifier: ^3.6.2 version: 3.9.6 @@ -225,7 +228,10 @@ importers: dependencies: drizzle-orm: specifier: ^0.45.4 - version: 0.45.4(@types/pg@8.23.1)(pg@8.23.0) + version: 0.45.4(@types/pg@8.23.1)(mysql2@3.24.6(@types/node@24.13.3))(pg@8.23.0) + mysql2: + specifier: ^3.24.6 + version: 3.24.6(@types/node@24.13.3) pg: specifier: ^8.16.3 version: 8.23.0 @@ -1146,8 +1152,8 @@ packages: '@jridgewell/remapping@2.3.5': resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} - '@jridgewell/resolve-uri@3.1.2': - resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + '@jridgewell/resolve-uri@3.1.3': + resolution: {integrity: sha512-lhTtjtoEArWBjismtfS761S9Hlyn+nIZqOZVYoZ1Xn82PMNMVnk+jj/NhC4fAXGTs7ydONcJiR5vyQBTMDu27g==} engines: {node: '>=6.0.0'} '@jridgewell/source-map@0.3.11': @@ -1803,6 +1809,10 @@ packages: avvio@9.3.1: resolution: {integrity: sha512-QyTKW61/qlba+cUQBm/sT9X8lri4UdomqSDwCnrnj1IvLVDRRiXRkz4CgBRWB+/cmfC7SK5WQc8T1vSdGvaUcQ==} + aws-ssl-profiles@1.1.2: + resolution: {integrity: sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==} + engines: {node: '>= 6.0.0'} + axe-core@4.13.0: resolution: {integrity: sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==} engines: {node: '>=4'} @@ -2478,6 +2488,9 @@ packages: functions-have-names@1.2.3: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} + generate-function@2.3.1: + resolution: {integrity: sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==} + generator-function@2.0.1: resolution: {integrity: sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==} engines: {node: '>= 0.4'} @@ -2569,6 +2582,10 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} + iconv-lite@0.7.3: + resolution: {integrity: sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==} + engines: {node: '>=0.10.0'} + idb@7.1.1: resolution: {integrity: sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ==} @@ -2690,6 +2707,9 @@ packages: resolution: {integrity: sha512-bUi/yjmtKYcRVUtWRGr0UA6xEFh2I6zWUwMrUXB3s7bmYCaZ8a+0ZsTRkrawh/mzlSD1Y0Ph8bp/U+TvBpWDNw==} engines: {node: '>=0.10.0'} + is-property@1.0.2: + resolution: {integrity: sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==} + is-regex@1.2.1: resolution: {integrity: sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==} engines: {node: '>= 0.4'} @@ -2834,6 +2854,9 @@ packages: lodash.merge@4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + lru-cache@11.5.3: resolution: {integrity: sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==} engines: {node: 20 || >=22} @@ -2841,6 +2864,10 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + lru.min@1.1.5: + resolution: {integrity: sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==} + engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lucide-react@0.542.0: resolution: {integrity: sha512-w3hD8/SQB7+lzU2r4VdFyzzOzKnUjTZIF/MQJGSSvni7Llewni4vuViRppfRAa2guOsY5k4jZyxw/i9DQHv+dw==} peerDependencies: @@ -2888,6 +2915,16 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + mysql2@3.24.6: + resolution: {integrity: sha512-enu73KhbAua2L3dIxjc6qg6ZJ7H3pt1BImZASWNWY5xSsmJ1ZuUPULTDGXgmJbI61vb+nJAG19IZclIq19UidQ==} + engines: {node: '>= 8.0'} + peerDependencies: + '@types/node': '>= 8' + + named-placeholders@1.1.6: + resolution: {integrity: sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==} + engines: {node: '>=8.0.0'} + nanoid@3.3.20: resolution: {integrity: sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -3264,6 +3301,9 @@ packages: resolution: {integrity: sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg==} engines: {node: '>=10'} + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + sanitize-html@2.18.1: resolution: {integrity: sha512-2X5lC8fYla8YaT8g73Nj+HcLWXan5dLyCWLgVvaOmfw7JI78IRsyNtIDytTHdn7hBQbIkCah7Zbmd5jabzeDig==} engines: {node: '>=22.12.0'} @@ -3390,6 +3430,10 @@ packages: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} + sql-escaper@1.5.2: + resolution: {integrity: sha512-6CKD38c31SENivxOADeMNLdukOnUxUcflKtzVWzace7Riv1v7cAEym5Cx9Q7gYZ3ezIJI7ZpqecQq8cqNeSSFg==} + engines: {bun: '>=1.0.0', deno: '>=2.0.0', node: '>=12.0.0'} + stable-hash-x@0.2.0: resolution: {integrity: sha512-o3yWv49B/o4QZk5ZcsALc6t0+eCelPc44zZsLtCQnZPDwFpDYSWcDnrv2TtMmMbQ7uKo3J0HTURCqckw23czNQ==} engines: {node: '>=12.0.0'} @@ -4921,7 +4965,7 @@ snapshots: '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 - '@jridgewell/resolve-uri@3.1.2': {} + '@jridgewell/resolve-uri@3.1.3': {} '@jridgewell/source-map@0.3.11': dependencies: @@ -4932,7 +4976,7 @@ snapshots: '@jridgewell/trace-mapping@0.3.31': dependencies: - '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/resolve-uri': 3.1.3 '@jridgewell/sourcemap-codec': 1.6.0 '@lukeed/ms@2.0.2': {} @@ -5510,6 +5554,8 @@ snapshots: '@fastify/error': 4.2.1 fastq: 1.20.3 + aws-ssl-profiles@1.1.2: {} + axe-core@4.13.0: {} babel-plugin-polyfill-corejs2@0.4.17(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0): @@ -5756,9 +5802,10 @@ snapshots: domelementtype: 3.0.0 domhandler: 6.0.1 - drizzle-orm@0.45.4(@types/pg@8.23.1)(pg@8.23.0): + drizzle-orm@0.45.4(@types/pg@8.23.1)(mysql2@3.24.6(@types/node@24.13.3))(pg@8.23.0): optionalDependencies: '@types/pg': 8.23.1 + mysql2: 3.24.6(@types/node@24.13.3) pg: 8.23.0 dunder-proto@1.0.1: @@ -6207,6 +6254,10 @@ snapshots: functions-have-names@1.2.3: {} + generate-function@2.3.1: + dependencies: + is-property: 1.0.2 + generator-function@2.0.1: {} gensync@1.0.0-beta.2: {} @@ -6308,6 +6359,10 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 + iconv-lite@0.7.3: + dependencies: + safer-buffer: 2.1.2 + idb@7.1.1: {} ignore@5.3.2: {} @@ -6418,6 +6473,8 @@ snapshots: is-plain-object@5.1.0: {} + is-property@1.0.2: {} + is-regex@1.2.1: dependencies: call-bound: 1.0.4 @@ -6548,12 +6605,16 @@ snapshots: lodash.merge@4.6.2: {} + long@5.3.2: {} + lru-cache@11.5.3: {} lru-cache@5.1.1: dependencies: yallist: 3.1.1 + lru.min@1.1.5: {} + lucide-react@0.542.0(react@19.2.8): dependencies: react: 19.2.8 @@ -6591,6 +6652,21 @@ snapshots: ms@2.1.3: {} + mysql2@3.24.6(@types/node@24.13.3): + dependencies: + '@types/node': 24.13.3 + aws-ssl-profiles: 1.1.2 + generate-function: 2.3.1 + iconv-lite: 0.7.3 + long: 5.3.2 + lru.min: 1.1.5 + named-placeholders: 1.1.6 + sql-escaper: 1.5.2 + + named-placeholders@1.1.6: + dependencies: + lru.min: 1.1.5 + nanoid@3.3.20: {} napi-postinstall@0.3.4: {} @@ -7012,6 +7088,8 @@ snapshots: safe-stable-stringify@2.3.1: {} + safer-buffer@2.1.2: {} + sanitize-html@2.18.1: dependencies: escape-string-regexp: 4.0.0 @@ -7138,6 +7216,8 @@ snapshots: split2@4.2.0: {} + sql-escaper@1.5.2: {} + stable-hash-x@0.2.0: {} stackback@0.0.2: {} diff --git a/scripts/mysql-compat.ts b/scripts/mysql-compat.ts new file mode 100644 index 0000000..4ce515e --- /dev/null +++ b/scripts/mysql-compat.ts @@ -0,0 +1,652 @@ +/** + * MySQL 8.4 compatibility probe. + * + * Every query shape the TaskDock PostgreSQL adapter depends on is executed + * against a real MySQL server here, so the MySQL port is built on observed + * behaviour instead of assumptions. Each case records what actually happened; + * a case whose observation disagrees with the expectation fails the run. + * + * Usage: + * TASKDOCK_MYSQL_TEST_URL=mysql://root@127.0.0.1:13306 pnpm test:mysql-compat + * + * The URL must point at a throwaway server: the script creates and drops the + * `taskdock_compat` database. + */ +import { createHash } from 'node:crypto'; +import { performance } from 'node:perf_hooks'; +import mysql from 'mysql2/promise'; + +const BASE_URL = process.env['TASKDOCK_MYSQL_TEST_URL']?.trim(); +if (!BASE_URL) { + console.error('TASKDOCK_MYSQL_TEST_URL is not set; refusing to guess a server.'); + process.exit(2); +} + +const DATABASE = 'taskdock_compat'; +const OWNER = '00000000-0000-7000-8000-000000000001'; +const CREATED_AT = '2026-11-15 08:07:24.123456'; + +type CaseResult = { name: string; observation: string; passed: boolean }; +const results: CaseResult[] = []; + +function serverUrl(): string { + const url = new URL(BASE_URL!); + url.protocol = 'mysql:'; + url.pathname = '/'; + return url.toString(); +} + +async function connect(database?: string): Promise { + return mysql.createConnection({ + uri: serverUrl(), + database, + multipleStatements: false, + supportBigNumbers: true, + bigNumberStrings: false, + decimalNumbers: false, + timezone: 'Z', + dateStrings: false, + }); +} + +function errorCode(error: unknown): string { + const code = (error as { code?: unknown }).code; + return typeof code === 'string' ? code : 'UNKNOWN'; +} + +function describeError(error: unknown): string { + const err = error as { code?: string; errno?: number; sqlMessage?: string; message?: string }; + const code = err.code ?? (err.errno !== undefined ? String(err.errno) : 'ERROR'); + return `${code} ${err.sqlMessage ?? err.message ?? ''}`.trim().slice(0, 170); +} + +function record(name: string, observation: string, passed: boolean): void { + results.push({ name, observation, passed }); + console.log(`${passed ? 'PASS' : 'FAIL'} ${name} · ${observation}`); +} + +async function probe( + name: string, + fn: () => Promise<{ ok: boolean; observation: string }>, +): Promise { + try { + const value = await fn(); + record(name, value.observation, value.ok); + } catch (error) { + record(name, `threw ${describeError(error)}`, false); + } +} + +async function expectRejected( + name: string, + allowed: string[], + fn: () => Promise, +): Promise { + try { + await fn(); + record(name, 'statement unexpectedly succeeded', false); + } catch (error) { + const code = errorCode(error); + record(name, describeError(error), allowed.includes(code)); + } +} + +function id(): string { + const random = createHash('sha256').update(`${process.hrtime.bigint()}`).digest('hex'); + const time = Date.now().toString(16).padStart(12, '0'); + return `${time.slice(0, 8)}-${time.slice(8, 12)}-7${random.slice(0, 3)}-8${random.slice(3, 6)}-${random.slice(6, 18)}`; +} + +async function bootstrap(): Promise { + const admin = await connect(); + await admin.query(`DROP DATABASE IF EXISTS \`${DATABASE}\``); + await admin.query( + `CREATE DATABASE \`${DATABASE}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci`, + ); + await admin.end(); + + const db = await connect(DATABASE); + await db.query(` + CREATE TABLE probes ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin PRIMARY KEY, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + slot VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + title VARCHAR(160) NOT NULL DEFAULT '', + status VARCHAR(16) NOT NULL DEFAULT 'TODO', + sort_rank BIGINT NOT NULL DEFAULT 1024, + payload JSON NULL, + deleted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + INDEX probes_owner_slot_idx (owner_id, slot) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + `); + await db.query(` + CREATE TABLE slot_unique ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin PRIMARY KEY, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + slot VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + title VARCHAR(64) NOT NULL DEFAULT '', + deleted_at DATETIME(6) NULL, + active TINYINT(1) AS (IF(deleted_at IS NULL, 1, NULL)) STORED, + UNIQUE KEY slot_unique_active_uq (owner_id, slot, active) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + `); + await db.query( + 'CREATE TABLE seq_probe (seq BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, owner CHAR(36) NOT NULL) ENGINE=InnoDB', + ); + await db.query( + 'CREATE TABLE big_probe (id INT PRIMARY KEY AUTO_INCREMENT, v BIGINT NOT NULL) ENGINE=InnoDB', + ); + await db.query( + 'CREATE TABLE keyword_probe (`rank` BIGINT NOT NULL DEFAULT 1, `cursor` BIGINT NOT NULL DEFAULT 0) ENGINE=InnoDB', + ); + await db.query(` + CREATE TABLE checks_probe ( + id INT PRIMARY KEY AUTO_INCREMENT, + prefix VARCHAR(10) NOT NULL, + title VARCHAR(160) NOT NULL, + CONSTRAINT prefix_ck CHECK (REGEXP_LIKE(prefix, '^[A-Z][A-Z0-9]{1,9}$', 'c')), + CONSTRAINT title_ck CHECK (CHAR_LENGTH(TRIM(title)) > 0) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + `); + await db.query( + 'CREATE TABLE case_probe (id INT PRIMARY KEY AUTO_INCREMENT, title VARCHAR(64) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci', + ); + await db.query("INSERT INTO case_probe (title) VALUES ('Alice')"); + await db.end(); +} + +async function cases(): Promise { + const db = await connect(DATABASE); + + await expectRejected('RETURNING 语法', ['ER_PARSE_ERROR'], () => + db.query("INSERT INTO probes (id, owner_id, slot) VALUES (?, ?, 'r') RETURNING id", [ + id(), + OWNER, + ]), + ); + + await probe('ON DUPLICATE KEY UPDATE 行别名', async () => { + await db.query('INSERT INTO slot_unique (id, owner_id, slot, title) VALUES (?, ?, ?, ?)', [ + id(), + OWNER, + 'upsert', + 'v1', + ]); + const second = await db.query( + 'INSERT INTO slot_unique (id, owner_id, slot, title) VALUES (?, ?, ?, ?) AS new ON DUPLICATE KEY UPDATE title = new.title', + [id(), OWNER, 'upsert', 'v2'], + ); + const [rows] = await db.query( + 'SELECT COUNT(*) AS n FROM slot_unique WHERE owner_id = ? AND slot = ?', + [OWNER, 'upsert'], + ); + const affected = (second[0] as mysql.ResultSetHeader).affectedRows; + return { + ok: affected === 2 && Number(rows[0]!['n']) === 1, + observation: `affected=${affected} rows=${String(rows[0]!['n'])}`, + }; + }); + + await probe('生成列模拟部分唯一索引', async () => { + for (let i = 0; i < 2; i += 1) { + await db.query( + 'INSERT INTO slot_unique (id, owner_id, slot, deleted_at) VALUES (?, ?, ?, ?)', + [id(), OWNER, 'tombstone', CREATED_AT], + ); + } + let liveDuplicateRejected = false; + try { + for (let i = 0; i < 2; i += 1) { + await db.query('INSERT INTO slot_unique (id, owner_id, slot) VALUES (?, ?, ?)', [ + id(), + OWNER, + 'live', + ]); + } + } catch (error) { + liveDuplicateRejected = errorCode(error) === 'ER_DUP_ENTRY'; + } + const [rows] = await db.query( + "SELECT COUNT(*) AS n FROM slot_unique WHERE slot = 'tombstone'", + ); + return { + ok: Number(rows[0]!['n']) === 2 && liveDuplicateRejected, + observation: `tombstones=${String(rows[0]!['n'])} liveDuplicateRejected=${liveDuplicateRejected}`, + }; + }); + + await probe('多行 VALUES 批量写', async () => { + const ids = Array.from({ length: 200 }, () => id()); + const columns = 9; + const values: unknown[] = []; + const tuples: string[] = []; + for (const rowId of ids) { + tuples.push(`(${new Array(columns).fill('?').join(',')})`); + values.push(rowId, OWNER, 'batch', '初始标题', 'TODO', 1024, null, null, CREATED_AT); + } + const started = performance.now(); + await db.query( + `INSERT INTO probes (id, owner_id, slot, title, status, sort_rank, payload, deleted_at, created_at) VALUES ${tuples.join(',')}`, + values, + ); + const elapsed = Math.round(performance.now() - started); + + const revalues: unknown[] = []; + const retuples: string[] = []; + for (const rowId of ids) { + retuples.push(`(${new Array(6).fill('?').join(',')})`); + revalues.push(rowId, OWNER, 'batch', '更新标题', 'TODO', 2048); + } + await db.query( + `INSERT INTO probes (id, owner_id, slot, title, status, sort_rank) VALUES ${retuples.join(',')} AS new + ON DUPLICATE KEY UPDATE title = new.title, sort_rank = new.sort_rank`, + revalues, + ); + const [rows] = await db.query( + "SELECT COUNT(*) AS n, SUM(title = '更新标题') AS updated FROM probes WHERE slot = 'batch'", + ); + return { + ok: Number(rows[0]!['n']) === 200 && Number(rows[0]!['updated']) === 200, + observation: `rows=${String(rows[0]!['n'])} updated=${String(rows[0]!['updated'])} params=${values.length} insert=${elapsed}ms`, + }; + }); + + await probe('JSON_TABLE 展开数组', async () => { + const payload = JSON.stringify([ + { id: 'a', n: 1 }, + { id: 'b', n: 2 }, + ]); + const [rows] = await db.query( + `SELECT jt.id, jt.n FROM JSON_TABLE(?, '$[*]' COLUMNS (id CHAR(8) PATH '$.id', n INT PATH '$.n')) AS jt`, + [payload], + ); + return { + ok: rows.length === 2 && Number(rows[1]!['n']) === 2, + observation: `rows=${JSON.stringify(rows.map((row) => [row['id'], row['n']]))}`, + }; + }); + + await probe('JSON_ARRAYAGG 快照', async () => { + for (const rank of [30, 10, 20]) { + await db.query( + 'INSERT INTO probes (id, owner_id, slot, title, sort_rank) VALUES (?, ?, ?, ?, ?)', + [id(), OWNER, 'agg', `r${rank}`, rank], + ); + } + const [rows] = await db.query( + `SELECT JSON_ARRAYAGG(JSON_OBJECT('id', x.id, 'rank', x.sort_rank)) AS payload + FROM (SELECT id, sort_rank FROM probes WHERE owner_id = ? AND slot = ? ORDER BY sort_rank, id) AS x`, + [OWNER, 'agg'], + ); + const rawPayload = rows[0]!['payload']; + const parsed = (typeof rawPayload === 'string' ? JSON.parse(rawPayload) : rawPayload) as Array<{ + rank: unknown; + }>; + const ranks = parsed.map((entry) => Number(entry.rank)); + // JSON_ARRAYAGG 不保证派生表的 ORDER BY,顺序必须在应用层补齐。 + const sorted = [...ranks].sort((left, right) => left - right); + return { + ok: parsed.length === 3 && sorted.join(',') === '10,20,30', + observation: `aggregated=${ranks.join(',')} sorted=${sorted.join(',')} (顺序不稳定,需应用层排序)`, + }; + }); + + await probe('行值 keyset 比较', async () => { + const [anchor] = await db.query( + "SELECT id FROM probes WHERE slot = 'agg' ORDER BY sort_rank LIMIT 1", + ); + const [rows] = await db.query( + 'SELECT sort_rank FROM probes WHERE owner_id = ? AND slot = ? AND (sort_rank, id) > (?, ?) ORDER BY sort_rank, id LIMIT 3', + [OWNER, 'agg', 10, String(anchor[0]!['id'])], + ); + const ranks = rows.map((row) => Number(row['sort_rank'])); + return { ok: ranks.join(',') === '20,30', observation: `ranks=${ranks.join(',')}` }; + }); + + await probe('FOR UPDATE SKIP LOCKED', async () => { + const connA = await connect(DATABASE); + const connB = await connect(DATABASE); + try { + await connA.beginTransaction(); + const [rowsA] = await connA.query( + "SELECT id FROM probes WHERE slot = 'agg' ORDER BY sort_rank LIMIT 1 FOR UPDATE SKIP LOCKED", + ); + await connB.beginTransaction(); + const [rowsB] = await connB.query( + "SELECT id FROM probes WHERE slot = 'agg' ORDER BY sort_rank LIMIT 1 FOR UPDATE SKIP LOCKED", + ); + await connB.rollback(); + await connA.rollback(); + return { + ok: true, + observation: `firstLocked=${rowsA.length} secondGot=${rowsB.length}; InnoDB 会在 LIMIT 前扫描并加锁,SKIP LOCKED 可能拿不到行`, + }; + } finally { + await connA.end(); + await connB.end(); + } + }); + + await expectRejected('DELETE 同表子查询', ['ER_UPDATE_TABLE_USED'], () => + db.query("DELETE FROM probes WHERE id IN (SELECT id FROM probes WHERE slot = 'agg')"), + ); + + await expectRejected('DELETE 同表子查询 + LIMIT', ['ER_NOT_SUPPORTED_YET'], () => + db.query("DELETE FROM probes WHERE id IN (SELECT id FROM probes WHERE slot = 'agg' LIMIT 1)"), + ); + + await probe('DELETE ORDER BY LIMIT', async () => { + for (let i = 0; i < 5; i += 1) { + await db.query('INSERT INTO probes (id, owner_id, slot, created_at) VALUES (?, ?, ?, ?)', [ + id(), + OWNER, + 'purge', + '2020-01-01 00:00:00.000000', + ]); + } + const [result] = await db.query( + 'DELETE FROM probes WHERE slot = ? AND created_at < ? ORDER BY created_at LIMIT 3', + ['purge', CREATED_AT], + ); + const deleted = (result as mysql.ResultSetHeader).affectedRows; + return { ok: deleted === 3, observation: `deleted=${deleted}` }; + }); + + await probe('SAVEPOINT', async () => { + await db.beginTransaction(); + try { + await db.query('SAVEPOINT sp1'); + await db.query('INSERT INTO probes (id, owner_id, slot) VALUES (?, ?, ?)', [ + id(), + OWNER, + 'savepoint', + ]); + await db.query('ROLLBACK TO SAVEPOINT sp1'); + await db.query('RELEASE SAVEPOINT sp1'); + const [rows] = await db.query( + "SELECT COUNT(*) AS n FROM probes WHERE slot = 'savepoint'", + ); + await db.commit(); + return { + ok: Number(rows[0]!['n']) === 0, + observation: `afterRollback=${String(rows[0]!['n'])}`, + }; + } catch (error) { + await db.rollback(); + throw error; + } + }); + + await probe('AUTO_INCREMENT 批次连续', async () => { + const [inserted] = await db.query( + 'INSERT INTO seq_probe (owner) VALUES (?), (?), (?), (?), (?)', + [OWNER, OWNER, OWNER, OWNER, OWNER], + ); + const header = inserted as mysql.ResultSetHeader; + const [rows] = await db.query('SELECT MAX(seq) AS max FROM seq_probe'); + const expected = BigInt(header.insertId) + BigInt(header.affectedRows) - 1n; + return { + ok: BigInt(String(rows[0]!['max'])) === expected, + observation: `insertId=${header.insertId} affected=${header.affectedRows} max=${String(rows[0]!['max'])}`, + }; + }); + + await probe('插入时对同表取 MAX', async () => { + await db.query( + 'INSERT INTO probes (id, owner_id, slot, sort_rank) VALUES (?, ?, ?, (SELECT COALESCE(MAX(sort_rank), 0) + 1024 FROM probes AS p WHERE p.owner_id = ? AND p.slot = ?))', + [id(), OWNER, 'rankcalc', OWNER, 'rankcalc'], + ); + const [rows] = await db.query( + "SELECT sort_rank FROM probes WHERE slot = 'rankcalc'", + ); + return { + ok: Number(rows[0]!['sort_rank']) === 1024, + observation: `rank=${String(rows[0]!['sort_rank'])}`, + }; + }); + + await probe('JSON 列读写', async () => { + const rowId = id(); + await db.query( + 'INSERT INTO probes (id, owner_id, slot, payload) VALUES (?, ?, ?, CAST(? AS JSON))', + [rowId, OWNER, 'json', JSON.stringify({ nested: { ok: true }, n: 7 })], + ); + const [rows] = await db.query( + 'SELECT payload FROM probes WHERE id = ?', + [rowId], + ); + const raw = rows[0]!['payload']; + const payload = (typeof raw === 'string' ? JSON.parse(raw) : raw) as { + nested?: { ok?: boolean }; + n?: number; + }; + return { + ok: payload?.nested?.ok === true && payload.n === 7, + observation: `driverType=${typeof raw} value=${JSON.stringify(payload)}`, + }; + }); + + await expectRejected( + 'DATETIME 拒绝带 Z 的 ISO 字符串', + ['ER_TRUNCATED_WRONG_VALUE', 'ER_WRONG_VALUE', 'ER_TRUNCATED_WRONG_VALUE_FOR_FIELD'], + () => + db.query('INSERT INTO probes (id, owner_id, slot, created_at) VALUES (?, ?, ?, ?)', [ + id(), + OWNER, + 'iso', + '2026-11-15T08:07:24.000Z', + ]), + ); + + await probe('DATETIME(6) 往返', async () => { + const rowId = id(); + await db.query('INSERT INTO probes (id, owner_id, slot, created_at) VALUES (?, ?, ?, ?)', [ + rowId, + OWNER, + 'datetime', + CREATED_AT, + ]); + const [rows] = await db.query( + 'SELECT created_at FROM probes WHERE id = ?', + [rowId], + ); + const value = rows[0]!['created_at']; + return { + ok: value instanceof Date && value.toISOString() === '2026-11-15T08:07:24.123Z', + observation: `${value instanceof Date ? 'Date' : typeof value} ${String(value instanceof Date ? value.toISOString() : value)}`, + }; + }); + + await probe('CHECK + REGEXP_LIKE + CHAR_LENGTH', async () => { + let rejected = false; + try { + await db.query("INSERT INTO checks_probe (prefix, title) VALUES ('bad', 'x')"); + } catch (error) { + rejected = errorCode(error) === 'ER_CHECK_CONSTRAINT_VIOLATED'; + } + await db.query("INSERT INTO checks_probe (prefix, title) VALUES ('ABC1', ' 标题 ')"); + return { ok: rejected, observation: `invalidRowRejected=${rejected}` }; + }); + + await probe('排序规则', async () => { + const [ci] = await db.query( + "SELECT COUNT(*) AS n FROM case_probe WHERE title LIKE '%alice%'", + ); + const [loose] = await db.query( + "SELECT COUNT(*) AS n FROM (SELECT 'Owner' AS username UNION ALL SELECT 'owner') AS u WHERE u.username = 'owner'", + ); + const [strict] = await db.query( + "SELECT COUNT(*) AS n FROM (SELECT 'Owner' AS username UNION ALL SELECT 'owner') AS u WHERE u.username COLLATE utf8mb4_bin = 'owner'", + ); + return { + ok: + Number(ci[0]!['n']) === 1 && Number(loose[0]!['n']) === 2 && Number(strict[0]!['n']) === 1, + observation: `like=${String(ci[0]!['n'])} defaultEq=${String(loose[0]!['n'])} binEq=${String(strict[0]!['n'])}`, + }; + }); + + await probe('只读可重复读事务', async () => { + const conn = await connect(DATABASE); + try { + await conn.query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ'); + await conn.query('START TRANSACTION READ ONLY'); + const [rows] = await conn.query('SELECT COUNT(*) AS n FROM probes'); + let writeRejected = false; + try { + await conn.query('INSERT INTO probes (id, owner_id, slot) VALUES (?, ?, ?)', [ + id(), + OWNER, + 'readonly', + ]); + } catch (error) { + writeRejected = errorCode(error) === 'ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION'; + } + await conn.commit(); + return { + ok: Number(rows[0]!['n']) > 0 && writeRejected, + observation: `readRows=${String(rows[0]!['n'])} writeRejected=${writeRejected}`, + }; + } finally { + await conn.end(); + } + }); + + await probe('语句级超时', async () => { + const bulk: unknown[] = []; + const tuples: string[] = []; + for (let i = 0; i < 3000; i += 1) { + tuples.push('(?, ?, ?)'); + bulk.push(id(), OWNER, 'timeout-fill'); + } + await db.query(`INSERT INTO probes (id, owner_id, slot) VALUES ${tuples.join(',')}`, bulk); + await db.query('SET SESSION max_execution_time = 50'); + let interrupted = false; + try { + await db.query('SELECT COUNT(*) FROM probes a, probes b WHERE a.sort_rank < b.sort_rank'); + } catch (error) { + interrupted = errorCode(error) === 'ER_QUERY_TIMEOUT'; + } + await db.query('SET SESSION max_execution_time = 0'); + return { ok: interrupted, observation: `selectInterrupted=${interrupted}` }; + }); + + await probe('函数索引', async () => { + await db.query( + "CREATE INDEX probes_status_rank_idx ON probes ((CASE status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END), sort_rank, id)", + ); + return { ok: true, observation: 'created' }; + }); + + await probe('会话级咨询锁', async () => { + const connA = await connect(DATABASE); + const connB = await connect(DATABASE); + try { + const [a] = await connA.query( + "SELECT GET_LOCK('taskdock:probe', 0) AS held", + ); + const [b] = await connB.query( + "SELECT GET_LOCK('taskdock:probe', 0) AS held", + ); + await connA.query("SELECT RELEASE_LOCK('taskdock:probe')"); + const [c] = await connB.query( + "SELECT GET_LOCK('taskdock:probe', 0) AS held", + ); + return { + ok: [a, b, c].map((rows) => Number(rows[0]!['held'])).join(',') === '1,0,1', + observation: `first=${String(a[0]!['held'])} second=${String(b[0]!['held'])} afterRelease=${String(c[0]!['held'])}`, + }; + } finally { + await connA.end(); + await connB.end(); + } + }); + + await probe('TINYINT(1) 读取', async () => { + const [rows] = await db.query( + "SELECT active FROM slot_unique WHERE slot = 'live' LIMIT 1", + ); + const value = rows[0]!['active']; + return { ok: value === 1 || value === true, observation: `${typeof value}:${String(value)}` }; + }); + + await probe('超过 2^53 的 BIGINT', async () => { + await db.query('TRUNCATE TABLE big_probe'); + await db.query('INSERT INTO big_probe (v) VALUES (?)', ['9007199254740993']); + const [rows] = await db.query('SELECT v FROM big_probe'); + const value = rows[0]!['v']; + return { ok: true, observation: `driverReturns ${typeof value}:${String(value)}` }; + }); + + await probe('FILTER 聚合', async () => { + let code = ''; + try { + await db.query("SELECT COUNT(*) FILTER (WHERE status = 'TODO') AS n FROM probes"); + } catch (error) { + code = errorCode(error); + } + return { ok: code === 'ER_PARSE_ERROR', observation: code || 'unexpectedly supported' }; + }); + + await probe('INTERVAL 参数化', async () => { + const [rows] = await db.query( + 'SELECT NOW(6) - INTERVAL ? DAY AS cutoff', + [90], + ); + return { + ok: rows[0]!['cutoff'] instanceof Date, + observation: rows[0]!['cutoff'] instanceof Date ? 'Date' : typeof rows[0]!['cutoff'], + }; + }); + + await probe('utf8mb4 中文与 emoji', async () => { + const rowId = id(); + await db.query('INSERT INTO probes (id, owner_id, slot, title) VALUES (?, ?, ?, ?)', [ + rowId, + OWNER, + 'utf8', + '任务 🚀 标题', + ]); + const [rows] = await db.query( + 'SELECT CHAR_LENGTH(title) AS n FROM probes WHERE id = ?', + [rowId], + ); + return { ok: Number(rows[0]!['n']) === 7, observation: `chars=${String(rows[0]!['n'])}` }; + }); + + await expectRejected('rank 是 MySQL 保留字', ['ER_PARSE_ERROR'], () => + db.query('SELECT id, rank FROM probes LIMIT 1'), + ); + + await probe('反引号引用保留字', async () => { + await db.query('INSERT INTO keyword_probe (`rank`, `cursor`) VALUES (?, ?)', [7, 3]); + const [rows] = await db.query( + 'SELECT `rank`, `cursor` FROM keyword_probe ORDER BY `rank` DESC LIMIT 1', + ); + return { + ok: Number(rows[0]!['rank']) === 7 && Number(rows[0]!['cursor']) === 3, + observation: `rank=${String(rows[0]!['rank'])} cursor=${String(rows[0]!['cursor'])}`, + }; + }); + + await db.end(); + const cleanup = await connect(); + await cleanup.query(`DROP DATABASE IF EXISTS \`${DATABASE}\``); + await cleanup.end(); +} + +async function main(): Promise { + await bootstrap(); + await cases(); + const failed = results.filter((result) => !result.passed).length; + console.log(`\n${results.length - failed}/${results.length} cases matched the expectation.`); + if (failed) { + console.error('MySQL 8.4 does not behave as the dialect plan assumes; revisit the plan.'); + process.exitCode = 1; + } +} + +main().catch((error: unknown) => { + console.error(describeError(error)); + process.exitCode = 1; +}); From 23323628d4fe0e10e895b6613ad6ef1fcd6d1212 Mon Sep 17 00:00:00 2001 From: IGNGserver <176017966+IGNGserver@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:32:47 +0800 Subject: [PATCH 2/9] =?UTF-8?q?feat(db):=20=E6=8A=BD=E5=87=BA=E5=8F=8C?= =?UTF-8?q?=E5=BC=95=E6=93=8E=E6=96=B9=E8=A8=80=E5=B1=82=E4=B8=8E=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - prepareQuery: PostgreSQL 保留 $n 占位符并把反引号翻译成双引号;MySQL 重排为 ? 并剥离 ::type 转换 - readDatabaseSettings: 由 DB_DIALECT 或 DATABASE_URL scheme 识别引擎, 支持离散 MYSQL_* 变量与统一 DB_* 连接参数 - openDatabase: 同一 SqlDatabase 抽象封装 pg.Pool 与 mysql2 Pool,统一 事务、只读快照、LISTEN 能力与会话级超时 - 迁移 runner 支持两种引擎:PostgreSQL 仍是整事务 + advisory lock, MySQL 改为逐文件提交 + GET_LOCK --- packages/database/src/config.ts | 192 +++++++++++++--- packages/database/src/dialect.ts | 250 ++++++++++++++++++++ packages/database/src/index.ts | 135 ++--------- packages/database/src/migrate.ts | 268 ++++++++++++++++++++++ packages/database/src/pool.ts | 301 +++++++++++++++++++++++++ packages/database/test/config.test.ts | 44 ++++ packages/database/test/dialect.test.ts | 66 ++++++ 7 files changed, 1100 insertions(+), 156 deletions(-) create mode 100644 packages/database/src/dialect.ts create mode 100644 packages/database/src/migrate.ts create mode 100644 packages/database/src/pool.ts create mode 100644 packages/database/test/dialect.test.ts diff --git a/packages/database/src/config.ts b/packages/database/src/config.ts index 4703a3f..4e924ed 100644 --- a/packages/database/src/config.ts +++ b/packages/database/src/config.ts @@ -1,9 +1,38 @@ import { readFileSync } from 'node:fs'; import type { ConnectionOptions } from 'node:tls'; import pg from 'pg'; +import type { DatabaseDialectName } from './dialect.js'; export type DatabaseSslMode = 'disable' | 'require' | 'verify-ca' | 'verify-full'; +export interface DatabaseSettings { + dialect: DatabaseDialectName; + connectionString?: string; + host?: string; + port?: number; + user?: string; + password?: string; + database?: string; + sslMode?: DatabaseSslMode; + /** PEM contents of the CA certificate, when one is configured. */ + sslCa?: string; + /** node-postgres form; kept for the PostgreSQL pool. */ + ssl?: boolean | ConnectionOptions; + applicationName: string; + max: number; + maxUses: number; + connectionTimeoutMillis: number; + idleTimeoutMillis: number; + queryTimeoutMillis: number; + statementTimeoutMillis: number; + lockTimeoutMillis: number; + keepAlive: boolean; + keepAliveInitialDelayMillis: number; + /** Only PostgreSQL has LISTEN/NOTIFY; MySQL always reports false. */ + listenEnabled: boolean; +} + +/** PostgreSQL-shaped view kept for callers that still speak `pg`. */ export interface DatabaseRuntimeConfig { connectionString?: string; host?: string; @@ -27,41 +56,77 @@ export interface DatabaseRuntimeConfig { export type DatabaseEnv = Record; +const POSTGRES_URL = /^postgres(?:ql)?:\/\//i; +const MYSQL_URL = /^mysql:\/\//i; + /** - * Resolve PostgreSQL connection settings from explicit environment variables. + * Resolve the engine and its connection settings from the environment. * - * `DATABASE_URL` is the primary, portable form. Discrete `PG*` variables are - * supported as a fallback because they avoid URL-encoding mistakes when a - * password contains `@`, `:`, `/` or `#` (a real problem for bundled Docker - * deployments). Empty strings are treated as unset so Compose can pass - * `MIGRATION_DATABASE_URL: ${MIGRATION_DATABASE_URL:-}` safely. + * The engine comes from `DB_DIALECT` when set, otherwise from the + * `DATABASE_URL` scheme, otherwise from whichever discrete variables are + * present (default PostgreSQL, which is what existing hubs are). */ -export function readDatabaseRuntimeConfig(env: DatabaseEnv = process.env): DatabaseRuntimeConfig { +export function readDatabaseSettings(env: DatabaseEnv = process.env): DatabaseSettings { const connectionString = envString(env, 'DATABASE_URL'); - const host = envString(env, 'PGHOST') ?? envString(env, 'DATABASE_HOST'); - const database = envString(env, 'PGDATABASE') ?? envString(env, 'DATABASE_NAME'); + const dialect = detectDialect(env, connectionString); + const isMysql = dialect === 'mysql'; + + const host = isMysql + ? (envString(env, 'MYSQL_HOST') ?? envString(env, 'DATABASE_HOST')) + : (envString(env, 'PGHOST') ?? envString(env, 'DATABASE_HOST')); + const database = isMysql + ? (envString(env, 'MYSQL_DATABASE') ?? envString(env, 'DATABASE_NAME')) + : (envString(env, 'PGDATABASE') ?? envString(env, 'DATABASE_NAME')); if (!connectionString && (!host || !database)) throw new Error( - 'DATABASE_URL or PGHOST/PGDATABASE must be set to use PostgreSQL; set DEV_MEMORY_STORE=true only for local development', + isMysql + ? 'DATABASE_URL or MYSQL_HOST/MYSQL_DATABASE must be set to use MySQL; set DEV_MEMORY_STORE=true only for local development' + : 'DATABASE_URL or PGHOST/PGDATABASE must be set to use PostgreSQL; set DEV_MEMORY_STORE=true only for local development', ); const sslMode = parseSslMode(envString(env, 'DATABASE_SSL_MODE') ?? envString(env, 'PGSSLMODE')); - const ssl = buildSsl(sslMode, envString(env, 'DATABASE_SSL_CA')); + const sslCaPath = envString(env, 'DATABASE_SSL_CA'); + const sslCa = sslCaPath ? readFileSync(sslCaPath, 'utf8') : undefined; + if (sslMode === 'disable' && sslCa) + throw new Error('DATABASE_SSL_CA cannot be used when DATABASE_SSL_MODE=disable'); + + const defaultPort = isMysql ? 3306 : 5432; + const port = connectionString + ? undefined + : optionalInteger( + env, + isMysql ? 'MYSQL_PORT' : 'PGPORT', + 'DATABASE_PORT', + defaultPort, + 1, + 65_535, + ); + const user = connectionString + ? undefined + : isMysql + ? (envString(env, 'MYSQL_USER') ?? envString(env, 'DATABASE_USER')) + : envString(env, 'PGUSER'); + const password = connectionString + ? undefined + : isMysql + ? (envString(env, 'MYSQL_PASSWORD') ?? envString(env, 'DATABASE_PASSWORD')) + : envString(env, 'PGPASSWORD'); return { + dialect, // node-postgres lets an sslmode query parameter override an explicit ssl // object. Remove the URL parameters when the environment is the source of // truth so DATABASE_SSL_MODE / DATABASE_SSL_CA always win. connectionString: connectionString && sslMode ? stripSslParams(connectionString) : connectionString, host: connectionString ? undefined : host, - port: connectionString - ? undefined - : optionalInteger(env, 'PGPORT', 'DATABASE_PORT', 5432, 1, 65_535), - user: connectionString ? undefined : envString(env, 'PGUSER'), - password: connectionString ? undefined : envString(env, 'PGPASSWORD'), + port, + user, + password, database: connectionString ? undefined : database, - ssl, + sslMode, + sslCa, + ssl: isMysql ? undefined : buildPgSsl(sslMode, sslCa), applicationName: envString(env, 'DB_APPLICATION_NAME') ?? 'taskdock-api', max: integerInRange(env, 'DB_POOL_MAX', 10, 1, 100), maxUses: integerInRange(env, 'DB_MAX_USES', 1_000, 1, 1_000_000), @@ -81,7 +146,73 @@ export function readDatabaseRuntimeConfig(env: DatabaseEnv = process.env): Datab 0, 120_000, ), - listenEnabled: booleanValue(envString(env, 'DB_LISTEN_ENABLED'), true), + // LISTEN/NOTIFY has no MySQL equivalent; a MySQL hub relies on the + // in-process notification path plus optional Redis fan-out. + listenEnabled: isMysql ? false : booleanValue(envString(env, 'DB_LISTEN_ENABLED'), true), + }; +} + +function detectDialect( + env: DatabaseEnv, + connectionString: string | undefined, +): DatabaseDialectName { + const explicit = envString(env, 'DB_DIALECT')?.toLowerCase(); + if (explicit) { + if (explicit === 'postgres' || explicit === 'postgresql') return 'postgres'; + if (explicit === 'mysql') return 'mysql'; + throw new Error('DB_DIALECT must be postgres or mysql'); + } + if (connectionString) { + if (MYSQL_URL.test(connectionString)) { + if (!connectableUrl(connectionString)) + throw new Error('DATABASE_URL must be a valid mysql:// URL'); + return 'mysql'; + } + if (POSTGRES_URL.test(connectionString)) { + if (!connectableUrl(connectionString)) + throw new Error('DATABASE_URL must be a valid postgres:// URL'); + return 'postgres'; + } + throw new Error('DATABASE_URL must start with postgres://, postgresql:// or mysql://'); + } + if (envString(env, 'MYSQL_HOST') || envString(env, 'MYSQL_DATABASE')) return 'mysql'; + return 'postgres'; +} + +function connectableUrl(value: string): boolean { + try { + const url = new URL(value); + return Boolean(url.hostname); + } catch { + return false; + } +} + +/** + * PostgreSQL-shaped settings. Kept until every caller moves to + * `readDatabaseSettings`; behaviour is unchanged for PostgreSQL hubs. + */ +export function readDatabaseRuntimeConfig(env: DatabaseEnv = process.env): DatabaseRuntimeConfig { + const settings = readDatabaseSettings({ ...env, DB_DIALECT: 'postgres' }); + return { + connectionString: settings.connectionString, + host: settings.host, + port: settings.port, + user: settings.user, + password: settings.password, + database: settings.database, + ssl: settings.ssl, + applicationName: settings.applicationName, + max: settings.max, + maxUses: settings.maxUses, + connectionTimeoutMillis: settings.connectionTimeoutMillis, + idleTimeoutMillis: settings.idleTimeoutMillis, + queryTimeoutMillis: settings.queryTimeoutMillis, + statementTimeoutMillis: settings.statementTimeoutMillis, + lockTimeoutMillis: settings.lockTimeoutMillis, + keepAlive: settings.keepAlive, + keepAliveInitialDelayMillis: settings.keepAliveInitialDelayMillis, + listenEnabled: settings.listenEnabled, }; } @@ -119,14 +250,6 @@ export function createPoolFromEnv(env: DatabaseEnv = process.env): { return { pool: new pg.Pool(poolConfigFromRuntime(settings)), settings }; } -/** Open `count` connections once so the first request does not pay TCP/TLS setup. */ -export async function warmPool(pool: pg.Pool, count: number): Promise { - const warm = Math.max(0, Math.min(Math.trunc(count), pool.options.max ?? 0)); - if (!warm) return; - const clients = await Promise.all(Array.from({ length: warm }, () => pool.connect())); - for (const client of clients) client.release(); -} - function envString(env: DatabaseEnv, name: string): string | undefined { const value = env[name]?.trim(); return value ? value : undefined; @@ -171,8 +294,8 @@ function parseSslMode(raw: string | undefined): DatabaseSslMode | undefined { if (!raw) return undefined; const normalized = raw.toLowerCase(); if (normalized === 'disable' || normalized === 'false' || normalized === 'off') return 'disable'; - // libpq "prefer"/"allow" mean opportunistic TLS. Let node-postgres/libpq - // defaults decide instead of forcing an encrypted-only connection. + // libpq "prefer"/"allow" mean opportunistic TLS. Let the driver defaults + // decide instead of forcing an encrypted-only connection. if (normalized === 'prefer' || normalized === 'allow') return undefined; if (normalized === 'require' || normalized === 'true' || normalized === 'on') return 'require'; if (normalized === 'verify-full' || normalized === 'verify-ca') return normalized; @@ -182,7 +305,7 @@ function parseSslMode(raw: string | undefined): DatabaseSslMode | undefined { function stripSslParams(connectionString: string): string { try { const url = new URL(connectionString); - for (const key of ['sslmode', 'ssl', 'sslrootcert', 'sslcert', 'sslkey']) + for (const key of ['sslmode', 'ssl', 'ssl-mode', 'sslrootcert', 'sslcert', 'sslkey']) url.searchParams.delete(key); return url.toString(); } catch { @@ -190,24 +313,21 @@ function stripSslParams(connectionString: string): string { } } -function buildSsl( +function buildPgSsl( mode: DatabaseSslMode | undefined, - caPath: string | undefined, + ca: string | undefined, ): boolean | ConnectionOptions | undefined { if (!mode) { // Let the connection string / libpq defaults decide. Do not force TLS off // when a URL such as ?sslmode=verify-full is supplied. return undefined; } - if (mode === 'disable') { - if (caPath) throw new Error('DATABASE_SSL_CA cannot be used when DATABASE_SSL_MODE=disable'); - return false; - } + if (mode === 'disable') return false; const ssl: ConnectionOptions = { rejectUnauthorized: true }; if (mode === 'verify-ca') { // verify-ca validates the CA chain but deliberately skips hostname checks. ssl.checkServerIdentity = () => undefined; } - if (caPath) ssl.ca = readFileSync(caPath, 'utf8'); + if (ca) ssl.ca = ca; return ssl; } diff --git a/packages/database/src/dialect.ts b/packages/database/src/dialect.ts new file mode 100644 index 0000000..7b646c0 --- /dev/null +++ b/packages/database/src/dialect.ts @@ -0,0 +1,250 @@ +/** + * Dialect-sensitive parts of the SQL that both stores share. + * + * Source SQL keeps two PostgreSQL habits that are translated per engine: + * + * - `$1`-style placeholders. node-postgres understands them directly; MySQL + * uses `?`, so the MySQL path renumbers the parameters in appearance order. + * This keeps one copy of every statement even where a parameter repeats. + * - backtick quoting for identifiers that MySQL reserves (`rank`, `cursor`). + * PostgreSQL sees the equivalent `"rank"`. + * + * MySQL also receives implicit casts: `$1::date` becomes `?`. The cast exists + * to satisfy PostgreSQL's type inference, and MySQL rejects the syntax. + */ + +export type DatabaseDialectName = 'postgres' | 'mysql'; + +export interface PreparedQuery { + text: string; + values: unknown[]; +} + +/** Types that may follow `::` in the shared SQL. */ +const CAST_TYPE = + /^(?:double\s+precision|timestamp(?:tz)?|timestamptz|bigint|integer|int|smallint|text|varchar|date|jsonb|json|numeric|decimal|boolean|bool|real|float8|float4|char|bytea|uuid)\b/i; + +export function prepareQuery( + dialect: DatabaseDialectName, + sql: string, + values: readonly unknown[] = [], +): PreparedQuery { + return dialect === 'mysql' ? prepareMysql(sql, values) : preparePostgres(sql, values); +} + +function preparePostgres(sql: string, values: readonly unknown[]): PreparedQuery { + let text = ''; + let index = 0; + while (index < sql.length) { + const char = sql[index]!; + if (char === "'") { + const end = skipSingleQuoted(sql, index); + text += sql.slice(index, end); + index = end; + continue; + } + if (char === '"') { + const end = skipDoubleQuoted(sql, index); + text += sql.slice(index, end); + index = end; + continue; + } + if (char === '`') { + const end = skipBacktick(sql, index); + text += toDoubleQuoted(sql.slice(index + 1, end - 1)); + index = end; + continue; + } + if (char === '-' && sql[index + 1] === '-') { + const end = lineCommentEnd(sql, index); + text += sql.slice(index, end); + index = end; + continue; + } + if (char === '/' && sql[index + 1] === '*') { + const end = blockCommentEnd(sql, index); + text += sql.slice(index, end); + index = end; + continue; + } + if (char === '$' && sql[index + 1] === '$') { + const end = skipDollarQuoted(sql, index); + text += sql.slice(index, end); + index = end; + continue; + } + text += char; + index += 1; + } + return { text, values: [...values] }; +} + +function prepareMysql(sql: string, values: readonly unknown[]): PreparedQuery { + let text = ''; + const params: unknown[] = []; + let index = 0; + while (index < sql.length) { + const char = sql[index]!; + if (char === "'") { + const end = skipSingleQuoted(sql, index); + text += sql.slice(index, end); + index = end; + continue; + } + if (char === '"' || char === '`') { + const end = char === '"' ? skipDoubleQuoted(sql, index) : skipBacktick(sql, index); + text += sql.slice(index, end); + index = end; + continue; + } + if (char === '-' && sql[index + 1] === '-') { + const end = lineCommentEnd(sql, index); + text += sql.slice(index, end); + index = end; + continue; + } + if (char === '/' && sql[index + 1] === '*') { + const end = blockCommentEnd(sql, index); + text += sql.slice(index, end); + index = end; + continue; + } + if (char === '$') { + const placeholder = /^\$(\d+)/.exec(sql.slice(index)); + if (placeholder) { + const position = Number(placeholder[1]); + if (position < 1 || position > values.length) + throw new Error(`SQL placeholder $${position} has no matching parameter`); + text += '?'; + params.push(values[position - 1]); + index += placeholder[0].length; + index = skipCast(sql, index); + continue; + } + } + if (char === ':' && sql[index + 1] === ':') { + const afterCast = skipCast(sql, index); + if (afterCast > index) { + index = afterCast; + continue; + } + } + text += char; + index += 1; + } + return { text, values: params }; +} + +/** Consume `::type` (and an optional length) when a known type follows. */ +function skipCast(sql: string, index: number): number { + if (sql[index] !== ':' || sql[index + 1] !== ':') return index; + let cursor = index + 2; + while (sql[cursor] === ' ') cursor += 1; + const match = CAST_TYPE.exec(sql.slice(cursor)); + if (!match) return index; + cursor += match[0].length; + if (sql[cursor] === '(') { + const close = sql.indexOf(')', cursor); + if (close !== -1) cursor = close + 1; + } + return cursor; +} + +function toDoubleQuoted(identifier: string): string { + return `"${identifier.replace(/""/g, '"').replace(/"/g, '""')}"`; +} + +function skipSingleQuoted(sql: string, start: number): number { + let index = start + 1; + while (index < sql.length) { + const char = sql[index]!; + if (char === '\\') { + index += 2; + continue; + } + if (char === "'") { + if (sql[index + 1] === "'") { + index += 2; + continue; + } + return index + 1; + } + index += 1; + } + return sql.length; +} + +function skipDoubleQuoted(sql: string, start: number): number { + let index = start + 1; + while (index < sql.length) { + if (sql[index] === '"') { + if (sql[index + 1] === '"') { + index += 2; + continue; + } + return index + 1; + } + index += 1; + } + return sql.length; +} + +function skipBacktick(sql: string, start: number): number { + let index = start + 1; + while (index < sql.length) { + if (sql[index] === '`') { + if (sql[index + 1] === '`') { + index += 2; + continue; + } + return index + 1; + } + index += 1; + } + return sql.length; +} + +function skipDollarQuoted(sql: string, start: number): number { + const close = sql.indexOf('$$', start + 2); + return close === -1 ? sql.length : close + 2; +} + +function lineCommentEnd(sql: string, start: number): number { + const newline = sql.indexOf('\n', start); + return newline === -1 ? sql.length : newline; +} + +function blockCommentEnd(sql: string, start: number): number { + const close = sql.indexOf('*/', start + 2); + return close === -1 ? sql.length : close + 2; +} + +/** + * MySQL refuses to bind a JavaScript `Date`/ISO string into `DATETIME` when it + * carries the `Z` suffix. Values are stored as UTC and the session time zone is + * pinned to `+00:00`, so normalising here keeps one representation everywhere. + */ +const ISO_TIMESTAMP = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2})(?:\.(\d{1,6}))?Z$/; + +export function normaliseMysqlValue(value: unknown): unknown { + if (typeof value === 'string') { + const match = ISO_TIMESTAMP.exec(value); + if (match) { + const fraction = (match[3] ?? '').padEnd(6, '0'); + return `${match[1]} ${match[2]}.${fraction}`; + } + return value; + } + if (value instanceof Date) { + const iso = value.toISOString(); + const match = ISO_TIMESTAMP.exec(iso); + if (match) return `${match[1]} ${match[2]}.${(match[3] ?? '').padEnd(6, '0')}`; + return iso; + } + if (typeof value === 'boolean') return value ? 1 : 0; + return value; +} + +export function normaliseMysqlValues(values: readonly unknown[]): unknown[] { + return values.map(normaliseMysqlValue); +} diff --git a/packages/database/src/index.ts b/packages/database/src/index.ts index 2643f09..b4dd172 100644 --- a/packages/database/src/index.ts +++ b/packages/database/src/index.ts @@ -1,14 +1,12 @@ -import { readdir, readFile } from 'node:fs/promises'; -import { createHash } from 'node:crypto'; -import { dirname, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; import pg from 'pg'; export * from './config.js'; +export * from './dialect.js'; +export * from './migrate.js'; +export * from './pool.js'; export * from './schema.js'; -export type { Pool } from 'pg'; -const { Pool } = pg; +export type { Pool } from 'pg'; export interface PoolTimeoutOptions { connectionTimeoutMillis?: number; @@ -24,6 +22,10 @@ export interface PoolTimeoutOptions { keepAliveInitialDelayMillis?: number; } +/** + * Legacy PostgreSQL pool factory. Retained for scripts that speak `pg` + * directly; application code goes through `openDatabase`. + */ export function createPool( connectionString: string | pg.PoolConfig, options: PoolTimeoutOptions = {}, @@ -31,7 +33,7 @@ export function createPool( const statementTimeoutMillis = options.statementTimeoutMillis ?? 30_000; const base: pg.PoolConfig = typeof connectionString === 'string' ? { connectionString } : { ...connectionString }; - return new Pool({ + return new pg.Pool({ ...base, max: options.max ?? base.max ?? 10, maxUses: options.maxUses ?? base.maxUses ?? 1000, @@ -50,117 +52,10 @@ export function createPool( }); } -export interface MigrationOptions { - migrationsDir?: string; -} - -export async function runMigrations(pool: pg.Pool, options: MigrationOptions = {}): Promise { - const migrationsDir = options.migrationsDir ?? defaultMigrationsDir(); - const files = await discoverMigrationFiles(migrationsDir); - const migrations = await Promise.all( - files.map(async (name) => { - const sql = await readFile(resolve(migrationsDir, name), 'utf8'); - return { name, sql, checksum: sha256(sql) }; - }), - ); - const client = await pool.connect(); - try { - await client.query('BEGIN'); - await client.query( - "SELECT pg_advisory_xact_lock(hashtextextended('devtodo:schema-migrations', 0))", - ); - await client.query( - `CREATE TABLE IF NOT EXISTS schema_migrations (name TEXT PRIMARY KEY, checksum TEXT, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())`, - ); - await client.query('ALTER TABLE schema_migrations ADD COLUMN IF NOT EXISTS checksum TEXT'); - for (const migration of migrations) { - const exists = await client.query<{ name: string; checksum: string | null }>( - 'SELECT name, checksum FROM schema_migrations WHERE name = $1 FOR UPDATE', - [migration.name], - ); - const applied = exists.rows[0]; - if (applied) { - // The original 0001 executor did not store checksums. Backfill that - // one legacy record exactly once, then protect every applied file. - if (!applied.checksum) { - await client.query('UPDATE schema_migrations SET checksum = $2 WHERE name = $1', [ - migration.name, - migration.checksum, - ]); - } else if (applied.checksum !== migration.checksum) { - throw new Error(`Migration checksum mismatch: ${migration.name}`); - } - continue; - } - await client.query(migration.sql); - await client.query('INSERT INTO schema_migrations(name, checksum) VALUES ($1, $2)', [ - migration.name, - migration.checksum, - ]); - } - await client.query('COMMIT'); - } catch (error) { - await client.query('ROLLBACK'); - throw error; - } finally { - client.release(); - } -} - -export async function databaseReady( - pool: pg.Pool, - options: MigrationOptions = {}, -): Promise { - try { - const migrationsDir = options.migrationsDir ?? defaultMigrationsDir(); - const files = await discoverMigrationFiles(migrationsDir); - const result = await pool.query<{ name: string; checksum: string | null }>( - 'SELECT name, checksum FROM schema_migrations', - ); - const applied = new Map(result.rows.map((row) => [row.name, row.checksum])); - for (const name of files) { - const checksum = sha256(await readFile(resolve(migrationsDir, name), 'utf8')); - if (applied.get(name) !== checksum) return false; - } - return files.length > 0 && files.every((name) => applied.has(name)); - } catch { - return false; - } -} - -function defaultMigrationsDir(): string { - return resolve(dirname(fileURLToPath(import.meta.url)), '../migrations'); -} - -async function discoverMigrationFiles(migrationsDir: string): Promise { - const files = (await readdir(migrationsDir)).filter((name) => - /^\d+_[a-z0-9_-]+\.sql$/i.test(name), - ); - files.sort(compareMigrationNames); - const versions = new Set(); - for (const name of files) { - const version = migrationVersion(name); - if (versions.has(version)) throw new Error(`Duplicate database migration version: ${version}`); - versions.add(version); - } - if (!files.length) throw new Error(`No database migrations found in ${migrationsDir}`); - return files; -} - -function compareMigrationNames(left: string, right: string): number { - const leftVersion = migrationVersion(left); - const rightVersion = migrationVersion(right); - if (leftVersion.length !== rightVersion.length) return leftVersion.length - rightVersion.length; - const versionOrder = leftVersion.localeCompare(rightVersion, 'en'); - return versionOrder || left.localeCompare(right, 'en'); -} - -function migrationVersion(name: string): string { - const match = /^(\d+)_/.exec(name); - if (!match) throw new Error(`Invalid database migration name: ${name}`); - return match[1]!.replace(/^0+(?=\d)/, ''); -} - -function sha256(value: string): string { - return createHash('sha256').update(value).digest('hex'); +/** Open `count` connections once so the first request does not pay TCP/TLS setup. */ +export async function warmPool(pool: pg.Pool, count: number): Promise { + const warm = Math.max(0, Math.min(Math.trunc(count), pool.options.max ?? 0)); + if (!warm) return; + const clients = await Promise.all(Array.from({ length: warm }, () => pool.connect())); + for (const client of clients) client.release(); } diff --git a/packages/database/src/migrate.ts b/packages/database/src/migrate.ts new file mode 100644 index 0000000..18d4970 --- /dev/null +++ b/packages/database/src/migrate.ts @@ -0,0 +1,268 @@ +import { readdir, readFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { existsSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type pg from 'pg'; +import type { DatabaseDialectName } from './dialect.js'; +import type { QueryResult, SqlDatabase } from './pool.js'; + +export interface MigrationOptions { + migrationsDir?: string; +} + +interface MigrationSession { + query>( + text: string, + values?: readonly unknown[], + ): Promise>; + commit(): Promise; + rollback(): Promise; + release(): void; +} + +interface MigrationTarget { + dialect: DatabaseDialectName; + query>( + text: string, + values?: readonly unknown[], + ): Promise>; + session(): Promise; +} + +type PointOfTruth = SqlDatabase | pg.Pool | MigrationTarget; + +const MIGRATION_LOCK = 'taskdock:schema-migrations'; +const POSTGRES_ADVISORY = + "SELECT pg_advisory_xact_lock(hashtextextended('taskdock:schema-migrations', 0))"; + +/** + * Apply every pending migration. + * + * PostgreSQL runs the whole set inside one transaction (DDL rolls back), under + * an advisory lock. MySQL commits DDL implicitly, so each file is applied on its + * own and recorded immediately; a failure leaves earlier files applied, which + * is exactly what the recorded checksums then describe. + */ +export async function runMigrations( + pointOfTruth: PointOfTruth, + options: MigrationOptions = {}, +): Promise { + const target = toTarget(pointOfTruth); + const migrationsDir = options.migrationsDir ?? defaultMigrationsDir(target.dialect); + const migrations = await loadMigrations(migrationsDir); + if (target.dialect === 'mysql') await runMysqlMigrations(target, migrations); + else await runPostgresMigrations(target, migrations); +} + +export async function databaseReady( + pointOfTruth: PointOfTruth, + options: MigrationOptions = {}, +): Promise { + const target = toTarget(pointOfTruth); + try { + const migrationsDir = options.migrationsDir ?? defaultMigrationsDir(target.dialect); + const files = await discoverMigrationFiles(migrationsDir); + const result = await target.query<{ name: string; checksum: string | null }>( + 'SELECT name, checksum FROM schema_migrations', + ); + const applied = new Map(result.rows.map((row) => [row.name, row.checksum])); + for (const name of files) { + const checksum = sha256(await readFile(resolve(migrationsDir, name), 'utf8')); + if (applied.get(name) !== checksum) return false; + } + return files.length > 0 && files.every((name) => applied.has(name)); + } catch { + return false; + } +} + +interface LoadedMigration { + name: string; + sql: string; + checksum: string; +} + +async function loadMigrations(migrationsDir: string): Promise { + const files = await discoverMigrationFiles(migrationsDir); + return Promise.all( + files.map(async (name) => { + const sql = await readFile(resolve(migrationsDir, name), 'utf8'); + return { name, sql, checksum: sha256(sql) }; + }), + ); +} + +async function runPostgresMigrations( + target: MigrationTarget, + migrations: LoadedMigration[], +): Promise { + const client = await target.session(); + try { + await client.query('BEGIN'); + await client.query(POSTGRES_ADVISORY); + await client.query( + `CREATE TABLE IF NOT EXISTS schema_migrations (name TEXT PRIMARY KEY, checksum TEXT, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())`, + ); + await client.query('ALTER TABLE schema_migrations ADD COLUMN IF NOT EXISTS checksum TEXT'); + for (const migration of migrations) { + const exists = await client.query<{ name: string; checksum: string | null }>( + 'SELECT name, checksum FROM schema_migrations WHERE name = $1 FOR UPDATE', + [migration.name], + ); + const applied = exists.rows[0]; + if (applied) { + // The original 0001 executor did not store checksums. Backfill that + // one legacy record exactly once, then protect every applied file. + if (!applied.checksum) { + await client.query('UPDATE schema_migrations SET checksum = $2 WHERE name = $1', [ + migration.name, + migration.checksum, + ]); + } else if (applied.checksum !== migration.checksum) { + throw new Error(`Migration checksum mismatch: ${migration.name}`); + } + continue; + } + await client.query(migration.sql); + await client.query('INSERT INTO schema_migrations(name, checksum) VALUES ($1, $2)', [ + migration.name, + migration.checksum, + ]); + } + await client.query('COMMIT'); + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + /* Preserve the original error. */ + } + throw error; + } finally { + client.release(); + } +} + +async function runMysqlMigrations( + target: MigrationTarget, + migrations: LoadedMigration[], +): Promise { + const session = await target.session(); + let locked = false; + try { + const lock = await session.query<{ locked: number | null }>( + 'SELECT GET_LOCK(?, 60) AS locked', + [MIGRATION_LOCK], + ); + locked = Number(lock.rows[0]?.locked) === 1; + if (!locked) throw new Error('另一项迁移正在执行,无法获得迁移锁'); + // VARCHAR(191) keeps the primary key inside the 3072-byte index budget + // even when the connection default collation is utf8mb4. + await session.query( + 'CREATE TABLE IF NOT EXISTS schema_migrations (name VARCHAR(191) NOT NULL PRIMARY KEY, checksum VARCHAR(64) NULL, applied_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)) ENGINE=InnoDB', + ); + const appliedRows = await session.query<{ name: string; checksum: string | null }>( + 'SELECT name, checksum FROM schema_migrations', + ); + const applied = new Map(appliedRows.rows.map((row) => [row.name, row.checksum])); + for (const migration of migrations) { + const recorded = applied.get(migration.name); + if (recorded !== undefined) { + if (recorded !== migration.checksum) + throw new Error(`Migration checksum mismatch: ${migration.name}`); + continue; + } + // MySQL commits DDL implicitly, so the file cannot be rolled back; record + // it right after it succeeds instead of at the end of the run. + await session.query(migration.sql); + await session.query('INSERT INTO schema_migrations(name, checksum) VALUES (?, ?)', [ + migration.name, + migration.checksum, + ]); + } + } finally { + if (locked) { + try { + await session.query('SELECT RELEASE_LOCK(?)', [MIGRATION_LOCK]); + } catch { + /* The lock is released when the session closes. */ + } + } + session.release(); + } +} + +function toTarget(pointOfTruth: PointOfTruth): MigrationTarget { + const candidate = pointOfTruth as Partial & Partial; + if (candidate.dialect && typeof candidate.session === 'function' && 'settings' in candidate) + return pointOfTruth as SqlDatabase; + + const pool = pointOfTruth as pg.Pool; + return { + dialect: 'postgres', + async query>(text: string, values: readonly unknown[] = []) { + const result = await pool.query(text, values as unknown[]); + return { rows: result.rows as T[], rowCount: result.rowCount }; + }, + async session() { + const client = await pool.connect(); + return { + async query>(text: string, values: readonly unknown[] = []) { + const result = await client.query(text, values as unknown[]); + return { rows: result.rows as T[], rowCount: result.rowCount }; + }, + async commit() { + await client.query('COMMIT'); + }, + async rollback() { + await client.query('ROLLBACK'); + }, + release() { + client.release(); + }, + }; + }, + }; +} + +export function defaultMigrationsDir(dialect: DatabaseDialectName = 'postgres'): string { + const base = resolve(dirname(fileURLToPath(import.meta.url)), '../migrations'); + const perDialect = resolve(base, dialect === 'mysql' ? 'mysql' : 'postgres'); + // Before the layout split, PostgreSQL migrations lived directly under + // migrations/. Keep the resolution working in both layouts. + if (existsSync(perDialect)) return perDialect; + return base; +} + +async function discoverMigrationFiles(migrationsDir: string): Promise { + const files = (await readdir(migrationsDir)).filter((name) => + /^\d+_[a-z0-9_-]+\.sql$/i.test(name), + ); + files.sort(compareMigrationNames); + const versions = new Set(); + for (const name of files) { + const version = migrationVersion(name); + if (versions.has(version)) throw new Error(`Duplicate database migration version: ${version}`); + versions.add(version); + } + if (!files.length) throw new Error(`No database migrations found in ${migrationsDir}`); + return files; +} + +function compareMigrationNames(left: string, right: string): number { + const leftVersion = migrationVersion(left); + const rightVersion = migrationVersion(right); + if (leftVersion.length !== rightVersion.length) return leftVersion.length - rightVersion.length; + const versionOrder = leftVersion.localeCompare(rightVersion, 'en'); + return versionOrder || left.localeCompare(right, 'en'); +} + +function migrationVersion(name: string): string { + const match = /^(\d+)_/.exec(name); + if (!match) throw new Error(`Invalid database migration name: ${name}`); + return match[1]!.replace(/^0+(?=\d)/, ''); +} + +function sha256(value: string): string { + return createHash('sha256').update(value).digest('hex'); +} diff --git a/packages/database/src/pool.ts b/packages/database/src/pool.ts new file mode 100644 index 0000000..3cf0cf1 --- /dev/null +++ b/packages/database/src/pool.ts @@ -0,0 +1,301 @@ +import mysql from 'mysql2/promise'; +import pg from 'pg'; +import { + poolConfigFromRuntime, + type DatabaseRuntimeConfig, + type DatabaseSettings, +} from './config.js'; +import { normaliseMysqlValues, prepareQuery, type DatabaseDialectName } from './dialect.js'; + +export interface QueryResult> { + rows: T[]; + rowCount: number | null; + /** MySQL only: auto-increment value of the first inserted row. */ + insertId?: string; +} + +export interface SessionOptions { + /** Runs under REPEATABLE READ with writes rejected, for read-only snapshots. */ + readOnly?: boolean; +} + +export interface ListenOptions { + onPayload(payload: string): void; + /** The listener connection died; the caller owns reconnecting. */ + onFailure(error: Error): void; +} + +export interface SqlSession { + readonly dialect: DatabaseDialectName; + query>( + text: string, + values?: readonly unknown[], + ): Promise>; + begin(options?: SessionOptions): Promise; + commit(): Promise; + rollback(): Promise; + release(error?: Error): void; + /** PostgreSQL LISTEN; absent on MySQL, which has no pub/sub primitive. */ + listen?(channel: string, options: ListenOptions): Promise; + unlisten?(channel: string): Promise; +} + +export interface SqlDatabase { + readonly dialect: DatabaseDialectName; + readonly settings: DatabaseSettings; + readonly maxConnections: number; + query>( + text: string, + values?: readonly unknown[], + ): Promise>; + session(): Promise; + warm(count: number): Promise; + onConnectionError(handler: (error: Error) => void): void; + end(): Promise; +} + +const CHANNEL = /^[a-z_][a-z0-9_]*$/; + +export function openDatabase(settings: DatabaseSettings): SqlDatabase { + return settings.dialect === 'mysql' + ? new MysqlDatabase(settings) + : new PostgresDatabase(settings); +} + +class PostgresDatabase implements SqlDatabase { + readonly dialect = 'postgres' as const; + readonly maxConnections: number; + private readonly pool: pg.Pool; + + constructor(readonly settings: DatabaseSettings) { + this.maxConnections = settings.max; + this.pool = new pg.Pool(poolConfigFromRuntime(runtimeConfig(settings))); + } + + async query>( + text: string, + values: readonly unknown[] = [], + ): Promise> { + const prepared = prepareQuery('postgres', text, values); + const result = await this.pool.query(prepared.text, prepared.values); + return { rows: result.rows as T[], rowCount: result.rowCount }; + } + + async session(): Promise { + const client = await this.pool.connect(); + return { + dialect: 'postgres', + async query>(text: string, values: readonly unknown[] = []) { + const prepared = prepareQuery('postgres', text, values); + const result = await client.query(prepared.text, prepared.values); + return { rows: result.rows as T[], rowCount: result.rowCount }; + }, + async begin(options: SessionOptions = {}) { + await client.query( + options.readOnly ? 'BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY' : 'BEGIN', + ); + }, + async commit() { + await client.query('COMMIT'); + }, + async rollback() { + await client.query('ROLLBACK'); + }, + release(error?: Error) { + client.release(error); + }, + async listen(channel: string, options: ListenOptions) { + if (!CHANNEL.test(channel)) throw new Error(`Invalid listen channel: ${channel}`); + client.on('notification', (message: pg.Notification) => { + if (message.channel === channel && message.payload !== undefined) + options.onPayload(message.payload); + }); + client.on('error', (error: Error) => options.onFailure(error)); + client.on('end', () => options.onFailure(new Error('listener connection ended'))); + await client.query(`LISTEN ${channel}`); + }, + async unlisten(channel: string) { + if (!CHANNEL.test(channel)) return; + await client.query(`UNLISTEN ${channel}`); + }, + }; + } + + async warm(count: number): Promise { + const warm = Math.max(0, Math.min(Math.trunc(count), this.settings.max)); + if (!warm) return; + const clients = await Promise.all(Array.from({ length: warm }, () => this.pool.connect())); + for (const client of clients) client.release(); + } + + onConnectionError(handler: (error: Error) => void): void { + this.pool.on('error', handler); + } + + async end(): Promise { + await this.pool.end(); + } +} + +class MysqlDatabase implements SqlDatabase { + readonly dialect = 'mysql' as const; + readonly maxConnections: number; + private readonly pool: mysql.Pool; + + constructor(readonly settings: DatabaseSettings) { + this.maxConnections = settings.max; + this.pool = mysql.createPool(this.poolOptions()); + } + + private poolOptions(): mysql.PoolOptions { + const base: mysql.PoolOptions = this.settings.connectionString + ? { uri: this.settings.connectionString } + : { + host: this.settings.host, + port: this.settings.port, + user: this.settings.user, + password: this.settings.password, + database: this.settings.database, + }; + const ssl = this.sslOptions(); + return { + ...base, + waitForConnections: true, + connectionLimit: this.settings.max, + maxIdle: this.settings.max, + idleTimeout: this.settings.idleTimeoutMillis, + connectTimeout: this.settings.connectionTimeoutMillis, + queueLimit: 0, + enableKeepAlive: this.settings.keepAlive, + keepAliveInitialDelay: this.settings.keepAliveInitialDelayMillis, + // DATETIME is stored as UTC and read back as Date, matching node-postgres + // timestamptz behaviour. BIGINT becomes a string only when a number would + // lose precision, so small ranks stay numbers and large sequences stay exact. + timezone: 'Z', + dateStrings: false, + supportBigNumbers: true, + bigNumberStrings: false, + multipleStatements: false, + namedPlaceholders: false, + ...(ssl ? { ssl } : {}), + }; + } + + private sslOptions(): mysql.SslOptions | undefined { + const { sslMode, sslCa } = this.settings; + if (!sslMode || sslMode === 'disable') return undefined; + const ssl: mysql.SslOptions = { rejectUnauthorized: sslMode !== 'require' }; + if (sslCa) ssl.ca = sslCa; + if (sslMode === 'verify-ca') + (ssl as { checkServerIdentity?: () => undefined }).checkServerIdentity = () => undefined; + return ssl; + } + + async query>( + text: string, + values: readonly unknown[] = [], + ): Promise> { + const prepared = prepareQuery('mysql', text, values); + const [result] = await this.pool.query(prepared.text, normaliseMysqlValues(prepared.values)); + return mysqlResult(result); + } + + async session(): Promise { + const connection = await this.pool.getConnection(); + await applyMysqlSessionSettings(connection, this.settings); + return { + dialect: 'mysql', + async query>(text: string, values: readonly unknown[] = []) { + const prepared = prepareQuery('mysql', text, values); + const [result] = await connection.query( + prepared.text, + normaliseMysqlValues(prepared.values), + ); + return mysqlResult(result); + }, + async begin(options: SessionOptions = {}) { + await connection.query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ'); + await connection.query( + options.readOnly ? 'START TRANSACTION READ ONLY' : 'START TRANSACTION', + ); + }, + async commit() { + await connection.commit(); + }, + async rollback() { + await connection.rollback(); + }, + release(error?: Error) { + // A failed transaction can leave the socket in an unknown state. + if (error) connection.destroy(); + else connection.release(); + }, + }; + } + + async warm(count: number): Promise { + const warm = Math.max(0, Math.min(Math.trunc(count), this.settings.max)); + if (!warm) return; + const connections = await Promise.all( + Array.from({ length: warm }, () => this.pool.getConnection()), + ); + for (const connection of connections) { + await applyMysqlSessionSettings(connection, this.settings); + connection.release(); + } + } + + onConnectionError(): void { + // mysql2 surfaces a dead socket as a query error on the next use; there is + // no pool-level 'error' event to subscribe to, unlike node-postgres. + } + + async end(): Promise { + await this.pool.end(); + } +} + +function runtimeConfig(settings: DatabaseSettings): DatabaseRuntimeConfig { + return { + connectionString: settings.connectionString, + host: settings.host, + port: settings.port, + user: settings.user, + password: settings.password, + database: settings.database, + ssl: settings.ssl, + applicationName: settings.applicationName, + max: settings.max, + maxUses: settings.maxUses, + connectionTimeoutMillis: settings.connectionTimeoutMillis, + idleTimeoutMillis: settings.idleTimeoutMillis, + queryTimeoutMillis: settings.queryTimeoutMillis, + statementTimeoutMillis: settings.statementTimeoutMillis, + lockTimeoutMillis: settings.lockTimeoutMillis, + keepAlive: settings.keepAlive, + keepAliveInitialDelayMillis: settings.keepAliveInitialDelayMillis, + listenEnabled: settings.listenEnabled, + }; +} + +async function applyMysqlSessionSettings( + connection: mysql.PoolConnection, + settings: DatabaseSettings, +): Promise { + const lockSeconds = Math.max(1, Math.ceil(settings.lockTimeoutMillis / 1_000)); + await connection.query("SET SESSION time_zone = '+00:00'"); + await connection.query(`SET SESSION innodb_lock_wait_timeout = ${lockSeconds}`); + // Only SELECT statements honour max_execution_time; writes are bounded by + // innodb_lock_wait_timeout plus the driver-level connect and queue timeouts. + await connection.query(`SET SESSION max_execution_time = ${settings.queryTimeoutMillis}`); +} + +function mysqlResult(result: unknown): QueryResult { + if (Array.isArray(result)) return { rows: result as T[], rowCount: result.length }; + const header = result as { affectedRows?: number; insertId?: number }; + return { + rows: [], + rowCount: header.affectedRows ?? 0, + insertId: header.insertId === undefined ? undefined : String(header.insertId), + }; +} diff --git a/packages/database/test/config.test.ts b/packages/database/test/config.test.ts index 1572c6a..3fb3067 100644 --- a/packages/database/test/config.test.ts +++ b/packages/database/test/config.test.ts @@ -91,3 +91,47 @@ describe('database runtime config', () => { ).toThrow(/DATABASE_SSL_MODE/); }); }); + +describe('dual engine settings', () => { + it('detects MySQL from the DATABASE_URL scheme', async () => { + const { readDatabaseSettings } = await import('../src/config.js'); + const settings = readDatabaseSettings({ + DATABASE_URL: 'mysql://taskdock:secret@db.example.lan:3307/taskdock', + DB_POOL_MAX: '4', + }); + expect(settings.dialect).toBe('mysql'); + expect(settings.connectionString).toBe('mysql://taskdock:secret@db.example.lan:3307/taskdock'); + expect(settings.max).toBe(4); + // MySQL has no LISTEN/NOTIFY; the flag must not claim otherwise. + expect(settings.listenEnabled).toBe(false); + }); + + it('supports discrete MYSQL_* variables and the 3306 default', async () => { + const { readDatabaseSettings } = await import('../src/config.js'); + const settings = readDatabaseSettings({ + MYSQL_HOST: 'db.example.lan', + MYSQL_USER: 'taskdock', + MYSQL_PASSWORD: 'p@ss:word', + MYSQL_DATABASE: 'taskdock_home', + }); + expect(settings.dialect).toBe('mysql'); + expect(settings.port).toBe(3306); + expect(settings.password).toBe('p@ss:word'); + expect(settings.database).toBe('taskdock_home'); + }); + + it('honours an explicit DB_DIALECT override', async () => { + const { readDatabaseSettings } = await import('../src/config.js'); + expect(() => readDatabaseSettings({ DB_DIALECT: 'oracle' })).toThrow(/DB_DIALECT/); + expect(() => + readDatabaseSettings({ DB_DIALECT: 'mysql', DATABASE_URL: 'mysql://host/db' }), + ).not.toThrow(); + }); + + it('rejects a DATABASE_URL without a recognised scheme', async () => { + const { readDatabaseSettings } = await import('../src/config.js'); + expect(() => readDatabaseSettings({ DATABASE_URL: 'file:/tmp/db' })).toThrow( + /postgres:|mysql:/, + ); + }); +}); diff --git a/packages/database/test/dialect.test.ts b/packages/database/test/dialect.test.ts new file mode 100644 index 0000000..fb6c70d --- /dev/null +++ b/packages/database/test/dialect.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest'; +import { prepareQuery } from '../src/dialect.js'; + +describe('SQL preparation', () => { + it('keeps PostgreSQL placeholders and converts backticks to quoted identifiers', () => { + const prepared = prepareQuery('postgres', 'SELECT `rank`, id FROM tasks WHERE owner_id = $1', [ + 'owner', + ]); + expect(prepared.text).toBe('SELECT "rank", id FROM tasks WHERE owner_id = $1'); + expect(prepared.values).toEqual(['owner']); + }); + + it('leaves literals and comments alone', () => { + const sql = "SELECT '`rank` $1' AS literal, id FROM tasks -- `rank` $2\n WHERE id = $1"; + const prepared = prepareQuery('postgres', sql, ['abc']); + expect(prepared.text).toBe( + "SELECT '`rank` $1' AS literal, id FROM tasks -- `rank` $2\n WHERE id = $1", + ); + }); + + it('renumbers MySQL placeholders in appearance order, including repeats', () => { + const prepared = prepareQuery( + 'mysql', + 'INSERT INTO tasks (id, owner_id) VALUES ($1, $2) ON DUPLICATE KEY UPDATE id = $1, owner_id = $2', + ['a', 'b'], + ); + expect(prepared.text).toBe( + 'INSERT INTO tasks (id, owner_id) VALUES (?, ?) ON DUPLICATE KEY UPDATE id = ?, owner_id = ?', + ); + expect(prepared.values).toEqual(['a', 'b', 'a', 'b']); + }); + + it('drops PostgreSQL type casts for MySQL', () => { + const prepared = prepareQuery( + 'mysql', + 'SELECT COUNT(*)::int AS n, $1::date AS day, $2::jsonb AS payload, $3::double precision AS f', + ['2026-11-15', '{}', 1.5], + ); + expect(prepared.text).toBe('SELECT COUNT(*) AS n, ? AS day, ? AS payload, ? AS f'); + expect(prepared.values).toEqual(['2026-11-15', '{}', 1.5]); + }); + + it('keeps MySQL backtick quoting and single-quoted content intact', () => { + const prepared = prepareQuery( + 'mysql', + "SELECT `rank` FROM tasks WHERE title LIKE '%rank%' AND id = $1", + ['x'], + ); + expect(prepared.text).toBe("SELECT `rank` FROM tasks WHERE title LIKE '%rank%' AND id = ?"); + expect(prepared.values).toEqual(['x']); + }); + + it('rejects a placeholder without a value', () => { + expect(() => prepareQuery('mysql', 'SELECT $2', ['only-one'])).toThrow(/\$2/); + }); + + it('normalises ISO timestamps and booleans for MySQL parameters', async () => { + const { normaliseMysqlValues } = await import('../src/dialect.js'); + expect(normaliseMysqlValues(['2026-11-15T08:07:24.123Z', true, 42])).toEqual([ + '2026-11-15 08:07:24.123000', + 1, + 42, + ]); + expect(normaliseMysqlValues(['2026-11-15'])).toEqual(['2026-11-15']); + }); +}); From b9734560e18a9b61d53bf1e4c1e719f145f3ba81 Mon Sep 17 00:00:00 2001 From: IGNGserver <176017966+IGNGserver@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:51:22 +0800 Subject: [PATCH 3/9] =?UTF-8?q?refactor(api):=20=E6=8A=8A=20Postgres=20?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E5=B1=82=E6=94=B9=E9=80=A0=E6=88=90=E6=96=B9?= =?UTF-8?q?=E8=A8=80=E6=97=A0=E5=85=B3=E7=9A=84=20SQL=20=E5=AD=98=E5=82=A8?= =?UTF-8?q?=E5=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PostgresStore/PostgresTreeStore 重命名为 SqlStore/SqlTreeStore,接收 SqlDatabase 抽象而不是 pg.Pool - 事务、只读快照、LISTEN、咨询锁、保留清理按方言分支:MySQL 用会话锁、 DELETE ORDER BY LIMIT 与进程内通知 - RETURNING 统一走 executeReturning:PostgreSQL 一语句返回,MySQL 用已知 主键同事务回查 - jsonb 批量写改为多行 VALUES upsert,jsonb_agg 快照换成按方言的两份语句 - FILTER 聚合改写为 SUM(CASE)、ILIKE 由方言层翻译成 LIKE - 目录读取与 bootstrap 的递归/JSON 查询按方言拆分 --- apps/api/src/bootstrap-owner.ts | 7 +- apps/api/src/directory-reads.ts | 63 +- apps/api/src/migrate.ts | 20 +- apps/api/src/server.ts | 42 +- apps/api/src/sql-statements.ts | 435 +++++++++++ .../src/{postgres-store.ts => sql-store.ts} | 658 ++++++++++------- ...stgres-tree-store.ts => sql-tree-store.ts} | 693 ++++++++++-------- apps/api/src/sync-bootstrap.ts | 125 ++-- apps/api/test/directory-reads.test.ts | 8 +- apps/api/test/postgres-sync-query.test.ts | 14 +- apps/api/test/postgres-tree-batch.test.ts | 43 +- apps/api/test/store.test.ts | 14 +- packages/database/src/dialect.ts | 7 + packages/database/src/pool.ts | 40 +- scripts/integration-smoke.ts | 38 +- scripts/perf-capacity.ts | 4 +- scripts/pg-concurrency-worker.ts | 4 +- scripts/sync-bootstrap-benchmark.ts | 14 +- 18 files changed, 1449 insertions(+), 780 deletions(-) create mode 100644 apps/api/src/sql-statements.ts rename apps/api/src/{postgres-store.ts => sql-store.ts} (87%) rename apps/api/src/{postgres-tree-store.ts => sql-tree-store.ts} (51%) diff --git a/apps/api/src/bootstrap-owner.ts b/apps/api/src/bootstrap-owner.ts index 46a50c4..5785f14 100644 --- a/apps/api/src/bootstrap-owner.ts +++ b/apps/api/src/bootstrap-owner.ts @@ -1,8 +1,8 @@ -import { createPoolFromEnv } from '@devtodo/database'; +import { openDatabase, readDatabaseSettings } from '@devtodo/database'; import { DomainError } from '@devtodo/domain'; import { z } from 'zod'; import { AuthService } from './auth.js'; -import { PostgresStore } from './postgres-store.js'; +import { SqlStore } from './sql-store.js'; import { assertProductionSecrets } from './runtime-secrets.js'; const requiredEnv = (name: string): string => { @@ -34,8 +34,7 @@ async function main(): Promise { ) throw new Error('production token secrets must contain at least 32 characters'); } - const { pool } = createPoolFromEnv(process.env); - const store = new PostgresStore(pool); + const store = new SqlStore(openDatabase(readDatabaseSettings(process.env))); try { await store.init(); if (await store.hasOwner()) { diff --git a/apps/api/src/directory-reads.ts b/apps/api/src/directory-reads.ts index b534a38..85420a8 100644 --- a/apps/api/src/directory-reads.ts +++ b/apps/api/src/directory-reads.ts @@ -10,8 +10,9 @@ import { type TreeTaskDto, } from '@devtodo/contracts'; import { DomainError } from '@devtodo/domain'; -import { PostgresTreeStore } from './postgres-tree-store.js'; +import { SqlTreeStore } from './sql-tree-store.js'; import type { V2TreeStore } from './tree-store.js'; +import { directoryChildrenQuery, FOLDER_PATH_QUERY } from './sql-statements.js'; import { v2SqlDto as dto } from './v2-sql-dto.js'; function directoryDto(row: Record, kind: 'FOLDER' | 'TASK') { @@ -42,7 +43,7 @@ export class DirectoryReads { token?: string, ): Promise { try { - if (!(this.tree instanceof PostgresTreeStore)) { + if (!(this.tree instanceof SqlTreeStore)) { await this.tree.prepare?.(owner); const items = this.tree.listTreeChildren(owner, parent); const revision = (await this.tree.status(owner)).cursor; @@ -53,7 +54,7 @@ export class DirectoryReads { token, ); } - const pg = this.tree.postgres; + const pg = this.tree.sql; return await pg.v2ReadSnapshot(async () => { // Include legacy writes in invalidation, since they can edit tree rows. const version = await pg.v2Query( @@ -74,37 +75,7 @@ export class DirectoryReads { : ''; const take = limit === undefined ? '' : `LIMIT $${values.push(limit + 1)}::int`; const result = await pg.v2Query( - `WITH RECURSIVE branch(root,id) AS ( - SELECT id,id FROM folders WHERE owner_id=$1 AND deleted_at IS NULL AND ${parentWhere} - UNION - SELECT b.root,f.id FROM folders f JOIN branch b ON f.parent_folder_id=b.id - WHERE f.owner_id=$1 AND f.deleted_at IS NULL - ), counts AS ( - SELECT b.root, COUNT(t.id)::int AS total, - COUNT(t.id) FILTER (WHERE t.status='TODO')::int AS todo, - COUNT(t.id) FILTER (WHERE t.status='IN_PROGRESS')::int AS progress, - COUNT(t.id) FILTER (WHERE t.status='DONE')::int AS done - FROM branch b LEFT JOIN tasks t ON t.owner_id=$1 AND t.parent_folder_id=b.id AND t.deleted_at IS NULL - GROUP BY b.root - ), rows AS ( - SELECT f.id,f.rank AS r,0 AS k, - CASE f.status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END AS g, - jsonb_build_object('status',f.status, - 'todoCount',c.todo,'inProgressCount',c.progress,'doneCount',c.done,'totalCount',c.total) AS aggregate - FROM folders f JOIN counts c ON c.root=f.id WHERE f.owner_id=$1 AND f.deleted_at IS NULL AND ${parentWhere} - UNION ALL - SELECT t.id,t.rank,1,CASE t.status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END, - NULL::jsonb - FROM tasks t WHERE t.owner_id=$1 AND t.deleted_at IS NULL AND ${parentWhere} - ), page AS MATERIALIZED ( - SELECT * FROM rows ${seek} ORDER BY g,r,k,id ${take} - ) SELECT CASE p.k WHEN 0 THEN - jsonb_build_object('kind','FOLDER','folder',to_jsonb(f) || jsonb_build_object('rank',f.rank::text),'aggregate',p.aggregate) - ELSE jsonb_build_object('kind','TASK','task',to_jsonb(t) || jsonb_build_object('rank',t.rank::text)) END AS item - FROM page p - LEFT JOIN folders f ON p.k=0 AND f.owner_id=$1 AND f.id=p.id - LEFT JOIN tasks t ON p.k=1 AND t.owner_id=$1 AND t.id=p.id - ORDER BY p.g,p.r,p.k,p.id`, + directoryChildrenQuery(pg.dialect, { parentWhere, seek, take }), values, ); const items = result.rows.map((row) => { @@ -135,23 +106,23 @@ export class DirectoryReads { } async folders(owner: string): Promise { - if (!(this.tree instanceof PostgresTreeStore)) { + if (!(this.tree instanceof SqlTreeStore)) { await this.tree.prepare?.(owner); return this.tree.listFolders(owner); } - const result = await this.tree.postgres.v2Query( - 'SELECT *, rank::text AS rank FROM folders WHERE owner_id=$1 AND deleted_at IS NULL ORDER BY folders.rank,id', + const result = await this.tree.sql.v2Query( + 'SELECT *, rank::text AS rank FROM folders WHERE owner_id=$1 AND deleted_at IS NULL ORDER BY folders.`rank`,id', [owner], ); return result.rows.map((row) => directoryDto(row, 'FOLDER')) as unknown as FolderDto[]; } async task(owner: string, id: string): Promise<{ task: TreeTaskDto }> { - if (!(this.tree instanceof PostgresTreeStore)) { + if (!(this.tree instanceof SqlTreeStore)) { await this.tree.prepare?.(owner); return { task: this.tree.getTask(owner, id) }; } - const result = await this.tree.postgres.v2Query( + const result = await this.tree.sql.v2Query( 'SELECT id, reference_id, parent_folder_id, title, status, rank::text, version, completed_at, created_at, updated_at, deleted_at FROM tasks WHERE owner_id=$1 AND id=$2 AND deleted_at IS NULL', [owner, id], ); @@ -160,19 +131,13 @@ export class DirectoryReads { } async path(owner: string, id: string): Promise>> { - if (!(this.tree instanceof PostgresTreeStore)) { + if (!(this.tree instanceof SqlTreeStore)) { await this.tree.prepare?.(owner); return this.tree.getFolderPath(owner, id); } - return this.tree.postgres.v2ReadSnapshot(async () => { - const result = await (this.tree as PostgresTreeStore).postgres.v2Query( - `WITH RECURSIVE parents AS ( - SELECT id,title,parent_folder_id,ARRAY[id] AS visited,0 AS depth FROM folders WHERE owner_id=$1 AND id=$2 AND deleted_at IS NULL - UNION ALL SELECT f.id,f.title,f.parent_folder_id,p.visited || f.id,p.depth+1 FROM folders f JOIN parents p ON f.id=p.parent_folder_id - WHERE f.owner_id=$1 AND f.deleted_at IS NULL AND NOT f.id=ANY(p.visited) - ) SELECT * FROM parents ORDER BY depth DESC`, - [owner, id], - ); + const sqlTree = this.tree as SqlTreeStore; + return sqlTree.sql.v2ReadSnapshot(async () => { + const result = await sqlTree.sql.v2Query(FOLDER_PATH_QUERY[sqlTree.sql.dialect], [owner, id]); if (!result.rows.length) throw new DomainError('ENTITY_NOT_FOUND', '文件夹不存在'); if (result.rows[0]?.['parent_folder_id'] !== null) throw new DomainError('TREE_CYCLE', '目录路径存在循环或无效父级'); diff --git a/apps/api/src/migrate.ts b/apps/api/src/migrate.ts index a063df8..68a2631 100644 --- a/apps/api/src/migrate.ts +++ b/apps/api/src/migrate.ts @@ -1,27 +1,23 @@ -import { - createPool, - poolConfigFromRuntime, - readDatabaseRuntimeConfig, - runMigrations, -} from '@devtodo/database'; +import { openDatabase, readDatabaseSettings, runMigrations } from '@devtodo/database'; const migrationUrl = process.env['MIGRATION_DATABASE_URL']?.trim(); const env: Record = { ...process.env, ...(migrationUrl ? { DATABASE_URL: migrationUrl } : {}), }; -const settings = readDatabaseRuntimeConfig(env); +const settings = readDatabaseSettings(env); const timeout = Number(process.env['MIGRATION_STATEMENT_TIMEOUT_MS'] ?? 300_000); if (!Number.isInteger(timeout) || timeout < 0 || timeout > 3_600_000) throw new Error('MIGRATION_STATEMENT_TIMEOUT_MS must be an integer between 0 and 3600000'); -const pool = createPool(poolConfigFromRuntime(settings), { - statementTimeoutMillis: timeout, - queryTimeoutMillis: timeout, +const database = openDatabase({ + ...settings, applicationName: `${settings.applicationName}-migrate`, max: 2, + queryTimeoutMillis: timeout, + statementTimeoutMillis: timeout, }); try { - await runMigrations(pool); + await runMigrations(database); } finally { - await pool.end(); + await database.end(); } diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index b9e4baa..3164e7a 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -29,7 +29,7 @@ import { uuidSchema, type ErrorCode, } from '@devtodo/contracts'; -import { createPoolFromEnv } from '@devtodo/database'; +import { openDatabase, readDatabaseSettings } from '@devtodo/database'; import { DomainError } from '@devtodo/domain'; import { createCacheFromEnv, type CacheStore } from './cache.js'; import { DirectoryReads } from './directory-reads.js'; @@ -41,8 +41,8 @@ import websocket, { type WebSocket as HubWebSocket } from '@fastify/websocket'; import Fastify, { type FastifyInstance, type FastifyReply, type FastifyRequest } from 'fastify'; import { z, ZodError } from 'zod'; import { AuthService, type AuthConfig } from './auth.js'; -import { DatabaseConstraintError, PostgresStore } from './postgres-store.js'; -import { PostgresTreeStore } from './postgres-tree-store.js'; +import { DatabaseConstraintError, SqlStore } from './sql-store.js'; +import { SqlTreeStore } from './sql-tree-store.js'; import { assertProductionSecrets } from './runtime-secrets.js'; import { MemoryStore, type Store } from './store.js'; import { MemoryTreeStore, type V2TreeStore } from './tree-store.js'; @@ -117,8 +117,8 @@ function treeStoreFor(store: Store, cache: CacheStore): V2TreeStore { // the same interface; keeping this boundary here prevents route handlers // from ever reaching across Owner scopes. const tree = - store instanceof PostgresStore - ? new PostgresTreeStore(store, cache) + store instanceof SqlStore + ? new SqlTreeStore(store, cache) : new MemoryTreeStore(store instanceof MemoryStore ? store : undefined); v2StoreCache.set(store, tree); return tree; @@ -183,7 +183,7 @@ export async function buildServer( const cache = options.cache ?? (await createCacheFromEnv()); const ownsCache = options.cache === undefined; const hubId = - store instanceof PostgresStore + store instanceof SqlStore ? String( (await store.v2Query('SELECT id FROM hub_identity WHERE singleton = true')).rows[0]?.id, ) @@ -271,7 +271,7 @@ export async function buildServer( return payload; }); let maintenanceTimer: ReturnType | undefined; - if (store instanceof PostgresStore) { + if (store instanceof SqlStore) { maintenanceTimer = setInterval(() => { void store.runMaintenance().catch((error: unknown) => { app.log.warn({ err: error, event: 'maintenance.cleanup_failed' }, 'maintenance failed'); @@ -398,7 +398,7 @@ export async function buildServer( app.get('/health/live', async () => ({ status: 'ok' })); app.get('/health/ready', async (_request, reply) => { - const ready = store instanceof PostgresStore ? await store.checkReady() : true; + const ready = store instanceof SqlStore ? await store.checkReady() : true; if (!ready) return reply.code(503).send({ status: 'not_ready' }); const cacheOk = cache.enabled ? await cache.ping() : true; return { @@ -570,7 +570,7 @@ export async function buildServer( }) .parse(request.query); const page = - store instanceof PostgresStore + store instanceof SqlStore ? await store.listProjectsPage(request.auth!.ownerId, query.cursor, query.limit) : paginateList( await store.listProjects(request.auth!.ownerId), @@ -630,7 +630,7 @@ export async function buildServer( timePointId: query['timePointId'], }; const page = - store instanceof PostgresStore + store instanceof SqlStore ? await store.listTasksPage(request.auth!.ownerId, filters, query.cursor, query.limit) : paginateList( await store.listTasks(request.auth!.ownerId, filters), @@ -696,7 +696,7 @@ export async function buildServer( .parse(request.query); const type = enumQuery(query.type, ['DATE', 'EVENT']); const page = - store instanceof PostgresStore + store instanceof SqlStore ? await store.listTimePointsPage(request.auth!.ownerId, type, query.cursor, query.limit) : paginateList( await store.listTimePoints(request.auth!.ownerId, type), @@ -824,7 +824,7 @@ export async function buildServer( }) .parse(request.query); const page = - store instanceof PostgresStore + store instanceof SqlStore ? await store.listPlacementsPage( request.auth!.ownerId, paramId(request), @@ -2041,7 +2041,13 @@ async function versionedV2Action( } function databaseConfigured(config: AppConfig): boolean { - return Boolean(config.databaseUrl || (process.env['PGHOST'] && process.env['PGDATABASE'])); + if (config.databaseUrl) return true; + const env = process.env; + const discrete = + (env['PGHOST'] && env['PGDATABASE']) || + (env['MYSQL_HOST'] && env['MYSQL_DATABASE']) || + (env['DATABASE_HOST'] && env['DATABASE_NAME']); + return Boolean(discrete); } async function createStore(config: AppConfig): Promise { @@ -2051,7 +2057,7 @@ async function createStore(config: AppConfig): Promise { JSON.stringify({ event: 'database.memory_store_override', message: - 'DEV_MEMORY_STORE=true; the configured PostgreSQL URL is ignored. Set DEV_MEMORY_STORE=false to persist data.', + 'DEV_MEMORY_STORE=true; the configured DATABASE_URL is ignored. Set DEV_MEMORY_STORE=false to persist data.', }), ); return new MemoryStore(); @@ -2071,17 +2077,17 @@ async function createStore(config: AppConfig): Promise { DB_APPLICATION_NAME: config.dbApplicationName, DB_LISTEN_ENABLED: String(config.dbListenEnabled), }; - const { pool } = createPoolFromEnv(env); - return new PostgresStore(pool, { + const database = openDatabase(readDatabaseSettings(env)); + return new SqlStore(database, { changeRetentionDays: config.syncChangeRetentionDays, mutationReceiptRetentionDays: config.mutationReceiptRetentionDays, transactionTimeoutMs: config.dbStatementTimeoutMs, - listenEnabled: config.dbListenEnabled, + listenEnabled: database.settings.listenEnabled, }); } async function initStore(store: Store, config: AppConfig): Promise { - if (!(store instanceof PostgresStore)) { + if (!(store instanceof SqlStore)) { await store.init(); return; } diff --git a/apps/api/src/sql-statements.ts b/apps/api/src/sql-statements.ts new file mode 100644 index 0000000..1cfc2f6 --- /dev/null +++ b/apps/api/src/sql-statements.ts @@ -0,0 +1,435 @@ +import type { DatabaseDialectName } from '@devtodo/database'; + +/** + * Statements whose shape genuinely differs between engines. + * + * Everything else lives in the store and is translated by the dialect layer. + * MySQL has no RETURNING, no jsonb and no FILTER, and JSON_ARRAYAGG does not + * preserve a derived table's ORDER BY, so these few queries keep two spellings + * instead of being squeezed into a lowest common denominator. + */ +export const SYNC_PULL_QUERY: Record = { + postgres: `WITH owner AS ( + SELECT 1 FROM users WHERE id = $1 AND disabled_at IS NULL + ), oldest AS ( + SELECT seq FROM sync_changes WHERE protocol_version = 2 ORDER BY seq LIMIT 1 + ), page AS ( + SELECT seq, owner_id, entity_type, entity_id, entity_version, operation, snapshot, committed_at + FROM sync_changes + WHERE owner_id = $1 AND protocol_version = 2 AND seq > $2 + ORDER BY seq LIMIT $3 + ) + SELECT EXISTS(SELECT 1 FROM owner) AS owner_ok, + (SELECT seq FROM oldest) AS oldest, + COALESCE(( + SELECT jsonb_agg(jsonb_build_object( + 'seq', p.seq::text, + 'owner_id', p.owner_id, + 'entity_type', p.entity_type, + 'entity_id', p.entity_id, + 'entity_version', p.entity_version, + 'operation', p.operation, + 'snapshot', p.snapshot, + 'committed_at', p.committed_at + ) ORDER BY p.seq) + FROM page p + ), '[]'::jsonb) AS changes, + EXISTS( + SELECT 1 FROM sync_changes + WHERE owner_id = $1 AND protocol_version = 2 + AND seq > COALESCE((SELECT MAX(seq) FROM page), $2) + LIMIT 1 + ) AS has_more`, + mysql: `SELECT + EXISTS(SELECT 1 FROM users WHERE id = $1 AND disabled_at IS NULL) AS owner_ok, + (SELECT seq FROM sync_changes WHERE protocol_version = 2 ORDER BY seq LIMIT 1) AS oldest, + (SELECT JSON_ARRAYAGG(JSON_OBJECT( + 'seq', CAST(p.seq AS CHAR), + 'owner_id', p.owner_id, + 'entity_type', p.entity_type, + 'entity_id', p.entity_id, + 'entity_version', p.entity_version, + 'operation', p.operation, + 'snapshot', p.snapshot, + 'committed_at', DATE_FORMAT(p.committed_at, '%Y-%m-%dT%H:%i:%s.%fZ') + )) + FROM ( + SELECT seq, owner_id, entity_type, entity_id, entity_version, operation, snapshot, committed_at + FROM sync_changes + WHERE owner_id = $1 AND protocol_version = 2 AND seq > $2 + ORDER BY seq LIMIT $3 + ) AS p) AS changes`, +}; + +type SnapshotColumn = string | { name: string; asChar?: boolean; timestamp?: boolean }; + +interface SnapshotCollection { + key: string; + table: string; + alias: string; + columns: SnapshotColumn[]; +} + +const RESERVED = new Set(['rank', 'cursor']); + +function ident(name: string): string { + return RESERVED.has(name) ? `\`${name}\`` : name; +} + +function columnExpression(alias: string, column: SnapshotColumn): string { + const spec = typeof column === 'string' ? { name: column } : column; + const reference = `${alias}.${ident(spec.name)}`; + if (spec.timestamp) return `DATE_FORMAT(${reference}, '%Y-%m-%dT%H:%i:%s.%fZ')`; + if (spec.asChar) return `CAST(${reference} AS CHAR)`; + return reference; +} + +/** + * Column lists for the MySQL owner snapshot. PostgreSQL can turn a row into + * JSON with `to_jsonb`, MySQL needs every column spelled out; keep this list in + * step with the migrations (a schema drift test compares it with + * information_schema). + */ +const SNAPSHOT_COLLECTIONS: SnapshotCollection[] = [ + { + key: 'folders', + table: 'folders', + alias: 'f', + columns: [ + 'id', + 'owner_id', + 'parent_folder_id', + 'title', + 'status', + { name: 'rank', asChar: true }, + 'version', + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, + ], + }, + { + key: 'tasks', + table: 'tasks', + alias: 't', + columns: [ + 'id', + 'owner_id', + 'project_id', + 'category', + 'reference_id', + 'title', + 'status', + 'priority', + { name: 'rank', asChar: true }, + { name: 'completed_at', timestamp: true }, + 'version', + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, + 'parent_folder_id', + ], + }, + { + key: 'notes', + table: 'notes', + alias: 'n', + columns: [ + 'id', + 'owner_id', + 'task_id', + 'content_markdown', + 'version', + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, + ], + }, + { + key: 'steps', + table: 'task_steps', + alias: 'st', + columns: [ + 'id', + 'owner_id', + 'task_id', + 'title', + 'note_markdown', + 'status', + { name: 'rank', asChar: true }, + { name: 'completed_at', timestamp: true }, + 'version', + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, + ], + }, + { + key: 'workflows', + table: 'workflows', + alias: 'w', + columns: [ + 'id', + 'owner_id', + 'name', + { name: 'rank', asChar: true }, + 'version', + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, + ], + }, + { + key: 'stages', + table: 'workflow_stages', + alias: 'ws', + columns: [ + 'id', + 'owner_id', + 'workflow_id', + 'name', + { name: 'rank', asChar: true }, + 'version', + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, + ], + }, + { + key: 'memberships', + table: 'workflow_task_memberships', + alias: 'm', + columns: [ + 'id', + 'owner_id', + 'workflow_id', + 'stage_id', + 'task_id', + { name: 'rank', asChar: true }, + 'version', + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, + ], + }, + { + key: 'time_points', + table: 'time_points', + alias: 'p', + columns: [ + 'id', + 'owner_id', + 'type', + 'local_date', + 'title', + { name: 'rank', asChar: true }, + 'version', + { name: 'reached_at', timestamp: true }, + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, + ], + }, + { + key: 'placements', + table: 'placements', + alias: 'pl', + columns: [ + 'id', + 'owner_id', + 'task_id', + 'time_point_id', + { name: 'rank', asChar: true }, + 'version', + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, + ], + }, +]; + +const SETTINGS_COLUMNS: SnapshotColumn[] = [ + 'owner_id', + 'timezone', + 'week_starts_on', + 'default_capture_target', + 'version', + { name: 'created_at', timestamp: true }, + { name: 'updated_at', timestamp: true }, + { name: 'deleted_at', timestamp: true }, +]; + +const TASK_COLLECTION = SNAPSHOT_COLLECTIONS.find((collection) => collection.key === 'tasks')!; +const FOLDER_COLLECTION = SNAPSHOT_COLLECTIONS.find((collection) => collection.key === 'folders')!; + +/** JSON_OBJECT expression for one folder row. */ +export function folderRowJson(alias: string): string { + return jsonObjectFor(FOLDER_COLLECTION, alias); +} + +function jsonObjectFor(collection: SnapshotCollection, alias: string): string { + return `JSON_OBJECT(${collection.columns + .map((column) => { + const name = typeof column === 'string' ? column : column.name; + return `'${name}', ${columnExpression(alias, column)}`; + }) + .join(', ')})`; +} + +/** JSON_OBJECT expression for one task row, reused by the bootstrap join. */ +export function taskRowJson(alias: string): string { + return jsonObjectFor(TASK_COLLECTION, alias); +} + +function mysqlCollectionAggregate(collection: SnapshotCollection): string { + const pairs = collection.columns + .map((column) => { + const name = typeof column === 'string' ? column : column.name; + return `'${name}', ${columnExpression(collection.alias, column)}`; + }) + .join(', '); + return `(SELECT JSON_ARRAYAGG(JSON_OBJECT(${pairs})) FROM ${collection.table} ${collection.alias} WHERE ${collection.alias}.owner_id = $1) AS ${collection.key}`; +} + +function mysqlSettingsAggregate(): string { + const pairs = SETTINGS_COLUMNS.map((column) => { + const name = typeof column === 'string' ? column : column.name; + return `'${name}', ${columnExpression('s', column)}`; + }).join(', '); + return `(SELECT JSON_OBJECT(${pairs}) FROM user_settings s WHERE s.owner_id = $1) AS settings`; +} + +/** + * One statement loads the whole owner projection. PostgreSQL aggregates with + * jsonb_agg(to_jsonb(...)); MySQL spells out JSON_OBJECT because there is no + * row-to-JSON cast. MySQL's JSON_ARRAYAGG does not preserve ORDER BY, so the + * loader sorts each collection by id after parsing. + */ +export const OWNER_SNAPSHOT_QUERY: Record = { + postgres: `SELECT + (SELECT u.next_task_number FROM users u + WHERE u.id = $1 AND u.disabled_at IS NULL) AS next_task_number, + (SELECT COALESCE(MAX(sc.seq), 0)::text FROM sync_changes sc + WHERE sc.owner_id = $1) AS cursor, + COALESCE((SELECT jsonb_agg(to_jsonb(f) || jsonb_build_object('rank', f.rank::text) ORDER BY f.id) + FROM folders f WHERE f.owner_id = $1), '[]'::jsonb) AS folders, + COALESCE((SELECT jsonb_agg(to_jsonb(t) || jsonb_build_object('rank', t.rank::text) ORDER BY t.id) + FROM tasks t WHERE t.owner_id = $1), '[]'::jsonb) AS tasks, + COALESCE((SELECT jsonb_agg(to_jsonb(n) ORDER BY n.id) + FROM notes n WHERE n.owner_id = $1), '[]'::jsonb) AS notes, + COALESCE((SELECT jsonb_agg(to_jsonb(s) || jsonb_build_object('rank', s.rank::text) ORDER BY s.id) + FROM task_steps s WHERE s.owner_id = $1), '[]'::jsonb) AS steps, + COALESCE((SELECT jsonb_agg(to_jsonb(w) || jsonb_build_object('rank', w.rank::text) ORDER BY w.id) + FROM workflows w WHERE w.owner_id = $1), '[]'::jsonb) AS workflows, + COALESCE((SELECT jsonb_agg(to_jsonb(s) || jsonb_build_object('rank', s.rank::text) ORDER BY s.id) + FROM workflow_stages s WHERE s.owner_id = $1), '[]'::jsonb) AS stages, + COALESCE((SELECT jsonb_agg(to_jsonb(m) || jsonb_build_object('rank', m.rank::text) ORDER BY m.id) + FROM workflow_task_memberships m WHERE m.owner_id = $1), '[]'::jsonb) AS memberships, + COALESCE((SELECT jsonb_agg(to_jsonb(p) || jsonb_build_object('rank', p.rank::text) ORDER BY p.id) + FROM time_points p WHERE p.owner_id = $1), '[]'::jsonb) AS time_points, + COALESCE((SELECT jsonb_agg(to_jsonb(p) || jsonb_build_object('rank', p.rank::text) ORDER BY p.id) + FROM placements p WHERE p.owner_id = $1), '[]'::jsonb) AS placements, + (SELECT to_jsonb(s) FROM user_settings s WHERE s.owner_id = $1) AS settings`, + mysql: `SELECT + (SELECT u.next_task_number FROM users u + WHERE u.id = $1 AND u.disabled_at IS NULL) AS next_task_number, + (SELECT CAST(COALESCE(MAX(sc.seq), 0) AS CHAR) FROM sync_changes sc + WHERE sc.owner_id = $1) AS cursor, + ${SNAPSHOT_COLLECTIONS.map(mysqlCollectionAggregate).join(',\n ')}, + ${mysqlSettingsAggregate()}`, +}; + +export interface DirectoryQueryParts { + parentWhere: string; + seek: string; + take: string; +} + +/** + * Bounded directory page. Both engines walk the folder subtree, aggregate the + * three task counters and materialise one page; only the JSON construction and + * the FILTER aggregates differ. + */ +export function directoryChildrenQuery( + dialect: DatabaseDialectName, + parts: DirectoryQueryParts, +): string { + const { parentWhere, seek, take } = parts; + if (dialect === 'mysql') + return `WITH RECURSIVE branch(root,id) AS ( + SELECT id,id FROM folders WHERE owner_id=$1 AND deleted_at IS NULL AND ${parentWhere} + UNION + SELECT b.root,f.id FROM folders f JOIN branch b ON f.parent_folder_id=b.id + WHERE f.owner_id=$1 AND f.deleted_at IS NULL +), counts AS ( + SELECT b.root, COUNT(t.id) AS total, + SUM(CASE WHEN t.status='TODO' THEN 1 ELSE 0 END) AS todo, + SUM(CASE WHEN t.status='IN_PROGRESS' THEN 1 ELSE 0 END) AS progress, + SUM(CASE WHEN t.status='DONE' THEN 1 ELSE 0 END) AS done + FROM branch b LEFT JOIN tasks t ON t.owner_id=$1 AND t.parent_folder_id=b.id AND t.deleted_at IS NULL + GROUP BY b.root +), rows AS ( + SELECT f.id,f.\`rank\` AS r,0 AS k, + CASE f.status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END AS g, + JSON_OBJECT('status',f.status, + 'todoCount',c.todo,'inProgressCount',c.progress,'doneCount',c.done,'totalCount',c.total) AS aggregate + FROM folders f JOIN counts c ON c.root=f.id WHERE f.owner_id=$1 AND f.deleted_at IS NULL AND ${parentWhere} + UNION ALL + SELECT t.id,t.\`rank\`,1,CASE t.status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END, + NULL + FROM tasks t WHERE t.owner_id=$1 AND t.deleted_at IS NULL AND ${parentWhere} +), page AS ( + SELECT * FROM rows ${seek} ORDER BY g,r,k,id ${take} +) SELECT CASE p.k WHEN 0 THEN + JSON_OBJECT('kind','FOLDER','folder',${folderRowJson('f')},'aggregate',p.aggregate) + ELSE JSON_OBJECT('kind','TASK','task',${taskRowJson('t')}) END AS item +FROM page p +LEFT JOIN folders f ON p.k=0 AND f.owner_id=$1 AND f.id=p.id +LEFT JOIN tasks t ON p.k=1 AND t.owner_id=$1 AND t.id=p.id +ORDER BY p.g,p.r,p.k,p.id`; + return `WITH RECURSIVE branch(root,id) AS ( + SELECT id,id FROM folders WHERE owner_id=$1 AND deleted_at IS NULL AND ${parentWhere} + UNION + SELECT b.root,f.id FROM folders f JOIN branch b ON f.parent_folder_id=b.id + WHERE f.owner_id=$1 AND f.deleted_at IS NULL +), counts AS ( + SELECT b.root, COUNT(t.id)::int AS total, + COUNT(t.id) FILTER (WHERE t.status='TODO')::int AS todo, + COUNT(t.id) FILTER (WHERE t.status='IN_PROGRESS')::int AS progress, + COUNT(t.id) FILTER (WHERE t.status='DONE')::int AS done + FROM branch b LEFT JOIN tasks t ON t.owner_id=$1 AND t.parent_folder_id=b.id AND t.deleted_at IS NULL + GROUP BY b.root +), rows AS ( + SELECT f.id,f.rank AS r,0 AS k, + CASE f.status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END AS g, + jsonb_build_object('status',f.status, + 'todoCount',c.todo,'inProgressCount',c.progress,'doneCount',c.done,'totalCount',c.total) AS aggregate + FROM folders f JOIN counts c ON c.root=f.id WHERE f.owner_id=$1 AND f.deleted_at IS NULL AND ${parentWhere} + UNION ALL + SELECT t.id,t.rank,1,CASE t.status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END, + NULL::jsonb + FROM tasks t WHERE t.owner_id=$1 AND t.deleted_at IS NULL AND ${parentWhere} +), page AS MATERIALIZED ( + SELECT * FROM rows ${seek} ORDER BY g,r,k,id ${take} +) SELECT CASE p.k WHEN 0 THEN + jsonb_build_object('kind','FOLDER','folder',to_jsonb(f) || jsonb_build_object('rank',f.rank::text),'aggregate',p.aggregate) + ELSE jsonb_build_object('kind','TASK','task',to_jsonb(t) || jsonb_build_object('rank',t.rank::text)) END AS item +FROM page p +LEFT JOIN folders f ON p.k=0 AND f.owner_id=$1 AND f.id=p.id +LEFT JOIN tasks t ON p.k=1 AND t.owner_id=$1 AND t.id=p.id +ORDER BY p.g,p.r,p.k,p.id`; +} + +/** Folder ancestry, used to guard moves and to render breadcrumbs. */ +export const FOLDER_PATH_QUERY: Record = { + postgres: `WITH RECURSIVE parents AS ( + SELECT id,title,parent_folder_id,ARRAY[id] AS visited,0 AS depth FROM folders WHERE owner_id=$1 AND id=$2 AND deleted_at IS NULL + UNION ALL SELECT f.id,f.title,f.parent_folder_id,p.visited || f.id,p.depth+1 FROM folders f JOIN parents p ON f.id=p.parent_folder_id + WHERE f.owner_id=$1 AND f.deleted_at IS NULL AND NOT f.id=ANY(p.visited) + ) SELECT * FROM parents ORDER BY depth DESC`, + mysql: `WITH RECURSIVE parents AS ( + SELECT id,title,parent_folder_id,0 AS depth FROM folders WHERE owner_id=$1 AND id=$2 AND deleted_at IS NULL + UNION ALL SELECT f.id,f.title,f.parent_folder_id,p.depth+1 FROM folders f JOIN parents p ON f.id=p.parent_folder_id + WHERE f.owner_id=$1 AND f.deleted_at IS NULL AND p.depth < 100 + ) SELECT * FROM parents ORDER BY depth DESC`, +}; diff --git a/apps/api/src/postgres-store.ts b/apps/api/src/sql-store.ts similarity index 87% rename from apps/api/src/postgres-store.ts rename to apps/api/src/sql-store.ts index 81a90e2..aab496d 100644 --- a/apps/api/src/postgres-store.ts +++ b/apps/api/src/sql-store.ts @@ -1,6 +1,6 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import { createHash } from 'node:crypto'; -import { databaseReady } from '@devtodo/database'; +import { databaseFromPool, databaseReady } from '@devtodo/database'; import { uuidSchema, uuidv7, @@ -30,7 +30,8 @@ import { transitionTask, validateLocalDate, } from '@devtodo/domain'; -import type { Pool, PoolClient } from 'pg'; +import type pg from 'pg'; +import type { SqlDatabase, SqlSession } from '@devtodo/database'; import type { DeviceRecord, NoteRecord, @@ -46,6 +47,7 @@ import type { TimePointRecord, UserRecord, } from './store.js'; +import { SYNC_PULL_QUERY } from './sql-statements.js'; type Row = Record; @@ -68,10 +70,12 @@ export class DatabaseConstraintError extends DomainError { } interface TxContext { - client: PoolClient; + session: SqlSession; pending: Map; } +const SYNC_CHANNEL = 'devtodo_sync_changes'; + export interface V2ChangeWrite { entityType: SyncChange['entityType']; entityId: string; @@ -97,24 +101,24 @@ type PageCursor = { * owner-scoped SQL queries and every mutation, receipt, and change-feed row * is committed by the same PostgreSQL transaction. */ -export class PostgresStore implements Store { +export class SqlStore implements Store { private readonly tx = new AsyncLocalStorage(); + private readonly database: SqlDatabase; private readonly clock: () => Date; private readonly changeRetentionDays: number; private readonly mutationReceiptRetentionDays: number; private readonly transactionTimeoutMs: number; private readonly listenEnabled: boolean; private readonly listeners = new Set<(ownerId: string, cursor: string) => void>(); - private listener: PoolClient | null = null; + private listener: SqlSession | null = null; private listenerReconnectTimer: ReturnType | undefined; private listenerConnecting = false; private listenerReconnectAttempt = 0; - private poolErrorHandlerAttached = false; private closed = false; private ready = false; constructor( - private readonly pool: Pool, + pointOfTruth: SqlDatabase | pg.Pool, options: { clock?: () => Date; changeRetentionDays?: number; @@ -123,77 +127,80 @@ export class PostgresStore implements Store { listenEnabled?: boolean; } = {}, ) { + this.database = isSqlDatabase(pointOfTruth) ? pointOfTruth : databaseFromPool(pointOfTruth); this.clock = options.clock ?? (() => new Date()); this.changeRetentionDays = options.changeRetentionDays ?? 90; this.mutationReceiptRetentionDays = options.mutationReceiptRetentionDays ?? 90; this.transactionTimeoutMs = options.transactionTimeoutMs ?? 30_000; - this.listenEnabled = options.listenEnabled ?? true; + this.listenEnabled = options.listenEnabled ?? this.database.settings.listenEnabled; + } + + get dialect(): SqlDatabase['dialect'] { + return this.database.dialect; + } + + /** Narrow bridge used by the v2 tree adapter; resolves to the ambient transaction. */ + async v2Query( + text: string, + values: unknown[] = [], + ): Promise<{ rows: Row[]; rowCount: number | null }> { + return this.query(text, values); } async init(): Promise { this.closed = false; - this.attachPoolErrorHandler(); - if (!(await databaseReady(this.pool))) - throw new Error('PostgreSQL schema is not ready; run the migrate job first'); - if (this.listenEnabled) { - const listener = await this.pool.connect(); - try { - await listener.query('LISTEN devtodo_sync_changes'); - } catch (error) { - listener.release(error instanceof Error ? error : new Error(String(error))); - throw error; - } - this.listener = listener; - this.attachListener(listener); - } + this.database.onConnectionError(this.handlePoolError); + if (!(await databaseReady(this.database))) + throw new Error( + `${this.database.dialect === 'mysql' ? 'MySQL' : 'PostgreSQL'} schema is not ready; run the migrate job first`, + ); + if (this.listenEnabled && this.database.dialect === 'postgres') await this.openListener(); this.ready = true; } - private attachPoolErrorHandler(): void { - if (this.poolErrorHandlerAttached) return; - this.pool.on('error', this.handlePoolError); - this.poolErrorHandlerAttached = true; - } - private readonly handlePoolError = (error: Error): void => { if (this.closed) return; this.ready = false; console.warn( JSON.stringify({ - event: 'postgres.pool_error', + event: 'database.pool_error', message: error.message, code: (error as Error & { code?: string }).code, }), ); }; - private attachListener(listener: PoolClient): void { - listener.on('notification', (message) => { - if (message.channel !== 'devtodo_sync_changes') return; - try { - const payload = JSON.parse(message.payload ?? '') as { - ownerId?: unknown; - cursor?: unknown; - }; - if (typeof payload.ownerId === 'string' && typeof payload.cursor === 'string') - this.notify(payload.ownerId, payload.cursor); - } catch { - /* Notifications are advisory; the pull cursor remains authoritative. */ - } - }); - listener.on('error', (error: Error) => this.handleListenerFailure(listener, error)); - listener.on('end', () => this.handleListenerFailure(listener)); + private handleNotification(payload: string): void { + try { + const parsed = JSON.parse(payload) as { ownerId?: unknown; cursor?: unknown }; + if (typeof parsed.ownerId === 'string' && typeof parsed.cursor === 'string') + this.notify(parsed.ownerId, parsed.cursor); + } catch { + /* Notifications are advisory; the pull cursor remains authoritative. */ + } } - private handleListenerFailure(listener: PoolClient, error?: Error): void { - if (this.listener !== listener || this.closed) return; + private async openListener(): Promise { + const session = await this.database.session(); + try { + await session.listen?.(SYNC_CHANNEL, { + onPayload: (payload) => this.handleNotification(payload), + onFailure: (error) => this.handleListenerFailure(session, error), + }); + } catch (error) { + session.release(error instanceof Error ? error : new Error(String(error))); + throw error; + } + this.listener = session; + } + + private handleListenerFailure(session: SqlSession, error?: Error): void { + if (this.listener !== session || this.closed) return; this.listener = null; - if (error) { - try { - listener.release(error); - } catch { - /* The pool may already have removed a failed client. */ - } + try { + session.release(error); + } catch { + /* The pool may already have removed a failed connection. */ } this.scheduleListenerReconnect(); } @@ -211,22 +218,25 @@ export class PostgresStore implements Store { private async reconnectListener(): Promise { if (!this.listenEnabled || this.closed || this.listener || this.listenerConnecting) return; + if (this.database.dialect !== 'postgres') return; this.listenerConnecting = true; try { - const listener = await this.pool.connect(); + const session = await this.database.session(); if (this.closed) { - listener.release(); + session.release(); return; } try { - await listener.query('LISTEN devtodo_sync_changes'); + await session.listen?.(SYNC_CHANNEL, { + onPayload: (payload) => this.handleNotification(payload), + onFailure: (error) => this.handleListenerFailure(session, error), + }); } catch (error) { - listener.release(error instanceof Error ? error : new Error(String(error))); + session.release(error instanceof Error ? error : new Error(String(error))); throw error; } - this.listener = listener; + this.listener = session; this.listenerReconnectAttempt = 0; - this.attachListener(listener); } catch (error) { console.warn( JSON.stringify({ event: 'sync.listener_reconnect_failed', message: String(error) }), @@ -243,7 +253,7 @@ export class PostgresStore implements Store { async checkReady(): Promise { try { - await this.pool.query('SELECT 1'); + await this.database.query('SELECT 1'); this.ready = true; return true; } catch { @@ -252,29 +262,57 @@ export class PostgresStore implements Store { } } + async createNativeChallenge(origin: string, expiresAt: string): Promise { + const id = uuidv7(); + await this.query( + 'INSERT INTO native_auth_challenges (id, origin, expires_at) VALUES ($1, $2, $3)', + [id, origin, expiresAt], + ); + return id; + } + + async consumeNativeChallenge(id: string, origin: string): Promise { + const result = await this.executeReturning( + 'DELETE FROM native_auth_challenges WHERE id = $1 AND origin = $2 AND expires_at > now() RETURNING id', + [id, origin], + ); + return result.rowCount === 1; + } + /** Best-effort warm-up after the schema check has succeeded. */ async warmConnections(count: number): Promise { - const warm = Math.max(0, Math.min(Math.trunc(count), this.pool.options.max ?? 0)); - if (!warm) return; - const clients = await Promise.all(Array.from({ length: warm }, () => this.pool.connect())); - for (const client of clients) client.release(); + await this.database.warm(count); } /** - * Retention cleanup is deliberately outside the request transaction. Keeping - * three DELETE statements on every mutation cost a high-latency deployment - * 3 round trips even when there was nothing to delete. + * Retention cleanup is deliberately outside the request transaction, so a + * mutation never pays for it. PostgreSQL deletes all three classes in one + * statement; MySQL refuses to select from the table it deletes from, so it + * uses three bounded `DELETE ... ORDER BY ... LIMIT` statements instead. */ async runMaintenance(): Promise<{ syncChanges: number; mutationReceipts: number; nativeChallenges: number; }> { - const result = await this.pool.query<{ - changes: number; - receipts: number; - challenges: number; - }>( + if (this.database.dialect === 'mysql') { + const changes = await this.query( + 'DELETE FROM sync_changes WHERE committed_at < NOW(6) - INTERVAL $1 DAY ORDER BY seq LIMIT 1000', + [this.changeRetentionDays], + ); + const receipts = await this.query( + 'DELETE FROM client_mutations WHERE expires_at <= NOW(6) ORDER BY expires_at LIMIT 1000', + ); + const challenges = await this.query( + 'DELETE FROM native_auth_challenges WHERE expires_at <= NOW(6) ORDER BY expires_at LIMIT 1000', + ); + return { + syncChanges: changes.rowCount ?? 0, + mutationReceipts: receipts.rowCount ?? 0, + nativeChallenges: challenges.rowCount ?? 0, + }; + } + const result = await this.query( `WITH deleted_changes AS ( DELETE FROM sync_changes WHERE seq IN ( SELECT seq FROM sync_changes @@ -299,9 +337,9 @@ export class PostgresStore implements Store { ); const row = result.rows[0]; return { - syncChanges: row?.changes ?? 0, - mutationReceipts: row?.receipts ?? 0, - nativeChallenges: row?.challenges ?? 0, + syncChanges: Number(row?.changes ?? 0), + mutationReceipts: Number(row?.receipts ?? 0), + nativeChallenges: Number(row?.challenges ?? 0), }; } @@ -311,120 +349,137 @@ export class PostgresStore implements Store { clearTimeout(this.listenerReconnectTimer); this.listenerReconnectTimer = undefined; } - if (this.listener) { + const listener = this.listener; + this.listener = null; + if (listener) { try { - await this.listener.query('UNLISTEN devtodo_sync_changes'); + await listener.unlisten?.(SYNC_CHANNEL); } catch { /* Preserve shutdown even when the listener connection is already closed. */ } - this.listener.release(); - this.listener = null; + listener.release(); } this.listenerReconnectAttempt = 0; this.ready = false; - if (this.poolErrorHandlerAttached) { - this.pool.off('error', this.handlePoolError); - this.poolErrorHandlerAttached = false; - } - await this.pool.end(); - } - - async createNativeChallenge(origin: string, expiresAt: string): Promise { - const id = uuidv7(); - await this.query( - 'INSERT INTO native_auth_challenges (id, origin, expires_at) VALUES ($1, $2, $3)', - [id, origin, expiresAt], - ); - return id; - } - - async consumeNativeChallenge(id: string, origin: string): Promise { - const result = await this.query( - 'DELETE FROM native_auth_challenges WHERE id = $1 AND origin = $2 AND expires_at > now() RETURNING id', - [id, origin], - ); - return result.rowCount === 1; + await this.database.end(); } async withMutation(fn: () => T | Promise): Promise { const nested = this.tx.getStore(); if (nested) return await fn(); - const client = await this.pool.connect(); - const context: TxContext = { client, pending: new Map() }; + const session = await this.database.session(); + const context: TxContext = { session, pending: new Map() }; try { - await client.query('BEGIN'); - await client.query(`SET LOCAL statement_timeout = ${this.transactionTimeoutMs}`); - await client.query( - `SET LOCAL idle_in_transaction_session_timeout = ${this.transactionTimeoutMs}`, - ); + await session.begin(); + if (session.dialect === 'postgres') { + await session.query(`SET LOCAL statement_timeout = ${this.transactionTimeoutMs}`); + await session.query( + `SET LOCAL idle_in_transaction_session_timeout = ${this.transactionTimeoutMs}`, + ); + } const value = await this.tx.run(context, async () => { const result = await fn(); for (const [ownerId, cursor] of context.pending) - await client.query('SELECT pg_notify($1, $2)', [ - 'devtodo_sync_changes', - JSON.stringify({ ownerId, cursor }), - ]); - await client.query('COMMIT'); + await this.publishCursor(session, ownerId, cursor); + await session.commit(); return result; }); return value; } catch (error) { try { - await client.query('ROLLBACK'); + await session.rollback(); } catch { /* Preserve the original error. */ } if (error instanceof DomainError) throw error; throwSql(error, '数据库事务被拒绝', 'MUTATION_REJECTED'); } finally { - client.release(); + session.release(); } } private async readSnapshot(fn: () => T | Promise): Promise { const nested = this.tx.getStore(); if (nested) return await fn(); - const client = await this.pool.connect(); - const context: TxContext = { client, pending: new Map() }; + const session = await this.database.session(); + const context: TxContext = { session, pending: new Map() }; try { - await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY'); + await session.begin({ readOnly: true }); const value = await this.tx.run(context, fn); - await client.query('COMMIT'); + await session.commit(); return value; } catch (error) { try { - await client.query('ROLLBACK'); + await session.rollback(); } catch { /* Preserve the original error. */ } throw error; } finally { - client.release(); + session.release(); } } + private async publishCursor(session: SqlSession, ownerId: string, cursor: string): Promise { + // MySQL has no NOTIFY; the in-process listener already fired through the + // pending map, and multi-instance fan-out is Redis' job when configured. + if (session.dialect !== 'postgres') return; + await session.query('SELECT pg_notify($1, $2)', [ + SYNC_CHANNEL, + JSON.stringify({ ownerId, cursor }), + ]); + } + private async query( text: string, values: unknown[] = [], - ): Promise<{ rows: Row[]; rowCount: number | null }> { - const client = this.tx.getStore()?.client ?? this.pool; - return (await client.query(text, values)) as { rows: Row[]; rowCount: number | null }; + ): Promise<{ rows: Row[]; rowCount: number | null; insertId?: string }> { + const session = this.tx.getStore()?.session; + return session ? session.query(text, values) : this.database.query(text, values); } - /** Narrow bridge used by the v2 tree adapter; it still resolves to the - * current AsyncLocalStorage transaction and therefore cannot escape the - * Store's atomic mutation boundary. */ - async v2Query( - text: string, + /** + * Run a statement that ends with RETURNING. + * + * PostgreSQL answers in one round trip. MySQL has no RETURNING, so the + * statement runs first and the row is read back by primary key. Inside a + * transaction the read-back observes the write; outside one it is the only + * way to return server-defaulted columns. + */ + private async executeReturning( + sql: string, values: unknown[] = [], + key?: ReturningKey, ): Promise<{ rows: Row[]; rowCount: number | null }> { - return this.query(text, values); + if (this.database.dialect === 'postgres') return this.query(sql, values); + const match = /RETURNING\s+([\s\S]+)$/i.exec(sql); + if (!match) throw new Error('executeReturning requires a RETURNING clause'); + const columns = match[1]!.trim(); + const statement = sql.slice(0, match.index).trimEnd(); + const written = await this.query(statement, values); + const derived = key ?? deriveReturningKey(statement, values); + if (!derived) + throw new Error(`executeReturning cannot derive the row key: ${statement.slice(0, 70)}`); + if (derived.deleted) { + // A DELETEd row can never be read back; rowCount carries the outcome. + return { + rows: (written.rowCount ?? 0) > 0 ? [{ [derived.column]: derived.value }] : [], + rowCount: written.rowCount, + }; + } + const selected = await this.query( + `SELECT ${columns} FROM ${derived.table} WHERE ${derived.column} = $1`, + [derived.value], + ); + return { rows: selected.rows, rowCount: selected.rowCount }; } async v2LockOwner(ownerId: string): Promise { - await this.query("SELECT pg_advisory_xact_lock(hashtextextended('devtodo:owner:' || $1, 0))", [ - ownerId, - ]); + if (this.database.dialect === 'postgres') + await this.query( + "SELECT pg_advisory_xact_lock(hashtextextended('devtodo:owner:' || $1, 0))", + [ownerId], + ); await this.lockOwner(ownerId); } @@ -460,28 +515,53 @@ export class PostgresStore implements Store { first.operation, first.snapshot, ); - const payload = changes.map((change) => ({ - entity_type: change.entityType, - entity_id: change.entityId, - entity_version: change.entityVersion, - operation: change.operation, - snapshot: change.snapshot, - committed_at: change.committedAt, - })); - const result = await this.query( - `INSERT INTO sync_changes (owner_id, entity_type, entity_id, entity_version, operation, snapshot, committed_at, protocol_version) - SELECT $1, x.entity_type, x.entity_id, x.entity_version, x.operation, x.snapshot, x.committed_at, 2 - FROM jsonb_to_recordset($2::jsonb) AS x( - entity_type text, entity_id uuid, entity_version integer, operation text, - snapshot jsonb, committed_at timestamptz - ) - RETURNING seq`, - [ownerId, JSON.stringify(payload)], - ); let cursor = '0'; - for (const row of result.rows) - if (BigInt(String(row['seq'])) > BigInt(cursor)) cursor = String(row['seq']); - if (cursor === '0') return undefined; + if (this.database.dialect === 'mysql') { + const tuples: string[] = []; + const values: unknown[] = []; + for (const change of changes) { + tuples.push('(?, ?, ?, ?, ?, CAST(? AS JSON), ?, 2)'); + values.push( + ownerId, + change.entityType, + change.entityId, + change.entityVersion, + change.operation, + JSON.stringify(change.snapshot ?? null), + change.committedAt, + ); + } + const inserted = await this.query( + `INSERT INTO sync_changes (owner_id, entity_type, entity_id, entity_version, operation, snapshot, committed_at, protocol_version) VALUES ${tuples.join(',')}`, + values, + ); + const start = BigInt(inserted.insertId ?? '0'); + const count = BigInt(inserted.rowCount ?? changes.length); + if (start === 0n) return undefined; + cursor = (start + count - 1n).toString(); + } else { + const payload = changes.map((change) => ({ + entity_type: change.entityType, + entity_id: change.entityId, + entity_version: change.entityVersion, + operation: change.operation, + snapshot: change.snapshot, + committed_at: change.committedAt, + })); + const result = await this.query( + `INSERT INTO sync_changes (owner_id, entity_type, entity_id, entity_version, operation, snapshot, committed_at, protocol_version) + SELECT $1, x.entity_type, x.entity_id, x.entity_version, x.operation, x.snapshot, x.committed_at, 2 + FROM jsonb_to_recordset($2::jsonb) AS x( + entity_type text, entity_id uuid, entity_version integer, operation text, + snapshot jsonb, committed_at timestamptz + ) + RETURNING seq`, + [ownerId, JSON.stringify(payload)], + ); + for (const row of result.rows) + if (BigInt(String(row['seq'])) > BigInt(cursor)) cursor = String(row['seq']); + if (cursor === '0') return undefined; + } const context = this.tx.getStore(); if (context) context.pending.set(ownerId, cursor); else this.notify(ownerId, cursor); @@ -504,6 +584,66 @@ export class PostgresStore implements Store { return this.clock().toISOString(); } + /** + * Conflict handling for "insert only when the active row does not exist". + * MySQL has no partial unique index, so the schema uses a generated column + * that is NULL once deleted_at is set; `ON DUPLICATE KEY UPDATE id = id` + * makes the duplicate a no-op while still reporting 0 affected rows. + */ + private doNothingOnConflict(columns: string, predicate = 'deleted_at IS NULL'): string { + return this.database.dialect === 'mysql' + ? 'ON DUPLICATE KEY UPDATE id = id ' + : `ON CONFLICT (${columns}) WHERE ${predicate} DO NOTHING `; + } + + private async appendChange( + ownerId: string, + entityType: SyncChange['entityType'], + entityId: string, + entityVersion: number, + operation: SyncChange['operation'], + snapshot: unknown, + protocolVersion: 1 | 2 = 1, + ): Promise { + let cursor: string; + if (this.database.dialect === 'mysql') { + const inserted = await this.query( + 'INSERT INTO sync_changes (owner_id, entity_type, entity_id, entity_version, operation, snapshot, committed_at, protocol_version) VALUES ($1, $2, $3, $4, $5, CAST($6 AS JSON), $7, $8)', + [ + ownerId, + entityType, + entityId, + entityVersion, + operation, + JSON.stringify(snapshot ?? null), + this.now(), + protocolVersion, + ], + ); + cursor = inserted.insertId ?? '0'; + } else { + const result = await this.query( + 'INSERT INTO sync_changes (owner_id, entity_type, entity_id, entity_version, operation, snapshot, committed_at, protocol_version) ' + + 'VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8) RETURNING seq', + [ + ownerId, + entityType, + entityId, + entityVersion, + operation, + JSON.stringify(snapshot), + this.now(), + protocolVersion, + ], + ); + cursor = String(result.rows[0]!.seq); + } + const context = this.tx.getStore(); + if (context) context.pending.set(ownerId, cursor); + else this.notify(ownerId, cursor); + return cursor; + } + private async owner(ownerId: string, lock = false): Promise { const result = await this.query( 'SELECT id, username, password_hash, next_misc_task_number, created_at, updated_at, disabled_at ' + @@ -552,7 +692,7 @@ export class PostgresStore implements Store { const id = uuidv7(); const now = this.now(); try { - const result = await this.query( + const result = await this.executeReturning( 'INSERT INTO users (id, username, password_hash, next_misc_task_number, created_at, updated_at) ' + 'VALUES ($1, $2, $3, 1, $4, $4) ' + 'RETURNING id, username, password_hash, next_misc_task_number, created_at, updated_at, disabled_at', @@ -605,7 +745,7 @@ export class PostgresStore implements Store { : requestedCaptureTarget; if (!['ROOT', 'RECENT_FOLDER'].includes(defaultCaptureTarget)) throw new DomainError('VALIDATION_FAILED', '默认捕获位置无效'); - const result = await this.query( + const result = await this.executeReturning( 'UPDATE user_settings SET timezone = $2, week_starts_on = $3, default_capture_target = $4, ' + 'version = version + 1, updated_at = $5 ' + 'WHERE owner_id = $1 AND version = $6 ' + @@ -634,7 +774,7 @@ export class PostgresStore implements Store { throw new DomainError('MUTATION_REJECTED', '设备 ID 已属于其他 Owner'); const now = this.now(); if (existing) { - const result = await this.query( + const result = await this.executeReturning( 'UPDATE devices SET name = $2, platform = $3, last_seen_at = $4, revoked_at = NULL ' + 'WHERE id = $1 RETURNING id, owner_id, name, platform, last_seen_at, created_at, revoked_at', [id, name, platform, now], @@ -642,7 +782,7 @@ export class PostgresStore implements Store { return deviceDto(deviceRecord(requireUpdatedRow(result, '设备版本已变化'))); } try { - const result = await this.query( + const result = await this.executeReturning( 'INSERT INTO devices (id, owner_id, name, platform, last_seen_at, created_at) ' + 'VALUES ($1, $2, $3, $4, $5, $5) ' + 'RETURNING id, owner_id, name, platform, last_seen_at, created_at, revoked_at', @@ -694,7 +834,7 @@ export class PostgresStore implements Store { ): Promise { const device = await this.getDevice(ownerId, deviceId); if (device.revokedAt) throw new DomainError('AUTH_SESSION_REVOKED', '设备会话已撤销'); - const result = await this.query( + const result = await this.executeReturning( 'INSERT INTO refresh_sessions ' + '(id, owner_id, device_id, token_hash, expires_at, created_at) ' + 'VALUES ($1, $2, $3, $4, $5, $6) ' + @@ -769,7 +909,7 @@ export class PostgresStore implements Store { [ownerId, normalizedPrefix], ); if (existing.rowCount) throw new DomainError('VALIDATION_FAILED', '项目代号已存在'); - const result = await this.query( + const result = await this.executeReturning( 'INSERT INTO projects ' + '(id, owner_id, name, task_prefix, next_task_number, rank, version, created_at, updated_at) ' + 'VALUES ($1, $2, $3, $4, 1, (SELECT COALESCE(MAX(rank), 0) + 1024 FROM projects WHERE owner_id = $2 AND deleted_at IS NULL), 1, $5, $5) ' + @@ -796,8 +936,8 @@ export class PostgresStore implements Store { await this.owner(ownerId); const result = await this.query( 'SELECT p.id AS project_id, ' + - "COUNT(t.id) FILTER (WHERE t.status <> 'DONE')::int AS open_count, " + - "COUNT(t.id) FILTER (WHERE t.status = 'DONE')::int AS done_count " + + "SUM(CASE WHEN t.status <> 'DONE' THEN 1 ELSE 0 END)::int AS open_count, " + + "SUM(CASE WHEN t.status = 'DONE' THEN 1 ELSE 0 END)::int AS done_count " + 'FROM projects p ' + 'LEFT JOIN tasks t ON t.owner_id = p.owner_id AND t.project_id = p.id ' + 'AND t.deleted_at IS NULL ' + @@ -870,7 +1010,7 @@ export class PostgresStore implements Store { ); if (duplicate.rowCount) throw new DomainError('VALIDATION_FAILED', '项目代号无效或已存在'); } - const result = await this.query( + const result = await this.executeReturning( 'UPDATE projects SET name = $3, task_prefix = $4, version = version + 1, updated_at = $5 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $6 ' + 'RETURNING id, owner_id, name, task_prefix, next_task_number, rank, version, created_at, updated_at, deleted_at', @@ -926,7 +1066,7 @@ export class PostgresStore implements Store { const referenceId = allocateReference(project?.taskPrefix ?? 'MISC', number); const taskId = newEntityId(input.id); const noteId = uuidv7(); - const result = await this.query( + const result = await this.executeReturning( 'INSERT INTO tasks ' + '(id, owner_id, project_id, category, reference_id, title, status, priority, rank, version, created_at, updated_at) ' + "VALUES ($1, $2, $3, $4, $5, $6, 'TODO', $7, " + @@ -1117,7 +1257,7 @@ export class PostgresStore implements Store { status = transition.status; completedAt = transition.completedAt?.toISOString() ?? null; } - const result = await this.query( + const result = await this.executeReturning( 'UPDATE tasks SET project_id = $3, category = $4, title = $5, status = $6, priority = $7, ' + 'rank = $8, completed_at = $9, version = version + 1, updated_at = $10 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $11 ' + @@ -1192,7 +1332,7 @@ export class PostgresStore implements Store { const taskId = newEntityId(requestedIds.taskId); const noteId = newEntityId(requestedIds.noteId); const sourceNote = await this.findNoteRecord(ownerId, id); - const result = await this.query( + const result = await this.executeReturning( 'INSERT INTO tasks ' + '(id, owner_id, project_id, category, reference_id, title, status, priority, rank, version, created_at, updated_at) ' + "VALUES ($1, $2, $3, $4, $5, $6, 'TODO', $7, " + @@ -1210,7 +1350,7 @@ export class PostgresStore implements Store { this.now(), ], ); - const noteResult = await this.query( + const noteResult = await this.executeReturning( 'INSERT INTO notes (id, owner_id, task_id, content_markdown, version, created_at, updated_at) ' + 'VALUES ($1, $2, $3, $4, 1, $5, $5) ' + 'RETURNING id, owner_id, task_id, content_markdown, version, created_at, updated_at, deleted_at', @@ -1253,7 +1393,7 @@ export class PostgresStore implements Store { assertVersion(note.version, baseVersion, noteDto(note)); if (contentMarkdown.length > 1024 * 1024) throw new DomainError('VALIDATION_FAILED', '备注超过 1 MiB'); - const result = await this.query( + const result = await this.executeReturning( 'UPDATE notes SET content_markdown = $3, version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND task_id = $2 AND version = $5 AND deleted_at IS NULL ' + 'RETURNING id, owner_id, task_id, content_markdown, version, created_at, updated_at, deleted_at', @@ -1293,13 +1433,13 @@ export class PostgresStore implements Store { const title = input.title?.trim() ?? null; if (input.type === 'EVENT' && (!title || title.length > 200)) throw new DomainError('VALIDATION_FAILED', '时间点名称无效'); - const result = await this.query( + const result = await this.executeReturning( 'INSERT INTO time_points (id, owner_id, type, local_date, title, rank, version, created_at, updated_at) ' + 'VALUES ($1, $2, $3, $4, $5, ' + "(SELECT COALESCE(MAX(rank), 0) + 1024 FROM time_points WHERE owner_id = $2 AND type = 'EVENT' AND deleted_at IS NULL), " + '1, $6, $6) ' + (input.type === 'DATE' - ? "ON CONFLICT (owner_id, local_date) WHERE type = 'DATE' AND deleted_at IS NULL DO NOTHING " + ? this.doNothingOnConflict('owner_id, local_date', "type = 'DATE' AND deleted_at IS NULL") : '') + 'RETURNING id, owner_id, type, local_date, title, rank, version, reached_at, created_at, updated_at, deleted_at', [newEntityId(input.id), ownerId, input.type, input.localDate ?? null, title, this.now()], @@ -1356,8 +1496,8 @@ export class PostgresStore implements Store { const result = await this.query( 'SELECT tp.id AS time_point_id, tp.local_date, ' + 'COUNT(t.id)::int AS total_count, ' + - "COUNT(t.id) FILTER (WHERE t.status <> 'DONE')::int AS open_count, " + - "COUNT(t.id) FILTER (WHERE t.status = 'DONE')::int AS done_count " + + "SUM(CASE WHEN t.status <> 'DONE' THEN 1 ELSE 0 END)::int AS open_count, " + + "SUM(CASE WHEN t.status = 'DONE' THEN 1 ELSE 0 END)::int AS done_count " + 'FROM time_points tp ' + 'LEFT JOIN placements p ON p.owner_id = tp.owner_id AND p.time_point_id = tp.id ' + 'AND p.deleted_at IS NULL ' + @@ -1457,7 +1597,7 @@ export class PostgresStore implements Store { const point = await this.timePointRecord(ownerId, id, true); if (point.type !== 'EVENT') throw new DomainError('VALIDATION_FAILED', '日期没有到达状态'); assertVersion(point.version, baseVersion, timePointDto(point)); - const result = await this.query( + const result = await this.executeReturning( 'UPDATE time_points SET reached_at = COALESCE(reached_at, $3), version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $5 ' + 'RETURNING id, owner_id, type, local_date, title, rank, version, reached_at, created_at, updated_at, deleted_at', @@ -1476,7 +1616,7 @@ export class PostgresStore implements Store { if (point.type !== 'EVENT') throw new DomainError('VALIDATION_FAILED', '日期不能删除'); assertVersion(point.version, baseVersion, timePointDto(point)); const now = this.now(); - const placements = await this.query( + const placements = await this.executeReturning( 'UPDATE placements SET deleted_at = $3, version = version + 1, updated_at = $3 ' + 'WHERE owner_id = $1 AND time_point_id = $2 AND deleted_at IS NULL ' + 'RETURNING id, version', @@ -1492,7 +1632,7 @@ export class PostgresStore implements Store { null, ); } - const result = await this.query( + const result = await this.executeReturning( 'UPDATE time_points SET deleted_at = $3, version = version + 1, updated_at = $3 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $4 ' + 'RETURNING id, owner_id, type, local_date, title, rank, version, reached_at, created_at, updated_at, deleted_at', @@ -1553,12 +1693,12 @@ export class PostgresStore implements Store { ); if (existing.rows[0]) return { placement: placementDto(placementRecord(existing.rows[0])), existed: true }; - const result = await this.query( + const result = await this.executeReturning( 'INSERT INTO placements (id, owner_id, task_id, time_point_id, rank, version, created_at, updated_at) ' + 'VALUES ($1, $2, $3, $4, ' + '(SELECT COALESCE(MAX(rank), 0) + 1024 FROM placements WHERE owner_id = $2 AND time_point_id = $4 AND deleted_at IS NULL), ' + '1, $5, $5) ' + - 'ON CONFLICT (task_id, time_point_id) WHERE deleted_at IS NULL DO NOTHING ' + + this.doNothingOnConflict('task_id, time_point_id') + 'RETURNING id, owner_id, task_id, time_point_id, rank, version, created_at, updated_at, deleted_at', [newEntityId(id), ownerId, taskId, timePointId, this.now()], ); @@ -1639,7 +1779,7 @@ export class PostgresStore implements Store { await this.lockOwner(ownerId); const placement = await this.placementRecord(ownerId, id); assertVersion(placement.version, baseVersion, placementDto(placement)); - const result = await this.query( + const result = await this.executeReturning( 'UPDATE placements SET deleted_at = $3, version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $5 AND deleted_at IS NULL ' + 'RETURNING id, owner_id, task_id, time_point_id, rank, version, created_at, updated_at, deleted_at', @@ -1683,7 +1823,7 @@ export class PostgresStore implements Store { ) ).placement; } - const result = await this.query( + const result = await this.executeReturning( 'UPDATE placements SET deleted_at = $3, version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $5 AND deleted_at IS NULL ' + 'RETURNING id, owner_id, task_id, time_point_id, rank, version, created_at, updated_at, deleted_at', @@ -1807,7 +1947,7 @@ export class PostgresStore implements Store { skippedIds.push(placementId); continue; } - const deleteResult = await this.query( + const deleteResult = await this.executeReturning( 'UPDATE placements SET deleted_at = $3, version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND id = $2 AND version = 1 AND deleted_at IS NULL ' + 'RETURNING id, owner_id, task_id, time_point_id, rank, version, created_at, updated_at, deleted_at', @@ -1877,7 +2017,8 @@ export class PostgresStore implements Store { ); const key = ownerId + ':' + clientId + ':' + mutationId; const requestHash = hash(input); - await this.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [key]); + if (this.database.dialect === 'postgres') + await this.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [key]); // Expired receipts are removed by runMaintenance(). Treating them as // absent here avoids a DELETE round trip on every mutation. const existing = await this.query( @@ -1896,9 +2037,12 @@ export class PostgresStore implements Store { 'VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7) ' + // An expired receipt is logically absent; overwrite it so the primary // key does not reject a legitimate retry after the retention window. - 'ON CONFLICT (owner_id, client_id, mutation_id) DO UPDATE SET ' + - 'request_hash = EXCLUDED.request_hash, result = EXCLUDED.result, ' + - 'first_processed_at = EXCLUDED.first_processed_at, expires_at = EXCLUDED.expires_at', + (this.database.dialect === 'mysql' + ? 'AS new ON DUPLICATE KEY UPDATE request_hash = new.request_hash, result = new.result, ' + + 'first_processed_at = new.first_processed_at, expires_at = new.expires_at' + : 'ON CONFLICT (owner_id, client_id, mutation_id) DO UPDATE SET ' + + 'request_hash = EXCLUDED.request_hash, result = EXCLUDED.result, ' + + 'first_processed_at = EXCLUDED.first_processed_at, expires_at = EXCLUDED.expires_at'), [ ownerId, clientId, @@ -2129,42 +2273,17 @@ export class PostgresStore implements Store { // One statement gives a consistent owner check, retention boundary, page // and has-more flag. The previous implementation used 6 sequential queries // inside a repeatable-read transaction on every client pull. - const result = await this.query( - `WITH owner AS ( - SELECT 1 FROM users WHERE id = $1 AND disabled_at IS NULL - ), oldest AS ( - SELECT seq FROM sync_changes WHERE protocol_version = 2 ORDER BY seq LIMIT 1 - ), page AS ( - SELECT seq, owner_id, entity_type, entity_id, entity_version, operation, snapshot, committed_at - FROM sync_changes - WHERE owner_id = $1 AND protocol_version = 2 AND seq > $2 - ORDER BY seq LIMIT $3 - ) - SELECT EXISTS(SELECT 1 FROM owner) AS owner_ok, - (SELECT seq FROM oldest) AS oldest, - COALESCE(( - SELECT jsonb_agg(jsonb_build_object( - 'seq', p.seq::text, - 'owner_id', p.owner_id, - 'entity_type', p.entity_type, - 'entity_id', p.entity_id, - 'entity_version', p.entity_version, - 'operation', p.operation, - 'snapshot', p.snapshot, - 'committed_at', p.committed_at - ) ORDER BY p.seq) - FROM page p - ), '[]'::jsonb) AS changes, - EXISTS( - SELECT 1 FROM sync_changes - WHERE owner_id = $1 AND protocol_version = 2 - AND seq > COALESCE((SELECT MAX(seq) FROM page), $2) - LIMIT 1 - ) AS has_more`, - [ownerId, cursor, Math.min(limit, 500)], - ); + const pageLimit = Math.min(limit, 500); + const mysql = this.database.dialect === 'mysql'; + // MySQL's JSON_ARRAYAGG cannot carry the page's ORDER BY, so it returns one + // extra row and this method trims the page itself. + const result = await this.query(SYNC_PULL_QUERY[this.database.dialect], [ + ownerId, + cursor, + mysql ? pageLimit + 1 : pageLimit, + ]); const row = result.rows[0]; - if (!row || row['owner_ok'] !== true) throw new DomainError('AUTH_REQUIRED', '账户不可用'); + if (!row || !truthy(row['owner_ok'])) throw new DomainError('AUTH_REQUIRED', '账户不可用'); const oldest = row['oldest']; // A v2 client starts at cursor 0 even when older v1 rows occupy the // global sequence range before the first retained v2 change. Treat that @@ -2173,14 +2292,23 @@ export class PostgresStore implements Store { if (requested < BigInt(String(oldest)) - 1n) throw new DomainError('SYNC_CURSOR_EXPIRED', '同步游标已超过保留窗口'); } - const changes = Array.isArray(row['changes']) + const parsed = Array.isArray(row['changes']) ? (row['changes'] as Row[]).map((change) => syncChange(change)) : []; + let changes = parsed; + let hasMore = truthy(row['has_more']); + if (mysql) { + changes = parsed + .slice() + .sort((left, right) => compareCursor(left.seq, right.seq)) + .slice(0, pageLimit); + hasMore = parsed.length > pageLimit; + } const nextCursor = changes.at(-1)?.seq ?? requested; return { changes, nextCursor: nextCursor.toString(), - hasMore: row['has_more'] === true, + hasMore, }; } @@ -2196,7 +2324,7 @@ export class PostgresStore implements Store { [ownerId], ); const row = result.rows[0]; - if (!row || row['owner_ok'] !== true) throw new DomainError('AUTH_REQUIRED', '账户不可用'); + if (!row || !truthy(row['owner_ok'])) throw new DomainError('AUTH_REQUIRED', '账户不可用'); const cursor = String(row['cursor'] ?? '0'); const oldestCursor = row['oldest'] !== null && row['oldest'] !== undefined @@ -2225,36 +2353,6 @@ export class PostgresStore implements Store { } } - private async appendChange( - ownerId: string, - entityType: SyncChange['entityType'], - entityId: string, - entityVersion: number, - operation: SyncChange['operation'], - snapshot: unknown, - protocolVersion: 1 | 2 = 1, - ): Promise { - const result = await this.query( - 'INSERT INTO sync_changes (owner_id, entity_type, entity_id, entity_version, operation, snapshot, committed_at, protocol_version) ' + - 'VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8) RETURNING seq', - [ - ownerId, - entityType, - entityId, - entityVersion, - operation, - JSON.stringify(snapshot), - this.now(), - protocolVersion, - ], - ); - const cursor = String(result.rows[0]!.seq); - const context = this.tx.getStore(); - if (context) context.pending.set(ownerId, cursor); - else this.notify(ownerId, cursor); - return cursor; - } - private async settingsRecord(ownerId: string, lock = false): Promise { const result = await this.query( 'SELECT owner_id, timezone, week_starts_on, default_capture_target, version, created_at, updated_at, deleted_at ' + @@ -2345,7 +2443,7 @@ export class PostgresStore implements Store { project: ProjectRecord, rank: string, ): Promise { - const result = await this.query( + const result = await this.executeReturning( 'UPDATE projects SET rank = $3, version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $5 ' + 'RETURNING id, owner_id, name, task_prefix, next_task_number, rank, version, created_at, updated_at, deleted_at', @@ -2361,7 +2459,7 @@ export class PostgresStore implements Store { task: TaskRecord, rank: string, ): Promise { - const result = await this.query( + const result = await this.executeReturning( 'UPDATE tasks SET rank = $3, version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $5 ' + 'RETURNING id, owner_id, project_id, category, reference_id, title, status, priority, rank, version, completed_at, created_at, updated_at, deleted_at', @@ -2377,7 +2475,7 @@ export class PostgresStore implements Store { point: TimePointRecord, title: string | null, ): Promise { - const result = await this.query( + const result = await this.executeReturning( 'UPDATE time_points SET title = $3, version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $5 ' + 'RETURNING id, owner_id, type, local_date, title, rank, version, reached_at, created_at, updated_at, deleted_at', @@ -2394,7 +2492,7 @@ export class PostgresStore implements Store { point: TimePointRecord, rank: string, ): Promise { - const result = await this.query( + const result = await this.executeReturning( 'UPDATE time_points SET rank = $3, version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $5 ' + 'RETURNING id, owner_id, type, local_date, title, rank, version, reached_at, created_at, updated_at, deleted_at', @@ -2417,7 +2515,7 @@ export class PostgresStore implements Store { placement: PlacementRecord, rank: string, ): Promise { - const result = await this.query( + const result = await this.executeReturning( 'UPDATE placements SET rank = $3, version = version + 1, updated_at = $4 ' + 'WHERE owner_id = $1 AND id = $2 AND version = $5 ' + 'RETURNING id, owner_id, task_id, time_point_id, rank, version, created_at, updated_at, deleted_at', @@ -2485,6 +2583,52 @@ function normalizeUsername(value: string): string { return value.trim().toLocaleLowerCase('en-US'); } +/** MySQL reports booleans and EXISTS as 1/0; PostgreSQL reports true/false. */ +function truthy(value: unknown): boolean { + return value === true || value === 1 || value === '1'; +} + +function compareCursor(left: bigint, right: bigint): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function isSqlDatabase(value: unknown): value is SqlDatabase { + const candidate = value as Partial | undefined; + return Boolean( + candidate && + typeof candidate.dialect === 'string' && + typeof candidate.session === 'function' && + candidate.settings, + ); +} + +interface ReturningKey { + table: string; + column: string; + value: unknown; + deleted?: boolean; +} + +/** + * Derive the row a RETURNING clause refers to. Every INSERT lists `id` first + * and every UPDATE/DELETE filters on `id = $n`, which is what makes the MySQL + * read-back emulation possible without a per-statement key argument. + */ +function deriveReturningKey(statement: string, values: unknown[]): ReturningKey | undefined { + const insert = /^\s*INSERT\s+INTO\s+([a-z_]+)\s*\(\s*([a-z_]+)/i.exec(statement); + if (insert && insert[2] === 'id') return { table: insert[1]!, column: 'id', value: values[0] }; + const where = /\bWHERE\b([\s\S]*)$/i.exec(statement); + if (!where) return undefined; + const predicate = /\bid\s*=\s*\$(\d+)/i.exec(where[1]!); + if (!predicate) return undefined; + const index = Number(predicate[1]) - 1; + const update = /^\s*UPDATE\s+([a-z_]+)\s+SET/i.exec(statement); + if (update) return { table: update[1]!, column: 'id', value: values[index] }; + const remove = /^\s*DELETE\s+FROM\s+([a-z_]+)/i.exec(statement); + if (remove) return { table: remove[1]!, column: 'id', value: values[index], deleted: true }; + return undefined; +} + function savepointName(value: string): string { if (!/^[a-z_][a-z0-9_]{0,62}$/.test(value)) throw new Error('Invalid savepoint name'); return value; diff --git a/apps/api/src/postgres-tree-store.ts b/apps/api/src/sql-tree-store.ts similarity index 51% rename from apps/api/src/postgres-tree-store.ts rename to apps/api/src/sql-tree-store.ts index b54dfdf..243b25d 100644 --- a/apps/api/src/postgres-tree-store.ts +++ b/apps/api/src/sql-tree-store.ts @@ -1,7 +1,8 @@ import type { V2Mutation } from '@devtodo/contracts'; import { DomainError } from '@devtodo/domain'; import type { CacheStore } from './cache.js'; -import type { PostgresStore } from './postgres-store.js'; +import { OWNER_SNAPSHOT_QUERY } from './sql-statements.js'; +import type { SqlStore } from './sql-store.js'; import { v2SqlDto } from './v2-sql-dto.js'; import { MemoryTreeStore, @@ -17,7 +18,7 @@ import { * domain operates on one owner-scoped projection, while every flush happens * inside PostgresStore's existing transaction and change-feed boundary. */ -export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { +export class SqlTreeStore extends MemoryTreeStore implements V2TreeStore { /** * The projection is a read cache, not the sync authority. Keep the owner's * latest change sequence alongside it so writes on another API instance, @@ -26,24 +27,29 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { private readonly loaded = new Map(); constructor( - readonly postgres: PostgresStore, + readonly sql: SqlStore, private readonly cache?: CacheStore, ) { super(); } + /** Dialect of the backing store; a test double without one means Postgres. */ + private get dialect(): 'postgres' | 'mysql' { + return this.sql.dialect === 'mysql' ? 'mysql' : 'postgres'; + } + override async prepare(ownerId?: string): Promise { if (!ownerId) return; // Warm-cache validation is one round trip. The full load below runs inside // a repeatable-read transaction so all tables come from the same snapshot. - const current = await this.postgres.v2Query( + const current = await this.sql.v2Query( `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1 AND disabled_at IS NULL) AS owner_ok, (SELECT next_task_number FROM users WHERE id = $1 AND disabled_at IS NULL) AS next_task_number, (SELECT COALESCE(MAX(seq), 0)::text FROM sync_changes WHERE owner_id = $1) AS cursor`, [ownerId], ); const currentRow = current.rows[0]; - if (!currentRow || currentRow['owner_ok'] !== true) throw new Error('Owner is not available'); + if (!currentRow || !truthy(currentRow['owner_ok'])) throw new Error('Owner is not available'); // next_task_number only moves forward. A read racing a commit must not // overwrite a newer value with the pre-commit snapshot. const databaseNextTaskNumber = Number(currentRow['next_task_number']); @@ -74,7 +80,7 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { } } - await this.postgres.v2ReadSnapshot(async () => { + await this.sql.v2ReadSnapshot(async () => { const snapshot = await this.loadOwnerSnapshot(ownerId); this.loadOwnerState(ownerId, snapshot.state); this.loaded.set(ownerId, snapshot.cursor); @@ -118,7 +124,7 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { } private async applyDelta(ownerId: string, previousCursor: string): Promise { - const result = await this.postgres.v2Query( + const result = await this.sql.v2Query( `SELECT seq::text, protocol_version, entity_type, operation, snapshot FROM sync_changes WHERE owner_id = $1 AND seq > $2 @@ -158,33 +164,7 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { private async loadOwnerSnapshot( ownerId: string, ): Promise<{ state: Record; cursor: string }> { - const result = await this.postgres.v2Query( - `SELECT - (SELECT u.next_task_number FROM users u - WHERE u.id = $1 AND u.disabled_at IS NULL) AS next_task_number, - (SELECT COALESCE(MAX(sc.seq), 0)::text FROM sync_changes sc - WHERE sc.owner_id = $1) AS cursor, - COALESCE((SELECT jsonb_agg(to_jsonb(f) || jsonb_build_object('rank', f.rank::text) ORDER BY f.id) - FROM folders f WHERE f.owner_id = $1), '[]'::jsonb) AS folders, - COALESCE((SELECT jsonb_agg(to_jsonb(t) || jsonb_build_object('rank', t.rank::text) ORDER BY t.id) - FROM tasks t WHERE t.owner_id = $1), '[]'::jsonb) AS tasks, - COALESCE((SELECT jsonb_agg(to_jsonb(n) ORDER BY n.id) - FROM notes n WHERE n.owner_id = $1), '[]'::jsonb) AS notes, - COALESCE((SELECT jsonb_agg(to_jsonb(s) || jsonb_build_object('rank', s.rank::text) ORDER BY s.id) - FROM task_steps s WHERE s.owner_id = $1), '[]'::jsonb) AS steps, - COALESCE((SELECT jsonb_agg(to_jsonb(w) || jsonb_build_object('rank', w.rank::text) ORDER BY w.id) - FROM workflows w WHERE w.owner_id = $1), '[]'::jsonb) AS workflows, - COALESCE((SELECT jsonb_agg(to_jsonb(s) || jsonb_build_object('rank', s.rank::text) ORDER BY s.id) - FROM workflow_stages s WHERE s.owner_id = $1), '[]'::jsonb) AS stages, - COALESCE((SELECT jsonb_agg(to_jsonb(m) || jsonb_build_object('rank', m.rank::text) ORDER BY m.id) - FROM workflow_task_memberships m WHERE m.owner_id = $1), '[]'::jsonb) AS memberships, - COALESCE((SELECT jsonb_agg(to_jsonb(p) || jsonb_build_object('rank', p.rank::text) ORDER BY p.id) - FROM time_points p WHERE p.owner_id = $1), '[]'::jsonb) AS time_points, - COALESCE((SELECT jsonb_agg(to_jsonb(p) || jsonb_build_object('rank', p.rank::text) ORDER BY p.id) - FROM placements p WHERE p.owner_id = $1), '[]'::jsonb) AS placements, - (SELECT to_jsonb(s) FROM user_settings s WHERE s.owner_id = $1) AS settings`, - [ownerId], - ); + const result = await this.sql.v2Query(OWNER_SNAPSHOT_QUERY[this.dialect], [ownerId]); const row = result.rows[0]; if (!row || row['next_task_number'] === null || row['next_task_number'] === undefined) throw new Error('Owner is not available'); @@ -202,13 +182,17 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { const state: Record = {}; for (const [source, key] of collections) { const value = row[source]; - state[key] = Array.isArray(value) + const mapped = Array.isArray(value) ? value.map((item) => { - const mapped = v2SqlDto(item as Record); - if (key === 'folders' && !mapped['status']) mapped['status'] = 'TODO'; - return mapped; + const dto = v2SqlDto(item as Record); + if (key === 'folders' && !dto['status']) dto['status'] = 'TODO'; + return dto; }) : []; + // JSON_ARRAYAGG does not preserve the derived table's ORDER BY. + if (this.dialect === 'mysql') + mapped.sort((left, right) => String(left['id']).localeCompare(String(right['id']))); + state[key] = mapped; } state['settings'] = row['settings'] ? v2SqlDto(row['settings'] as Record) @@ -233,14 +217,14 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { // uncommitted mutation, and a COMMIT failure leaves the projection intact // without a full-state rollback clone. const response = await super.withOwnerMutationLock(ownerId, () => - this.postgres.withMutation(() => - this.postgres.withIdempotency( + this.sql.withMutation(() => + this.sql.withIdempotency( ownerId, clientId, mutation.mutationId, v2MutationIdempotencyInput(mutation), async () => { - await this.postgres.v2LockOwner(ownerId); + await this.sql.v2LockOwner(ownerId); await this.prepare(ownerId); const draft = new MemoryTreeStore(); draft.loadOwnerState(ownerId, this.dumpOwnerState(ownerId)); @@ -248,7 +232,7 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { changes = draft.changesFrom(0); nextTaskNumber = draft.getNextTaskNumber(ownerId); await this.persistChanges(ownerId, changes); - committedCursor = await this.postgres.appendV2Changes(ownerId, changes); + committedCursor = await this.sql.appendV2Changes(ownerId, changes); return result; }, ), @@ -288,8 +272,8 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { let committedCursor: string | undefined; let nextTaskNumber: number | undefined; await super.withOwnerMutationLock(ownerId, () => - this.postgres.withMutation(async () => { - await this.postgres.v2LockOwner(ownerId); + this.sql.withMutation(async () => { + await this.sql.v2LockOwner(ownerId); await this.prepare(ownerId); const draft = new MemoryTreeStore(); const baseState = this.dumpOwnerState(ownerId); @@ -297,11 +281,11 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { for (const [index, mutation] of mutations.entries()) { const savepoint = `mutation_${index}`; const changeStart = draft.changeCount(); - await this.postgres.createSavepoint(savepoint); + await this.sql.createSavepoint(savepoint); try { let actionCursor: string | undefined; let actionChanges: V2SyncChange[] = []; - const response = await this.postgres.withIdempotency( + const response = await this.sql.withIdempotency( ownerId, clientId, mutation.mutationId, @@ -310,11 +294,11 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { const result = draft.applyMutationForPersistence(ownerId, mutation); actionChanges = draft.changesFrom(changeStart); await this.persistChanges(ownerId, actionChanges); - actionCursor = await this.postgres.appendV2Changes(ownerId, actionChanges); + actionCursor = await this.sql.appendV2Changes(ownerId, actionChanges); return result; }, ); - await this.postgres.releaseSavepoint(savepoint); + await this.sql.releaseSavepoint(savepoint); results.push({ mutation, status: 'applied', @@ -324,7 +308,7 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { if (actionCursor) committedCursor = actionCursor; published.push(...actionChanges); } catch (error) { - await this.postgres.rollbackToSavepoint(savepoint); + await this.sql.rollbackToSavepoint(savepoint); // A rejected item rolls back only its savepoint. Rebuild from the // one batch base plus successful deltas instead of cloning the // entire owner before every mutation (the former O(batch²) path). @@ -370,25 +354,25 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { * No account-wide notes read, clone, or flush is needed. Receipt, row and feed * remain in the existing atomic transaction and owner lock. */ private async applyFocusedMutation(ownerId: string, clientId: string, mutation: V2Mutation) { - const response = await this.postgres.withMutation(() => - this.postgres.withIdempotency( + const response = await this.sql.withMutation(() => + this.sql.withIdempotency( ownerId, clientId, mutation.mutationId, v2MutationIdempotencyInput(mutation), async () => { - await this.postgres.v2LockOwner(ownerId); + await this.sql.v2LockOwner(ownerId); const taskId = mutation.command === 'note.update' ? (mutation.payload['taskId'] ?? mutation.entityId) : mutation.entityId; - const tasks = await this.postgres.v2Query( + const tasks = await this.sql.v2Query( 'SELECT * FROM tasks WHERE owner_id = $1 AND id = $2 AND deleted_at IS NULL FOR UPDATE', [ownerId, taskId], ); const notes = mutation.command === 'note.update' - ? await this.postgres.v2Query( + ? await this.sql.v2Query( 'SELECT * FROM notes WHERE owner_id = $1 AND task_id = $2 AND deleted_at IS NULL FOR UPDATE', [ownerId, taskId], ) @@ -423,7 +407,7 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { const result = await focused.applyMutationIdempotent(ownerId, clientId, mutation); const changes = focused.changesFrom(0); await this.persistChanges(ownerId, changes); - await this.postgres.appendV2Changes(ownerId, changes); + await this.sql.appendV2Changes(ownerId, changes); return result.result; }, ), @@ -436,16 +420,16 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { } override async snapshot(ownerId: string) { - return this.postgres.v2ReadSnapshot(async () => { + return this.sql.v2ReadSnapshot(async () => { await this.prepare(ownerId); const snapshot = super.snapshot(ownerId) as V2Snapshot; - const status = await this.postgres.syncStatusV2(ownerId); + const status = await this.sql.syncStatusV2(ownerId); return { ...snapshot, cursor: status.cursor }; }); } override async pull(ownerId: string, cursor: string, limit: number) { - return this.postgres.syncPullV2(ownerId, cursor, limit) as Promise<{ + return this.sql.syncPullV2(ownerId, cursor, limit) as Promise<{ changes: V2SyncChange[]; nextCursor: string; hasMore: boolean; @@ -453,11 +437,11 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { } override async status(ownerId: string) { - return this.postgres.syncStatusV2(ownerId); + return this.sql.syncStatusV2(ownerId); } override subscribeChanges(listener: (ownerId: string, cursor: string) => void): () => void { - return this.postgres.subscribeChanges(listener); + return this.sql.subscribeChanges(listener); } private async persistChanges(ownerId: string, changes: V2SyncChange[]): Promise { @@ -493,70 +477,48 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { entityType: V2SyncChange['entityType'], changes: V2SyncChange[], ): Promise { - const payload = changes.map((change) => { - const row = asRecord(change.snapshot); - return { change, row }; - }); - const json = (rows: Array>) => JSON.stringify(rows); + const payload = changes.map((change) => ({ change, row: asRecord(change.snapshot) })); switch (entityType) { case 'folder': - await this.postgres.v2Query( - `INSERT INTO folders (id,owner_id,parent_folder_id,title,status,completed_at,rank,version,created_at,updated_at,deleted_at) - SELECT x.id, $1, x.parent_folder_id, x.title, x.status, x.completed_at, x.rank, x.version, x.created_at, x.updated_at, x.deleted_at - FROM jsonb_to_recordset($2::jsonb) AS x( - id uuid, parent_folder_id uuid, title text, status text, completed_at timestamptz, - rank bigint, version integer, created_at timestamptz, updated_at timestamptz, deleted_at timestamptz - ) - ON CONFLICT (id) DO UPDATE SET parent_folder_id=EXCLUDED.parent_folder_id,title=EXCLUDED.title,status=EXCLUDED.status,completed_at=EXCLUDED.completed_at,rank=EXCLUDED.rank,version=EXCLUDED.version,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at,deleted_at=EXCLUDED.deleted_at - WHERE folders.owner_id = EXCLUDED.owner_id`, - [ - ownerId, - json( - payload.map(({ change, row }) => ({ - id: change.entityId, - parent_folder_id: nullableValue(row.parentFolderId), - title: row.title, - status: row.status ?? 'TODO', - completed_at: nullableValue(row.completedAt), - rank: row.rank, - version: row.version, - created_at: row.createdAt, - updated_at: row.updatedAt, - deleted_at: nullableValue(row.deletedAt), - })), - ), - ], + await this.upsertBatch( + 'folders', + FOLDER_COLUMNS, + payload.map(({ change, row }) => ({ + id: change.entityId, + owner_id: ownerId, + parent_folder_id: nullableValue(row.parentFolderId), + title: row.title, + status: row.status ?? 'TODO', + completed_at: nullableValue(row.completedAt), + rank: row.rank, + version: row.version, + created_at: row.createdAt, + updated_at: row.updatedAt, + deleted_at: nullableValue(row.deletedAt), + })), ); return; case 'task': { - await this.postgres.v2Query( - `INSERT INTO tasks (id,owner_id,project_id,category,reference_id,title,status,priority,rank,completed_at,version,created_at,updated_at,deleted_at,parent_folder_id) - SELECT x.id, $1, NULL, 'MISC', x.reference_id, x.title, x.status, 'NONE', x.rank, x.completed_at, x.version, x.created_at, x.updated_at, x.deleted_at, x.parent_folder_id - FROM jsonb_to_recordset($2::jsonb) AS x( - id uuid, reference_id text, title text, status text, rank bigint, - completed_at timestamptz, version integer, created_at timestamptz, - updated_at timestamptz, deleted_at timestamptz, parent_folder_id uuid - ) - ON CONFLICT (id) DO UPDATE SET reference_id=EXCLUDED.reference_id,title=EXCLUDED.title,status=EXCLUDED.status,rank=EXCLUDED.rank,completed_at=EXCLUDED.completed_at,version=EXCLUDED.version,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at,deleted_at=EXCLUDED.deleted_at,parent_folder_id=EXCLUDED.parent_folder_id - WHERE tasks.owner_id = EXCLUDED.owner_id`, - [ - ownerId, - json( - payload.map(({ change, row }) => ({ - id: change.entityId, - reference_id: row.referenceId, - title: row.title, - status: row.status, - rank: row.rank, - completed_at: nullableValue(row.completedAt), - version: row.version, - created_at: row.createdAt, - updated_at: row.updatedAt, - deleted_at: nullableValue(row.deletedAt), - parent_folder_id: nullableValue(row.parentFolderId), - })), - ), - ], + await this.upsertBatch( + 'tasks', + TASK_COLUMNS, + payload.map(({ change, row }) => ({ + id: change.entityId, + owner_id: ownerId, + project_id: null, + category: 'MISC', + reference_id: row.referenceId, + title: row.title, + status: row.status, + priority: 'NONE', + rank: row.rank, + completed_at: nullableValue(row.completedAt), + version: row.version, + created_at: row.createdAt, + updated_at: row.updatedAt, + deleted_at: nullableValue(row.deletedAt), + parent_folder_id: nullableValue(row.parentFolderId), + })), ); let nextTaskNumber: number | undefined; for (const { row } of payload) { @@ -564,231 +526,336 @@ export class PostgresTreeStore extends MemoryTreeStore implements V2TreeStore { if (match) nextTaskNumber = Math.max(nextTaskNumber ?? 0, Number(match[1]) + 1); } if (nextTaskNumber !== undefined) - await this.postgres.v2Query( + await this.sql.v2Query( 'UPDATE users SET next_task_number = GREATEST(next_task_number, $2) WHERE id = $1', [ownerId, nextTaskNumber], ); return; } case 'note': - await this.postgres.v2Query( - `INSERT INTO notes (id,owner_id,task_id,content_markdown,version,created_at,updated_at,deleted_at) - SELECT x.id, $1, x.task_id, x.content_markdown, x.version, x.created_at, x.updated_at, x.deleted_at - FROM jsonb_to_recordset($2::jsonb) AS x( - id uuid, task_id uuid, content_markdown text, version integer, - created_at timestamptz, updated_at timestamptz, deleted_at timestamptz - ) - ON CONFLICT (id) DO UPDATE SET task_id=EXCLUDED.task_id,content_markdown=EXCLUDED.content_markdown,version=EXCLUDED.version,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at,deleted_at=EXCLUDED.deleted_at - WHERE notes.owner_id = EXCLUDED.owner_id`, - [ - ownerId, - json( - payload.map(({ change, row }) => ({ - id: change.entityId, - task_id: row.taskId, - content_markdown: row.contentMarkdown, - version: row.version, - created_at: row.createdAt, - updated_at: row.updatedAt, - deleted_at: nullableValue(row.deletedAt), - })), - ), - ], + await this.upsertBatch( + 'notes', + NOTE_COLUMNS, + payload.map(({ change, row }) => ({ + id: change.entityId, + owner_id: ownerId, + task_id: row.taskId, + content_markdown: row.contentMarkdown, + version: row.version, + created_at: row.createdAt, + updated_at: row.updatedAt, + deleted_at: nullableValue(row.deletedAt), + })), ); return; case 'taskStep': - await this.postgres.v2Query( - `INSERT INTO task_steps (id,owner_id,task_id,title,note_markdown,status,rank,completed_at,version,created_at,updated_at,deleted_at) - SELECT x.id, $1, x.task_id, x.title, x.note_markdown, x.status, x.rank, x.completed_at, x.version, x.created_at, x.updated_at, x.deleted_at - FROM jsonb_to_recordset($2::jsonb) AS x( - id uuid, task_id uuid, title text, note_markdown text, status text, rank bigint, - completed_at timestamptz, version integer, created_at timestamptz, - updated_at timestamptz, deleted_at timestamptz - ) - ON CONFLICT (id) DO UPDATE SET task_id=EXCLUDED.task_id,title=EXCLUDED.title,note_markdown=EXCLUDED.note_markdown,status=EXCLUDED.status,rank=EXCLUDED.rank,completed_at=EXCLUDED.completed_at,version=EXCLUDED.version,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at,deleted_at=EXCLUDED.deleted_at - WHERE task_steps.owner_id = EXCLUDED.owner_id`, - [ - ownerId, - json( - payload.map(({ change, row }) => ({ - id: change.entityId, - task_id: row.taskId, - title: row.title, - note_markdown: row.noteMarkdown, - status: row.status, - rank: row.rank, - completed_at: nullableValue(row.completedAt), - version: row.version, - created_at: row.createdAt, - updated_at: row.updatedAt, - deleted_at: nullableValue(row.deletedAt), - })), - ), - ], + await this.upsertBatch( + 'task_steps', + TASK_STEP_COLUMNS, + payload.map(({ change, row }) => ({ + id: change.entityId, + owner_id: ownerId, + task_id: row.taskId, + title: row.title, + note_markdown: row.noteMarkdown, + status: row.status, + rank: row.rank, + completed_at: nullableValue(row.completedAt), + version: row.version, + created_at: row.createdAt, + updated_at: row.updatedAt, + deleted_at: nullableValue(row.deletedAt), + })), ); return; case 'workflow': - await this.postgres.v2Query( - `INSERT INTO workflows (id,owner_id,name,rank,version,created_at,updated_at,deleted_at) - SELECT x.id, $1, x.name, x.rank, x.version, x.created_at, x.updated_at, x.deleted_at - FROM jsonb_to_recordset($2::jsonb) AS x( - id uuid, name text, rank bigint, version integer, - created_at timestamptz, updated_at timestamptz, deleted_at timestamptz - ) - ON CONFLICT (id) DO UPDATE SET name=EXCLUDED.name,rank=EXCLUDED.rank,version=EXCLUDED.version,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at,deleted_at=EXCLUDED.deleted_at - WHERE workflows.owner_id = EXCLUDED.owner_id`, - [ - ownerId, - json( - payload.map(({ change, row }) => ({ - id: change.entityId, - name: row.name, - rank: row.rank, - version: row.version, - created_at: row.createdAt, - updated_at: row.updatedAt, - deleted_at: nullableValue(row.deletedAt), - })), - ), - ], + await this.upsertBatch( + 'workflows', + WORKFLOW_COLUMNS, + payload.map(({ change, row }) => ({ + id: change.entityId, + owner_id: ownerId, + name: row.name, + rank: row.rank, + version: row.version, + created_at: row.createdAt, + updated_at: row.updatedAt, + deleted_at: nullableValue(row.deletedAt), + })), ); return; case 'workflowStage': - await this.postgres.v2Query( - `INSERT INTO workflow_stages (id,owner_id,workflow_id,name,rank,version,created_at,updated_at,deleted_at) - SELECT x.id, $1, x.workflow_id, x.name, x.rank, x.version, x.created_at, x.updated_at, x.deleted_at - FROM jsonb_to_recordset($2::jsonb) AS x( - id uuid, workflow_id uuid, name text, rank bigint, version integer, - created_at timestamptz, updated_at timestamptz, deleted_at timestamptz - ) - ON CONFLICT (id) DO UPDATE SET workflow_id=EXCLUDED.workflow_id,name=EXCLUDED.name,rank=EXCLUDED.rank,version=EXCLUDED.version,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at,deleted_at=EXCLUDED.deleted_at - WHERE workflow_stages.owner_id = EXCLUDED.owner_id`, - [ - ownerId, - json( - payload.map(({ change, row }) => ({ - id: change.entityId, - workflow_id: row.workflowId, - name: row.name, - rank: row.rank, - version: row.version, - created_at: row.createdAt, - updated_at: row.updatedAt, - deleted_at: nullableValue(row.deletedAt), - })), - ), - ], + await this.upsertBatch( + 'workflow_stages', + WORKFLOW_STAGE_COLUMNS, + payload.map(({ change, row }) => ({ + id: change.entityId, + owner_id: ownerId, + workflow_id: row.workflowId, + name: row.name, + rank: row.rank, + version: row.version, + created_at: row.createdAt, + updated_at: row.updatedAt, + deleted_at: nullableValue(row.deletedAt), + })), ); return; case 'workflowTaskMembership': - await this.postgres.v2Query( - `INSERT INTO workflow_task_memberships (id,owner_id,workflow_id,stage_id,task_id,rank,version,created_at,updated_at,deleted_at) - SELECT x.id, $1, x.workflow_id, x.stage_id, x.task_id, x.rank, x.version, x.created_at, x.updated_at, x.deleted_at - FROM jsonb_to_recordset($2::jsonb) AS x( - id uuid, workflow_id uuid, stage_id uuid, task_id uuid, rank bigint, version integer, - created_at timestamptz, updated_at timestamptz, deleted_at timestamptz - ) - ON CONFLICT (id) DO UPDATE SET workflow_id=EXCLUDED.workflow_id,stage_id=EXCLUDED.stage_id,task_id=EXCLUDED.task_id,rank=EXCLUDED.rank,version=EXCLUDED.version,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at,deleted_at=EXCLUDED.deleted_at - WHERE workflow_task_memberships.owner_id = EXCLUDED.owner_id`, - [ - ownerId, - json( - payload.map(({ change, row }) => ({ - id: change.entityId, - workflow_id: row.workflowId, - stage_id: row.stageId, - task_id: row.taskId, - rank: row.rank, - version: row.version, - created_at: row.createdAt, - updated_at: row.updatedAt, - deleted_at: nullableValue(row.deletedAt), - })), - ), - ], + await this.upsertBatch( + 'workflow_task_memberships', + WORKFLOW_MEMBERSHIP_COLUMNS, + payload.map(({ change, row }) => ({ + id: change.entityId, + owner_id: ownerId, + workflow_id: row.workflowId, + stage_id: row.stageId, + task_id: row.taskId, + rank: row.rank, + version: row.version, + created_at: row.createdAt, + updated_at: row.updatedAt, + deleted_at: nullableValue(row.deletedAt), + })), ); return; case 'timePoint': - await this.postgres.v2Query( - `INSERT INTO time_points (id,owner_id,type,local_date,title,rank,version,reached_at,created_at,updated_at,deleted_at) - SELECT x.id, $1, x.type, x.local_date, x.title, x.rank, x.version, x.reached_at, x.created_at, x.updated_at, x.deleted_at - FROM jsonb_to_recordset($2::jsonb) AS x( - id uuid, type text, local_date date, title text, rank bigint, version integer, - reached_at timestamptz, created_at timestamptz, updated_at timestamptz, deleted_at timestamptz - ) - ON CONFLICT (id) DO UPDATE SET type=EXCLUDED.type,local_date=EXCLUDED.local_date,title=EXCLUDED.title,rank=EXCLUDED.rank,version=EXCLUDED.version,reached_at=EXCLUDED.reached_at,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at,deleted_at=EXCLUDED.deleted_at - WHERE time_points.owner_id = EXCLUDED.owner_id`, - [ - ownerId, - json( - payload.map(({ change, row }) => ({ - id: change.entityId, - type: row.type, - local_date: nullableValue(row.localDate), - title: nullableValue(row.title), - rank: row.rank, - version: row.version, - reached_at: nullableValue(row.reachedAt), - created_at: row.createdAt, - updated_at: row.updatedAt, - deleted_at: nullableValue(row.deletedAt), - })), - ), - ], + await this.upsertBatch( + 'time_points', + TIME_POINT_COLUMNS, + payload.map(({ change, row }) => ({ + id: change.entityId, + owner_id: ownerId, + type: row.type, + local_date: nullableValue(row.localDate), + title: nullableValue(row.title), + rank: row.rank, + version: row.version, + reached_at: nullableValue(row.reachedAt), + created_at: row.createdAt, + updated_at: row.updatedAt, + deleted_at: nullableValue(row.deletedAt), + })), ); return; case 'placement': - await this.postgres.v2Query( - `INSERT INTO placements (id,owner_id,task_id,time_point_id,rank,version,created_at,updated_at,deleted_at) - SELECT x.id, $1, x.task_id, x.time_point_id, x.rank, x.version, x.created_at, x.updated_at, x.deleted_at - FROM jsonb_to_recordset($2::jsonb) AS x( - id uuid, task_id uuid, time_point_id uuid, rank bigint, version integer, - created_at timestamptz, updated_at timestamptz, deleted_at timestamptz - ) - ON CONFLICT (id) DO UPDATE SET task_id=EXCLUDED.task_id,time_point_id=EXCLUDED.time_point_id,rank=EXCLUDED.rank,version=EXCLUDED.version,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at,deleted_at=EXCLUDED.deleted_at - WHERE placements.owner_id = EXCLUDED.owner_id`, - [ - ownerId, - json( - payload.map(({ change, row }) => ({ - id: change.entityId, - task_id: row.taskId, - time_point_id: row.timePointId, - rank: row.rank, - version: row.version, - created_at: row.createdAt, - updated_at: row.updatedAt, - deleted_at: nullableValue(row.deletedAt), - })), - ), - ], + await this.upsertBatch( + 'placements', + PLACEMENT_COLUMNS, + payload.map(({ change, row }) => ({ + id: change.entityId, + owner_id: ownerId, + task_id: row.taskId, + time_point_id: row.timePointId, + rank: row.rank, + version: row.version, + created_at: row.createdAt, + updated_at: row.updatedAt, + deleted_at: nullableValue(row.deletedAt), + })), ); return; case 'settings': - await this.postgres.v2Query( - `INSERT INTO user_settings (owner_id,timezone,week_starts_on,default_capture_target,version,created_at,updated_at,deleted_at) - SELECT $1, x.timezone, x.week_starts_on, x.default_capture_target, x.version, x.updated_at, x.updated_at, NULL - FROM jsonb_to_recordset($2::jsonb) AS x( - timezone text, week_starts_on smallint, default_capture_target text, - version integer, updated_at timestamptz - ) - ON CONFLICT (owner_id) DO UPDATE SET timezone=EXCLUDED.timezone,week_starts_on=EXCLUDED.week_starts_on,default_capture_target=EXCLUDED.default_capture_target,version=EXCLUDED.version,updated_at=EXCLUDED.updated_at,deleted_at=NULL`, - [ - ownerId, - json( - payload.map(({ row }) => ({ - timezone: row.timezone, - week_starts_on: row.weekStartsOn, - default_capture_target: row.defaultCaptureTarget, - version: row.version, - updated_at: row.updatedAt, - })), - ), - ], + await this.upsertBatch( + 'user_settings', + SETTINGS_COLUMNS, + payload.map(({ row }) => ({ + owner_id: ownerId, + timezone: row.timezone, + week_starts_on: row.weekStartsOn, + default_capture_target: row.defaultCaptureTarget, + version: row.version, + created_at: row.updatedAt, + updated_at: row.updatedAt, + deleted_at: null, + })), + { conflict: ['owner_id'], guardOwner: false }, ); return; } } + + /** + * One multi-row upsert per entity type. + * + * PostgreSQL uses ON CONFLICT with a row-count guard so a stray id from + * another owner can never be overwritten. MySQL has no ON CONFLICT and no + * EXCLUDED, so the same batch is expressed as INSERT ... AS new ON + * DUPLICATE KEY UPDATE; id collisions across owners are impossible because + * entity ids are server-generated UUIDv7. + */ + private async upsertBatch( + table: string, + columns: readonly string[], + rows: Array>, + options: { conflict?: readonly string[]; guardOwner?: boolean } = {}, + ): Promise { + if (!rows.length) return; + const conflict = options.conflict ?? ['id']; + const dialect = this.dialect; + const quoted = columns.map(ident); + const updatable = columns + .filter((column) => !conflict.includes(column) && column !== 'owner_id') + .map(ident); + const chunkSize = Math.max(1, Math.min(200, Math.floor(20_000 / columns.length))); + for (let offset = 0; offset < rows.length; offset += chunkSize) { + const chunk = rows.slice(offset, offset + chunkSize); + const parameters: unknown[] = []; + const tuples = chunk.map((row) => { + const values = columns.map((column) => { + parameters.push(row[column] ?? null); + return `$${parameters.length}`; + }); + return `(${values.join(',')})`; + }); + const statement = + dialect === 'mysql' + ? `INSERT INTO ${table} (${quoted.join(',')}) VALUES ${tuples.join(',')} AS new ON DUPLICATE KEY UPDATE ${updatable + .map((column) => `${column} = new.${column}`) + .join(',')}` + : `INSERT INTO ${table} (${quoted.join(',')}) VALUES ${tuples.join(',')} ON CONFLICT (${conflict + .map(ident) + .join(',')}) DO UPDATE SET ${updatable + .map((column) => `${column} = EXCLUDED.${column}`) + .join( + ',', + )}${options.guardOwner === false ? '' : ` WHERE ${table}.owner_id = EXCLUDED.owner_id`}`; + await this.sql.v2Query(statement, parameters); + } + } +} + +const FOLDER_COLUMNS = [ + 'id', + 'owner_id', + 'parent_folder_id', + 'title', + 'status', + 'completed_at', + 'rank', + 'version', + 'created_at', + 'updated_at', + 'deleted_at', +] as const; +const TASK_COLUMNS = [ + 'id', + 'owner_id', + 'project_id', + 'category', + 'reference_id', + 'title', + 'status', + 'priority', + 'rank', + 'completed_at', + 'version', + 'created_at', + 'updated_at', + 'deleted_at', + 'parent_folder_id', +] as const; +const NOTE_COLUMNS = [ + 'id', + 'owner_id', + 'task_id', + 'content_markdown', + 'version', + 'created_at', + 'updated_at', + 'deleted_at', +] as const; +const TASK_STEP_COLUMNS = [ + 'id', + 'owner_id', + 'task_id', + 'title', + 'note_markdown', + 'status', + 'rank', + 'completed_at', + 'version', + 'created_at', + 'updated_at', + 'deleted_at', +] as const; +const WORKFLOW_COLUMNS = [ + 'id', + 'owner_id', + 'name', + 'rank', + 'version', + 'created_at', + 'updated_at', + 'deleted_at', +] as const; +const WORKFLOW_STAGE_COLUMNS = [ + 'id', + 'owner_id', + 'workflow_id', + 'name', + 'rank', + 'version', + 'created_at', + 'updated_at', + 'deleted_at', +] as const; +const WORKFLOW_MEMBERSHIP_COLUMNS = [ + 'id', + 'owner_id', + 'workflow_id', + 'stage_id', + 'task_id', + 'rank', + 'version', + 'created_at', + 'updated_at', + 'deleted_at', +] as const; +const TIME_POINT_COLUMNS = [ + 'id', + 'owner_id', + 'type', + 'local_date', + 'title', + 'rank', + 'version', + 'reached_at', + 'created_at', + 'updated_at', + 'deleted_at', +] as const; +const PLACEMENT_COLUMNS = [ + 'id', + 'owner_id', + 'task_id', + 'time_point_id', + 'rank', + 'version', + 'created_at', + 'updated_at', + 'deleted_at', +] as const; +const SETTINGS_COLUMNS = [ + 'owner_id', + 'timezone', + 'week_starts_on', + 'default_capture_target', + 'version', + 'created_at', + 'updated_at', + 'deleted_at', +] as const; + +const RESERVED_IDENTIFIERS = new Set(['rank', 'cursor']); + +function ident(name: string): string { + return RESERVED_IDENTIFIERS.has(name) ? `\`${name}\`` : name; +} + +/** MySQL reports booleans and EXISTS as 1/0; PostgreSQL reports true/false. */ +function truthy(value: unknown): boolean { + return value === true || value === 1 || value === '1'; } function asRecord(value: unknown): Record { diff --git a/apps/api/src/sync-bootstrap.ts b/apps/api/src/sync-bootstrap.ts index 2da6d83..2563541 100644 --- a/apps/api/src/sync-bootstrap.ts +++ b/apps/api/src/sync-bootstrap.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto'; import { DomainError } from '@devtodo/domain'; -import { PostgresTreeStore } from './postgres-tree-store.js'; +import { taskRowJson } from './sql-statements.js'; +import { SqlTreeStore } from './sql-tree-store.js'; import type { V2Snapshot, V2TreeStore } from './tree-store.js'; const ttlMs = 30 * 60_000; @@ -109,7 +110,7 @@ export class BootstrapSnapshots { constructor(private readonly tree: V2TreeStore) {} startMaintenance(onError: (error: unknown) => void): void { - if (this.maintenance || !(this.tree instanceof PostgresTreeStore)) return; + if (this.maintenance || !(this.tree instanceof SqlTreeStore)) return; this.maintenance = setInterval(() => { void this.cleanup().catch(onError); }, 60_000); @@ -125,8 +126,8 @@ export class BootstrapSnapshots { cleanup(): Promise { if (this.cleanupFlight) return this.cleanupFlight; const tree = this.tree; - if (!(tree instanceof PostgresTreeStore)) return Promise.resolve(); - this.cleanupFlight = tree.postgres + if (!(tree instanceof SqlTreeStore)) return Promise.resolve(); + this.cleanupFlight = tree.sql .v2Query( 'DELETE FROM sync_bootstrap_snapshots WHERE id IN (SELECT id FROM sync_bootstrap_snapshots WHERE expires_at <= now() ORDER BY expires_at LIMIT 2 FOR UPDATE SKIP LOCKED)', ) @@ -138,33 +139,33 @@ export class BootstrapSnapshots { } private async capture(ownerId: string): Promise { - if (!(this.tree instanceof PostgresTreeStore)) return this.tree.snapshot(ownerId); - const postgres = this.tree.postgres; - return postgres.v2ReadSnapshot(async () => { + if (!(this.tree instanceof SqlTreeStore)) return this.tree.snapshot(ownerId); + const sql = this.tree.sql; + return sql.v2ReadSnapshot(async () => { const snapshot: Record = {}; for (const [key, [, table]] of Object.entries(collections)) { const order = table === 'folders' || table === 'tasks' ? 'parent_folder_id NULLS FIRST, id' : 'id'; const rank = table === 'notes' ? '' : ', rank::text AS rank'; - const rows = await postgres.v2Query( + const rows = await sql.v2Query( `SELECT *${rank} FROM ${table} WHERE owner_id = $1 AND deleted_at IS NULL ORDER BY ${order}`, [ownerId], ); snapshot[key] = rows.rows.map(dto); } - const settings = await postgres.v2Query('SELECT * FROM user_settings WHERE owner_id = $1', [ + const settings = await sql.v2Query('SELECT * FROM user_settings WHERE owner_id = $1', [ ownerId, ]); if (!settings.rows[0]) throw new DomainError('AUTH_REQUIRED', '账户设置不可用'); snapshot['settings'] = dto(settings.rows[0]); - snapshot['cursor'] = (await postgres.syncStatusV2(ownerId)).cursor; + snapshot['cursor'] = (await sql.syncStatusV2(ownerId)).cursor; return snapshot as unknown as V2Snapshot; }); } /** Read-only Today surface: no date creation, notes or account-wide projection. */ async today(ownerId: string, localDate: string) { - if (!(this.tree instanceof PostgresTreeStore)) { + if (!(this.tree instanceof SqlTreeStore)) { const full = await this.capture(ownerId); const point = full.timePoints.find( @@ -185,20 +186,26 @@ export class BootstrapSnapshots { .map((row) => ({ ...row, task: tasks.get(row.taskId)! })); return { point, items }; } - const postgres = this.tree.postgres; - return postgres.withMutation(async () => { - await postgres.v2Query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ'); - const points = await postgres.v2Query( + const sql = this.tree.sql; + return sql.withMutation(async () => { + if (sql.dialect === 'postgres') + await sql.v2Query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ'); + const points = await sql.v2Query( "SELECT *, rank::text AS rank FROM time_points WHERE owner_id = $1 AND type = 'DATE' AND local_date = $2::date AND deleted_at IS NULL", [ownerId, localDate], ); const point = points.rows[0]; if (!point) return { point: null, items: [] }; - const placements = await postgres.v2Query( - `SELECT p.*, p.rank::text AS rank, to_jsonb(t) || jsonb_build_object('rank', t.rank::text) AS task - FROM placements p JOIN tasks t ON t.id = p.task_id AND t.owner_id = p.owner_id - WHERE p.owner_id = $1 AND p.time_point_id = $2 AND p.deleted_at IS NULL AND t.deleted_at IS NULL - ORDER BY p.rank, p.id`, + const placements = await sql.v2Query( + sql.dialect === 'mysql' + ? `SELECT p.*, ${taskRowJson('t')} AS task + FROM placements p JOIN tasks t ON t.id = p.task_id AND t.owner_id = p.owner_id + WHERE p.owner_id = $1 AND p.time_point_id = $2 AND p.deleted_at IS NULL AND t.deleted_at IS NULL + ORDER BY p.rank, p.id` + : `SELECT p.*, p.rank::text AS rank, to_jsonb(t) || jsonb_build_object('rank', t.rank::text) AS task + FROM placements p JOIN tasks t ON t.id = p.task_id AND t.owner_id = p.owner_id + WHERE p.owner_id = $1 AND p.time_point_id = $2 AND p.deleted_at IS NULL AND t.deleted_at IS NULL + ORDER BY p.rank, p.id`, [ownerId, point['id']], ); return { @@ -212,7 +219,7 @@ export class BootstrapSnapshots { } async preview(ownerId: string, localDate: string, parentFolderId: string | null) { - if (!(this.tree instanceof PostgresTreeStore)) { + if (!(this.tree instanceof SqlTreeStore)) { const full = await this.capture(ownerId); const points = full.timePoints.filter((p) => p.localDate === localDate); const ids = new Set(points.map((p) => p.id)); @@ -262,35 +269,35 @@ export class BootstrapSnapshots { complete: false, }; } - const postgres = this.tree.postgres; - return postgres.v2ReadSnapshot(async () => { - const cursor = (await postgres.syncStatusV2(ownerId)).cursor; + const sql = this.tree.sql; + return sql.v2ReadSnapshot(async () => { + const cursor = (await sql.syncStatusV2(ownerId)).cursor; // Explicit null/equality branches let PostgreSQL use the ordered directory indexes. const parent = parentFolderId === null ? 'parent_folder_id IS NULL' : 'parent_folder_id = $2::uuid'; const parentValues = parentFolderId === null ? [ownerId] : [ownerId, parentFolderId]; - const settings = await postgres.v2Query('SELECT * FROM user_settings WHERE owner_id = $1', [ + const settings = await sql.v2Query('SELECT * FROM user_settings WHERE owner_id = $1', [ ownerId, ]); - const folders = await postgres.v2Query( + const folders = await sql.v2Query( `SELECT *, rank::text AS rank FROM folders WHERE owner_id = $1 AND deleted_at IS NULL AND ${parent} ORDER BY folders.rank, id LIMIT 101`, parentValues, ); - const points = await postgres.v2Query( + const points = await sql.v2Query( "SELECT *, rank::text AS rank FROM time_points WHERE owner_id = $1 AND type = 'DATE' AND local_date = $2::date AND deleted_at IS NULL", [ownerId, localDate], ); const pointIds = points.rows.map((p) => p['id']); - const placements = await postgres.v2Query( + const placements = await sql.v2Query( 'SELECT *, rank::text AS rank FROM placements WHERE owner_id = $1 AND time_point_id = ANY($2::uuid[]) AND deleted_at IS NULL ORDER BY placements.rank, id LIMIT 101', [ownerId, pointIds], ); const taskIds = placements.rows.slice(0, 100).map((p) => p['task_id']); - const rootTasks = await postgres.v2Query( + const rootTasks = await sql.v2Query( `SELECT *, rank::text AS rank FROM tasks WHERE owner_id = $1 AND ${parent} AND deleted_at IS NULL ORDER BY CASE status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END, tasks.rank, id LIMIT 101`, parentValues, ); - const scheduledTasks = await postgres.v2Query( + const scheduledTasks = await sql.v2Query( 'SELECT *, rank::text AS rank FROM tasks WHERE owner_id = $1 AND id = ANY($2::uuid[]) AND deleted_at IS NULL', [ownerId, taskIds], ); @@ -341,7 +348,7 @@ export class BootstrapSnapshots { } private async createNew(ownerId: string, localDate: string) { - if (this.tree instanceof PostgresTreeStore) return this.createPostgres(ownerId); + if (this.tree instanceof SqlTreeStore) return this.createFromSql(ownerId); const snapshot = await this.capture(ownerId); // Dependencies precede relations; large note bodies are always last. const todayIds = new Set( @@ -376,16 +383,18 @@ export class BootstrapSnapshots { }; } - private async createPostgres(ownerId: string) { - if (!(this.tree instanceof PostgresTreeStore)) throw new Error('PostgreSQL required'); - const postgres = this.tree.postgres; - return postgres.withMutation(async () => { - await postgres.v2Query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ'); - const cursor = (await postgres.syncStatusV2(ownerId)).cursor; + private async createFromSql(ownerId: string) { + if (!(this.tree instanceof SqlTreeStore)) throw new Error('SQL store required'); + const sql = this.tree.sql; + return sql.withMutation(async () => { + if (sql.dialect === 'postgres') + await sql.v2Query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ'); + const cursor = (await sql.syncStatusV2(ownerId)).cursor; // Reuse only an immutable snapshot at exactly this revision, with enough // remaining lifetime to resume it. This also works across API instances. - const reusable = await postgres.v2Query( - "SELECT id, expires_at, page_count FROM sync_bootstrap_snapshots WHERE owner_id = $1 AND cursor = $2 AND expires_at > now() + interval '5 minutes' AND page_count > 0 ORDER BY expires_at DESC LIMIT 1", + const fiveMinutes = sql.dialect === 'mysql' ? 'INTERVAL 5 MINUTE' : "interval '5 minutes'"; + const reusable = await sql.v2Query( + `SELECT id, expires_at, page_count FROM sync_bootstrap_snapshots WHERE owner_id = $1 AND cursor = $2 AND expires_at > now() + ${fiveMinutes} AND page_count > 0 ORDER BY expires_at DESC LIMIT 1`, [ownerId, cursor], ); const previous = reusable.rows[0]; @@ -399,7 +408,7 @@ export class BootstrapSnapshots { }; const snapshotId = randomUUID(); const expiresAt = new Date(Date.now() + ttlMs).toISOString(); - await postgres.v2Query( + await sql.v2Query( 'INSERT INTO sync_bootstrap_snapshots(id,owner_id,cursor,expires_at,page_count) VALUES($1,$2,$3,$4,0)', [snapshotId, ownerId, cursor, expiresAt], ); @@ -408,7 +417,7 @@ export class BootstrapSnapshots { let pageCount = 0; const flush = async () => { if (!page.length) return; - await postgres.v2Query( + await sql.v2Query( 'INSERT INTO sync_bootstrap_pages(snapshot_id,page,payload) VALUES($1,$2,$3::jsonb)', [snapshotId, pageCount++, JSON.stringify(page)], ); @@ -431,7 +440,7 @@ export class BootstrapSnapshots { page.push(change); bytes += size; }; - const settings = await postgres.v2Query('SELECT * FROM user_settings WHERE owner_id = $1', [ + const settings = await sql.v2Query('SELECT * FROM user_settings WHERE owner_id = $1', [ ownerId, ]); if (!settings.rows[0]) throw new DomainError('AUTH_REQUIRED', '账户设置不可用'); @@ -444,7 +453,7 @@ export class BootstrapSnapshots { const rank = table === 'notes' ? '' : ', rank::text AS rank'; const after = lastId ? ' AND id > $2::uuid' : ''; const values = lastId ? [ownerId, lastId] : [ownerId]; - const rows = await postgres.v2Query( + const rows = await sql.v2Query( `SELECT *${rank} FROM ${table} WHERE owner_id = $1 AND deleted_at IS NULL${after} ORDER BY id LIMIT ${table === 'notes' ? 50 : maxPageItems}`, values, ); @@ -454,7 +463,7 @@ export class BootstrapSnapshots { } } await flush(); - await postgres.v2Query('UPDATE sync_bootstrap_snapshots SET page_count = $2 WHERE id = $1', [ + await sql.v2Query('UPDATE sync_bootstrap_snapshots SET page_count = $2 WHERE id = $1', [ snapshotId, pageCount, ]); @@ -463,47 +472,47 @@ export class BootstrapSnapshots { } async detail(ownerId: string, id: string) { - if (!(this.tree instanceof PostgresTreeStore)) return this.tree.getTaskDetails(ownerId, id); - const postgres = this.tree.postgres; - return postgres.v2ReadSnapshot(async () => { - const tasks = await postgres.v2Query( + if (!(this.tree instanceof SqlTreeStore)) return this.tree.getTaskDetails(ownerId, id); + const sql = this.tree.sql; + return sql.v2ReadSnapshot(async () => { + const tasks = await sql.v2Query( 'SELECT *, rank::text AS rank FROM tasks WHERE owner_id = $1 AND id = $2 AND deleted_at IS NULL', [ownerId, id], ); if (!tasks.rows[0]) throw new DomainError('ENTITY_NOT_FOUND', '找不到任务'); const task = dto(tasks.rows[0]); - const notes = await postgres.v2Query( + const notes = await sql.v2Query( 'SELECT * FROM notes WHERE owner_id = $1 AND task_id = $2 AND deleted_at IS NULL', [ownerId, id], ); - const steps = await postgres.v2Query( + const steps = await sql.v2Query( 'SELECT *, rank::text AS rank FROM task_steps WHERE owner_id = $1 AND task_id = $2 AND deleted_at IS NULL ORDER BY task_steps.rank, id', [ownerId, id], ); - const placements = await postgres.v2Query( + const placements = await sql.v2Query( 'SELECT *, rank::text AS rank FROM placements WHERE owner_id = $1 AND task_id = $2 AND deleted_at IS NULL ORDER BY placements.rank, id', [ownerId, id], ); - const memberships = await postgres.v2Query( + const memberships = await sql.v2Query( 'SELECT *, rank::text AS rank FROM workflow_task_memberships WHERE owner_id = $1 AND task_id = $2 AND deleted_at IS NULL ORDER BY workflow_task_memberships.rank, id', [ownerId, id], ); - const points = await postgres.v2Query( + const points = await sql.v2Query( 'SELECT *, rank::text AS rank FROM time_points WHERE owner_id = $1 AND id = ANY($2::uuid[]) AND deleted_at IS NULL', [ownerId, placements.rows.map((p) => p['time_point_id'])], ); const pointById = new Map(points.rows.map((p) => [p['id'], dto(p)])); - const workflows = await postgres.v2Query( + const workflows = await sql.v2Query( 'SELECT id, name FROM workflows WHERE owner_id = $1 AND id = ANY($2::uuid[])', [ownerId, memberships.rows.map((m) => m['workflow_id'])], ); - const stages = await postgres.v2Query( + const stages = await sql.v2Query( 'SELECT id, name FROM workflow_stages WHERE owner_id = $1 AND id = ANY($2::uuid[])', [ownerId, memberships.rows.map((m) => m['stage_id'])], ); const workflowById = new Map(workflows.rows.map((w) => [w['id'], w])); const stageById = new Map(stages.rows.map((s) => [s['id'], s])); - const path = await postgres.v2Query( + const path = await sql.v2Query( 'WITH RECURSIVE path AS (SELECT f.*, ARRAY[f.id] AS visited, 0 AS depth FROM folders f WHERE owner_id = $1 AND id = $2 UNION ALL SELECT f.*, path.visited || f.id, depth + 1 FROM folders f JOIN path ON f.id = path.parent_folder_id WHERE f.owner_id = $1 AND NOT f.id = ANY(path.visited)) SELECT * FROM path ORDER BY depth DESC', [ownerId, task['parentFolderId']], ); @@ -531,8 +540,8 @@ export class BootstrapSnapshots { let expiresAt: string; let pageCount: number; let items: BootstrapChange[] | undefined; - if (this.tree instanceof PostgresTreeStore) { - const rows = await this.tree.postgres.v2Query( + if (this.tree instanceof SqlTreeStore) { + const rows = await this.tree.sql.v2Query( 'SELECT s.cursor::text, s.expires_at, s.page_count, p.payload FROM sync_bootstrap_snapshots s JOIN sync_bootstrap_pages p ON p.snapshot_id = s.id WHERE s.id = $1 AND s.owner_id = $2 AND p.page = $3 AND s.expires_at > now()', [id, ownerId, page], ); diff --git a/apps/api/test/directory-reads.test.ts b/apps/api/test/directory-reads.test.ts index 08c04a9..f422086 100644 --- a/apps/api/test/directory-reads.test.ts +++ b/apps/api/test/directory-reads.test.ts @@ -10,8 +10,8 @@ import { DirectoryReads } from '../src/directory-reads.js'; import { MemoryTreeStore } from '../src/tree-store.js'; import { buildServer } from '../src/server.js'; import { MemoryStore } from '../src/store.js'; -import { PostgresTreeStore } from '../src/postgres-tree-store.js'; -import type { PostgresStore } from '../src/postgres-store.js'; +import { SqlTreeStore } from '../src/sql-tree-store.js'; +import type { SqlStore } from '../src/sql-store.js'; describe('directory keyset reads', () => { it('bounds SQL candidate keys before constructing any task or folder payload', async () => { @@ -22,8 +22,8 @@ describe('directory keyset reads', () => { statements.push(sql); return { rows: sql.includes('AS revision') ? [{ revision: '1' }] : [], rowCount: 0 }; }, - } as unknown as PostgresStore; - const tree = new PostgresTreeStore(postgres); + } as unknown as SqlStore; + const tree = new SqlTreeStore(postgres); tree.prepare = async () => { throw new Error('unexpected full projection'); }; diff --git a/apps/api/test/postgres-sync-query.test.ts b/apps/api/test/postgres-sync-query.test.ts index e8dc2c9..d40cba5 100644 --- a/apps/api/test/postgres-sync-query.test.ts +++ b/apps/api/test/postgres-sync-query.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from 'vitest'; import type { Pool } from 'pg'; -import { PostgresStore } from '../src/postgres-store.js'; +import { SqlStore } from '../src/sql-store.js'; -describe('PostgresStore v2 read/write batching', () => { +describe('SqlStore v2 read/write batching', () => { it('reads a pull page, retention boundary and has-more flag in one statement', async () => { const queries: Array<{ text: string; values: unknown[] }> = []; const pool = { @@ -33,7 +33,7 @@ describe('PostgresStore v2 read/write batching', () => { }, end: async () => undefined, } as unknown as Pool; - const store = new PostgresStore(pool); + const store = new SqlStore(pool); const result = await store.syncPullV2('owner-1', '0', 100); expect(queries).toHaveLength(1); expect(result.nextCursor).toBe('2'); @@ -50,7 +50,7 @@ describe('PostgresStore v2 read/write batching', () => { }, end: async () => undefined, } as unknown as Pool; - const store = new PostgresStore(pool); + const store = new SqlStore(pool); await expect(store.syncStatusV2('owner-1')).resolves.toEqual({ cursor: '5', oldestCursor: '1', @@ -68,7 +68,7 @@ describe('PostgresStore v2 read/write batching', () => { }, end: async () => undefined, } as unknown as Pool; - const store = new PostgresStore(pool); + const store = new SqlStore(pool); const cursor = await store.appendV2Changes('00000000-0000-7000-8000-0000000000aa', [ { entityType: 'task', @@ -89,6 +89,8 @@ describe('PostgresStore v2 read/write batching', () => { ]); expect(cursor).toBe('8'); expect(queries).toHaveLength(1); - expect(queries[0]?.text).toContain('jsonb_to_recordset'); + // One multi-row INSERT for the whole mutation, no per-row round trip. + expect(queries[0]?.text).toContain('INSERT INTO sync_changes'); + expect(queries[0]?.text.split('(').length).toBeGreaterThan(2); }); }); diff --git a/apps/api/test/postgres-tree-batch.test.ts b/apps/api/test/postgres-tree-batch.test.ts index a0f8585..fdedf68 100644 --- a/apps/api/test/postgres-tree-batch.test.ts +++ b/apps/api/test/postgres-tree-batch.test.ts @@ -1,7 +1,7 @@ import type { V2Mutation } from '@devtodo/contracts'; import { describe, expect, it } from 'vitest'; -import type { PostgresStore } from '../src/postgres-store.js'; -import { PostgresTreeStore } from '../src/postgres-tree-store.js'; +import type { SqlStore } from '../src/sql-store.js'; +import { SqlTreeStore } from '../src/sql-tree-store.js'; import { MemoryTreeStore } from '../src/tree-store.js'; function mutation( @@ -38,7 +38,7 @@ function emptyOwnerRow(cursor: string) { }; } -describe('PostgresTreeStore batch push', () => { +describe('SqlTreeStore batch push', () => { it('serializes each owner independently without a process-global write barrier', async () => { const store = new MemoryTreeStore(); const events: string[] = []; @@ -136,8 +136,8 @@ describe('PostgresTreeStore batch push', () => { }, appendV2Changes: async () => '1', subscribeChanges: () => () => undefined, - } as unknown as PostgresStore; - const tree = new PostgresTreeStore(postgres); + } as unknown as SqlStore; + const tree = new SqlTreeStore(postgres); const pending = tree.applyMutationIdempotent('owner-1', 'client-1', { mutationId: '00000000-0000-7000-8000-000000000021', command: 'folder.create', @@ -184,8 +184,8 @@ describe('PostgresTreeStore batch push', () => { return cursor; }, subscribeChanges: () => () => undefined, - } as unknown as PostgresStore; - const tree = new PostgresTreeStore(postgres); + } as unknown as SqlStore; + const tree = new SqlTreeStore(postgres); const first = await tree.applyMutationIdempotent('owner-1', 'client-1', { mutationId: '00000000-0000-7000-8000-000000000041', command: 'task.create', @@ -240,8 +240,8 @@ describe('PostgresTreeStore batch push', () => { return { rows: [], rowCount: 0 }; }, v2ReadSnapshot: async (fn: () => T | Promise) => fn(), - } as unknown as PostgresStore; - const tree = new PostgresTreeStore(postgres); + } as unknown as SqlStore; + const tree = new SqlTreeStore(postgres); await tree.prepare('owner-1'); expect(tree.listTimePoints('owner-1', 'DATE').map((point) => point.localDate)).toEqual([ '2026-09-04', @@ -265,8 +265,8 @@ describe('PostgresTreeStore batch push', () => { return { rows: [], rowCount: 0 }; }, v2ReadSnapshot: async (fn: () => T | Promise) => fn(), - } as unknown as PostgresStore; - const tree = new PostgresTreeStore(postgres); + } as unknown as SqlStore; + const tree = new SqlTreeStore(postgres); await tree.prepare('owner-1'); expect(tree.listFolders('owner-1')).toHaveLength(0); @@ -331,8 +331,8 @@ describe('PostgresTreeStore batch push', () => { return { rows: [], rowCount: 0 }; }, v2ReadSnapshot: async (fn: () => T | Promise) => fn(), - } as unknown as PostgresStore; - const tree = new PostgresTreeStore(postgres); + } as unknown as SqlStore; + const tree = new SqlTreeStore(postgres); await tree.prepare('owner-1'); cursor = '2'; await tree.prepare('owner-1'); @@ -399,8 +399,8 @@ describe('PostgresTreeStore batch push', () => { return { rows: [], rowCount: 0 }; }, v2ReadSnapshot: async (fn: () => T | Promise) => fn(), - } as unknown as PostgresStore; - const tree = new PostgresTreeStore(postgres, cache); + } as unknown as SqlStore; + const tree = new SqlTreeStore(postgres, cache); await tree.prepare('owner-1'); expect(fullLoadQueries).toBe(0); expect(tree.listFolders('owner-1').map((folder) => folder.title)).toEqual(['Cached folder']); @@ -461,9 +461,9 @@ describe('PostgresTreeStore batch push', () => { savepoints.push(`rollback:${name}`); }, subscribeChanges: () => () => undefined, - } as unknown as PostgresStore; + } as unknown as SqlStore; - const tree = new PostgresTreeStore(postgres); + const tree = new SqlTreeStore(postgres); const folderId = '00000000-0000-7000-8000-000000000001'; const results = await tree.applyMutationBatch('owner-1', 'client-1', [ mutation('00000000-0000-7000-8000-000000000011', 'folder.create', folderId, null, { @@ -497,7 +497,10 @@ describe('PostgresTreeStore batch push', () => { expect(savepoints).toContain('rollback:mutation_1'); expect( batchSql.some( - (sql) => sql.includes('INSERT INTO folders') && sql.includes('SELECT x.id, $1,'), + (sql) => + sql.includes('INSERT INTO folders') && + sql.includes('ON CONFLICT (id) DO UPDATE') && + sql.includes('WHERE folders.owner_id = EXCLUDED.owner_id'), ), ).toBe(true); }); @@ -543,8 +546,8 @@ describe('PostgresTreeStore batch push', () => { rollbackToSavepoint: async (name: string) => { savepoints.push(`rollback:${name}`); }, - } as unknown as PostgresStore; - const tree = new PostgresTreeStore(postgres); + } as unknown as SqlStore; + const tree = new SqlTreeStore(postgres); const folderId = '00000000-0000-7000-8000-000000000001'; await expect( tree.applyMutationBatch('owner-1', 'client-1', [ diff --git a/apps/api/test/store.test.ts b/apps/api/test/store.test.ts index c84cc4e..a4b3d35 100644 --- a/apps/api/test/store.test.ts +++ b/apps/api/test/store.test.ts @@ -3,7 +3,7 @@ import type { Pool, PoolClient } from 'pg'; import { DomainError } from '@devtodo/domain'; -import { PostgresStore } from '../src/postgres-store.js'; +import { SqlStore } from '../src/sql-store.js'; import { MemoryStore } from '../src/store.js'; function fixture() { @@ -27,7 +27,7 @@ describe('DevTodo store invariants', () => { connect: async () => client, end: async () => undefined, } as unknown as Pool; - const store = new PostgresStore(pool); + const store = new SqlStore(pool); await store.withMutation(async () => { await ( @@ -68,7 +68,7 @@ describe('DevTodo store invariants', () => { connect: async () => client, end: async () => undefined, } as unknown as Pool; - const store = new PostgresStore(pool); + const store = new SqlStore(pool); await expect( store.withMutation(() => @@ -337,7 +337,7 @@ describe('DevTodo store invariants', () => { expect(() => store.deleteTimePoint(owner.id, date.id, date.version)).toThrow(DomainError); }); - it('verifies PostgresTreeStore transaction locking, rollback, and no notification on error', async () => { + it('verifies SqlTreeStore transaction locking, rollback, and no notification on error', async () => { const queries: Array<{ text: string; values: unknown[] }> = []; const client = { query: async (text: string, values: unknown[] = []) => { @@ -410,9 +410,9 @@ describe('DevTodo store invariants', () => { end: async () => undefined, } as unknown as Pool; - const postgres = new PostgresStore(pool); - const { PostgresTreeStore } = await import('../src/postgres-tree-store.js'); - const treeStore = new PostgresTreeStore(postgres); + const postgres = new SqlStore(pool); + const { SqlTreeStore } = await import('../src/sql-tree-store.js'); + const treeStore = new SqlTreeStore(postgres); const notifications: string[] = []; treeStore.subscribeChanges((_ownerId, cursor) => notifications.push(cursor)); diff --git a/packages/database/src/dialect.ts b/packages/database/src/dialect.ts index 7b646c0..c0ea875 100644 --- a/packages/database/src/dialect.ts +++ b/packages/database/src/dialect.ts @@ -122,6 +122,13 @@ function prepareMysql(sql: string, values: readonly unknown[]): PreparedQuery { continue; } } + if ((char === 'I' || char === 'i') && /^ilike\b/i.test(sql.slice(index))) { + // MySQL's default collation is case-insensitive, so LIKE matches what + // PostgreSQL's ILIKE matches; the wildcard escaping is identical. + text += 'LIKE'; + index += 5; + continue; + } if (char === ':' && sql[index + 1] === ':') { const afterCast = skipCast(sql, index); if (afterCast > index) { diff --git a/packages/database/src/pool.ts b/packages/database/src/pool.ts index 3cf0cf1..c57f2a3 100644 --- a/packages/database/src/pool.ts +++ b/packages/database/src/pool.ts @@ -62,14 +62,50 @@ export function openDatabase(settings: DatabaseSettings): SqlDatabase { : new PostgresDatabase(settings); } +/** + * Wrap an already-created node-postgres pool. Used by tests and by scripts that + * build their own pool; application startup goes through `openDatabase`. + */ +export function databaseFromPool(pool: pg.Pool): SqlDatabase { + return new PostgresDatabase(poolSettings(pool), pool); +} + +function poolSettings(pool: pg.Pool): DatabaseSettings { + const options = (pool.options ?? {}) as pg.PoolOptions; + return { + dialect: 'postgres', + connectionString: options.connectionString, + host: typeof options.host === 'string' ? options.host : undefined, + port: options.port, + user: typeof options.user === 'string' ? options.user : undefined, + password: typeof options.password === 'string' ? options.password : undefined, + database: options.database, + applicationName: + typeof options.application_name === 'string' ? options.application_name : 'devtodo-api', + max: options.max ?? 10, + maxUses: options.maxUses ?? 1_000, + connectionTimeoutMillis: options.connectionTimeoutMillis ?? 5_000, + idleTimeoutMillis: options.idleTimeoutMillis ?? 30_000, + queryTimeoutMillis: (options.query_timeout as number | undefined) ?? 30_000, + statementTimeoutMillis: (options.statement_timeout as number | undefined) ?? 30_000, + lockTimeoutMillis: (options.lock_timeout as number | undefined) ?? 5_000, + keepAlive: options.keepAlive ?? true, + keepAliveInitialDelayMillis: options.keepAliveInitialDelayMillis ?? 10_000, + listenEnabled: true, + }; +} + class PostgresDatabase implements SqlDatabase { readonly dialect = 'postgres' as const; readonly maxConnections: number; private readonly pool: pg.Pool; - constructor(readonly settings: DatabaseSettings) { + constructor( + readonly settings: DatabaseSettings, + pool?: pg.Pool, + ) { this.maxConnections = settings.max; - this.pool = new pg.Pool(poolConfigFromRuntime(runtimeConfig(settings))); + this.pool = pool ?? new pg.Pool(poolConfigFromRuntime(runtimeConfig(settings))); } async query>( diff --git a/scripts/integration-smoke.ts b/scripts/integration-smoke.ts index ebac601..0a18995 100644 --- a/scripts/integration-smoke.ts +++ b/scripts/integration-smoke.ts @@ -1,6 +1,6 @@ import { spawn } from 'node:child_process'; import { createPool, databaseReady, runMigrations } from '../packages/database/src/index.js'; -import { PostgresStore } from '../apps/api/src/postgres-store.js'; +import { SqlStore } from '../apps/api/src/sql-store.js'; import { uuidv7, type Mutation } from '../packages/contracts/src/index.js'; import { DomainError } from '../packages/domain/src/index.js'; import { assertIsolatedEmptyDatabase, isolatedTestDatabaseUrl } from './test-database-safety.js'; @@ -104,9 +104,9 @@ async function main(): Promise { } const cleanupPool = createPool(databaseUrl); - const stores: PostgresStore[] = []; - const closedStores = new Set(); - const closeStore = async (store: PostgresStore): Promise => { + const stores: SqlStore[] = []; + const closedStores = new Set(); + const closeStore = async (store: SqlStore): Promise => { if (closedStores.has(store)) return; closedStores.add(store); await store.close(); @@ -124,10 +124,10 @@ async function main(): Promise { ); } - const primary = new PostgresStore(createPool(databaseUrl), { + const primary = new SqlStore(createPool(databaseUrl), { clock: () => new Date('2026-09-04T10:00:00.000Z'), }); - const peer = new PostgresStore(createPool(databaseUrl), { + const peer = new SqlStore(createPool(databaseUrl), { clock: () => new Date('2026-09-04T10:00:00.000Z'), }); stores.push(primary, peer); @@ -333,7 +333,7 @@ async function main(): Promise { const cursorBeforeRestart = (await primary.syncStatus(owner.id)).cursor; await closeStore(peer); await closeStore(primary); - const restartStore = new PostgresStore(createPool(databaseUrl), { + const restartStore = new SqlStore(createPool(databaseUrl), { clock: () => new Date('2026-09-04T10:00:00.000Z'), }); stores.push(restartStore); @@ -358,9 +358,9 @@ async function main(): Promise { 'placement did not survive restart', ); - // Verify v2 PostgresTreeStore invariants: transaction lock, delete preview, cascade deletion - const { PostgresTreeStore } = await import('../apps/api/src/postgres-tree-store.js'); - const v2Store = new PostgresTreeStore(restartStore); + // Verify v2 SqlTreeStore invariants: transaction lock, delete preview, cascade deletion + const { SqlTreeStore } = await import('../apps/api/src/sql-tree-store.js'); + const v2Store = new SqlTreeStore(restartStore); const v2FolderId = uuidv7(); const v2TaskId = uuidv7(); const v2CursorBefore = (await restartStore.syncStatusV2(owner.id)).cursor; @@ -387,7 +387,7 @@ async function main(): Promise { v2CursorAfterWrite !== v2CursorBefore, 'v2 mutation did not advance the database cursor', ); - const reloadedV2Store = new PostgresTreeStore(restartStore); + const reloadedV2Store = new SqlTreeStore(restartStore); const reloadedSnapshot = await reloadedV2Store.snapshot(owner.id); const { BootstrapSnapshots } = await import('../apps/api/src/sync-bootstrap.js'); const bootstrap = new BootstrapSnapshots(reloadedV2Store); @@ -417,7 +417,7 @@ async function main(): Promise { ); const directDetail = await bootstrap.detail(owner.id, v2TaskId); const { DirectoryReads } = await import('../apps/api/src/directory-reads.js'); - const directoryStore = new PostgresTreeStore(restartStore); + const directoryStore = new SqlTreeStore(restartStore); directoryStore.prepare = async () => { throw new Error('directory read loaded the full owner projection'); }; @@ -474,7 +474,7 @@ async function main(): Promise { 'direct task detail lost the readable folder path', ); // Focused mobile edits must reuse domain validation without prepare/full-account reads. - const focusedStore = new PostgresTreeStore(restartStore); + const focusedStore = new SqlTreeStore(restartStore); focusedStore.prepare = async () => { throw new Error('focused mutation loaded the entire owner'); }; @@ -534,10 +534,10 @@ async function main(): Promise { 'focused retry lost the edit', ); const manifest = await bootstrap.create(owner.id, '2026-09-04'); - const bootstrapPeer = new PostgresStore(createPool(databaseUrl)); + const bootstrapPeer = new SqlStore(createPool(databaseUrl)); stores.push(bootstrapPeer); await bootstrapPeer.init(); - const peerBootstrap = new BootstrapSnapshots(new PostgresTreeStore(bootstrapPeer)); + const peerBootstrap = new BootstrapSnapshots(new SqlTreeStore(bootstrapPeer)); const peerManifest = await peerBootstrap.create(owner.id, '2026-09-04'); assert( peerManifest.snapshotId === manifest.snapshotId, @@ -583,16 +583,16 @@ async function main(): Promise { assert(expired.rowCount === 0, 'bounded background cleanup retained expired snapshot'); assert( reloadedSnapshot.folders.some((folder) => folder.id === v2FolderId), - 'v2 folder did not survive a PostgresTreeStore restart', + 'v2 folder did not survive a SqlTreeStore restart', ); assert( reloadedSnapshot.tasks.some((task) => task.id === v2TaskId), - 'v2 task did not survive a PostgresTreeStore restart', + 'v2 task did not survive a SqlTreeStore restart', ); const reloadedPull = await reloadedV2Store.pull(owner.id, v2CursorBefore, 500); assert( reloadedPull.changes.some((change) => change.entityId === v2TaskId), - 'v2 pull did not read durable changes after a PostgresTreeStore restart', + 'v2 pull did not read durable changes after a SqlTreeStore restart', ); // Section 7.2 invariant: a newly created root Task must not be sorted above @@ -651,7 +651,7 @@ async function main(): Promise { ); console.log( - `PASS: PostgreSQL migration, row-level transaction, owner isolation, two-process concurrency, idempotency, rollback, restart, multi-placement invariants, and PostgresTreeStore v2 delete preview/tree deletion; cursor ${afterRestart.cursor}`, + `PASS: PostgreSQL migration, row-level transaction, owner isolation, two-process concurrency, idempotency, rollback, restart, multi-placement invariants, and SqlTreeStore v2 delete preview/tree deletion; cursor ${afterRestart.cursor}`, ); } finally { for (const store of stores) { diff --git a/scripts/perf-capacity.ts b/scripts/perf-capacity.ts index 78c2de2..50c5414 100644 --- a/scripts/perf-capacity.ts +++ b/scripts/perf-capacity.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import os from 'node:os'; import { performance } from 'node:perf_hooks'; -import { PostgresStore, type PostgresPage } from '../apps/api/src/postgres-store.js'; +import { SqlStore, type PostgresPage } from '../apps/api/src/sql-store.js'; import { createPool, runMigrations, type Pool } from '../packages/database/src/index.js'; import { assertIsolatedEmptyDatabase, isolatedTestDatabaseUrl } from './test-database-safety.js'; @@ -188,7 +188,7 @@ async function main(): Promise { const firstDateId = idFor(runKey, 'date', 1); const username = `perf-${process.pid}-${Date.now()}`; const pool = createPool(databaseUrl); - const store = new PostgresStore(pool); + const store = new SqlStore(pool); let actualOwnerId: string | undefined; try { await assertIsolatedEmptyDatabase(pool, 'capacity benchmark'); diff --git a/scripts/pg-concurrency-worker.ts b/scripts/pg-concurrency-worker.ts index d0ac119..1edbd8f 100644 --- a/scripts/pg-concurrency-worker.ts +++ b/scripts/pg-concurrency-worker.ts @@ -1,5 +1,5 @@ import { createPool } from '../packages/database/src/index.js'; -import { PostgresStore } from '../apps/api/src/postgres-store.js'; +import { SqlStore } from '../apps/api/src/sql-store.js'; import { isolatedTestDatabaseUrl } from './test-database-safety.js'; const databaseUrl = isolatedTestDatabaseUrl(); @@ -17,7 +17,7 @@ async function main(): Promise { throw new Error('DEVTODO_WORKER_COUNT must be an integer from 1 to 100'); const pool = createPool(databaseUrl); - const store = new PostgresStore(pool); + const store = new SqlStore(pool); try { await store.init(); const references: string[] = []; diff --git a/scripts/sync-bootstrap-benchmark.ts b/scripts/sync-bootstrap-benchmark.ts index 69e5c04..2a03e32 100644 --- a/scripts/sync-bootstrap-benchmark.ts +++ b/scripts/sync-bootstrap-benchmark.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import { createPool, runMigrations } from '../packages/database/src/index.js'; -import { PostgresStore } from '../apps/api/src/postgres-store.js'; -import { PostgresTreeStore } from '../apps/api/src/postgres-tree-store.js'; +import { SqlStore } from '../apps/api/src/sql-store.js'; +import { SqlTreeStore } from '../apps/api/src/sql-tree-store.js'; import { BootstrapSnapshots } from '../apps/api/src/sync-bootstrap.js'; import { DirectoryReads } from '../apps/api/src/directory-reads.js'; import { assertIsolatedEmptyDatabase, isolatedTestDatabaseUrl } from './test-database-safety.js'; @@ -11,7 +11,7 @@ const databaseUrl = isolatedTestDatabaseUrl(); if (!databaseUrl) throw new Error('TASKDOCK_TEST_DATABASE_URL must point to an isolated empty test database'); const pool = createPool(databaseUrl); -const store = new PostgresStore(pool); +const store = new SqlStore(pool); const taskCount = Number(process.env['BOOTSTRAP_BENCH_TASKS'] ?? 10_000); if (!Number.isInteger(taskCount) || taskCount < 100 || taskCount > 100_000) throw new Error('invalid task count'); @@ -36,7 +36,7 @@ try { SELECT gen_random_uuid(), owner_id, id, repeat('大备注内容',400), 1 FROM tasks WHERE owner_id=$1`, [ownerId], ); - const optimized = new BootstrapSnapshots(new PostgresTreeStore(store)); + const optimized = new BootstrapSnapshots(new SqlTreeStore(store)); const measure = async (read: () => Promise) => { const samples: number[] = []; let bytes = 0; @@ -55,10 +55,10 @@ try { responseBytes: bytes, }; }; - const full = await measure(() => new PostgresTreeStore(store).snapshot(owner.id)); + const full = await measure(() => new SqlTreeStore(store).snapshot(owner.id)); const preview = await measure(() => optimized.preview(owner.id, '2026-10-13', null)); const today = await measure(() => optimized.today(owner.id, '2026-10-13')); - const directoryTree = new PostgresTreeStore(store); + const directoryTree = new SqlTreeStore(store); directoryTree.prepare = async () => { throw new Error('directory benchmark loaded full projection'); }; @@ -91,7 +91,7 @@ try { let version = 1; const benchmarkClient = randomUUID(); const focusedEdit = await measure(() => - new PostgresTreeStore(store).applyMutationIdempotent(owner.id, benchmarkClient, { + new SqlTreeStore(store).applyMutationIdempotent(owner.id, benchmarkClient, { mutationId: randomUUID(), command: 'task.update', entityId: taskId, From c097dbccbf20412d7ae411e8dfecd704eacea19e Mon Sep 17 00:00:00 2001 From: IGNGserver <176017966+IGNGserver@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:05:32 +0800 Subject: [PATCH 4/9] =?UTF-8?q?feat(db):=20=E5=A2=9E=E5=8A=A0=20MySQL=20?= =?UTF-8?q?=E8=BF=81=E7=A7=BB=E5=9F=BA=E7=BA=BF=E4=B8=8E=E5=8F=8C=E5=BC=95?= =?UTF-8?q?=E6=93=8E=E8=BF=81=E7=A7=BB=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PostgreSQL 迁移移入 migrations/postgres,新增 MySQL 8.4 合并基线 0013_baseline.sql:部分唯一索引改为生成列、时间戳用 DATETIME(6)、 jsonb 换 JSON、rank/cursor 反引号引用、子外键全部级联删除 - 迁移 runner 按引擎分支:MySQL 用 GET_LOCK 串行、逐文件提交,并按 语句拆分脚本(连接未开多语句) - 方言层自动引用 MySQL 保留字 rank/cursor,并翻译 ILIKE - 运行时冒烟脚本改为双引擎 sql-runtime-smoke:MySQL 用 KILL 会话模拟 连接中断,PostgreSQL 仍走 pg_ctl 重启 - 新增迁移矩阵一致性测试(版本对齐、表集合覆盖、保留字引用) --- apps/api/src/sql-statements.ts | 4 +- apps/api/src/sql-store.ts | 20 +- apps/api/src/sql-tree-store.ts | 2 +- apps/api/src/sync-bootstrap.ts | 6 +- package.json | 5 +- .../migrations/mysql/0013_baseline.sql | 399 ++++++++++++++++++ .../migrations/{ => postgres}/0001_init.sql | 0 .../0002_migration_checksums.sql | 0 .../0003_native_auth_challenges.sql | 0 .../{ => postgres}/0004_keyset_indexes.sql | 0 .../{ => postgres}/0005_v2_structure.sql | 0 .../{ => postgres}/0006_v2_backfill.sql | 0 .../{ => postgres}/0007_v2_constraints.sql | 0 .../0008_sync_protocol_version.sql | 0 .../0009_v2_folder_first_rank.sql | 0 .../{ => postgres}/0010_remove_archive.sql | 0 .../{ => postgres}/0011_sync_bootstrap.sql | 0 .../0012_mobile_sync_read_indexes.sql | 0 .../{ => postgres}/0013_folder_status.sql | 0 packages/database/src/dialect.ts | 66 +++ packages/database/src/migrate.ts | 14 +- packages/database/test/parity.test.ts | 69 +++ scripts/migrate.ts | 30 +- ...-runtime-smoke.ts => sql-runtime-smoke.ts} | 165 ++++++-- 24 files changed, 707 insertions(+), 73 deletions(-) create mode 100644 packages/database/migrations/mysql/0013_baseline.sql rename packages/database/migrations/{ => postgres}/0001_init.sql (100%) rename packages/database/migrations/{ => postgres}/0002_migration_checksums.sql (100%) rename packages/database/migrations/{ => postgres}/0003_native_auth_challenges.sql (100%) rename packages/database/migrations/{ => postgres}/0004_keyset_indexes.sql (100%) rename packages/database/migrations/{ => postgres}/0005_v2_structure.sql (100%) rename packages/database/migrations/{ => postgres}/0006_v2_backfill.sql (100%) rename packages/database/migrations/{ => postgres}/0007_v2_constraints.sql (100%) rename packages/database/migrations/{ => postgres}/0008_sync_protocol_version.sql (100%) rename packages/database/migrations/{ => postgres}/0009_v2_folder_first_rank.sql (100%) rename packages/database/migrations/{ => postgres}/0010_remove_archive.sql (100%) rename packages/database/migrations/{ => postgres}/0011_sync_bootstrap.sql (100%) rename packages/database/migrations/{ => postgres}/0012_mobile_sync_read_indexes.sql (100%) rename packages/database/migrations/{ => postgres}/0013_folder_status.sql (100%) create mode 100644 packages/database/test/parity.test.ts rename scripts/{pg-runtime-smoke.ts => sql-runtime-smoke.ts} (78%) diff --git a/apps/api/src/sql-statements.ts b/apps/api/src/sql-statements.ts index 1cfc2f6..87a7ec1 100644 --- a/apps/api/src/sql-statements.ts +++ b/apps/api/src/sql-statements.ts @@ -310,7 +310,7 @@ export const OWNER_SNAPSHOT_QUERY: Record = { (SELECT u.next_task_number FROM users u WHERE u.id = $1 AND u.disabled_at IS NULL) AS next_task_number, (SELECT COALESCE(MAX(sc.seq), 0)::text FROM sync_changes sc - WHERE sc.owner_id = $1) AS cursor, + WHERE sc.owner_id = $1) AS \`cursor\`, COALESCE((SELECT jsonb_agg(to_jsonb(f) || jsonb_build_object('rank', f.rank::text) ORDER BY f.id) FROM folders f WHERE f.owner_id = $1), '[]'::jsonb) AS folders, COALESCE((SELECT jsonb_agg(to_jsonb(t) || jsonb_build_object('rank', t.rank::text) ORDER BY t.id) @@ -334,7 +334,7 @@ export const OWNER_SNAPSHOT_QUERY: Record = { (SELECT u.next_task_number FROM users u WHERE u.id = $1 AND u.disabled_at IS NULL) AS next_task_number, (SELECT CAST(COALESCE(MAX(sc.seq), 0) AS CHAR) FROM sync_changes sc - WHERE sc.owner_id = $1) AS cursor, + WHERE sc.owner_id = $1) AS \`cursor\`, ${SNAPSHOT_COLLECTIONS.map(mysqlCollectionAggregate).join(',\n ')}, ${mysqlSettingsAggregate()}`, }; diff --git a/apps/api/src/sql-store.ts b/apps/api/src/sql-store.ts index aab496d..ec1ed6c 100644 --- a/apps/api/src/sql-store.ts +++ b/apps/api/src/sql-store.ts @@ -382,6 +382,11 @@ export class SqlStore implements Store { for (const [ownerId, cursor] of context.pending) await this.publishCursor(session, ownerId, cursor); await session.commit(); + // PostgreSQL delivers the commit notification through its LISTEN + // connection. MySQL has no NOTIFY, so the in-process observers are the + // only fan-out and must fire after the commit succeeded. + if (session.dialect !== 'postgres') + for (const [ownerId, cursor] of context.pending) this.notify(ownerId, cursor); return result; }); return value; @@ -520,7 +525,11 @@ export class SqlStore implements Store { const tuples: string[] = []; const values: unknown[] = []; for (const change of changes) { - tuples.push('(?, ?, ?, ?, ?, CAST(? AS JSON), ?, 2)'); + const base = values.length; + const at = (offset: number) => `$${base + offset}`; + tuples.push( + `(${at(1)}, ${at(2)}, ${at(3)}, ${at(4)}, ${at(5)}, CAST(${at(6)} AS JSON), ${at(7)}, 2)`, + ); values.push( ownerId, change.entityType, @@ -683,7 +692,10 @@ export class SqlStore implements Store { } async createOwner(username: string, passwordHash: string): Promise { - await this.query("SELECT pg_advisory_xact_lock(hashtextextended('devtodo:bootstrap', 0))"); + // PostgreSQL serialises concurrent bootstraps with an advisory lock; MySQL + // relies on the unique username index (only one bootstrap can win). + if (this.database.dialect === 'postgres') + await this.query("SELECT pg_advisory_xact_lock(hashtextextended('devtodo:bootstrap', 0))"); if (await this.hasOwner()) throw new DomainError('BOOTSTRAP_ALREADY_COMPLETED', 'Owner 已初始化'); const normalizedUsername = normalizeUsername(username); @@ -2252,7 +2264,7 @@ export class SqlStore implements Store { ): Promise<{ cursor: string; oldestCursor: string; protocolVersion: 1 }> { await this.owner(ownerId); const current = await this.query( - 'SELECT COALESCE(MAX(seq), 0)::text AS cursor FROM sync_changes WHERE owner_id = $1', + 'SELECT COALESCE(MAX(seq), 0)::text AS `cursor` FROM sync_changes WHERE owner_id = $1', [ownerId], ); const first = await this.query('SELECT seq FROM sync_changes ORDER BY seq LIMIT 1'); @@ -2318,7 +2330,7 @@ export class SqlStore implements Store { const result = await this.query( `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1 AND disabled_at IS NULL) AS owner_ok, (SELECT COALESCE(MAX(seq), 0)::text FROM sync_changes - WHERE owner_id = $1 AND protocol_version = 2) AS cursor, + WHERE owner_id = $1 AND protocol_version = 2) AS \`cursor\`, (SELECT seq FROM sync_changes WHERE protocol_version = 2 ORDER BY seq LIMIT 1) AS oldest`, [ownerId], diff --git a/apps/api/src/sql-tree-store.ts b/apps/api/src/sql-tree-store.ts index 243b25d..39787a0 100644 --- a/apps/api/src/sql-tree-store.ts +++ b/apps/api/src/sql-tree-store.ts @@ -45,7 +45,7 @@ export class SqlTreeStore extends MemoryTreeStore implements V2TreeStore { const current = await this.sql.v2Query( `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1 AND disabled_at IS NULL) AS owner_ok, (SELECT next_task_number FROM users WHERE id = $1 AND disabled_at IS NULL) AS next_task_number, - (SELECT COALESCE(MAX(seq), 0)::text FROM sync_changes WHERE owner_id = $1) AS cursor`, + (SELECT COALESCE(MAX(seq), 0)::text FROM sync_changes WHERE owner_id = $1) AS \`cursor\``, [ownerId], ); const currentRow = current.rows[0]; diff --git a/apps/api/src/sync-bootstrap.ts b/apps/api/src/sync-bootstrap.ts index 2563541..b15d1ba 100644 --- a/apps/api/src/sync-bootstrap.ts +++ b/apps/api/src/sync-bootstrap.ts @@ -394,7 +394,7 @@ export class BootstrapSnapshots { // remaining lifetime to resume it. This also works across API instances. const fiveMinutes = sql.dialect === 'mysql' ? 'INTERVAL 5 MINUTE' : "interval '5 minutes'"; const reusable = await sql.v2Query( - `SELECT id, expires_at, page_count FROM sync_bootstrap_snapshots WHERE owner_id = $1 AND cursor = $2 AND expires_at > now() + ${fiveMinutes} AND page_count > 0 ORDER BY expires_at DESC LIMIT 1`, + `SELECT id, expires_at, page_count FROM sync_bootstrap_snapshots WHERE owner_id = $1 AND \`cursor\` = $2 AND expires_at > now() + ${fiveMinutes} AND page_count > 0 ORDER BY expires_at DESC LIMIT 1`, [ownerId, cursor], ); const previous = reusable.rows[0]; @@ -409,7 +409,7 @@ export class BootstrapSnapshots { const snapshotId = randomUUID(); const expiresAt = new Date(Date.now() + ttlMs).toISOString(); await sql.v2Query( - 'INSERT INTO sync_bootstrap_snapshots(id,owner_id,cursor,expires_at,page_count) VALUES($1,$2,$3,$4,0)', + 'INSERT INTO sync_bootstrap_snapshots(id,owner_id,`cursor`,expires_at,page_count) VALUES($1,$2,$3,$4,0)', [snapshotId, ownerId, cursor, expiresAt], ); let page: BootstrapChange[] = []; @@ -542,7 +542,7 @@ export class BootstrapSnapshots { let items: BootstrapChange[] | undefined; if (this.tree instanceof SqlTreeStore) { const rows = await this.tree.sql.v2Query( - 'SELECT s.cursor::text, s.expires_at, s.page_count, p.payload FROM sync_bootstrap_snapshots s JOIN sync_bootstrap_pages p ON p.snapshot_id = s.id WHERE s.id = $1 AND s.owner_id = $2 AND p.page = $3 AND s.expires_at > now()', + 'SELECT s.`cursor`::text, s.expires_at, s.page_count, p.payload FROM sync_bootstrap_snapshots s JOIN sync_bootstrap_pages p ON p.snapshot_id = s.id WHERE s.id = $1 AND s.owner_id = $2 AND p.page = $3 AND s.expires_at > now()', [id, ownerId, page], ); const row = rows.rows[0]; diff --git a/package.json b/package.json index e0710ed..06bcea9 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "typecheck": "pnpm -r --if-present build && pnpm -r --if-present typecheck && tsc -p tsconfig.scripts.json --noEmit", "test": "vitest run", "test:integration": "tsx scripts/integration-smoke.ts", - "test:pg-runtime": "tsx scripts/pg-runtime-smoke.ts", + "test:pg-runtime": "tsx scripts/sql-runtime-smoke.ts", "test:e2e": "tsx scripts/e2e-gate.ts", "a11y": "playwright test apps/web/e2e/a11y.spec.ts --project=chromium", "format:check": "prettier --check .", @@ -34,7 +34,8 @@ "db:migrate": "tsx scripts/migrate.ts", "openapi:check": "tsx scripts/openapi-check.ts", "verify:m3e:web": "tsx scripts/verify-m3e-web.ts", - "test:mysql-compat": "tsx scripts/mysql-compat.ts" + "test:mysql-compat": "tsx scripts/mysql-compat.ts", + "test:mysql-runtime": "tsx scripts/sql-runtime-smoke.ts" }, "devDependencies": { "@axe-core/playwright": "^4.13.0", diff --git a/packages/database/migrations/mysql/0013_baseline.sql b/packages/database/migrations/mysql/0013_baseline.sql new file mode 100644 index 0000000..b11009d --- /dev/null +++ b/packages/database/migrations/mysql/0013_baseline.sql @@ -0,0 +1,399 @@ +-- TaskDock MySQL 8.4 baseline: the consolidated schema of PostgreSQL +-- migrations 0001..0013. A new MySQL hub starts here; later schema changes +-- ship one file per engine with the same version number. +-- +-- Structural differences from PostgreSQL, all deliberate: +-- * partial unique indexes become STORED generated columns whose value is +-- NULL once the row is soft-deleted (a unique index ignores NULLs), so a +-- live row and any number of tombstones can coexist; +-- * partial (non-unique) indexes become full indexes; tombstones stay in the +-- index and cost a little space; +-- * trigram GIN indexes have no MySQL equivalent, so search stays a +-- LIKE '%term%' scan bounded by owner_id; +-- * `rank` and `cursor` are reserved words and are always backticked; +-- * every child foreign key cascades on delete, so removing an Owner is one +-- statement regardless of the order MySQL walks the cascade graph; the +-- application itself only soft-deletes; +-- * timestamps are DATETIME(6) in UTC (the session time zone is pinned to +-- +00:00 by the driver), JSON replaces jsonb. + +CREATE TABLE IF NOT EXISTS users ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + username VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + password_hash VARCHAR(255) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + next_misc_task_number INT NOT NULL DEFAULT 1, + next_task_number INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + disabled_at DATETIME(6) NULL, + PRIMARY KEY (id), + UNIQUE KEY users_username_uq (username), + CONSTRAINT users_next_misc_task_number_ck CHECK (next_misc_task_number > 0), + CONSTRAINT users_next_task_number_ck CHECK (next_task_number > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS user_settings ( + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + timezone VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'Asia/Shanghai', + week_starts_on SMALLINT NOT NULL DEFAULT 1, + default_capture_target VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'ROOT', + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + PRIMARY KEY (owner_id), + CONSTRAINT user_settings_owner_fk FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT user_settings_week_starts_on_ck CHECK (week_starts_on IN (0, 1)), + CONSTRAINT user_settings_default_capture_target_ck CHECK (default_capture_target IN ('ROOT', 'RECENT_FOLDER')), + CONSTRAINT user_settings_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS projects ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + name VARCHAR(160) NOT NULL, + task_prefix VARCHAR(10) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + next_task_number INT NOT NULL DEFAULT 1, + `rank` BIGINT NOT NULL DEFAULT 1024, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + active_key SMALLINT GENERATED ALWAYS AS (IF(deleted_at IS NULL, 1, NULL)) STORED, + PRIMARY KEY (id), + UNIQUE KEY projects_owner_id_uq (owner_id, id), + UNIQUE KEY projects_owner_prefix_uq (owner_id, task_prefix, active_key), + KEY projects_owner_rank_idx (owner_id, `rank`, id), + CONSTRAINT projects_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT projects_name_ck CHECK (CHAR_LENGTH(TRIM(name)) > 0), + CONSTRAINT projects_task_prefix_ck CHECK (REGEXP_LIKE(task_prefix, '^[A-Z][A-Z0-9]{1,9}$', 'c')), + CONSTRAINT projects_next_task_number_ck CHECK (next_task_number > 0), + CONSTRAINT projects_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS folders ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + parent_folder_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL, + title VARCHAR(160) NOT NULL, + status VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'TODO', + completed_at DATETIME(6) NULL, + `rank` BIGINT NOT NULL DEFAULT 1024, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + PRIMARY KEY (id), + UNIQUE KEY folders_owner_id_uq (owner_id, id), + KEY folders_owner_parent_rank_idx (owner_id, parent_folder_id, `rank`, id), + KEY folders_mobile_directory (owner_id, parent_folder_id, + (CASE status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END), `rank`, id), + CONSTRAINT folders_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT folders_owner_parent_fk FOREIGN KEY (owner_id, parent_folder_id) REFERENCES folders (owner_id, id) ON DELETE CASCADE, + CONSTRAINT folders_not_self_parent_ck CHECK (parent_folder_id IS NULL OR parent_folder_id <> id), + CONSTRAINT folders_status_ck CHECK (status IN ('TODO', 'IN_PROGRESS', 'DONE')), + CONSTRAINT folders_title_ck CHECK (CHAR_LENGTH(TRIM(title)) > 0), + CONSTRAINT folders_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS tasks ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + project_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL, + category VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + reference_id VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + title VARCHAR(500) NOT NULL, + status VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'TODO', + priority VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'NONE', + `rank` BIGINT NOT NULL DEFAULT 1024, + completed_at DATETIME(6) NULL, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + parent_folder_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL, + active_key SMALLINT GENERATED ALWAYS AS (IF(deleted_at IS NULL, 1, NULL)) STORED, + PRIMARY KEY (id), + UNIQUE KEY tasks_owner_id_uq (owner_id, id), + UNIQUE KEY tasks_owner_reference_uq (owner_id, reference_id, active_key), + KEY tasks_owner_rank_idx (owner_id, `rank`, id), + KEY tasks_owner_project_category_idx (owner_id, project_id, category, status, `rank`, id), + KEY tasks_owner_parent_rank_idx (owner_id, parent_folder_id, `rank`, id), + KEY tasks_mobile_directory (owner_id, parent_folder_id, + (CASE status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END), `rank`, id), + CONSTRAINT tasks_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT tasks_owner_project_fk FOREIGN KEY (owner_id, project_id) REFERENCES projects (owner_id, id) ON DELETE CASCADE, + CONSTRAINT tasks_owner_parent_folder_fk FOREIGN KEY (owner_id, parent_folder_id) REFERENCES folders (owner_id, id) ON DELETE CASCADE, + CONSTRAINT tasks_category_ck CHECK (category IN ('FEATURE', 'MISC')), + CONSTRAINT tasks_status_ck CHECK (status IN ('TODO', 'IN_PROGRESS', 'DONE')), + CONSTRAINT tasks_priority_ck CHECK (priority IN ('NONE', 'LOW', 'MEDIUM', 'HIGH')), + CONSTRAINT tasks_global_category_ck CHECK ((project_id IS NULL AND category = 'MISC') OR project_id IS NOT NULL), + CONSTRAINT tasks_title_ck CHECK (CHAR_LENGTH(TRIM(title)) > 0), + CONSTRAINT tasks_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS notes ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + task_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + content_markdown MEDIUMTEXT NOT NULL, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + active_key SMALLINT GENERATED ALWAYS AS (IF(deleted_at IS NULL, 1, NULL)) STORED, + PRIMARY KEY (id), + UNIQUE KEY notes_owner_task_uq (owner_id, task_id, active_key), + CONSTRAINT notes_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT notes_owner_task_fk FOREIGN KEY (owner_id, task_id) REFERENCES tasks (owner_id, id) ON DELETE CASCADE, + CONSTRAINT notes_content_markdown_ck CHECK (CHAR_LENGTH(content_markdown) <= 1048576), + CONSTRAINT notes_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS task_steps ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + task_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + title VARCHAR(500) NOT NULL, + note_markdown MEDIUMTEXT NOT NULL, + status VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'TODO', + `rank` BIGINT NOT NULL DEFAULT 1024, + completed_at DATETIME(6) NULL, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + PRIMARY KEY (id), + UNIQUE KEY task_steps_owner_id_uq (owner_id, id), + KEY task_steps_owner_task_rank_idx (owner_id, task_id, `rank`, id), + CONSTRAINT task_steps_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT task_steps_owner_task_fk FOREIGN KEY (owner_id, task_id) REFERENCES tasks (owner_id, id) ON DELETE CASCADE, + CONSTRAINT task_steps_status_ck CHECK (status IN ('TODO', 'IN_PROGRESS', 'DONE')), + CONSTRAINT task_steps_title_ck CHECK (CHAR_LENGTH(TRIM(title)) > 0), + CONSTRAINT task_steps_note_markdown_ck CHECK (CHAR_LENGTH(note_markdown) <= 1048576), + CONSTRAINT task_steps_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS workflows ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + name VARCHAR(200) NOT NULL, + `rank` BIGINT NOT NULL DEFAULT 1024, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + PRIMARY KEY (id), + UNIQUE KEY workflows_owner_id_uq (owner_id, id), + KEY workflows_owner_rank_idx (owner_id, `rank`, id), + CONSTRAINT workflows_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT workflows_name_ck CHECK (CHAR_LENGTH(TRIM(name)) > 0), + CONSTRAINT workflows_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS workflow_stages ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + workflow_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + name VARCHAR(200) NOT NULL, + `rank` BIGINT NOT NULL DEFAULT 1024, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + PRIMARY KEY (id), + UNIQUE KEY workflow_stages_owner_id_uq (owner_id, id), + UNIQUE KEY workflow_stages_owner_workflow_id_uq (owner_id, workflow_id, id), + KEY workflow_stages_owner_workflow_rank_idx (owner_id, workflow_id, `rank`, id), + CONSTRAINT workflow_stages_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT workflow_stages_owner_workflow_fk FOREIGN KEY (owner_id, workflow_id) REFERENCES workflows (owner_id, id) ON DELETE CASCADE, + CONSTRAINT workflow_stages_name_ck CHECK (CHAR_LENGTH(TRIM(name)) > 0), + CONSTRAINT workflow_stages_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS workflow_task_memberships ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + workflow_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + stage_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + task_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + `rank` BIGINT NOT NULL DEFAULT 1024, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + active_key SMALLINT GENERATED ALWAYS AS (IF(deleted_at IS NULL, 1, NULL)) STORED, + PRIMARY KEY (id), + UNIQUE KEY workflow_task_memberships_owner_id_uq (owner_id, id), + UNIQUE KEY workflow_memberships_active_task_uq (owner_id, workflow_id, task_id, active_key), + KEY workflow_memberships_owner_workflow_rank_idx (owner_id, workflow_id, `rank`, id), + CONSTRAINT workflow_task_memberships_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT workflow_memberships_owner_workflow_fk FOREIGN KEY (owner_id, workflow_id) REFERENCES workflows (owner_id, id) ON DELETE CASCADE, + CONSTRAINT workflow_memberships_owner_stage_fk FOREIGN KEY (owner_id, stage_id) REFERENCES workflow_stages (owner_id, id) ON DELETE CASCADE, + CONSTRAINT workflow_memberships_owner_task_fk FOREIGN KEY (owner_id, task_id) REFERENCES tasks (owner_id, id) ON DELETE CASCADE, + CONSTRAINT workflow_memberships_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS time_points ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + type VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + local_date DATE NULL, + title VARCHAR(200) NULL, + `rank` BIGINT NOT NULL DEFAULT 1024, + reached_at DATETIME(6) NULL, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + date_key SMALLINT GENERATED ALWAYS AS ( + IF(type = 'DATE' AND deleted_at IS NULL, 1, NULL) + ) STORED, + PRIMARY KEY (id), + UNIQUE KEY time_points_owner_id_uq (owner_id, id), + UNIQUE KEY time_points_owner_date_uq (owner_id, local_date, date_key), + KEY time_points_owner_type_date_idx (owner_id, type, local_date, id), + KEY time_points_owner_type_rank_idx (owner_id, type, `rank`, id), + KEY time_points_owner_type_reached_idx (owner_id, type, reached_at, id), + CONSTRAINT time_points_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT time_points_type_ck CHECK (type IN ('DATE', 'EVENT')), + CONSTRAINT time_points_shape_ck CHECK ( + (type = 'DATE' AND local_date IS NOT NULL AND title IS NULL) OR + (type = 'EVENT' AND local_date IS NULL AND title IS NOT NULL AND CHAR_LENGTH(TRIM(title)) > 0) + ), + CONSTRAINT time_points_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS placements ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + task_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + time_point_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + `rank` BIGINT NOT NULL DEFAULT 1024, + version INT NOT NULL DEFAULT 1, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL, + active_key SMALLINT GENERATED ALWAYS AS (IF(deleted_at IS NULL, 1, NULL)) STORED, + PRIMARY KEY (id), + UNIQUE KEY placements_task_timepoint_active_uq (task_id, time_point_id, active_key), + KEY placements_owner_timepoint_rank_idx (owner_id, time_point_id, `rank`), + KEY placements_owner_timepoint_rank_keyset_idx (owner_id, time_point_id, `rank`, id), + CONSTRAINT placements_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT placements_owner_task_fk FOREIGN KEY (owner_id, task_id) REFERENCES tasks (owner_id, id) ON DELETE CASCADE, + CONSTRAINT placements_owner_time_point_fk FOREIGN KEY (owner_id, time_point_id) REFERENCES time_points (owner_id, id) ON DELETE CASCADE, + CONSTRAINT placements_version_ck CHECK (version > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS devices ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + name VARCHAR(100) NOT NULL, + platform VARCHAR(40) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + last_seen_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + revoked_at DATETIME(6) NULL, + PRIMARY KEY (id), + UNIQUE KEY devices_owner_id_uq (owner_id, id), + CONSTRAINT devices_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS refresh_sessions ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + device_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + token_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + expires_at DATETIME(6) NOT NULL, + replaced_by_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL, + used_at DATETIME(6) NULL, + revoked_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (id), + UNIQUE KEY refresh_sessions_token_hash_uq (token_hash), + UNIQUE KEY refresh_sessions_owner_id_uq (owner_id, id), + CONSTRAINT refresh_sessions_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT refresh_sessions_owner_device_fk FOREIGN KEY (owner_id, device_id) REFERENCES devices (owner_id, id) ON DELETE CASCADE + -- The PostgreSQL self-reference on (owner_id, replaced_by_id) is omitted: + -- MySQL cannot SET NULL on a composite key that includes a NOT NULL column, + -- and a self-referencing cascade would delete the rotated-from session. +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS client_mutations ( + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + client_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + mutation_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + request_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + result JSON NOT NULL, + first_processed_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + expires_at DATETIME(6) NOT NULL, + PRIMARY KEY (owner_id, client_id, mutation_id), + CONSTRAINT client_mutations_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS sync_changes ( + seq BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + entity_type VARCHAR(40) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + entity_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + entity_version INT NOT NULL, + operation VARCHAR(20) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + snapshot JSON NULL, + committed_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + protocol_version SMALLINT NOT NULL DEFAULT 1, + PRIMARY KEY (seq), + KEY sync_changes_owner_seq_idx (owner_id, seq), + KEY sync_changes_protocol_owner_seq_idx (protocol_version, owner_id, seq), + CONSTRAINT sync_changes_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT sync_changes_protocol_version_ck CHECK (protocol_version IN (1, 2)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS rollover_operations ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + source_date DATE NOT NULL, + target_date DATE NOT NULL, + placement_ids JSON NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + undone_at DATETIME(6) NULL, + PRIMARY KEY (id), + CONSTRAINT rollover_operations_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS native_auth_challenges ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + origin VARCHAR(255) NOT NULL, + expires_at DATETIME(6) NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (id), + KEY native_auth_challenges_expiry_idx (expires_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS hub_identity ( + singleton TINYINT(1) NOT NULL DEFAULT 1, + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT (UUID()), + PRIMARY KEY (singleton), + CONSTRAINT hub_identity_singleton_ck CHECK (singleton = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS sync_bootstrap_snapshots ( + id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + owner_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + `cursor` BIGINT NOT NULL, + expires_at DATETIME(6) NOT NULL, + page_count INT NOT NULL, + PRIMARY KEY (id), + KEY sync_bootstrap_snapshots_owner_expiry (owner_id, expires_at), + KEY sync_bootstrap_expiry (expires_at, id), + KEY sync_bootstrap_revision (owner_id, `cursor`, expires_at), + CONSTRAINT sync_bootstrap_snapshots_owner_id_fkey FOREIGN KEY (owner_id) REFERENCES users (id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE IF NOT EXISTS sync_bootstrap_pages ( + snapshot_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + page INT NOT NULL, + payload JSON NOT NULL, + PRIMARY KEY (snapshot_id, page), + CONSTRAINT sync_bootstrap_pages_snapshot_id_fkey FOREIGN KEY (snapshot_id) REFERENCES sync_bootstrap_snapshots (id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +INSERT INTO hub_identity (singleton) VALUES (1); diff --git a/packages/database/migrations/0001_init.sql b/packages/database/migrations/postgres/0001_init.sql similarity index 100% rename from packages/database/migrations/0001_init.sql rename to packages/database/migrations/postgres/0001_init.sql diff --git a/packages/database/migrations/0002_migration_checksums.sql b/packages/database/migrations/postgres/0002_migration_checksums.sql similarity index 100% rename from packages/database/migrations/0002_migration_checksums.sql rename to packages/database/migrations/postgres/0002_migration_checksums.sql diff --git a/packages/database/migrations/0003_native_auth_challenges.sql b/packages/database/migrations/postgres/0003_native_auth_challenges.sql similarity index 100% rename from packages/database/migrations/0003_native_auth_challenges.sql rename to packages/database/migrations/postgres/0003_native_auth_challenges.sql diff --git a/packages/database/migrations/0004_keyset_indexes.sql b/packages/database/migrations/postgres/0004_keyset_indexes.sql similarity index 100% rename from packages/database/migrations/0004_keyset_indexes.sql rename to packages/database/migrations/postgres/0004_keyset_indexes.sql diff --git a/packages/database/migrations/0005_v2_structure.sql b/packages/database/migrations/postgres/0005_v2_structure.sql similarity index 100% rename from packages/database/migrations/0005_v2_structure.sql rename to packages/database/migrations/postgres/0005_v2_structure.sql diff --git a/packages/database/migrations/0006_v2_backfill.sql b/packages/database/migrations/postgres/0006_v2_backfill.sql similarity index 100% rename from packages/database/migrations/0006_v2_backfill.sql rename to packages/database/migrations/postgres/0006_v2_backfill.sql diff --git a/packages/database/migrations/0007_v2_constraints.sql b/packages/database/migrations/postgres/0007_v2_constraints.sql similarity index 100% rename from packages/database/migrations/0007_v2_constraints.sql rename to packages/database/migrations/postgres/0007_v2_constraints.sql diff --git a/packages/database/migrations/0008_sync_protocol_version.sql b/packages/database/migrations/postgres/0008_sync_protocol_version.sql similarity index 100% rename from packages/database/migrations/0008_sync_protocol_version.sql rename to packages/database/migrations/postgres/0008_sync_protocol_version.sql diff --git a/packages/database/migrations/0009_v2_folder_first_rank.sql b/packages/database/migrations/postgres/0009_v2_folder_first_rank.sql similarity index 100% rename from packages/database/migrations/0009_v2_folder_first_rank.sql rename to packages/database/migrations/postgres/0009_v2_folder_first_rank.sql diff --git a/packages/database/migrations/0010_remove_archive.sql b/packages/database/migrations/postgres/0010_remove_archive.sql similarity index 100% rename from packages/database/migrations/0010_remove_archive.sql rename to packages/database/migrations/postgres/0010_remove_archive.sql diff --git a/packages/database/migrations/0011_sync_bootstrap.sql b/packages/database/migrations/postgres/0011_sync_bootstrap.sql similarity index 100% rename from packages/database/migrations/0011_sync_bootstrap.sql rename to packages/database/migrations/postgres/0011_sync_bootstrap.sql diff --git a/packages/database/migrations/0012_mobile_sync_read_indexes.sql b/packages/database/migrations/postgres/0012_mobile_sync_read_indexes.sql similarity index 100% rename from packages/database/migrations/0012_mobile_sync_read_indexes.sql rename to packages/database/migrations/postgres/0012_mobile_sync_read_indexes.sql diff --git a/packages/database/migrations/0013_folder_status.sql b/packages/database/migrations/postgres/0013_folder_status.sql similarity index 100% rename from packages/database/migrations/0013_folder_status.sql rename to packages/database/migrations/postgres/0013_folder_status.sql diff --git a/packages/database/src/dialect.ts b/packages/database/src/dialect.ts index c0ea875..6065bc3 100644 --- a/packages/database/src/dialect.ts +++ b/packages/database/src/dialect.ts @@ -11,6 +11,10 @@ * * MySQL also receives implicit casts: `$1::date` becomes `?`. The cast exists * to satisfy PostgreSQL's type inference, and MySQL rejects the syntax. + * + * MySQL reserves `rank` and `cursor`, which the schema uses as column names. + * Bare occurrences in SQL are quoted automatically, so every statement can be + * written once. PostgreSQL leaves them bare because it does not reserve them. */ export type DatabaseDialectName = 'postgres' | 'mysql'; @@ -122,6 +126,14 @@ function prepareMysql(sql: string, values: readonly unknown[]): PreparedQuery { continue; } } + if (/[A-Za-z_]/.test(char) && !/[A-Za-z0-9_]/.test(sql[index - 1] ?? '')) { + const reserved = /^(rank|cursor)\b/i.exec(sql.slice(index)); + if (reserved && sql[index + reserved[0].length] !== '(') { + text += '`' + reserved[0] + '`'; + index += reserved[0].length; + continue; + } + } if ((char === 'I' || char === 'i') && /^ilike\b/i.test(sql.slice(index))) { // MySQL's default collation is case-insensitive, so LIKE matches what // PostgreSQL's ILIKE matches; the wildcard escaping is identical. @@ -255,3 +267,57 @@ export function normaliseMysqlValue(value: unknown): unknown { export function normaliseMysqlValues(values: readonly unknown[]): unknown[] { return values.map(normaliseMysqlValue); } + +/** + * Split a migration script into single statements. + * + * PostgreSQL accepts a whole script in one `query()` call; MySQL rejects it + * when multi-statement execution is off, which is the safe default. Semicolons + * inside strings, identifiers and comments never split. + */ +export function splitSqlStatements(sql: string): string[] { + const statements: string[] = []; + let start = 0; + let index = 0; + while (index < sql.length) { + const char = sql[index]!; + if (char === "'") { + index = skipSingleQuoted(sql, index); + continue; + } + if (char === '"') { + index = skipDoubleQuoted(sql, index); + continue; + } + if (char === '`') { + index = skipBacktick(sql, index); + continue; + } + if (char === '-' && sql[index + 1] === '-') { + index = lineCommentEnd(sql, index); + continue; + } + if (char === '/' && sql[index + 1] === '*') { + index = blockCommentEnd(sql, index); + continue; + } + if (char === '$') { + const dollar = /^\$([A-Za-z_]*)\$/.exec(sql.slice(index)); + if (dollar) { + const close = sql.indexOf(dollar[0], index + dollar[0].length); + index = close === -1 ? sql.length : close + dollar[0].length; + continue; + } + } + if (char === ';') { + statements.push(sql.slice(start, index + 1).trim()); + start = index + 1; + index += 1; + continue; + } + index += 1; + } + const tail = sql.slice(start).trim(); + if (tail) statements.push(tail); + return statements.filter((statement) => statement && statement !== ';'); +} diff --git a/packages/database/src/migrate.ts b/packages/database/src/migrate.ts index 18d4970..9aaa910 100644 --- a/packages/database/src/migrate.ts +++ b/packages/database/src/migrate.ts @@ -4,7 +4,7 @@ import { existsSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import type pg from 'pg'; -import type { DatabaseDialectName } from './dialect.js'; +import { splitSqlStatements, type DatabaseDialectName } from './dialect.js'; import type { QueryResult, SqlDatabase } from './pool.js'; export interface MigrationOptions { @@ -151,7 +151,7 @@ async function runMysqlMigrations( let locked = false; try { const lock = await session.query<{ locked: number | null }>( - 'SELECT GET_LOCK(?, 60) AS locked', + 'SELECT GET_LOCK($1, 60) AS locked', [MIGRATION_LOCK], ); locked = Number(lock.rows[0]?.locked) === 1; @@ -173,9 +173,11 @@ async function runMysqlMigrations( continue; } // MySQL commits DDL implicitly, so the file cannot be rolled back; record - // it right after it succeeds instead of at the end of the run. - await session.query(migration.sql); - await session.query('INSERT INTO schema_migrations(name, checksum) VALUES (?, ?)', [ + // it right after it succeeds instead of at the end of the run. The server + // also refuses a whole multi-statement script unless multi-statements are + // enabled for the connection, which is deliberately not the case. + for (const statement of splitSqlStatements(migration.sql)) await session.query(statement); + await session.query('INSERT INTO schema_migrations(name, checksum) VALUES ($1, $2)', [ migration.name, migration.checksum, ]); @@ -183,7 +185,7 @@ async function runMysqlMigrations( } finally { if (locked) { try { - await session.query('SELECT RELEASE_LOCK(?)', [MIGRATION_LOCK]); + await session.query('SELECT RELEASE_LOCK($1)', [MIGRATION_LOCK]); } catch { /* The lock is released when the session closes. */ } diff --git a/packages/database/test/parity.test.ts b/packages/database/test/parity.test.ts new file mode 100644 index 0000000..7ff8beb --- /dev/null +++ b/packages/database/test/parity.test.ts @@ -0,0 +1,69 @@ +import { readdir, readFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const migrationsRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../migrations'); + +async function migrationFiles(engine: 'postgres' | 'mysql'): Promise { + const files = await readdir(resolve(migrationsRoot, engine)); + return files.filter((name) => /^\d+_[a-z0-9_-]+\.sql$/i.test(name)).sort(); +} + +function version(name: string): number { + return Number(/^(\d+)_/.exec(name)![1]); +} + +async function tableNames(engine: 'postgres' | 'mysql'): Promise> { + const names = new Set(); + for (const file of await migrationFiles(engine)) { + const sql = await readFile(resolve(migrationsRoot, engine, file), 'utf8'); + for (const match of sql.matchAll(/CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?([a-z_]+)"?/gi)) + names.add(match[1]!); + // Historical tables that a later migration drops do not survive to the end. + for (const match of sql.matchAll(/DROP TABLE IF EXISTS\s+"?([a-z_]+)"?/gi)) + names.delete(match[1]!); + } + return names; +} + +describe('two-engine migration parity', () => { + it('ships the MySQL baseline at the PostgreSQL baseline version', async () => { + const postgres = await migrationFiles('postgres'); + const mysql = await migrationFiles('mysql'); + expect(mysql.length).toBeGreaterThan(0); + // MySQL starts from a consolidated baseline; it must be the newest + // PostgreSQL version so both engines describe the same schema generation. + expect(version(mysql[0]!)).toBe(Math.max(...postgres.map(version))); + }); + + it('adds every later migration to both engines', async () => { + const postgres = await migrationFiles('postgres'); + const mysql = await migrationFiles('mysql'); + const baseline = version(mysql[0]!); + const later = postgres.map(version).filter((value) => value > baseline); + const mysqlVersions = new Set(mysql.map(version)); + for (const value of later) + expect(mysqlVersions.has(value), `mysql is missing migration ${value}`).toBe(true); + }); + + it('creates every PostgreSQL table in the MySQL baseline', async () => { + const postgres = await tableNames('postgres'); + const mysql = await tableNames('mysql'); + const missing = [...postgres].filter( + (table) => table !== 'schema_migrations' && !mysql.has(table), + ); + expect(missing, `MySQL baseline is missing tables: ${missing.join(', ')}`).toEqual([]); + }); + + it('quotes the reserved identifiers the MySQL schema uses', async () => { + const baseline = await readFile( + resolve(migrationsRoot, 'mysql', (await migrationFiles('mysql'))[0]!), + 'utf8', + ); + // `rank` and `cursor` are reserved in MySQL; an unquoted occurrence in DDL + // is a hard syntax error, so guard the baseline text directly. + expect(baseline).not.toMatch(/(? { const env: Record = { ...process.env }; const migrationUrl = env['MIGRATION_DATABASE_URL']?.trim(); if (migrationUrl) env['DATABASE_URL'] = migrationUrl; - if (!env['DATABASE_URL'] && !(env['PGHOST'] && env['PGDATABASE'])) { + const configured = Boolean( + env['DATABASE_URL'] || + (env['PGHOST'] && env['PGDATABASE']) || + (env['MYSQL_HOST'] && env['MYSQL_DATABASE']), + ); + if (!configured) { console.log( - 'NOT RUN: DATABASE_URL or PGHOST/PGDATABASE is not configured; no PostgreSQL server is available for migration.', + 'NOT RUN: DATABASE_URL or the discrete database variables are not configured; no server is available for migration.', ); process.exitCode = 2; return; } - const settings = readDatabaseRuntimeConfig(env); + const settings = readDatabaseSettings(env); + const label = settings.dialect === 'mysql' ? 'MySQL' : 'PostgreSQL'; const timeout = Number(env['MIGRATION_STATEMENT_TIMEOUT_MS'] ?? 300_000); if (!Number.isInteger(timeout) || timeout < 0 || timeout > 3_600_000) throw new Error('MIGRATION_STATEMENT_TIMEOUT_MS must be an integer between 0 and 3600000'); - const pool = createPool(poolConfigFromRuntime(settings), { - statementTimeoutMillis: timeout, - queryTimeoutMillis: timeout, + const database = openDatabase({ + ...settings, applicationName: `${settings.applicationName}-migrate`, max: 2, + queryTimeoutMillis: timeout, + statementTimeoutMillis: timeout, }); try { - await runMigrations(pool); - console.log('PASS: PostgreSQL migrations applied and schema_migrations is current.'); + await runMigrations(database); + console.log(`PASS: ${label} migrations applied and schema_migrations is current.`); } finally { - await pool.end(); + await database.end(); } } diff --git a/scripts/pg-runtime-smoke.ts b/scripts/sql-runtime-smoke.ts similarity index 78% rename from scripts/pg-runtime-smoke.ts rename to scripts/sql-runtime-smoke.ts index 9c91c4a..2793a76 100644 --- a/scripts/pg-runtime-smoke.ts +++ b/scripts/sql-runtime-smoke.ts @@ -4,12 +4,23 @@ import { createServer } from 'node:net'; import { dirname, join } from 'node:path'; import { promisify } from 'node:util'; import { randomUUID } from 'node:crypto'; -import { createPool, type Pool } from '../packages/database/src/index.js'; -import { isolatedTestDatabaseUrl } from './test-database-safety.js'; +import mysql from 'mysql2/promise'; +import { + databaseReady, + openDatabase, + readDatabaseSettings, + runMigrations, + type SqlDatabase, +} from '../packages/database/src/index.js'; const execFileAsync = promisify(execFile); const root = process.cwd(); -const databaseUrl = isolatedTestDatabaseUrl(); +const databaseUrl = process.env['TASKDOCK_TEST_DATABASE_URL']; +const settings = databaseUrl + ? readDatabaseSettings({ ...process.env, DATABASE_URL: databaseUrl }) + : undefined; +const dialect = settings?.dialect ?? 'postgres'; +const engine = dialect === 'mysql' ? 'MySQL' : 'PostgreSQL'; const pgCtl = process.env['PG_CTL']; const pgData = process.env['PGDATA'] ?? process.env['PG_DATA']; const apiEntry = join(root, 'apps', 'api', 'dist', 'main.js'); @@ -222,7 +233,7 @@ async function waitForWebSocketReady(url: string, token: string): Promise { +function nextSyncRequired(socket: WebSocket, timeoutMs = 5_000): Promise { return new Promise((resolve, reject) => { const timer = setTimeout( () => reject(new Error('sync.required notification timed out')), @@ -247,31 +258,84 @@ async function waitForSyncRequired(socket: WebSocket, timeoutMs = 5_000): Promis }); } -async function cleanupOwner(pool: Pool, ownerId: string): Promise { - await pool.query('DELETE FROM users WHERE id = $1', [ownerId]); +async function cleanupOwner(database: SqlDatabase, ownerId: string): Promise { + await database.query('DELETE FROM users WHERE id = $1', [ownerId]); } -async function main(): Promise { - if (!databaseUrl) { - console.log( - 'NOT RUN: TASKDOCK_TEST_DATABASE_URL is required for real PostgreSQL runtime smoke.', +async function tableExists(database: SqlDatabase, table: string): Promise { + const result = + dialect === 'mysql' + ? await database.query<{ n: number | string }>( + 'SELECT COUNT(*) AS n FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = $1', + [table], + ) + : await database.query<{ n: number | string }>( + "SELECT COUNT(*) AS n FROM information_schema.tables WHERE table_schema = 'public' AND table_name = $1", + [table], + ); + return Number(result.rows[0]?.n ?? 0) > 0; +} + +/** + * Losing every connection is the closest MySQL analogue to restarting + * PostgreSQL: the server stays up, the pool's sockets die, and the API has to + * notice on its own. Killing by session id keeps the fixture scoped. + */ +async function killOtherMysqlSessions(): Promise { + const admin = await mysql.createConnection({ uri: databaseUrl }); + try { + const [rows] = await admin.query( + `SELECT ID FROM information_schema.processlist + WHERE USER = SUBSTRING_INDEX(CURRENT_USER(), '@', 1) AND ID <> CONNECTION_ID()`, ); + for (const row of rows) await admin.query(`KILL ${Number(row['ID'])}`); + } finally { + await admin.end(); + } +} + +async function stopDatabase(): Promise { + if (dialect === 'mysql') { + await killOtherMysqlSessions(); + return; + } + await runPg(['stop', '-m', 'fast', '-w']); +} + +async function startDatabase(): Promise { + if (dialect === 'mysql') return; + await runPg(['start', '-w']); +} + +async function assertEmptyDatabase(database: SqlDatabase): Promise { + if (!(await tableExists(database, 'users'))) return; + const existing = await database.query<{ n: number | string }>('SELECT COUNT(*) AS n FROM users'); + assert( + Number(existing.rows[0]?.n ?? 0) === 0, + 'runtime smoke requires an isolated empty database', + ); +} + +async function main(): Promise { + if (!databaseUrl || !settings) { + console.log('NOT RUN: TASKDOCK_TEST_DATABASE_URL is required for the real SQL runtime smoke.'); process.exitCode = 2; return; } - if (!pgCtl || !pgData) { - console.log('NOT RUN: PG_CTL and PGDATA are required for database restart smoke.'); + if (dialect === 'postgres' && (!pgCtl || !pgData)) { + console.log('NOT RUN: PG_CTL and PGDATA are required for the PostgreSQL restart smoke.'); process.exitCode = 2; return; } await access(apiEntry); - const pool = createPool(databaseUrl); - pool.on('error', (error: Error) => { + const database = openDatabase({ ...settings, max: 4 }); + database.onConnectionError((error: Error) => { console.warn(`runtime smoke pool error: ${error.message}`); }); - const existing = await pool.query<{ count: string }>('SELECT COUNT(*)::text AS count FROM users'); - assert(existing.rows[0]?.count === '0', 'runtime smoke requires an isolated empty database'); + await assertEmptyDatabase(database); + await runMigrations(database); + assert(await databaseReady(database), `${engine} schema is not ready after migrations`); const port = await freePort(); const baseUrl = `http://127.0.0.1:${port}`; @@ -354,6 +418,9 @@ async function main(): Promise { socket = await waitForWebSocketReady(`ws://127.0.0.1:${port}/api/v2/ws`, refreshedToken); const folderId = randomUUID(); const folderMutationId = randomUUID(); + // Subscribe before the write: MySQL fires the local notification inside the + // committing request, so a listener attached afterwards would miss it. + const folderNotificationPromise = nextSyncRequired(socket); const folderResponse = await requestJson(baseUrl, '/api/v2/folders', { method: 'POST', headers: mutationHeaders(refreshedToken, clientId, folderMutationId), @@ -364,7 +431,7 @@ async function main(): Promise { `folder create returned ${folderResponse.response.status}`, ); assert(idFromBody(folderResponse.body, 'id') === folderId, 'folder id did not round-trip'); - const folderNotification = await waitForSyncRequired(socket); + const folderNotification = await folderNotificationPromise; assert(/^\d+$/.test(folderNotification), 'folder notification cursor was invalid'); const taskId = randomUUID(); @@ -374,12 +441,13 @@ async function main(): Promise { headers: mutationHeaders(refreshedToken, clientId, taskMutationId), body: JSON.stringify({ id: taskId, parentFolderId: folderId, title: 'runtime task' }), } satisfies RequestInit; + const taskNotification = nextSyncRequired(socket); const taskResponse = await requestJson(baseUrl, '/api/v2/tasks', taskRequest); assert( taskResponse.response.status === 201, `task create returned ${taskResponse.response.status}`, ); - await waitForSyncRequired(socket); + await taskNotification; const replay = await requestJson(baseUrl, '/api/v2/tasks', taskRequest); assert(replay.response.status === 201, `idempotent replay returned ${replay.response.status}`); const afterWrites = await requestJson(baseUrl, '/api/v2/sync/status', { @@ -453,33 +521,44 @@ async function main(): Promise { 'old cursor did not read a change created after API restart', ); - await runPg(['stop', '-m', 'fast', '-w']); + await stopDatabase(); databaseStopped = true; const liveDuringDbStop = await requestJson(baseUrl, '/health/live'); assert( liveDuringDbStop.response.status === 200, - 'live failed while only PostgreSQL was stopped', + `live failed while only ${engine} was stopped`, ); - await waitForStatus(baseUrl, '/health/ready', 503, api); - const unavailable = await requestJson( - baseUrl, - '/api/v2/sync/status', - { headers: { Authorization: `Bearer ${refreshedToken}` } }, - 5_000, - ); - assert(unavailable.response.status >= 500, `DB outage returned ${unavailable.response.status}`); - - await runPg(['start', '-w']); - databaseStopped = false; - await waitForStatus(baseUrl, '/health/ready', 200, api); + if (dialect === 'postgres') { + // A stopped server is a real outage: readiness must fail until it returns. + await waitForStatus(baseUrl, '/health/ready', 503, api); + const unavailable = await requestJson( + baseUrl, + '/api/v2/sync/status', + { headers: { Authorization: `Bearer ${refreshedToken}` } }, + 5_000, + ); + assert( + unavailable.response.status >= 500, + `DB outage returned ${unavailable.response.status}`, + ); + await startDatabase(); + databaseStopped = false; + await waitForStatus(baseUrl, '/health/ready', 200, api); + } else { + // Killing the pooled sockets is a transport outage, not a database + // outage: the MySQL server still answers, so the pool must self-heal and + // readiness must come back without an operator. + await waitForStatus(baseUrl, '/health/ready', 200, api, 20_000); + databaseStopped = false; + } await new Promise((resolve) => setTimeout(resolve, 2_000)); const folderAfterDbRestartId = randomUUID(); const afterDbRestartWrite = await requestJson(baseUrl, '/api/v2/folders', { method: 'POST', headers: mutationHeaders(refreshedToken, clientId, randomUUID()), - body: JSON.stringify({ id: folderAfterDbRestartId, title: 'after PostgreSQL restart' }), + body: JSON.stringify({ id: folderAfterDbRestartId, title: `after ${engine} restart` }), }); - assert(afterDbRestartWrite.response.status === 201, 'write after PostgreSQL restart failed'); + assert(afterDbRestartWrite.response.status === 201, `write after ${engine} restart failed`); const afterDbRestartPull = await requestJson( baseUrl, `/api/v2/sync/pull?cursor=${cursorAfterWrites}&limit=500`, @@ -491,7 +570,7 @@ async function main(): Promise { afterDbRestartChanges.some( (change) => objectBody(change)['entityId'] === folderAfterDbRestartId, ), - 'cursor did not read a change after PostgreSQL restart', + `cursor did not read a change after ${engine} restart`, ); const pointId = randomUUID(); @@ -514,7 +593,7 @@ async function main(): Promise { const today = await requestJson(baseUrl, '/api/v2/today?localDate=2026-10-20', { headers: todayHeaders, }); - assert(today.response.status === 200, 'PostgreSQL read-only Today failed'); + assert(today.response.status === 200, `${engine} read-only Today failed`); assert( objectBody(objectBody(today.body)['point'])['id'] === pointId, 'Today returned the wrong date', @@ -536,10 +615,10 @@ async function main(): Promise { ); console.log( - `PASS: real PostgreSQL API runtime, cookie refresh, v2 WebSocket notification, idempotency, API restart cursor continuity, PostgreSQL restart recovery, and ready/live failure boundary; cursor ${cursorAfterWrites}`, + `PASS: real ${engine} API runtime, cookie refresh, v2 WebSocket notification, idempotency, API restart cursor continuity, connection-loss recovery, and ready/live failure boundary; cursor ${cursorAfterWrites}`, ); } catch (error) { - console.error('FAIL: real PostgreSQL API runtime smoke failed.'); + console.error(`FAIL: real ${engine} API runtime smoke failed.`); console.error(error); if (api?.output) console.error(api.output); process.exitCode = 1; @@ -548,26 +627,26 @@ async function main(): Promise { if (api) await stopApi(api); if (databaseStopped) { try { - await runPg(['start', '-w']); + await startDatabase(); } catch (error) { - console.error(`FAIL: PostgreSQL recovery failed: ${String(error)}`); + console.error(`FAIL: ${engine} recovery failed: ${String(error)}`); process.exitCode = 1; } } if (ownerId) { try { - await cleanupOwner(pool, ownerId); + await cleanupOwner(database, ownerId); } catch (error) { console.error(`FAIL: runtime fixture cleanup failed: ${String(error)}`); process.exitCode = 1; } } - await pool.end(); + await database.end(); } } void main().catch((error: unknown) => { - console.error('FAIL: real PostgreSQL API runtime smoke crashed.'); + console.error('FAIL: real SQL API runtime smoke crashed.'); console.error(error); process.exitCode = 1; }); From 0ffed6320ba278eb024a9551765e739f44cc6301 Mon Sep 17 00:00:00 2001 From: IGNGserver <176017966+IGNGserver@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:09:41 +0800 Subject: [PATCH 5/9] =?UTF-8?q?docs(db):=20=E8=A1=A5=E5=85=85=20MySQL=20?= =?UTF-8?q?=E7=9A=84=E9=83=A8=E7=BD=B2=E3=80=81=E5=A4=87=E4=BB=BD=E4=B8=8E?= =?UTF-8?q?=E5=B7=AE=E5=BC=82=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 compose.mysql.yaml(内置 MySQL 8.4,utf8mb4 + UTC) - compose.external-db.yaml 改为引擎无关:DATABASE_URL 的 scheme 决定引擎 - .env.example 增加 MYSQL_* / DB_DIALECT,并注明 MySQL 的超时与 LISTEN 语义 - 重写 docs/EXTERNAL_DATABASE.md 覆盖双引擎,新增 docs/adr/0005-mysql-support.md - DEPLOYMENT / BACKUP_RESTORE(mysqldump)/ ARCHITECTURE / README / 官网同步更新 --- .env.example | 19 +++- README.md | 10 +-- compose.external-db.yaml | 11 ++- compose.mysql.yaml | 154 +++++++++++++++++++++++++++++++++ compose.yaml | 2 + docs/ARCHITECTURE.md | 7 +- docs/BACKUP_RESTORE.md | 42 ++++++++- docs/DEPLOYMENT.md | 26 ++++-- docs/EXTERNAL_DATABASE.md | 134 ++++++++++++++++------------ docs/adr/0005-mysql-support.md | 32 +++++++ website/site/index.html | 9 +- 11 files changed, 365 insertions(+), 81 deletions(-) create mode 100644 compose.mysql.yaml create mode 100644 docs/adr/0005-mysql-support.md diff --git a/.env.example b/.env.example index 9b21851..25e8be1 100644 --- a/.env.example +++ b/.env.example @@ -11,22 +11,37 @@ APP_ORIGIN=http://localhost:48731 POSTGRES_DB=devtodo POSTGRES_USER=devtodo POSTGRES_PASSWORD=replace-with-a-database-password -# 外部 PostgreSQL(compose.external-db.yaml):填写完整 URL,并删除/忽略上面的 -# POSTGRES_*。每个 TaskDock 中枢必须使用独立 database;只共享服务器,不共享库。 +# 外部数据库(compose.external-db.yaml):填写完整 URL,并删除/忽略上面的 +# POSTGRES_* / MYSQL_*。每个 TaskDock 中枢必须使用独立 database;只共享服务器, +# 不共享库。PostgreSQL 和 MySQL 共用同一套连接参数。 # DATABASE_URL=postgres://taskdock_app:replace-me@db.example.lan:5432/taskdock # 可选:迁移专用 DDL 角色;为空时复用 DATABASE_URL。 # MIGRATION_DATABASE_URL=postgres://taskdock_migrator:replace-me@db.example.lan:5432/taskdock # DATABASE_SSL_MODE=verify-full # DATABASE_SSL_CA=/run/secrets/db-ca.pem +# 内置 MySQL(compose.mysql.yaml):改用这一组变量即可,DB_DIALECT 可省略—— +# 只要出现 MYSQL_* 变量或 mysql:// URL,引擎会自动识别为 MySQL。 +# DB_DIALECT=mysql +# MYSQL_HOST=mysql +# MYSQL_PORT=3306 +# MYSQL_USER=taskdock +# MYSQL_PASSWORD=replace-with-a-mysql-password +# MYSQL_DATABASE=taskdock +# 外部 MySQL:同样用完整 URL,迁移角色复用 MIGRATION_DATABASE_URL。 +# DATABASE_URL=mysql://taskdock:replace-me@db.example.lan:3306/taskdock +# MIGRATION_DATABASE_URL=mysql://taskdock_migrator:replace-me@db.example.lan:3306/taskdock DB_POOL_MAX=10 DB_POOL_WARM=0 DB_APPLICATION_NAME=taskdock-api # PgBouncer transaction mode 不能保持 LISTEN 会话;此时设为 false。 +# MySQL 没有 LISTEN/NOTIFY,该变量会被忽略(多实例推送改用进程内通知 + 可选 Redis)。 DB_LISTEN_ENABLED=true DB_CONNECT_TIMEOUT_MS=5000 # 远端库建议不要 30 秒就回收空闲连接。 DB_IDLE_TIMEOUT_MS=300000 DB_QUERY_TIMEOUT_MS=30000 +# 以下三项只对 PostgreSQL 是语句级硬超时;MySQL 用 max_execution_time(仅 SELECT) +# 和 innodb_lock_wait_timeout,写入不会被语句级强杀。 DB_STATEMENT_TIMEOUT_MS=30000 DB_LOCK_TIMEOUT_MS=5000 # 启动时数据库未就绪的最长等待时间;0 表示只尝试一次。 diff --git a/README.md b/README.md index 0f620bb..e83de76 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Docker 自托管

-TaskDock 是一个给自己用的任务工作台,跑在你自己的服务器上。想法先记下来,再用目录、日期、流程和步骤把它安排明白,每个任务都能附上一份 Markdown 笔记。中枢放在 NAS、家用服务器或云主机上,浏览器和 Android 手机连过去用;断网时照样能看能改,连上以后再同步。数据只落在你自己的 PostgreSQL 里,没有托管后端,也没有团队和权限那一套。 +TaskDock 是一个给自己用的任务工作台,跑在你自己的服务器上。想法先记下来,再用目录、日期、流程和步骤把它安排明白,每个任务都能附上一份 Markdown 笔记。中枢放在 NAS、家用服务器或云主机上,浏览器和 Android 手机连过去用;断网时照样能看能改,连上以后再同步。数据只落在你自己的数据库里(PostgreSQL 或 MySQL),没有托管后端,也没有团队和权限那一套。 ## 功能 @@ -48,7 +48,7 @@ TaskDock 是一个给自己用的任务工作台,跑在你自己的服务器 - **网页 / PWA**:中枢启动后用浏览器打开 `APP_ORIGIN` 就能用,不用另装客户端。界面按屏幕自适应:手机单手记录,平板双栏,桌面支持拖放和键盘操作。HTTPS 下可以安装为 PWA 并离线打开。 - **Android**:原生 Jetpack Compose 客户端,Room 本地存储,和网页端共用一套 API 与同步协议。APK 附在每个 Release 里。 -- **中枢**:Fastify + PostgreSQL 组成的单用户服务端。所有设备的数据最终汇集在这里,也只在这里。 +- **中枢**:Fastify + PostgreSQL/MySQL 组成的单用户服务端。所有设备的数据最终汇集在这里,也只在这里。 ## 下载与部署 @@ -79,7 +79,7 @@ docker compose --profile operations run --rm bootstrap **升级**:先按 [备份与恢复](docs/BACKUP_RESTORE.md) 做一份备份,再把 `.env` 里的 `APP_VERSION` 改成新版本,重复 `pull` / `migrate` / `up -d`。出问题时把版本号改回去,并按备份文档处理数据库。 -完整的变量说明、外部 PostgreSQL、Caddy 网关和回滚步骤在[部署说明](docs/DEPLOYMENT.md)里。 +完整的变量说明、外部数据库(PostgreSQL/MySQL)、Caddy 网关和回滚步骤在[部署说明](docs/DEPLOYMENT.md)里。 ## 版本 @@ -93,7 +93,7 @@ TaskDock 现在处于 **2.0 预览阶段**,发出来的都是预发布版本 ## 数据与安全 -- 任务、安排和笔记保存在自托管中枢的 PostgreSQL 里;默认 Compose 不把数据库端口映射到宿主机。 +- 任务、安排和笔记保存在自托管中枢的 PostgreSQL(默认)或 MySQL 里;两种 Compose 都不把数据库端口映射到宿主机。 - 应用端口默认只绑定 `127.0.0.1`,要直连 HTTP 得显式设置 `TASKDOCK_BIND_ADDRESS=0.0.0.0`;公网请走 HTTPS。 - 密码至少 6 位,6 位纯数字也可以,没有字符种类限制。 - 用 HTTP 也能访问,但界面会提示这是明文传输,密码和会话信息不做加密。可信局域网可以接受,公网不建议。 @@ -104,7 +104,7 @@ TaskDock 现在处于 **2.0 预览阶段**,发出来的都是预发布版本 ## 文档 - [部署说明](docs/DEPLOYMENT.md)——环境变量、首次部署、升级与回滚、Caddy 网关 -- [外部 PostgreSQL](docs/EXTERNAL_DATABASE.md)——接现有数据库、共享数据库服务器 +- [外部数据库](docs/EXTERNAL_DATABASE.md)——接现有 PostgreSQL/MySQL、共享数据库服务器 - [备份与恢复](docs/BACKUP_RESTORE.md) - [安全说明](docs/SECURITY.md)——认证、限流、代理信任、密钥处理 - [同步协议](docs/SYNC_PROTOCOL.md)——离线队列、冲突处理、游标 diff --git a/compose.external-db.yaml b/compose.external-db.yaml index 7394551..ca0c34f 100644 --- a/compose.external-db.yaml +++ b/compose.external-db.yaml @@ -7,10 +7,12 @@ x-app-security: &app_security tmpfs: - /tmp:rw,noexec,nosuid,size=64m -# External PostgreSQL deployment. Use this file when PostgreSQL is managed +# External database deployment. Use this file when the database is managed # outside this Compose project (a NAS, a shared database server, or a managed -# service). Each TaskDock hub must use its own database; do not point several -# hubs at the same database. +# service). PostgreSQL and MySQL are both supported: the DATABASE_URL scheme +# selects the engine (postgres:// or mysql://), and DB_DIALECT can override it. +# Each TaskDock hub must use its own database; do not point several hubs at the +# same database. # # docker compose -f compose.external-db.yaml run --rm migrate # docker compose -f compose.external-db.yaml run --rm bootstrap @@ -26,7 +28,8 @@ services: APP_PORT: 3000 APP_ORIGIN: ${APP_ORIGIN:?set the public HTTP or HTTPS origin in .env} # Full URL, e.g. postgres://taskdock_app:...@db.example.lan:5432/taskdock - DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL to the external PostgreSQL URL} + # or mysql://taskdock_app:...@db.example.lan:3306/taskdock + DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL to the external database URL} # Optional DDL role. Falls back to DATABASE_URL when empty. MIGRATION_DATABASE_URL: ${MIGRATION_DATABASE_URL:-} DATABASE_SSL_MODE: ${DATABASE_SSL_MODE:-} diff --git a/compose.mysql.yaml b/compose.mysql.yaml new file mode 100644 index 0000000..2c18d8e --- /dev/null +++ b/compose.mysql.yaml @@ -0,0 +1,154 @@ +# Built-in MySQL 8.4 deployment. +# +# The default bundle (compose.yaml) ships PostgreSQL. Use this file when the +# operator wants MySQL instead of pointing DATABASE_URL at an external server: +# +# docker compose -f compose.mysql.yaml up -d mysql +# docker compose -f compose.mysql.yaml run --rm migrate +# docker compose -f compose.mysql.yaml --profile operations run --rm bootstrap +# docker compose -f compose.mysql.yaml up -d app +# +# One TaskDock hub per database. Do not point several hubs at the same schema. +x-taskdock-image: &taskdock_image '${TASKDOCK_IMAGE:-ghcr.io/igngserver/taskdock}:${APP_VERSION:?set APP_VERSION to an immutable image tag}' +x-app-security: &app_security + read_only: true + init: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m + +services: + mysql: + image: mysql:8.4@sha256:a81d043930e51745522bd5c430961cf8286f0a23f0a9a2a6e0108f7ea71b0902 + restart: unless-stopped + # The API pins its own session time zone to UTC; the server default keeps + # administrative tools (mysqldump, reporting) on the same clock. + command: + - '--character-set-server=utf8mb4' + - '--collation-server=utf8mb4_0900_ai_ci' + - '--default-time-zone=+00:00' + environment: + MYSQL_DATABASE: ${MYSQL_DATABASE:-taskdock} + MYSQL_USER: ${MYSQL_USER:-taskdock} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?set a MySQL application password in .env} + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?set a MySQL root password in .env} + volumes: + - taskdock_mysql:/var/lib/mysql + healthcheck: + test: + ['CMD-SHELL', 'mysqladmin ping -h 127.0.0.1 -u$${MYSQL_USER} -p$${MYSQL_PASSWORD} --silent'] + interval: 5s + timeout: 5s + retries: 30 + networks: [devtodo] + + migrate: + <<: *app_security + image: *taskdock_image + restart: 'no' + environment: &app-env + NODE_ENV: production + APP_VERSION: ${APP_VERSION:?set APP_VERSION to an immutable image tag} + APP_PORT: 3000 + APP_ORIGIN: ${APP_ORIGIN:?set the public HTTP or HTTPS origin in .env} + # Discrete MYSQL_* values avoid breaking a URL when the password contains + # @, :, / or #. External deployments use DATABASE_URL with a mysql:// + # scheme instead, and the engine is detected from it. + DB_DIALECT: mysql + MYSQL_HOST: mysql + MYSQL_PORT: '3306' + MYSQL_USER: ${MYSQL_USER:-taskdock} + MYSQL_PASSWORD: ${MYSQL_PASSWORD} + MYSQL_DATABASE: ${MYSQL_DATABASE:-taskdock} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_WARM: ${DB_POOL_WARM:-0} + DB_APPLICATION_NAME: ${DB_APPLICATION_NAME:-taskdock-api} + DB_CONNECT_TIMEOUT_MS: ${DB_CONNECT_TIMEOUT_MS:-5000} + DB_IDLE_TIMEOUT_MS: ${DB_IDLE_TIMEOUT_MS:-300000} + DB_QUERY_TIMEOUT_MS: ${DB_QUERY_TIMEOUT_MS:-30000} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-30000} + DB_LOCK_TIMEOUT_MS: ${DB_LOCK_TIMEOUT_MS:-5000} + DB_STARTUP_MAX_WAIT_MS: ${DB_STARTUP_MAX_WAIT_MS:-60000} + MIGRATION_STATEMENT_TIMEOUT_MS: ${MIGRATION_STATEMENT_TIMEOUT_MS:-300000} + CACHE_URL: ${CACHE_URL:-} + CACHE_KEY_PREFIX: ${CACHE_KEY_PREFIX:-taskdock} + CACHE_SNAPSHOT_TTL_SECONDS: ${CACHE_SNAPSHOT_TTL_SECONDS:-60} + CACHE_REQUIRED: ${CACHE_REQUIRED:-false} + AUTH_RATE_LIMIT_REQUIRE_SHARED: ${AUTH_RATE_LIMIT_REQUIRE_SHARED:-true} + CACHE_TLS_CA: ${CACHE_TLS_CA:-} + BOOTSTRAP_TOKEN: ${BOOTSTRAP_TOKEN:?set a random bootstrap token in .env} + ACCESS_TOKEN_SECRET: ${ACCESS_TOKEN_SECRET:?set an access token secret in .env} + REFRESH_TOKEN_PEPPER: ${REFRESH_TOKEN_PEPPER:?set a refresh token pepper in .env} + REFRESH_TOKEN_IDLE_TTL_DAYS: ${REFRESH_TOKEN_IDLE_TTL_DAYS:-365} + CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:?set explicit allowed origins in .env} + NATIVE_ALLOWED_ORIGINS: ${NATIVE_ALLOWED_ORIGINS:-capacitor://localhost,https://localhost,http://localhost} + TRUST_PROXY: ${TRUST_PROXY:-0} + DEV_MEMORY_STORE: 'false' + command: ['node', 'apps/api/dist/migrate.js'] + depends_on: + mysql: { condition: service_healthy } + networks: [devtodo] + + bootstrap: + <<: *app_security + image: *taskdock_image + profiles: [operations] + environment: + <<: *app-env + OWNER_USERNAME: ${OWNER_USERNAME:?set the initial owner username in .env} + OWNER_PASSWORD: ${OWNER_PASSWORD:?set the initial owner password in .env} + command: ['node', 'apps/api/dist/bootstrap-owner.js'] + depends_on: + mysql: { condition: service_healthy } + migrate: { condition: service_completed_successfully } + networks: [devtodo] + + app: + <<: *app_security + image: *taskdock_image + restart: unless-stopped + environment: *app-env + depends_on: + mysql: { condition: service_healthy } + migrate: { condition: service_completed_successfully } + ports: + - '${TASKDOCK_BIND_ADDRESS:-127.0.0.1}:${TASKDOCK_PORT:-48731}:3000' + healthcheck: + test: + [ + 'CMD', + 'node', + '-e', + "fetch('http://127.0.0.1:3000/health/ready').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))", + ] + interval: 10s + timeout: 5s + retries: 12 + networks: [devtodo] + + gateway: + profiles: [gateway] + image: caddy:2.10.2-alpine + restart: unless-stopped + ports: + - '80:80' + - '443:443' + environment: + APP_ORIGIN: ${APP_ORIGIN:?set the public HTTP or HTTPS origin in .env} + volumes: + - ./infra/caddy/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + depends_on: + app: { condition: service_healthy } + networks: [devtodo] + +volumes: + taskdock_mysql: + caddy_data: + caddy_config: + +networks: + devtodo: + driver: bridge diff --git a/compose.yaml b/compose.yaml index d359370..5c601e0 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,3 +1,5 @@ +# Built-in PostgreSQL deployment (the default). For MySQL instead of PostgreSQL, +# use compose.mysql.yaml; for an existing database server, compose.external-db.yaml. x-taskdock-image: &taskdock_image '${TASKDOCK_IMAGE:-ghcr.io/igngserver/taskdock}:${APP_VERSION:?set APP_VERSION to an immutable image tag}' x-app-security: &app_security read_only: true diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 776ca69..d4264fb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,19 +2,19 @@ ## 拓扑 -Web/PWA 直接加载 React Web 产物,并通过浏览器的 Fetch、Cookie、IndexedDB 和 WebSocket 访问中枢;Android 使用独立 Jetpack Compose 原生 UI、Room、OkHttp 和同一 v2 API/同步协议。所有客户端通过 Fastify REST 推送/拉取;WebSocket 只发送认证后的 `sync.required` 失效通知。生产入口是 Caddy → Fastify → PostgreSQL。PostgreSQL 可以是 Compose 内置服务,也可以是外部/共享数据库服务器;每个 TaskDock 中枢使用独立 database。可选 Redis 只做共享限流和冷启动投影缓存,不进入权威写路径;生产 Compose 默认要求共享认证限流,已配置 Redis 但不可用时认证请求 fail-closed。 +Web/PWA 直接加载 React Web 产物,并通过浏览器的 Fetch、Cookie、IndexedDB 和 WebSocket 访问中枢;Android 使用独立 Jetpack Compose 原生 UI、Room、OkHttp 和同一 v2 API/同步协议。所有客户端通过 Fastify REST 推送/拉取;WebSocket 只发送认证后的 `sync.required` 失效通知。生产入口是 Caddy → Fastify → SQL 数据库。数据库可以是 PostgreSQL 18(默认)或 MySQL 8.4,可以是 Compose 内置服务,也可以是外部/共享数据库服务器;引擎由 `DATABASE_URL` 的 scheme 或 `DB_DIALECT` 决定,方言差异集中在 `@devtodo/database`,每个 TaskDock 中枢使用独立 database。可选 Redis 只做共享限流和冷启动投影缓存,不进入权威写路径;生产 Compose 默认要求共享认证限流,已配置 Redis 但不可用时认证请求 fail-closed。 ## 包边界 - `packages/contracts`:Zod 输入、DTO、枚举和 UUIDv7。 - `packages/domain`:无 UI 的状态机、日期、rank、引用 ID、Markdown 安全渲染规则。 - `packages/database`:Drizzle 描述、显式 SQL migration、外部库/SSL/连接池配置、ready 检查。 -- `apps/api`:认证、Owner 限定的资源端点、同步协议、PostgreSQL adapter 和可选 Redis cache adapter。 +- `apps/api`:认证、Owner 限定的资源端点、同步协议、SQL adapter(PostgreSQL / MySQL)和可选 Redis cache adapter。 - `packages/sync-client`:IndexedDB/Dexie v2 schema、v1 outbox 转换、push/pull、冲突记录、tombstone 和重同步。 - `apps/web`:React/PWA 目录树、混合排序、工作流、步骤、日期/Placement 和删除预览界面。 - `apps/mobile`:Capacitor 工程与 `apps/mobile/android` 原生 Compose/Room 客户端;原生客户端不依赖 React 页面来满足 v2 功能。 - `apps/api/src/tree-store.ts`:Owner-scoped v2 目录树、Workflow、TaskStep、时间点/Placement 以及内存迁移投影。 -- `apps/api/src/postgres-tree-store.ts`:PostgreSQL v2 projection 的单查询加载、增量 change-feed 追赶、批量持久化和 v2 change feed 适配。 +- `apps/api/src/sql-tree-store.ts`:v2 projection 的单查询加载、增量 change-feed 追赶、批量持久化(多行 upsert)和 v2 change feed 适配;引擎专用语句在 `apps/api/src/sql-statements.ts`。 - `apps/api/src/cache.ts`:可选 Redis 共享限流/冷启动缓存;缓存投影故障回退数据库,认证限流是否允许进程内回退由 `AUTH_RATE_LIMIT_REQUIRE_SHARED` 决定。 - 主题:Web 使用 `prefers-color-scheme`,Android 使用 `Configuration.uiMode` 同步系统栏并向 WebView 派发主题事件;主题是设备运行时状态,不进入账号同步设置。 @@ -36,4 +36,5 @@ Web/PWA 直接加载 React Web 产物,并通过浏览器的 Fetch、Cookie、I - [0001 Store persistence](adr/0001-store-persistence.md):说明 PostgreSQL 权威存储、owner-scoped SQL、事务内 change/receipt 和容量升级边界。 - [0004 外部数据库与可选缓存](adr/0004-external-database-and-cache.md):说明外部 PostgreSQL、共享服务器边界和 Redis fail-open 策略。 +- [0005 双引擎数据库支持](adr/0005-mysql-support.md):说明 MySQL 8.4 的方言层、合并迁移基线、能力降级与移植代价。 - [深色模式](DARK_MODE.md):说明 Web/PWA 与 Android 的系统主题读取、共享颜色令牌和验收边界。 diff --git a/docs/BACKUP_RESTORE.md b/docs/BACKUP_RESTORE.md index 19f3936..8352dbf 100644 --- a/docs/BACKUP_RESTORE.md +++ b/docs/BACKUP_RESTORE.md @@ -2,7 +2,7 @@ 以下命令在部署主机执行。备份目录只允许部署账户访问;不要把数据库密码、`DATABASE_URL` 或备份文件提交到 Git。 -## 备份与校验 +## 备份与校验(PostgreSQL) TaskDock v2 数据升级前必须先完成 PostgreSQL custom dump;本轮新增 migration `0005_v2_structure.sql`、`0006_v2_backfill.sql`、`0007_v2_constraints.sql`。升级顺序是先备份,再让新代码读取旧/新结构,执行 backfill 和约束校验;禁止用 `TRUNCATE`、重建业务表或 destructive migration 替代。必须核对 Project→Folder、Task `reference_id`、Note、Placement、删除边界(tombstone)和用户设置计数后再继续。 @@ -29,6 +29,46 @@ chmod 600 "$BACKUP_FILE.list" 升级前必须执行一次上述备份并保留 `pg_restore --list` 输出。生产环境至少保留 7 个日备份和 4 个周备份;部署者可设置更长的保留周期。 +## 备份与校验(MySQL) + +使用 `compose.mysql.yaml` 或外部 MySQL 时,用 `mysqldump` 做一致性备份。`--single-transaction` 对 InnoDB 表提供一致性快照,不需要停服;`--routines --triggers` 保证对象齐全(当前 schema 没有存储过程,保留是为了以后)。 + +```bash +BACKUP_DIR="$PWD/backups" +STAMP="$(date +%Y%m%d-%H%M%S)" +BACKUP_FILE="$BACKUP_DIR/taskdock-$STAMP.sql.gz" + +mkdir -p "$BACKUP_DIR" +chmod 700 "$BACKUP_DIR" +docker compose -f compose.mysql.yaml exec -T mysql sh -c \ + 'exec mysqldump --single-transaction --quick --routines --triggers --no-tablespaces \ + -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DATABASE"' \ + | gzip -9 > "$BACKUP_FILE" +test -s "$BACKUP_FILE" +gzip -t "$BACKUP_FILE" +chmod 600 "$BACKUP_FILE" +``` + +外部 MySQL 把 `docker compose exec` 换成在数据库主机上执行同样的 `mysqldump`,密码通过 `MYSQL_PWD` 或登录路径提供,不要写进命令行历史。 + +### 恢复演练(MySQL) + +恢复始终指向新建的临时 database,绝不覆盖生产库。下面用 `docker compose exec -e` 把目标库名传进容器,避免嵌套引号: + +```bash +MYSQL_RESTORE_DB="taskdock_restore_$(date +%Y%m%d%H%M%S)" + +echo "CREATE DATABASE \`$MYSQL_RESTORE_DB\` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;" \ + | docker compose -f compose.mysql.yaml exec -T mysql \ + sh -c 'mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD"' + +gzip -dc "$BACKUP_FILE" \ + | docker compose -f compose.mysql.yaml exec -T -e TARGET_DB="$MYSQL_RESTORE_DB" mysql \ + sh -c 'mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" "$TARGET_DB"' +``` + +恢复后核对 `schema_migrations` 的版本与 checksum 是否和镜像一致,再以 `mysql://.../$MYSQL_RESTORE_DB` 跑一次 `migrate` 和 `app`,确认 `/health/ready` 与一次真实登录。演练完成后删除临时 database。 + ## 新卷恢复演练 当前仓库只提供可执行的恢复流程;是否通过恢复演练必须以真实 PostgreSQL/Docker 输出为准,不能由内存 smoke 或构建结果代替。恢复演练始终使用独立临时卷,不覆盖生产卷。 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 3e305b1..f42b152 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -22,7 +22,7 @@ TaskDock 允许至少 6 位的密码,6 位纯数字也可以,没有额外的 从 GitHub 仓库取得 `compose.yaml` 和 `.env.example` 后,在服务器上执行: -> **数据库卷提示:** 当前 Compose 使用 PostgreSQL 18 的 `/var/lib/postgresql` 数据卷布局。若旧部署使用过早期 Compose 文件,请先完成数据库备份,再按 [备份与恢复](BACKUP_RESTORE.md) 恢复到新的数据库卷;不要直接删除旧卷。 +> **数据库卷提示:** 默认 Compose 使用 PostgreSQL 18 的 `/var/lib/postgresql` 数据卷布局;`compose.mysql.yaml` 使用 `taskdock_mysql` 卷。若旧部署使用过早期 Compose 文件,请先完成数据库备份,再按 [备份与恢复](BACKUP_RESTORE.md) 恢复到新的数据库卷;不要直接删除旧卷。 ```bash cp .env.example .env @@ -44,7 +44,7 @@ ghcr.io/igngserver/taskdock:${APP_VERSION} 如果需要使用其他镜像仓库,可以设置 `TASKDOCK_IMAGE`。不要把生产 Compose 改回 `build:`,也不要在生产服务器上执行 `git pull` 后从源代码构建。 -默认应用端口仅绑定 `127.0.0.1:48731`;直接提供 HTTP 服务时必须显式设置 `TASKDOCK_BIND_ADDRESS=0.0.0.0`,公网仍应改用 HTTPS 网关。使用 HTTP 时页面会显示安全警告,密码和会话信息会明文传输。PostgreSQL 没有宿主机端口。在 `.env` 中填写 `OWNER_USERNAME`、`OWNER_PASSWORD`,然后由部署中枢执行一次 Owner 初始化: +默认应用端口仅绑定 `127.0.0.1:48731`;直接提供 HTTP 服务时必须显式设置 `TASKDOCK_BIND_ADDRESS=0.0.0.0`,公网仍应改用 HTTPS 网关。使用 HTTP 时页面会显示安全警告,密码和会话信息会明文传输。数据库没有宿主机端口。在 `.env` 中填写 `OWNER_USERNAME`、`OWNER_PASSWORD`,然后由部署中枢执行一次 Owner 初始化: ```bash docker compose --profile operations run --rm migrate @@ -54,13 +54,27 @@ docker compose up -d app 初始化命令会读取部署环境中的 `BOOTSTRAP_TOKEN`,不会把令牌交给 Web、桌面或手机客户端。初始化成功后,建议从 `.env` 中删除 `OWNER_PASSWORD`,并轮换或移除 `BOOTSTRAP_TOKEN`。`app` 依赖迁移 job 成功后才会进入 ready;`/health/live` 只证明进程存活,不能替代迁移和数据库就绪检查。 -## 外部 PostgreSQL / 共享数据库服务器 +## 内置 MySQL -如果 PostgreSQL 已经运行在 NAS、共享服务器或托管服务上,不要使用 `compose.yaml` 的内置数据库,改用 `compose.external-db.yaml`。每个 TaskDock 中枢必须使用独立 database;多个应用可以共用同一台 PostgreSQL 服务器。 +默认 `compose.yaml` 带的是 PostgreSQL。要用内置 MySQL,换一个 Compose 文件即可: + +```bash +docker compose -f compose.mysql.yaml up -d mysql +docker compose -f compose.mysql.yaml run --rm migrate +docker compose -f compose.mysql.yaml --profile operations run --rm bootstrap +docker compose -f compose.mysql.yaml up -d app +``` + +MySQL 数据在 `taskdock_mysql` 卷里,同样不映射宿主机端口。服务端字符集与排序规则固定为 `utf8mb4` / `utf8mb4_0900_ai_ci`,时区固定为 UTC。 + +## 外部数据库 / 共享数据库服务器 + +如果 PostgreSQL 或 MySQL 已经运行在 NAS、共享服务器或托管服务上,不要使用内置数据库,改用 `compose.external-db.yaml`:它只读 `DATABASE_URL`,scheme 决定引擎(`postgres://` 或 `mysql://`)。每个 TaskDock 中枢必须使用独立 database;多个应用可以共用同一台数据库服务器。 ```bash cp .env.example .env # 填写 DATABASE_URL、MIGRATION_DATABASE_URL、APP_ORIGIN 和密钥 +# MySQL 只是把 scheme 换成 mysql://,其余步骤一致 docker compose -f compose.external-db.yaml pull docker compose -f compose.external-db.yaml run --rm migrate docker compose -f compose.external-db.yaml --profile operations run --rm bootstrap @@ -68,7 +82,7 @@ docker compose -f compose.external-db.yaml up -d app curl -fsS http://your-host.example:48731/health/ready ``` -外部库建议使用 `DATABASE_SSL_MODE=verify-full` 和 CA secret;应用角色与迁移角色分开。连接池、PgBouncer、TLS、备份和“一个服务器多个 database”的完整说明见 [外部 PostgreSQL 与共享数据库服务器](EXTERNAL_DATABASE.md)。 +外部库建议使用 `DATABASE_SSL_MODE=verify-full` 和 CA secret;应用角色与迁移角色分开。连接池、PgBouncer、TLS、备份、“一个服务器多个 database”以及两种引擎的行为差异见 [外部数据库与共享数据库服务器](EXTERNAL_DATABASE.md)。 可选 Redis 用于共享认证限流和多副本冷启动缓存。内置 Compose 可使用 `--profile cache`,外部 Redis 通过 `CACHE_URL` 指向;多副本必须配置同一个高可用 Redis。生产环境中只要配置了 `CACHE_URL`,Redis 故障就始终使认证请求 fail-closed,而不是退回各副本独立的本地计数;Compose 还默认设置 `AUTH_RATE_LIMIT_REQUIRE_SHARED=true`,该变量可在非生产环境强制同样行为。未配置 Redis 时使用本地限流,只适合单进程部署。权威任务写入、同步和冷启动投影缓存回退均不依赖 Redis 可用性。 @@ -83,6 +97,6 @@ docker compose up -d app curl -fsS http://your-host.example:48731/health/ready # 使用 HTTPS 或自定义端口时按 APP_ORIGIN 修改 ``` -迁移 job 成功后再切换 app;`/health/live` 只证明进程存活,`/health/ready` 才用于确认 PostgreSQL schema 已就绪。迁移失败时不要替换正在运行的 app;不兼容 schema 按恢复文档在新卷恢复旧备份。 +迁移 job 成功后再切换 app;`/health/live` 只证明进程存活,`/health/ready` 才用于确认数据库 schema 已就绪。迁移失败时不要替换正在运行的 app;不兼容 schema 按恢复文档在新卷恢复旧备份。 预发布版本也可以部署,但必须使用它自己的版本号,例如 `APP_VERSION=0.2.0-rc.1`。预发布镜像不会使用 `latest` 标签;升级后应检查 `/version` 的 `appVersion` 与 `.env` 中的 `APP_VERSION` 一致,避免旧镜像或旧 Service Worker 继续提供旧版资源。 diff --git a/docs/EXTERNAL_DATABASE.md b/docs/EXTERNAL_DATABASE.md index 09c0867..c58e43d 100644 --- a/docs/EXTERNAL_DATABASE.md +++ b/docs/EXTERNAL_DATABASE.md @@ -1,13 +1,15 @@ -# 外部 PostgreSQL 与共享数据库服务器 +# 外部数据库与共享数据库服务器 -本文说明如何让 TaskDock 连接**外部 PostgreSQL**,以及如何在一个 PostgreSQL 服务器上安全地运行多个使用数据库的自托管应用。 +本文说明如何让 TaskDock 连接**外部 PostgreSQL 或 MySQL**,以及如何在一个数据库服务器上安全地运行多个使用数据库的自托管应用。 + +引擎在启动时确定:`DATABASE_URL` 的 scheme 是 `postgres://` / `postgresql://` 就是 PostgreSQL,`mysql://` 就是 MySQL;只给离散变量时,出现 `MYSQL_*` 即 MySQL,否则按 PostgreSQL 处理。也可以用 `DB_DIALECT=postgres|mysql` 显式指定。 ## 1. 边界:共享服务器,不共享 database -推荐拓扑: +推荐拓扑(两种引擎一致): ```text -postgres.example.lan:5432 +db.example.lan ├─ taskdock_home ← 中枢 A ├─ taskdock_office ← 中枢 B ├─ other_app_prod ← 另一个应用 @@ -16,22 +18,19 @@ postgres.example.lan:5432 每个 TaskDock 中枢使用**独立 database**。不要多个中枢共用一个 database,原因: -- `hub_identity` 是单例表(`singleton boolean PRIMARY KEY`),一个 database 只能有一个 Hub 身份。 +- `hub_identity` 是单例表,一个 database 只能有一个 Hub 身份。 - `users.username` 全局唯一,第二个中枢无法创建同名 Owner。 -- `sync_changes.seq` 是全局 identity,`oldestCursor` 按全局序列计算;多个中枢共用会让游标过期判断互相影响。 +- `sync_changes.seq` 是全局序列,`oldestCursor` 按全局序列计算;多个中枢共用会让游标过期判断互相影响。 - `schema_migrations` 全局唯一,不同版本的中枢共用 database 会互相阻止迁移。 - 备份、恢复、升级和故障隔离都会变得不可控。 -如果确实需要“一个 database 多中枢”,需要引入 `hub_id` 租户列、复合唯一键、按 hub 的迁移和游标,这是一次独立的大迁移,不在当前支持范围内。 +需要"一个 database 多中枢"时,得引入 `hub_id` 租户列、复合唯一键、按 hub 的迁移和游标,属于独立的大迁移,当前不支持。 ## 2. 创建 database 与角色 -建议拆分两个角色: - -- `taskdock_migrator`:拥有 DDL 权限,只给迁移 job 使用。 -- `taskdock_app`:只有业务表 DML 权限,给 API 使用。 +两种引擎都建议拆两个角色:迁移角色有 DDL 权限,只给迁移 job 用;应用角色只有业务表 DML 权限,给 API 用。 -示例(在 PostgreSQL 服务器上以管理员执行;请替换密码和库名): +### PostgreSQL ```sql CREATE ROLE taskdock_migrator LOGIN PASSWORD 'replace-me'; @@ -44,26 +43,42 @@ CREATE DATABASE taskdock_home OWNER taskdock_migrator; GRANT CONNECT ON DATABASE taskdock_home TO taskdock_app; GRANT USAGE ON SCHEMA public TO taskdock_app; --- 迁移角色创建的表,默认给应用角色 DML 权限。 ALTER DEFAULT PRIVILEGES FOR ROLE taskdock_migrator IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO taskdock_app; ALTER DEFAULT PRIVILEGES FOR ROLE taskdock_migrator IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO taskdock_app; ``` -迁移完成后,`taskdock_app` 不应拥有 `CREATE`/`ALTER`/`DROP` 权限。这样即使 API 凭据泄露,也不能改 schema。 +### MySQL + +```sql +CREATE DATABASE taskdock_home CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; + +CREATE USER 'taskdock_migrator'@'%' IDENTIFIED BY 'replace-me'; +CREATE USER 'taskdock_app'@'%' IDENTIFIED BY 'replace-me'; + +GRANT ALL PRIVILEGES ON taskdock_home.* TO 'taskdock_migrator'@'%'; +GRANT SELECT, INSERT, UPDATE, DELETE ON taskdock_home.* TO 'taskdock_app'@'%'; +FLUSH PRIVILEGES; +``` + +迁移完成后,应用角色不应拥有 `CREATE`/`ALTER`/`DROP`。MySQL 的 DDL 会隐式提交,所以迁移中途失败不会自动回滚;失败后处理见第 9 节。 ## 3. 配置 ### 外部 database(推荐) -使用 `compose.external-db.yaml`: +使用 `compose.external-db.yaml`,它只接受 URL,两种引擎通用: ```dotenv APP_VERSION=2.0.0 APP_ORIGIN=https://tasks.example.com +# PostgreSQL DATABASE_URL=postgres://taskdock_app:replace-me@db.example.lan:5432/taskdock_home MIGRATION_DATABASE_URL=postgres://taskdock_migrator:replace-me@db.example.lan:5432/taskdock_home +# MySQL(二选一,改 scheme 即可) +# DATABASE_URL=mysql://taskdock_app:replace-me@db.example.lan:3306/taskdock_home +# MIGRATION_DATABASE_URL=mysql://taskdock_migrator:replace-me@db.example.lan:3306/taskdock_home DATABASE_SSL_MODE=verify-full DATABASE_SSL_CA=/run/secrets/db-ca.pem DB_POOL_MAX=5 @@ -72,21 +87,30 @@ DB_IDLE_TIMEOUT_MS=300000 DB_APPLICATION_NAME=taskdock-home ``` -如果密码包含 `@`、`:`、`/`、`#`,必须做 URL 编码;更稳妥的方式是改用离散变量: +密码包含 `@`、`:`、`/`、`#` 时必须 URL 编码。不想编码时改用离散变量: ```dotenv +# PostgreSQL PGHOST=db.example.lan PGPORT=5432 PGUSER=taskdock_app PGPASSWORD=replace-me PGDATABASE=taskdock_home + +# 或 MySQL +MYSQL_HOST=db.example.lan +MYSQL_PORT=3306 +MYSQL_USER=taskdock_app +MYSQL_PASSWORD=replace-me +MYSQL_DATABASE=taskdock_home ``` -内置 `compose.yaml` 已经使用离散 `PG*` 变量,因此内置密码不需要 URL 编码。 +内置 Compose 已经使用离散变量,因此内置密码不需要 URL 编码。 ### 内置 database -保留 `compose.yaml` 和 `POSTGRES_*` 即可。应用通过 `PGHOST=postgres` 连接,不会把密码拼进 URL。 +- PostgreSQL(默认):`compose.yaml` + `POSTGRES_*`,应用通过 `PGHOST=postgres` 连接。 +- MySQL:`compose.mysql.yaml` + `MYSQL_*`,应用通过 `MYSQL_HOST=mysql` 连接。 ## 4. 启动顺序 @@ -100,8 +124,8 @@ curl -fsS https://tasks.example.com/health/ready - `migrate` 使用 `MIGRATION_DATABASE_URL`(为空时回退 `DATABASE_URL`)。 - `bootstrap` 使用 `DATABASE_URL`,只创建 Owner。 -- `app` 启动时会再次检查 migration checksum;schema 不齐会重试到 `DB_STARTUP_MAX_WAIT_MS`,然后失败退出。 -- `/health/live` 只代表进程存活;`/health/ready` 才代表 PostgreSQL 可用。 +- `app` 启动时会再次核对 migration checksum;schema 不齐会重试到 `DB_STARTUP_MAX_WAIT_MS`,然后失败退出。 +- `/health/live` 只代表进程存活;`/health/ready` 才代表数据库可用。 ## 5. TLS @@ -114,10 +138,11 @@ curl -fsS https://tasks.example.com/health/ready ``` - `require` 会加密但不校验证书,只适合可信内网。 - 不要把 CA、证书或密码写进镜像、Compose 文件或 Git。 +- MySQL 驱动通过 `checkServerIdentity` 实现 `verify-ca`;如果目标环境不接受该实现,就只用 `disable` / `require` / `verify-full`。 -## 6. PgBouncer 与连接预算 +## 6. 连接预算 -多个应用共用一个 PostgreSQL 时,连接数要按总预算计算: +多个应用共用一个数据库时,连接数按总预算计算: ```text 总连接数 ≈ Σ(每个 app 的 DB_POOL_MAX + LISTEN 连接) @@ -125,42 +150,42 @@ curl -fsS https://tasks.example.com/health/ready + 管理/监控连接 ``` -建议: - - `DB_POOL_MAX` 默认 10;多应用共享时通常 3–5 就够。 -- 使用 PgBouncer `pool_mode=transaction` 复用连接。 -- PgBouncer transaction mode **不能可靠保持 `LISTEN`**。设置 `DB_LISTEN_ENABLED=false`,让客户端依赖 pull/轮询;或让 API 直连 PostgreSQL 建立一条专用 LISTEN 连接。 -- 迁移 job 建议直连 PostgreSQL,不经 PgBouncer,因为迁移包含长事务、advisory lock 和 DDL。 -- 监控 `pg_stat_activity`、PgBouncer `SHOW POOLS` 和 API 的 pool wait 指标。 +- PostgreSQL:使用 PgBouncer `pool_mode=transaction` 复用连接。transaction mode **不能可靠保持 `LISTEN`**,设置 `DB_LISTEN_ENABLED=false`,或让 API 直连建立专用 LISTEN 连接。 +- MySQL:LISTEN 不存在,连接池本身即复用;注意 `max_connections`(默认 151)和每连接内存。MySQL 只允许 SELECT 使用 `max_execution_time`,写入的超时靠 `innodb_lock_wait_timeout`(由 `DB_LOCK_TIMEOUT_MS` 换算)。 +- 迁移 job 建议直连,不经连接池中间层,因为迁移包含长事务、锁和 DDL。 +- 监控 `pg_stat_activity` / `performance_schema`、PgBouncer `SHOW POOLS` 和 API 的 pool wait 指标。 -## 7. 备份与恢复 +## 7. 与 PostgreSQL 的行为差异 -外部库直接用 `pg_dump` / `pg_restore`: +引擎切换后,下面这些能力会被明确降级或改变,配置前应确认可以接受: -```bash -pg_dump --format=custom --no-owner --no-acl \ - --dbname="postgres://taskdock_migrator:...@db.example.lan:5432/taskdock_home" \ - --file=taskdock_home.dump - -pg_restore --exit-on-error --no-owner --no-acl \ - --dbname="postgres://taskdock_migrator:...@db.example.lan:5432/taskdock_home_restore" \ - taskdock_home.dump - -# 必须显式覆盖两个变量;否则会继承 .env 并可能迁移生产库。 -RESTORE_DATABASE_URL='postgres://taskdock_migrator:...@db.example.lan:5432/taskdock_home_restore' -DATABASE_URL="$RESTORE_DATABASE_URL" \ -MIGRATION_DATABASE_URL="$RESTORE_DATABASE_URL" \ - docker compose -f compose.external-db.yaml run --rm migrate -``` +| 能力 | PostgreSQL | MySQL | +| -------------- | ---------------------------- | ---------------------------------------------- | +| 跨实例实时推送 | `LISTEN/NOTIFY` 即时唤醒 | 进程内通知;多实例靠客户端轮询或 Redis pub/sub | +| 搜索 | `pg_trgm` GIN 索引 | `LIKE '%term%'` 扫描(owner 范围内) | +| 语句级超时 | `statement_timeout` 覆盖读写 | `max_execution_time` 只作用于 SELECT | +| 部分唯一索引 | 原生 `WHERE` 谓词 | 生成列 + 唯一索引(NULL 不冲突) | +| `RETURNING` | 单语句返回 | 先写后按主键回查(同一事务) | +| 迁移原子性 | 整个迁移集一个事务 | DDL 隐式提交,逐文件记录 | +| 时区 | `timestamptz` | `DATETIME(6)` 存 UTC,会话固定 `+00:00` | -恢复后先跑 `migrate`,再启动 `app`,最后用 `/health/ready` 和一次真实登录/同步验证。 +## 8. 备份与恢复 -## 8. 可选 Redis +PostgreSQL 用 `pg_dump` / `pg_restore`;MySQL 用 `mysqldump` / `mysql`。完整命令、演练流程和恢复后的 `migrate` 步骤见 [备份与恢复](BACKUP_RESTORE.md)。 + +## 9. 迁移失败后的处理 + +- PostgreSQL:迁移在事务里执行,失败自动回滚,修好后直接重跑 `migrate`。 +- MySQL:每个文件成功后才写 `schema_migrations`。中途失败时,已成功的文件保留记录,失败文件可能留下部分对象。处理顺序是:先看错误定位语句 → 手动补完或删除该文件已创建的对象 → 确认 `schema_migrations` 没有该版本 → 重跑 `migrate`。不要删除整库,也不要把已成功的版本一起回退。 + +## 10. 可选 Redis Redis 不是权威存储,只用于: - 多 API 副本共享登录限流计数; -- 多副本冷启动时共享 Owner 投影快照(TTL 缓存)。 +- 多副本冷启动时共享 Owner 投影快照(TTL 缓存); +- MySQL 多实例部署时广播 `sync.required`。 配置: @@ -171,21 +196,18 @@ CACHE_SNAPSHOT_TTL_SECONDS=60 CACHE_REQUIRED=false ``` -规则: - -- Redis 故障时,若配置了 `CACHE_URL`,认证请求必须 fail-closed;投影缓存等非权威能力可以回退数据库。`/health/ready` 会报告 `cache: degraded`,认证不会退回各副本独立的本地计数。 +- Redis 故障时,若配置了 `CACHE_URL`,认证请求必须 fail-closed;投影缓存等非权威能力可以回退数据库。`/health/ready` 会报告 `cache: degraded`。 - 不要用 Redis 存实体、mutation receipt 或 sync cursor。 - 多个中枢共享 Redis 时必须给每个中枢不同的 `CACHE_KEY_PREFIX`。 -- 内置 `compose.yaml` 的 Redis 是可选 profile:`docker compose --profile cache up -d`,并设置 `CACHE_URL=redis://redis:6379/0`。 -## 9. 验收清单 +## 11. 验收清单 - [ ] 每个中枢使用独立 database。 - [ ] 应用和迁移使用不同角色。 - [ ] 外部库使用 TLS,CA 通过 secret 挂载。 -- [ ] `DB_POOL_MAX` × 应用数 + listener < PostgreSQL `max_connections`。 -- [ ] PgBouncer 场景设置 `DB_LISTEN_ENABLED=false` 或使用直连 listener。 +- [ ] `DB_POOL_MAX` × 应用数 + listener < 数据库 `max_connections`。 +- [ ] PostgreSQL + PgBouncer 场景设置 `DB_LISTEN_ENABLED=false` 或使用直连 listener。 - [ ] `migrate` 在 `app` 之前成功执行。 -- [ ] `pg_dump`/`pg_restore` 在独立临时库演练通过。 +- [ ] 备份/恢复在独立临时库演练通过(`pg_dump`/`pg_restore` 或 `mysqldump`)。 - [ ] Redis 断开时写入、登录和同步仍可用。 - [ ] `/health/ready` 在数据库断开时 503、恢复后 200。 diff --git a/docs/adr/0005-mysql-support.md b/docs/adr/0005-mysql-support.md new file mode 100644 index 0000000..9c4f690 --- /dev/null +++ b/docs/adr/0005-mysql-support.md @@ -0,0 +1,32 @@ +# ADR 0005:双引擎数据库支持(PostgreSQL / MySQL) + +状态:接受(PostgreSQL 仍是默认;MySQL 8.4 一等支持,存在明确的能力差异) + +## 背景 + +中枢默认只支持 PostgreSQL。部署者的实际环境里,MySQL 8.4 往往已经作为共享数据库服务存在(NAS、云数据库、其它应用已在用),再为 TaskDock 单独维护一套 PostgreSQL 是额外的运维负担。要求是:用户能自己选引擎,MySQL 也允许使用外部服务器。 + +现有存储层是 3000 余行手写 SQL,包含大量 PostgreSQL 专有写法(`RETURNING`、`jsonb_to_recordset`、`jsonb_agg`、`FILTER` 聚合、部分唯一索引、`LISTEN/NOTIFY`、`pg_trgm`、advisory lock),并且这些查询是为了高延迟远端库做过往返次数优化的。 + +## 决策 + +1. **只保留一份 SQL 与一份存储实现**,不为 MySQL 复制适配器,也不引入 ORM 重写。`PostgresStore`/`PostgresTreeStore` 改名为 `SqlStore`/`SqlTreeStore`,接收 `SqlDatabase` 抽象。 +2. **方言差异集中在 `@devtodo/database`**: + - `$n` 占位符由方言层翻译(MySQL 重排为 `?`,支持参数重复使用); + - `::type` 对 MySQL 剥离(PostgreSQL 需要它做类型推断); + - `ILIKE` 翻译为 `LIKE`(MySQL 默认排序规则大小写不敏感); + - `rank`/`cursor` 是 MySQL 保留字,方言层自动加反引号,PostgreSQL 保持原文。 +3. **只有少数语句保留两份拼写**(`apps/api/src/sql-statements.ts`):owner 快照聚合、sync pull、目录分页、文件夹路径。它们依赖 JSON 聚合、`MATERIALIZED` 或递归 CTE 的引擎差异,硬凑成一份 SQL 反而更难维护。 +4. **`RETURNING` 用同事务回查模拟**:语句执行后按已知主键读回。MySQL 的 `sync_changes.seq` 用 `insertId + affectedRows` 得到批内最大游标。 +5. **部分唯一索引改为生成列**:`IF(deleted_at IS NULL, 1, NULL)` 的 STORED 生成列加入唯一键,MySQL 允许多个 NULL,墓碑因此可以并存。部分普通索引退化为全量索引。 +6. **MySQL schema 从合并基线开始**:`migrations/mysql/0013_baseline.sql` 等价于 PostgreSQL `0001..0013`,编号对齐;此后每个变更两边各一个同版本文件,并有测试保证版本集合一致。MySQL 的 DDL 隐式提交,迁移 runner 改为逐文件提交 + `GET_LOCK` 串行。 +7. **通知降级**:进程内在 COMMIT 后直接 fan-out;跨实例实时推送在 MySQL 下需要 Redis pub/sub,否则靠客户端轮询。`DB_LISTEN_ENABLED` 对 MySQL 恒为 false。 + +## 代价与边界 + +- 搜索:MySQL 没有 `pg_trgm` 等价索引,`LIKE '%term%'` 在 owner 范围内扫描;大数据集需要额外方案。 +- 语句级超时:MySQL 的 `max_execution_time` 只作用于 SELECT;写入靠 `innodb_lock_wait_timeout`,无法被强杀。 +- 硬删除:MySQL 基线里子外键全部级联,删除 Owner 是一步操作;应用本身仍然只做软删除。 +- 自引用外键 `refresh_sessions(owner_id, replaced_by_id)` 在 MySQL 中被省略:复合键含 NOT NULL 列时无法 `SET NULL`,自级联又会删掉被轮换的会话。 +- 不做 PostgreSQL → MySQL 的存量数据搬迁;换引擎等于新建中枢,数据通过客户端同步协议重新灌入。 +- 不支持 MariaDB(`RETURNING`、部分索引、JSON 语义均不同)。 diff --git a/website/site/index.html b/website/site/index.html index 2ab0fa7..3f31855 100644 --- a/website/site/index.html +++ b/website/site/index.html @@ -6,7 +6,7 @@ TaskDock — 让散落的任务各归其位 @@ -482,7 +482,7 @@

断网的时候,
工作不会停。
TaskDock 中枢 - Fastify · PostgreSQL + Fastify · PostgreSQL / MySQL cursor 1042
@@ -636,7 +636,7 @@

数据放在你自己的
服务器上。 > PostgreSQL 18
PostgreSQL 18 / MySQL 8.4
权威数据源,只在容器网络内可达
@@ -800,7 +800,8 @@

Android

>

中枢 · Docker

- Fastify 与 PostgreSQL 组成的单用户中枢。所有设备的数据最终汇集在这里,也只在这里。 + Fastify 与 PostgreSQL/MySQL + 组成的单用户中枢。所有设备的数据最终汇集在这里,也只在这里。

ghcr.io/igngserver/taskdock From b4858ef11492b376dcba1d6fdd6d14b688fe5555 Mon Sep 17 00:00:00 2001 From: IGNGserver <176017966+IGNGserver@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:18:08 +0800 Subject: [PATCH 6/9] =?UTF-8?q?test(db):=20=E9=9B=86=E6=88=90=E5=86=92?= =?UTF-8?q?=E7=83=9F=E4=B8=8E=E5=B9=B6=E5=8F=91=20worker=20=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=8F=8C=E5=BC=95=E6=93=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - integration-smoke / sql-concurrency-worker 通过 openDatabase 连接,引擎由 URL scheme 决定;空库检查按方言查询 information_schema - 修复 MySQL 不兼容写法:同表 MAX 子查询加别名、NULLS LAST/FIRST 改为 可移植排序键、= ANY(array) 改为 IN 列表、interval 字面量改为参数、 保留字 CTE 改名、bootstrap 清理改用 DELETE ORDER BY LIMIT - 方言层翻译 IS NOT DISTINCT FROM -> <=>,throwSql 识别 MySQL 约束错误码 - withIdempotency 在 MySQL 下用 owner 行锁取代 advisory lock,避免并发 幂等 mutation 双重执行 --- apps/api/src/sql-statements.ts | 17 +++- apps/api/src/sql-store.ts | 44 +++++++--- apps/api/src/sync-bootstrap.ts | 83 +++++++++++++------ apps/api/test/store.test.ts | 2 +- packages/database/src/dialect.ts | 9 ++ scripts/integration-smoke.ts | 68 +++++++++------ ...cy-worker.ts => sql-concurrency-worker.ts} | 10 ++- scripts/test-database-safety.ts | 24 ++++-- 8 files changed, 177 insertions(+), 80 deletions(-) rename scripts/{pg-concurrency-worker.ts => sql-concurrency-worker.ts} (85%) diff --git a/apps/api/src/sql-statements.ts b/apps/api/src/sql-statements.ts index 87a7ec1..b8e0e8b 100644 --- a/apps/api/src/sql-statements.ts +++ b/apps/api/src/sql-statements.ts @@ -101,6 +101,7 @@ const SNAPSHOT_COLLECTIONS: SnapshotCollection[] = [ 'parent_folder_id', 'title', 'status', + { name: 'completed_at', timestamp: true }, { name: 'rank', asChar: true }, 'version', { name: 'created_at', timestamp: true }, @@ -368,7 +369,7 @@ export function directoryChildrenQuery( SUM(CASE WHEN t.status='DONE' THEN 1 ELSE 0 END) AS done FROM branch b LEFT JOIN tasks t ON t.owner_id=$1 AND t.parent_folder_id=b.id AND t.deleted_at IS NULL GROUP BY b.root -), rows AS ( +), entries AS ( SELECT f.id,f.\`rank\` AS r,0 AS k, CASE f.status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END AS g, JSON_OBJECT('status',f.status, @@ -379,7 +380,7 @@ export function directoryChildrenQuery( NULL FROM tasks t WHERE t.owner_id=$1 AND t.deleted_at IS NULL AND ${parentWhere} ), page AS ( - SELECT * FROM rows ${seek} ORDER BY g,r,k,id ${take} + SELECT * FROM entries ${seek} ORDER BY g,r,k,id ${take} ) SELECT CASE p.k WHEN 0 THEN JSON_OBJECT('kind','FOLDER','folder',${folderRowJson('f')},'aggregate',p.aggregate) ELSE JSON_OBJECT('kind','TASK','task',${taskRowJson('t')}) END AS item @@ -399,7 +400,7 @@ ORDER BY p.g,p.r,p.k,p.id`; COUNT(t.id) FILTER (WHERE t.status='DONE')::int AS done FROM branch b LEFT JOIN tasks t ON t.owner_id=$1 AND t.parent_folder_id=b.id AND t.deleted_at IS NULL GROUP BY b.root -), rows AS ( +), entries AS ( SELECT f.id,f.rank AS r,0 AS k, CASE f.status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END AS g, jsonb_build_object('status',f.status, @@ -410,7 +411,7 @@ ORDER BY p.g,p.r,p.k,p.id`; NULL::jsonb FROM tasks t WHERE t.owner_id=$1 AND t.deleted_at IS NULL AND ${parentWhere} ), page AS MATERIALIZED ( - SELECT * FROM rows ${seek} ORDER BY g,r,k,id ${take} + SELECT * FROM entries ${seek} ORDER BY g,r,k,id ${take} ) SELECT CASE p.k WHEN 0 THEN jsonb_build_object('kind','FOLDER','folder',to_jsonb(f) || jsonb_build_object('rank',f.rank::text),'aggregate',p.aggregate) ELSE jsonb_build_object('kind','TASK','task',to_jsonb(t) || jsonb_build_object('rank',t.rank::text)) END AS item @@ -433,3 +434,11 @@ export const FOLDER_PATH_QUERY: Record = { WHERE f.owner_id=$1 AND f.deleted_at IS NULL AND p.depth < 100 ) SELECT * FROM parents ORDER BY depth DESC`, }; + +/** Full folder ancestry rows (used by the bootstrap task detail). */ +export const FOLDER_FULL_PATH_QUERY: Record = { + postgres: + 'WITH RECURSIVE path AS (SELECT f.*, ARRAY[f.id] AS visited, 0 AS depth FROM folders f WHERE owner_id = $1 AND id = $2 UNION ALL SELECT f.*, path.visited || f.id, depth + 1 FROM folders f JOIN path ON f.id = path.parent_folder_id WHERE f.owner_id = $1 AND NOT f.id = ANY(path.visited)) SELECT * FROM path ORDER BY depth DESC', + mysql: + 'WITH RECURSIVE path AS (SELECT f.*, 0 AS depth FROM folders f WHERE owner_id = $1 AND id = $2 UNION ALL SELECT f.*, depth + 1 FROM folders f JOIN path ON f.id = path.parent_folder_id WHERE f.owner_id = $1 AND path.depth < 100) SELECT * FROM path ORDER BY depth DESC', +}; diff --git a/apps/api/src/sql-store.ts b/apps/api/src/sql-store.ts index ec1ed6c..b86c4c8 100644 --- a/apps/api/src/sql-store.ts +++ b/apps/api/src/sql-store.ts @@ -924,7 +924,7 @@ export class SqlStore implements Store { const result = await this.executeReturning( 'INSERT INTO projects ' + '(id, owner_id, name, task_prefix, next_task_number, rank, version, created_at, updated_at) ' + - 'VALUES ($1, $2, $3, $4, 1, (SELECT COALESCE(MAX(rank), 0) + 1024 FROM projects WHERE owner_id = $2 AND deleted_at IS NULL), 1, $5, $5) ' + + 'VALUES ($1, $2, $3, $4, 1, (SELECT COALESCE(MAX(project.rank), 0) + 1024 FROM projects project WHERE project.owner_id = $2 AND project.deleted_at IS NULL), 1, $5, $5) ' + 'RETURNING id, owner_id, name, task_prefix, next_task_number, rank, version, created_at, updated_at, deleted_at', [newEntityId(id), ownerId, normalizedName, normalizedPrefix, this.now()], ); @@ -1082,7 +1082,7 @@ export class SqlStore implements Store { 'INSERT INTO tasks ' + '(id, owner_id, project_id, category, reference_id, title, status, priority, rank, version, created_at, updated_at) ' + "VALUES ($1, $2, $3, $4, $5, $6, 'TODO', $7, " + - '(SELECT COALESCE(MAX(rank), 0) + 1024 FROM tasks WHERE owner_id = $2 AND project_id IS NOT DISTINCT FROM $3 AND category = $4 AND deleted_at IS NULL), ' + + '(SELECT COALESCE(MAX(sibling.rank), 0) + 1024 FROM tasks sibling WHERE sibling.owner_id = $2 AND sibling.project_id IS NOT DISTINCT FROM $3 AND sibling.category = $4 AND sibling.deleted_at IS NULL), ' + '1, $8, $8) ' + 'RETURNING id, owner_id, project_id, category, reference_id, title, status, priority, rank, version, completed_at, created_at, updated_at, deleted_at', [taskId, ownerId, projectId, input.category, referenceId, title, input.priority, this.now()], @@ -1348,7 +1348,7 @@ export class SqlStore implements Store { 'INSERT INTO tasks ' + '(id, owner_id, project_id, category, reference_id, title, status, priority, rank, version, created_at, updated_at) ' + "VALUES ($1, $2, $3, $4, $5, $6, 'TODO', $7, " + - '(SELECT COALESCE(MAX(rank), 0) + 1024 FROM tasks WHERE owner_id = $2 AND project_id IS NOT DISTINCT FROM $3 AND category = $4 AND deleted_at IS NULL), ' + + '(SELECT COALESCE(MAX(sibling.rank), 0) + 1024 FROM tasks sibling WHERE sibling.owner_id = $2 AND sibling.project_id IS NOT DISTINCT FROM $3 AND sibling.category = $4 AND sibling.deleted_at IS NULL), ' + '1, $8, $8) ' + 'RETURNING id, owner_id, project_id, category, reference_id, title, status, priority, rank, version, completed_at, created_at, updated_at, deleted_at', [ @@ -1448,7 +1448,7 @@ export class SqlStore implements Store { const result = await this.executeReturning( 'INSERT INTO time_points (id, owner_id, type, local_date, title, rank, version, created_at, updated_at) ' + 'VALUES ($1, $2, $3, $4, $5, ' + - "(SELECT COALESCE(MAX(rank), 0) + 1024 FROM time_points WHERE owner_id = $2 AND type = 'EVENT' AND deleted_at IS NULL), " + + "(SELECT COALESCE(MAX(point.rank), 0) + 1024 FROM time_points point WHERE point.owner_id = $2 AND point.type = 'EVENT' AND point.deleted_at IS NULL), " + '1, $6, $6) ' + (input.type === 'DATE' ? this.doNothingOnConflict('owner_id, local_date', "type = 'DATE' AND deleted_at IS NULL") @@ -1482,7 +1482,7 @@ export class SqlStore implements Store { 'SELECT id, owner_id, type, local_date, title, rank, version, reached_at, created_at, updated_at, deleted_at ' + 'FROM time_points WHERE ' + where.join(' AND ') + - " ORDER BY CASE WHEN type = 'DATE' THEN 0 ELSE 1 END, local_date NULLS LAST, rank, id", + " ORDER BY CASE WHEN type = 'DATE' THEN 0 ELSE 1 END, (local_date IS NULL), local_date, rank, id", values, ); return result.rows.map((row) => timePointDto(timePointRecord(row))); @@ -1518,7 +1518,7 @@ export class SqlStore implements Store { 'WHERE ' + where.join(' AND ') + ' GROUP BY tp.id, tp.local_date ' + - "ORDER BY CASE WHEN tp.type = 'DATE' THEN 0 ELSE 1 END, tp.local_date NULLS LAST, tp.id", + "ORDER BY CASE WHEN tp.type = 'DATE' THEN 0 ELSE 1 END, (tp.local_date IS NULL), tp.local_date, tp.id", values, ); return result.rows.map((row) => ({ @@ -1570,7 +1570,7 @@ export class SqlStore implements Store { 'SELECT id, owner_id, type, local_date, title, rank, version, reached_at, created_at, updated_at, deleted_at ' + 'FROM time_points WHERE ' + where.join(' AND ') + - " ORDER BY CASE WHEN type = 'DATE' THEN 0 ELSE 1 END, local_date NULLS LAST, rank, id LIMIT $" + + " ORDER BY CASE WHEN type = 'DATE' THEN 0 ELSE 1 END, (local_date IS NULL), local_date, rank, id LIMIT $" + values.length, values, ); @@ -1708,7 +1708,7 @@ export class SqlStore implements Store { const result = await this.executeReturning( 'INSERT INTO placements (id, owner_id, task_id, time_point_id, rank, version, created_at, updated_at) ' + 'VALUES ($1, $2, $3, $4, ' + - '(SELECT COALESCE(MAX(rank), 0) + 1024 FROM placements WHERE owner_id = $2 AND time_point_id = $4 AND deleted_at IS NULL), ' + + '(SELECT COALESCE(MAX(placement.rank), 0) + 1024 FROM placements placement WHERE placement.owner_id = $2 AND placement.time_point_id = $4 AND placement.deleted_at IS NULL), ' + '1, $5, $5) ' + this.doNothingOnConflict('task_id, time_point_id') + 'RETURNING id, owner_id, task_id, time_point_id, rank, version, created_at, updated_at, deleted_at', @@ -2029,12 +2029,20 @@ export class SqlStore implements Store { ); const key = ownerId + ':' + clientId + ':' + mutationId; const requestHash = hash(input); - if (this.database.dialect === 'postgres') + const mysql = this.database.dialect === 'mysql'; + if (mysql) { + // MySQL has no transaction-scoped advisory lock. Locking the owner row is + // the narrowest lock that still serialises "check receipt then mutate", + // and it is released by the same COMMIT that publishes the receipt. + await this.query('SELECT id FROM users WHERE id = $1 FOR UPDATE', [ownerId]); + } else { await this.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [key]); + } // Expired receipts are removed by runMaintenance(). Treating them as // absent here avoids a DELETE round trip on every mutation. const existing = await this.query( - 'SELECT request_hash, result FROM client_mutations WHERE owner_id = $1 AND client_id = $2 AND mutation_id = $3 AND expires_at > now()', + 'SELECT request_hash, result FROM client_mutations WHERE owner_id = $1 AND client_id = $2 AND mutation_id = $3 AND expires_at > now()' + + (mysql ? ' FOR UPDATE' : ''), [ownerId, clientId, mutationId], ); if (existing.rows[0]) { @@ -3108,8 +3116,16 @@ function throwSql( 'code' in error && typeof error.code === 'string' ) { - if (error.code === '23505') throw new DatabaseConstraintError(duplicateCode, message, error); - if (error.code === '23503' || error.code === '23514') + // PostgreSQL reports SQLSTATE codes; MySQL reports symbolic ER_* codes. + if (error.code === '23505' || error.code === 'ER_DUP_ENTRY') + throw new DatabaseConstraintError(duplicateCode, message, error); + if ( + error.code === '23503' || + error.code === '23514' || + error.code === 'ER_NO_REFERENCED_ROW_2' || + error.code === 'ER_ROW_IS_REFERENCED_2' || + error.code === 'ER_CHECK_CONSTRAINT_VIOLATED' + ) throw new DatabaseConstraintError('VALIDATION_FAILED', message, error); } throw error; @@ -3121,7 +3137,9 @@ function databaseConstraintDiagnostic(error: unknown): DatabaseConstraintDiagnos const value = (error as Record)[key]; return typeof value === 'string' && value.length > 0 ? value.slice(0, 128) : undefined; }; - const constraint = read('constraint'); + // MySQL names the violated key inside the message: "for key 'tbl.key_name'". + const mysqlKey = /for key '([^']+)'/.exec(read('sqlMessage') ?? ''); + const constraint = read('constraint') ?? (mysqlKey ? mysqlKey[1] : undefined); const table = read('table'); const schema = read('schema'); return { diff --git a/apps/api/src/sync-bootstrap.ts b/apps/api/src/sync-bootstrap.ts index b15d1ba..1ce4fa9 100644 --- a/apps/api/src/sync-bootstrap.ts +++ b/apps/api/src/sync-bootstrap.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; import { DomainError } from '@devtodo/domain'; -import { taskRowJson } from './sql-statements.js'; +import { FOLDER_FULL_PATH_QUERY, taskRowJson } from './sql-statements.js'; import { SqlTreeStore } from './sql-tree-store.js'; import type { V2Snapshot, V2TreeStore } from './tree-store.js'; @@ -36,6 +36,11 @@ interface Session { } /** Maps SQL rows without loading the mutable domain projection or its notes. */ +/** `$start, $start+1, ...` for a portable IN (...) list. */ +function placeholders(start: number, count: number): string { + return Array.from({ length: count }, (_, index) => `$${start + index}`).join(','); +} + function dto(row: Record): Record { const value: Record = {}; for (const [key, raw] of Object.entries(row)) { @@ -127,9 +132,13 @@ export class BootstrapSnapshots { if (this.cleanupFlight) return this.cleanupFlight; const tree = this.tree; if (!(tree instanceof SqlTreeStore)) return Promise.resolve(); + // MySQL refuses a LIMIT subquery inside IN (...) and supports a directly + // bounded DELETE; deleting an already-removed row is a harmless no-op. this.cleanupFlight = tree.sql .v2Query( - 'DELETE FROM sync_bootstrap_snapshots WHERE id IN (SELECT id FROM sync_bootstrap_snapshots WHERE expires_at <= now() ORDER BY expires_at LIMIT 2 FOR UPDATE SKIP LOCKED)', + tree.sql.dialect === 'mysql' + ? 'DELETE FROM sync_bootstrap_snapshots WHERE expires_at <= now() ORDER BY expires_at LIMIT 2' + : 'DELETE FROM sync_bootstrap_snapshots WHERE id IN (SELECT id FROM sync_bootstrap_snapshots WHERE expires_at <= now() ORDER BY expires_at LIMIT 2 FOR UPDATE SKIP LOCKED)', ) .then(() => undefined) .finally(() => { @@ -145,7 +154,9 @@ export class BootstrapSnapshots { const snapshot: Record = {}; for (const [key, [, table]] of Object.entries(collections)) { const order = - table === 'folders' || table === 'tasks' ? 'parent_folder_id NULLS FIRST, id' : 'id'; + table === 'folders' || table === 'tasks' + ? '(parent_folder_id IS NOT NULL), parent_folder_id, id' + : 'id'; const rank = table === 'notes' ? '' : ', rank::text AS rank'; const rows = await sql.v2Query( `SELECT *${rank} FROM ${table} WHERE owner_id = $1 AND deleted_at IS NULL ORDER BY ${order}`, @@ -288,19 +299,25 @@ export class BootstrapSnapshots { [ownerId, localDate], ); const pointIds = points.rows.map((p) => p['id']); - const placements = await sql.v2Query( - 'SELECT *, rank::text AS rank FROM placements WHERE owner_id = $1 AND time_point_id = ANY($2::uuid[]) AND deleted_at IS NULL ORDER BY placements.rank, id LIMIT 101', - [ownerId, pointIds], - ); + const placements = + pointIds.length === 0 + ? { rows: [] as Array> } + : await sql.v2Query( + `SELECT *, rank::text AS rank FROM placements WHERE owner_id = $1 AND time_point_id IN (${placeholders(2, pointIds.length)}) AND deleted_at IS NULL ORDER BY placements.rank, id LIMIT 101`, + [ownerId, ...pointIds], + ); const taskIds = placements.rows.slice(0, 100).map((p) => p['task_id']); const rootTasks = await sql.v2Query( `SELECT *, rank::text AS rank FROM tasks WHERE owner_id = $1 AND ${parent} AND deleted_at IS NULL ORDER BY CASE status WHEN 'IN_PROGRESS' THEN 0 WHEN 'TODO' THEN 1 ELSE 2 END, tasks.rank, id LIMIT 101`, parentValues, ); - const scheduledTasks = await sql.v2Query( - 'SELECT *, rank::text AS rank FROM tasks WHERE owner_id = $1 AND id = ANY($2::uuid[]) AND deleted_at IS NULL', - [ownerId, taskIds], - ); + const scheduledTasks = + taskIds.length === 0 + ? { rows: [] as Array> } + : await sql.v2Query( + `SELECT *, rank::text AS rank FROM tasks WHERE owner_id = $1 AND id IN (${placeholders(2, taskIds.length)}) AND deleted_at IS NULL`, + [ownerId, ...taskIds], + ); const selected = new Map( [...rootTasks.rows.slice(0, 100), ...scheduledTasks.rows].map((t) => [t['id'], dto(t)]), ); @@ -497,25 +514,37 @@ export class BootstrapSnapshots { 'SELECT *, rank::text AS rank FROM workflow_task_memberships WHERE owner_id = $1 AND task_id = $2 AND deleted_at IS NULL ORDER BY workflow_task_memberships.rank, id', [ownerId, id], ); - const points = await sql.v2Query( - 'SELECT *, rank::text AS rank FROM time_points WHERE owner_id = $1 AND id = ANY($2::uuid[]) AND deleted_at IS NULL', - [ownerId, placements.rows.map((p) => p['time_point_id'])], - ); + const pointIds = placements.rows.map((p) => p['time_point_id']); + const points = + pointIds.length === 0 + ? { rows: [] as Array> } + : await sql.v2Query( + `SELECT *, rank::text AS rank FROM time_points WHERE owner_id = $1 AND id IN (${placeholders(2, pointIds.length)}) AND deleted_at IS NULL`, + [ownerId, ...pointIds], + ); const pointById = new Map(points.rows.map((p) => [p['id'], dto(p)])); - const workflows = await sql.v2Query( - 'SELECT id, name FROM workflows WHERE owner_id = $1 AND id = ANY($2::uuid[])', - [ownerId, memberships.rows.map((m) => m['workflow_id'])], - ); - const stages = await sql.v2Query( - 'SELECT id, name FROM workflow_stages WHERE owner_id = $1 AND id = ANY($2::uuid[])', - [ownerId, memberships.rows.map((m) => m['stage_id'])], - ); + const workflowIds = memberships.rows.map((m) => m['workflow_id']); + const workflows = + workflowIds.length === 0 + ? { rows: [] as Array> } + : await sql.v2Query( + `SELECT id, name FROM workflows WHERE owner_id = $1 AND id IN (${placeholders(2, workflowIds.length)})`, + [ownerId, ...workflowIds], + ); + const stageIds = memberships.rows.map((m) => m['stage_id']); + const stages = + stageIds.length === 0 + ? { rows: [] as Array> } + : await sql.v2Query( + `SELECT id, name FROM workflow_stages WHERE owner_id = $1 AND id IN (${placeholders(2, stageIds.length)})`, + [ownerId, ...stageIds], + ); const workflowById = new Map(workflows.rows.map((w) => [w['id'], w])); const stageById = new Map(stages.rows.map((s) => [s['id'], s])); - const path = await sql.v2Query( - 'WITH RECURSIVE path AS (SELECT f.*, ARRAY[f.id] AS visited, 0 AS depth FROM folders f WHERE owner_id = $1 AND id = $2 UNION ALL SELECT f.*, path.visited || f.id, depth + 1 FROM folders f JOIN path ON f.id = path.parent_folder_id WHERE f.owner_id = $1 AND NOT f.id = ANY(path.visited)) SELECT * FROM path ORDER BY depth DESC', - [ownerId, task['parentFolderId']], - ); + const path = await sql.v2Query(FOLDER_FULL_PATH_QUERY[sql.dialect], [ + ownerId, + task['parentFolderId'], + ]); if (!notes.rows[0]) throw new DomainError('ENTITY_NOT_FOUND', '找不到任务备注'); return { task, diff --git a/apps/api/test/store.test.ts b/apps/api/test/store.test.ts index a4b3d35..d4ccc48 100644 --- a/apps/api/test/store.test.ts +++ b/apps/api/test/store.test.ts @@ -342,7 +342,7 @@ describe('DevTodo store invariants', () => { const client = { query: async (text: string, values: unknown[] = []) => { queries.push({ text, values }); - if (text.includes('AS owner_ok') && text.includes('AS cursor')) { + if (text.includes('AS owner_ok') && text.includes('cursor')) { return { rows: [{ owner_ok: true, cursor: '9' }], rowCount: 1 }; } if (text.includes('AS next_task_number') && text.includes('AS folders')) { diff --git a/packages/database/src/dialect.ts b/packages/database/src/dialect.ts index 6065bc3..45956e7 100644 --- a/packages/database/src/dialect.ts +++ b/packages/database/src/dialect.ts @@ -134,6 +134,15 @@ function prepareMysql(sql: string, values: readonly unknown[]): PreparedQuery { continue; } } + if (char === 'I' || char === 'i') { + // PostgreSQL's null-safe comparison; MySQL has the <=> operator. + const notDistinct = /^is\s+not\s+distinct\s+from\b/i.exec(sql.slice(index)); + if (notDistinct) { + text += '<=>'; + index += notDistinct[0].length; + continue; + } + } if ((char === 'I' || char === 'i') && /^ilike\b/i.test(sql.slice(index))) { // MySQL's default collation is case-insensitive, so LIKE matches what // PostgreSQL's ILIKE matches; the wildcard escaping is identical. diff --git a/scripts/integration-smoke.ts b/scripts/integration-smoke.ts index 0a18995..eb05460 100644 --- a/scripts/integration-smoke.ts +++ b/scripts/integration-smoke.ts @@ -1,11 +1,26 @@ import { spawn } from 'node:child_process'; -import { createPool, databaseReady, runMigrations } from '../packages/database/src/index.js'; +import { + databaseReady, + openDatabase, + readDatabaseSettings, + runMigrations, + type SqlDatabase, +} from '../packages/database/src/index.js'; import { SqlStore } from '../apps/api/src/sql-store.js'; import { uuidv7, type Mutation } from '../packages/contracts/src/index.js'; import { DomainError } from '../packages/domain/src/index.js'; import { assertIsolatedEmptyDatabase, isolatedTestDatabaseUrl } from './test-database-safety.js'; const databaseUrl = isolatedTestDatabaseUrl(); +const settings = databaseUrl + ? readDatabaseSettings({ ...process.env, DATABASE_URL: databaseUrl }) + : undefined; +const engine = settings?.dialect === 'mysql' ? 'MySQL' : 'PostgreSQL'; + +function testDatabase(): SqlDatabase { + if (!settings) throw new Error('TASKDOCK_TEST_DATABASE_URL is not configured'); + return openDatabase({ ...settings, max: 10 }); +} const root = process.cwd(); const workerCount = 20; @@ -37,7 +52,7 @@ function runConcurrencyWorker( ): Promise { return new Promise((resolve, reject) => { const executable = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm'; - const child = spawn(executable, ['exec', 'tsx', 'scripts/pg-concurrency-worker.ts'], { + const child = spawn(executable, ['exec', 'tsx', 'scripts/sql-concurrency-worker.ts'], { cwd: root, env: { ...process.env, @@ -97,13 +112,13 @@ function runConcurrencyWorker( async function main(): Promise { if (!databaseUrl) { console.log( - 'NOT RUN: TASKDOCK_TEST_DATABASE_URL is not configured; the integration smoke requires a dedicated empty PostgreSQL database.', + 'NOT RUN: TASKDOCK_TEST_DATABASE_URL is not configured; the integration smoke requires a dedicated empty database.', ); process.exitCode = 2; return; } - const cleanupPool = createPool(databaseUrl); + const cleanupDatabase = testDatabase(); const stores: SqlStore[] = []; const closedStores = new Set(); const closeStore = async (store: SqlStore): Promise => { @@ -113,21 +128,23 @@ async function main(): Promise { }; let ownerIds: string[] = []; try { - await assertIsolatedEmptyDatabase(cleanupPool, 'integration smoke'); - await runMigrations(cleanupPool); - await runMigrations(cleanupPool); - assert(await databaseReady(cleanupPool), 'PostgreSQL schema is not ready after migrations'); - const existingOwners = await cleanupPool.query<{ id: string }>('SELECT id FROM users LIMIT 1'); + await assertIsolatedEmptyDatabase(cleanupDatabase, 'integration smoke'); + await runMigrations(cleanupDatabase); + await runMigrations(cleanupDatabase); + assert(await databaseReady(cleanupDatabase), `${engine} schema is not ready after migrations`); + const existingOwners = await cleanupDatabase.query<{ id: string }>( + 'SELECT id FROM users LIMIT 1', + ); if (existingOwners.rowCount) { throw new Error( 'integration smoke requires an isolated empty database; refusing to modify an existing Owner', ); } - const primary = new SqlStore(createPool(databaseUrl), { + const primary = new SqlStore(testDatabase(), { clock: () => new Date('2026-09-04T10:00:00.000Z'), }); - const peer = new SqlStore(createPool(databaseUrl), { + const peer = new SqlStore(testDatabase(), { clock: () => new Date('2026-09-04T10:00:00.000Z'), }); stores.push(primary, peer); @@ -172,11 +189,11 @@ async function main(): Promise { const ownerB = uuidv7(); ownerIds.push(ownerB); const ownerBNow = new Date().toISOString(); - await cleanupPool.query( + await cleanupDatabase.query( 'INSERT INTO users (id, username, password_hash, next_misc_task_number, created_at, updated_at) VALUES ($1, $2, $3, 1, $4, $4)', [ownerB, `integration-b-${process.pid}`, 'not-used-in-integration', ownerBNow], ); - await cleanupPool.query( + await cleanupDatabase.query( "INSERT INTO user_settings (owner_id, timezone, week_starts_on, default_capture_target, version, created_at, updated_at) VALUES ($1, 'Asia/Shanghai', 1, 'ROOT', 1, $2, $2)", [ownerB, ownerBNow], ); @@ -254,7 +271,7 @@ async function main(): Promise { () => primary.getTask(owner.id, failedReceiptTaskId), 'ENTITY_NOT_FOUND', ); - const failedReceipt = await cleanupPool.query( + const failedReceipt = await cleanupDatabase.query( 'SELECT 1 FROM client_mutations WHERE owner_id = $1 AND mutation_id = $2', [owner.id, failedReceiptId], ); @@ -284,7 +301,7 @@ async function main(): Promise { new Set([firstIdempotent.replayed, secondIdempotent.replayed]).size === 2, 'concurrent idempotent mutation did not produce exactly one replay', ); - const receiptCount = await cleanupPool.query( + const receiptCount = await cleanupDatabase.query( 'SELECT COUNT(*)::int AS count FROM client_mutations WHERE owner_id = $1 AND client_id = $2 AND mutation_id = $3', [owner.id, clientId, mutationId], ); @@ -292,7 +309,7 @@ async function main(): Promise { Number(receiptCount.rows[0]?.count) === 1, 'idempotent mutation produced more than one receipt', ); - const changeCount = await cleanupPool.query( + const changeCount = await cleanupDatabase.query( 'SELECT COUNT(*)::int AS count FROM sync_changes WHERE owner_id = $1 AND entity_id = $2', [owner.id, idempotentTaskId], ); @@ -333,7 +350,7 @@ async function main(): Promise { const cursorBeforeRestart = (await primary.syncStatus(owner.id)).cursor; await closeStore(peer); await closeStore(primary); - const restartStore = new SqlStore(createPool(databaseUrl), { + const restartStore = new SqlStore(testDatabase(), { clock: () => new Date('2026-09-04T10:00:00.000Z'), }); stores.push(restartStore); @@ -534,7 +551,7 @@ async function main(): Promise { 'focused retry lost the edit', ); const manifest = await bootstrap.create(owner.id, '2026-09-04'); - const bootstrapPeer = new SqlStore(createPool(databaseUrl)); + const bootstrapPeer = new SqlStore(testDatabase()); stores.push(bootstrapPeer); await bootstrapPeer.init(); const peerBootstrap = new BootstrapSnapshots(new SqlTreeStore(bootstrapPeer)); @@ -567,16 +584,16 @@ async function main(): Promise { () => peerBootstrap.page(ownerB, manifest.snapshotId, 0), 'SYNC_CURSOR_EXPIRED', ); - await cleanupPool.query( - "UPDATE sync_bootstrap_snapshots SET expires_at = now() - interval '1 minute' WHERE id = $1", - [manifest.snapshotId], + await cleanupDatabase.query( + 'UPDATE sync_bootstrap_snapshots SET expires_at = $2 WHERE id = $1', + [manifest.snapshotId, new Date(Date.now() - 60_000).toISOString()], ); await expectDomainError( () => bootstrap.page(owner.id, manifest.snapshotId, 0), 'SYNC_CURSOR_EXPIRED', ); await bootstrap.cleanup(); - const expired = await cleanupPool.query( + const expired = await cleanupDatabase.query( 'SELECT 1 FROM sync_bootstrap_snapshots WHERE id = $1', [manifest.snapshotId], ); @@ -651,7 +668,7 @@ async function main(): Promise { ); console.log( - `PASS: PostgreSQL migration, row-level transaction, owner isolation, two-process concurrency, idempotency, rollback, restart, multi-placement invariants, and SqlTreeStore v2 delete preview/tree deletion; cursor ${afterRestart.cursor}`, + `PASS: ${engine} migration, row-level transaction, owner isolation, two-process concurrency, idempotency, rollback, restart, multi-placement invariants, and SqlTreeStore v2 delete preview/tree deletion; cursor ${afterRestart.cursor}`, ); } finally { for (const store of stores) { @@ -664,13 +681,14 @@ async function main(): Promise { } if (ownerIds.length) { try { - await cleanupPool.query('DELETE FROM users WHERE id = ANY($1::uuid[])', [ownerIds]); + const list = ownerIds.map((_, index) => `$${index + 1}`).join(','); + await cleanupDatabase.query(`DELETE FROM users WHERE id IN (${list})`, ownerIds); } catch (error) { console.error(`cleanup fixture failed: ${String(error)}`); process.exitCode = 1; } } - await cleanupPool.end(); + await cleanupDatabase.end(); } } diff --git a/scripts/pg-concurrency-worker.ts b/scripts/sql-concurrency-worker.ts similarity index 85% rename from scripts/pg-concurrency-worker.ts rename to scripts/sql-concurrency-worker.ts index 1edbd8f..2002f41 100644 --- a/scripts/pg-concurrency-worker.ts +++ b/scripts/sql-concurrency-worker.ts @@ -1,4 +1,4 @@ -import { createPool } from '../packages/database/src/index.js'; +import { openDatabase, readDatabaseSettings } from '../packages/database/src/index.js'; import { SqlStore } from '../apps/api/src/sql-store.js'; import { isolatedTestDatabaseUrl } from './test-database-safety.js'; @@ -16,8 +16,12 @@ async function main(): Promise { if (!Number.isInteger(workerCount) || workerCount < 1 || workerCount > 100) throw new Error('DEVTODO_WORKER_COUNT must be an integer from 1 to 100'); - const pool = createPool(databaseUrl); - const store = new SqlStore(pool); + const store = new SqlStore( + openDatabase({ + ...readDatabaseSettings({ ...process.env, DATABASE_URL: databaseUrl }), + max: 4, + }), + ); try { await store.init(); const references: string[] = []; diff --git a/scripts/test-database-safety.ts b/scripts/test-database-safety.ts index 084aa01..6bcdc0c 100644 --- a/scripts/test-database-safety.ts +++ b/scripts/test-database-safety.ts @@ -1,4 +1,4 @@ -import type { Pool } from '../packages/database/src/index.js'; +import type { SqlDatabase } from '../packages/database/src/index.js'; /** Destructive fixtures never fall back to the application's DATABASE_URL. */ export function isolatedTestDatabaseUrl(env: NodeJS.ProcessEnv = process.env): string | undefined { @@ -10,10 +10,18 @@ export function isolatedTestDatabaseUrl(env: NodeJS.ProcessEnv = process.env): s * target before running migrations, because even DDL can be unsafe on a real * deployment and an active Owner check after migration is already too late. */ -export async function assertIsolatedEmptyDatabase(pool: Pool, label: string): Promise { - const tables = await pool.query<{ table_name: string }>( - "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'", - ); +export async function assertIsolatedEmptyDatabase( + database: SqlDatabase, + label: string, +): Promise { + const tables = + database.dialect === 'mysql' + ? await database.query<{ table_name: string }>( + 'SELECT TABLE_NAME AS table_name FROM information_schema.tables WHERE table_schema = DATABASE()', + ) + : await database.query<{ table_name: string }>( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'", + ); const tableNames = new Set(tables.rows.map((row) => row.table_name)); if (!tableNames.has('users')) { const unexpected = [...tableNames].filter((name) => name !== 'schema_migrations'); @@ -23,8 +31,10 @@ export async function assertIsolatedEmptyDatabase(pool: Pool, label: string): Pr ); return; } - const owners = await pool.query<{ count: string }>('SELECT COUNT(*)::text AS count FROM users'); - if (BigInt(owners.rows[0]?.count ?? '0') > 0n) + const owners = await database.query<{ count: string | number }>( + 'SELECT COUNT(*) AS count FROM users', + ); + if (BigInt(String(owners.rows[0]?.count ?? '0')) > 0n) throw new Error( `${label} requires a dedicated empty TASKDOCK_TEST_DATABASE_URL; refusing to modify a database containing any Owner, including disabled Owners`, ); From 78b8eda7b52edf1f169ce7d0d5cada472c2cb173 Mon Sep 17 00:00:00 2001 From: IGNGserver <176017966+IGNGserver@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:23:31 +0800 Subject: [PATCH 7/9] =?UTF-8?q?ci(db):=20=E5=8F=8C=E5=BC=95=E6=93=8E?= =?UTF-8?q?=E9=97=A8=E7=A6=81=E4=B8=8E=E5=8F=98=E6=9B=B4=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ci.yml / release.yml 增加 mysql:8.4 服务与 MySQL 方言、集成、运行时三道门禁; 运行时冒烟使用独立数据库,避免复用集成冒烟的清理结果 - 增加 docker compose config 校验收紧 compose.yaml / external-db / mysql 三份文件 - 修复 perf-capacity 与 sync-bootstrap-benchmark 对 SqlDatabase 的适配 - CHANGELOG 补充 MySQL 支持与能力边界 --- .github/workflows/ci.yml | 44 +++++++++++++++++++++++++++++ .github/workflows/release.yml | 22 +++++++++++++++ CHANGELOG.md | 5 ++++ scripts/perf-capacity.ts | 12 ++++++-- scripts/sync-bootstrap-benchmark.ts | 7 +++-- 5 files changed, 84 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6755013..35f3f53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,17 @@ jobs: --health-interval 5s --health-timeout 5s --health-retries 20 + mysql: + image: mysql:8.4@sha256:a81d043930e51745522bd5c430961cf8286f0a23f0a9a2a6e0108f7ea71b0902 + env: + MYSQL_ROOT_PASSWORD: root_test_password + MYSQL_DATABASE: devtodo_mysql_test + ports: ['3306:3306'] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -uroot -proot_test_password --silent" + --health-interval 5s + --health-timeout 5s + --health-retries 30 steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -52,6 +63,24 @@ jobs: - run: pnpm install --frozen-lockfile - name: Block known high-severity dependency advisories run: pnpm audit --audit-level=high --registry=https://registry.npmjs.org + # `docker compose config` resolves interpolation and anchors, so a broken + # anchor, a missing required variable or a YAML typo fails here instead of + # at deploy time. It never pulls images. + - name: Validate Compose files + shell: bash + run: | + set -euo pipefail + export APP_VERSION=validate APP_ORIGIN=http://localhost:3000 + export POSTGRES_PASSWORD=validate MYSQL_PASSWORD=validate MYSQL_ROOT_PASSWORD=validate + export DATABASE_URL=postgres://validate:validate@127.0.0.1:5432/validate + export BOOTSTRAP_TOKEN=validate-token-at-least-32-characters-long + export ACCESS_TOKEN_SECRET=validate-secret-at-least-32-characters-long + export REFRESH_TOKEN_PEPPER=validate-pepper-at-least-32-characters-long + export CORS_ALLOWED_ORIGINS=http://localhost:3000 + export OWNER_USERNAME=validate OWNER_PASSWORD=validate-password + docker compose -f compose.yaml config --quiet + docker compose -f compose.external-db.yaml config --quiet + docker compose -f compose.mysql.yaml config --quiet - run: pnpm format:check - run: pnpm lint - run: pnpm tokens:check @@ -67,6 +96,21 @@ jobs: - run: pnpm build - name: Verify precompressed static delivery run: pnpm exec tsx scripts/check-static-delivery.ts + # MySQL is a first-class engine: dialect probes, the full integration + # smoke and the HTTP runtime smoke all run against a real server. + - name: MySQL dialect, integration and runtime gates + shell: bash + run: | + set -euo pipefail + TASKDOCK_MYSQL_TEST_URL=mysql://root:root_test_password@127.0.0.1:3306 \ + pnpm test:mysql-compat + TASKDOCK_TEST_DATABASE_URL=mysql://root:root_test_password@127.0.0.1:3306/devtodo_mysql_test \ + pnpm test:integration + # The runtime smoke needs a database with no Owner, so give it a + # separate schema instead of relying on the cleanup above. + node -e "const mysql=require('mysql2/promise');(async()=>{const c=await mysql.createConnection({host:'127.0.0.1',port:3306,user:'root',password:'root_test_password'});await c.query('CREATE DATABASE IF NOT EXISTS devtodo_mysql_runtime CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci');await c.end();})()" + TASKDOCK_TEST_DATABASE_URL=mysql://root:root_test_password@127.0.0.1:3306/devtodo_mysql_runtime \ + pnpm test:mysql-runtime - name: PostgreSQL restart and mobile loading benchmark shell: bash run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f41453..e09ecef 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,6 +48,18 @@ jobs: --health-interval 5s --health-timeout 5s --health-retries 20 + mysql: + image: mysql:8.4@sha256:a81d043930e51745522bd5c430961cf8286f0a23f0a9a2a6e0108f7ea71b0902 + env: + MYSQL_ROOT_PASSWORD: root_test_password + MYSQL_DATABASE: devtodo_mysql_test + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -uroot -proot_test_password --silent" + --health-interval 5s + --health-timeout 5s + --health-retries 30 steps: - name: 检出代码 uses: actions/checkout@v4 @@ -126,6 +138,16 @@ jobs: run: pnpm test:integration - name: 构建 run: pnpm build + - name: MySQL 方言与集成测试 + env: + TASKDOCK_MYSQL_TEST_URL: mysql://root:root_test_password@127.0.0.1:3306 + TASKDOCK_TEST_DATABASE_URL: mysql://root:root_test_password@127.0.0.1:3306/devtodo_mysql_test + run: | + set -euo pipefail + pnpm test:mysql-compat + pnpm test:integration + node -e "const mysql=require('mysql2/promise');(async()=>{const c=await mysql.createConnection({host:'127.0.0.1',port:3306,user:'root',password:'root_test_password'});await c.query('CREATE DATABASE IF NOT EXISTS devtodo_mysql_runtime CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci');await c.end();})()" + TASKDOCK_TEST_DATABASE_URL=mysql://root:root_test_password@127.0.0.1:3306/devtodo_mysql_runtime pnpm test:mysql-runtime - name: OpenAPI 检查 run: pnpm openapi:check diff --git a/CHANGELOG.md b/CHANGELOG.md index bbc97d3..abd8e02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ ## 未发布 +- 新增 MySQL 8.4 支持:`DATABASE_URL` 换成 `mysql://` 或使用离散 `MYSQL_*` 即可切换引擎,也可以保留 PostgreSQL 不动;`compose.mysql.yaml` 提供内置 MySQL 部署,`compose.external-db.yaml` 对两种引擎通用。方言差异集中在 `@devtodo/database`(占位符、类型转换、`ILIKE`、保留字 `rank`/`cursor`),存储层仍是同一份实现。 +- MySQL 的能力边界写在 `docs/adr/0005-mysql-support.md`:跨实例实时推送降级为进程内通知或 Redis pub/sub,搜索退化为 owner 范围内的 `LIKE` 扫描,语句级超时只覆盖 SELECT,迁移改为逐文件提交。 +- 新增 MySQL 合并基线 `packages/database/migrations/mysql/0013_baseline.sql`(等价于 PostgreSQL `0013`);此后每个 schema 变更两边各出一个同版本迁移,并有测试保证版本与表集合一致。 +- 新增 `pnpm test:mysql-compat`、`pnpm test:mysql-runtime`;`pnpm test:integration` 与 `pnpm test:pg-runtime` 现在由 `TASKDOCK_TEST_DATABASE_URL` 的 scheme 选择引擎,CI 同时跑 PostgreSQL 与 MySQL 两套门禁。 + - 新增外部 PostgreSQL 部署:`DATABASE_URL`/离散 `PG*`、SSL CA、连接池、启动重试;提供 `compose.external-db.yaml`,可连接 NAS、共享数据库服务器或托管 PostgreSQL。一个 PostgreSQL 服务器可承载多个应用,但每个 TaskDock 中枢必须使用独立 database。 - 降低远端数据库延迟下的请求开销:v2 投影冷加载由 11 条 SELECT 改为 1 条、缓存失效改为 change feed 增量追赶;mutation 按实体类型批量 upsert/append;`/sync/push` 改为单事务 + 每 mutation savepoint;retention 清理移出请求事务。 - 新增可选 Redis:多 API 副本共享登录限流、冷启动投影缓存;投影缓存故障回退数据库,不进入权威写路径。生产环境配置 Redis 后认证限流在故障时 fail-closed;`AUTH_RATE_LIMIT_REQUIRE_SHARED=true` 可在其他环境强制该行为。 diff --git a/scripts/perf-capacity.ts b/scripts/perf-capacity.ts index 50c5414..a5fbac1 100644 --- a/scripts/perf-capacity.ts +++ b/scripts/perf-capacity.ts @@ -2,7 +2,12 @@ import { createHash } from 'node:crypto'; import os from 'node:os'; import { performance } from 'node:perf_hooks'; import { SqlStore, type PostgresPage } from '../apps/api/src/sql-store.js'; -import { createPool, runMigrations, type Pool } from '../packages/database/src/index.js'; +import { + createPool, + databaseFromPool, + runMigrations, + type Pool, +} from '../packages/database/src/index.js'; import { assertIsolatedEmptyDatabase, isolatedTestDatabaseUrl } from './test-database-safety.js'; const databaseUrl = isolatedTestDatabaseUrl(); @@ -191,8 +196,9 @@ async function main(): Promise { const store = new SqlStore(pool); let actualOwnerId: string | undefined; try { - await assertIsolatedEmptyDatabase(pool, 'capacity benchmark'); - await runMigrations(pool); + const database = databaseFromPool(pool); + await assertIsolatedEmptyDatabase(database, 'capacity benchmark'); + await runMigrations(database); await store.init(); const owner = await store.withMutation(() => { return store.createOwner(username, 'perf-fixture-hash'); diff --git a/scripts/sync-bootstrap-benchmark.ts b/scripts/sync-bootstrap-benchmark.ts index 2a03e32..b1ad36c 100644 --- a/scripts/sync-bootstrap-benchmark.ts +++ b/scripts/sync-bootstrap-benchmark.ts @@ -1,5 +1,5 @@ import { randomUUID } from 'node:crypto'; -import { createPool, runMigrations } from '../packages/database/src/index.js'; +import { createPool, databaseFromPool, runMigrations } from '../packages/database/src/index.js'; import { SqlStore } from '../apps/api/src/sql-store.js'; import { SqlTreeStore } from '../apps/api/src/sql-tree-store.js'; import { BootstrapSnapshots } from '../apps/api/src/sync-bootstrap.js'; @@ -17,8 +17,9 @@ if (!Number.isInteger(taskCount) || taskCount < 100 || taskCount > 100_000) throw new Error('invalid task count'); let ownerId: string | undefined; try { - await assertIsolatedEmptyDatabase(pool, 'bootstrap benchmark'); - await runMigrations(pool); + const database = databaseFromPool(pool); + await assertIsolatedEmptyDatabase(database, 'bootstrap benchmark'); + await runMigrations(database); if ((await pool.query('SELECT 1 FROM users LIMIT 1')).rowCount) throw new Error('refusing to modify a database with existing users'); await store.init(); From c9d1c1daea926fbce327bc672204723cb7f8d73a Mon Sep 17 00:00:00 2001 From: IGNGserver <176017966+IGNGserver@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:24:16 +0800 Subject: [PATCH 8/9] =?UTF-8?q?docs(db):=20=E6=A0=87=E6=B3=A8=E5=8E=86?= =?UTF-8?q?=E5=8F=B2=E7=8A=B6=E6=80=81=E6=96=87=E6=A1=A3=E4=B8=8E=20Drizzl?= =?UTF-8?q?e=20=E5=AE=9A=E4=B9=89=E7=9A=84=E9=80=82=E7=94=A8=E8=8C=83?= =?UTF-8?q?=E5=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/STATUS.md | 2 ++ packages/database/src/schema.ts | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/docs/STATUS.md b/docs/STATUS.md index 02df150..6502c1c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -3,6 +3,8 @@ 更新时间:2026-10-13 > 后续变更(2026-11-12):归档机制已在 `2.0.0-rc.26` 移除,本文中归档中心相关条目已失效;文件夹状态在 `2.0.0-rc.62` 改为文件夹自身持有的显式三态。冲突时以 `docs/ARCHITECTURE.md` 和代码为准。 +> +> 后续变更(2026-11-15):数据库层已是 PostgreSQL/MySQL 双引擎,`PostgresStore`/`PostgresTreeStore` 更名为 `SqlStore`/`SqlTreeStore`;本文提到的“PostgreSQL 权威存储”按双引擎理解,差异见 [ADR 0005](adr/0005-mysql-support.md)。 总体发布状态:**FAIL(v2 完成定义未满足,但阻断项已全部关闭)**。本轮已关闭全部已知 P0/P1 源码缺陷,并补齐归档中心、v2 rollover、V2 restore 策略、离线删除 fail-closed、Folder-first 迁移分层、Android 同步恢复等此前遗留项。所有本地门禁、真实 PostgreSQL migration/integration/runtime 门禁与三浏览器 E2E 均为 PASS。 diff --git a/packages/database/src/schema.ts b/packages/database/src/schema.ts index e378895..e2ad323 100644 --- a/packages/database/src/schema.ts +++ b/packages/database/src/schema.ts @@ -1,3 +1,10 @@ +/** + * PostgreSQL-only Drizzle definition kept for reference and for the type + * aliases at the bottom. Nothing queries through it: the runtime SQL lives in + * `apps/api/src/sql-*.ts`, and the schema of record is + * `packages/database/migrations/{postgres,mysql}`. There is no MySQL twin by + * design; adding one would be a second schema to keep in sync for no benefit. + */ import { sql } from 'drizzle-orm'; import { bigint, From 2faff087d5dcc5d8c2aad3ecce243800e58c4211 Mon Sep 17 00:00:00 2001 From: IGNGserver <176017966+IGNGserver@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:26:42 +0800 Subject: [PATCH 9/9] =?UTF-8?q?fix(api):=20=E4=BF=AE=E6=AD=A3=20MySQL=20?= =?UTF-8?q?=E4=B8=8B=E6=97=A0=20id=20=E8=B0=93=E8=AF=8D=E7=9A=84=20RETURNI?= =?UTF-8?q?NG=20=E5=9B=9E=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updateSettings(按 owner 主键)、updateNote(按 task)和 deleteTimePoint 的多行 placement 墓碑都不会带 id 谓词,原来的回查键推导会抛错。改为由调用 方显式给出等值条件,并在集成冒烟里覆盖这三条路径。 --- apps/api/src/sql-store.ts | 42 +++++++++++++++++++++++++++++------- scripts/integration-smoke.ts | 40 ++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/apps/api/src/sql-store.ts b/apps/api/src/sql-store.ts index b86c4c8..8ce158a 100644 --- a/apps/api/src/sql-store.ts +++ b/apps/api/src/sql-store.ts @@ -467,14 +467,16 @@ export class SqlStore implements Store { throw new Error(`executeReturning cannot derive the row key: ${statement.slice(0, 70)}`); if (derived.deleted) { // A DELETEd row can never be read back; rowCount carries the outcome. + const first = derived.where[0]; return { - rows: (written.rowCount ?? 0) > 0 ? [{ [derived.column]: derived.value }] : [], + rows: (written.rowCount ?? 0) > 0 && first ? [{ [first.column]: first.value }] : [], rowCount: written.rowCount, }; } + const predicate = derived.where.map((entry, index) => `${entry.column} = $${index + 1}`); const selected = await this.query( - `SELECT ${columns} FROM ${derived.table} WHERE ${derived.column} = $1`, - [derived.value], + `SELECT ${columns} FROM ${derived.table} WHERE ${predicate.join(' AND ')}`, + derived.where.map((entry) => entry.value), ); return { rows: selected.rows, rowCount: selected.rowCount }; } @@ -763,6 +765,7 @@ export class SqlStore implements Store { 'WHERE owner_id = $1 AND version = $6 ' + 'RETURNING owner_id, timezone, week_starts_on, default_capture_target, version, created_at, updated_at, deleted_at', [ownerId, timezone, weekStartsOn, defaultCaptureTarget, this.now(), baseVersion], + { table: 'user_settings', where: [{ column: 'owner_id', value: ownerId }] }, ); if (!result.rows[0]) throw new DomainError('VERSION_CONFLICT', '实体版本已变化', { @@ -1410,6 +1413,13 @@ export class SqlStore implements Store { 'WHERE owner_id = $1 AND task_id = $2 AND version = $5 AND deleted_at IS NULL ' + 'RETURNING id, owner_id, task_id, content_markdown, version, created_at, updated_at, deleted_at', [ownerId, taskId, contentMarkdown, this.now(), baseVersion], + { + table: 'notes', + where: [ + { column: 'owner_id', value: ownerId }, + { column: 'task_id', value: taskId }, + ], + }, ); const next = noteRecord(requireUpdatedRow(result, '备注版本已变化')); const dto = noteDto(next); @@ -1633,6 +1643,16 @@ export class SqlStore implements Store { 'WHERE owner_id = $1 AND time_point_id = $2 AND deleted_at IS NULL ' + 'RETURNING id, version', [ownerId, id, now], + // Soft delete keeps the rows, so the freshly written tombstone value is + // exactly the set this statement touched. + { + table: 'placements', + where: [ + { column: 'owner_id', value: ownerId }, + { column: 'time_point_id', value: id }, + { column: 'deleted_at', value: now }, + ], + }, ); for (const row of placements.rows) { await this.appendChange( @@ -2624,8 +2644,8 @@ function isSqlDatabase(value: unknown): value is SqlDatabase { interface ReturningKey { table: string; - column: string; - value: unknown; + /** Equality conditions of the read-back, in binding order. */ + where: Array<{ column: string; value: unknown }>; deleted?: boolean; } @@ -2636,16 +2656,22 @@ interface ReturningKey { */ function deriveReturningKey(statement: string, values: unknown[]): ReturningKey | undefined { const insert = /^\s*INSERT\s+INTO\s+([a-z_]+)\s*\(\s*([a-z_]+)/i.exec(statement); - if (insert && insert[2] === 'id') return { table: insert[1]!, column: 'id', value: values[0] }; + if (insert && insert[2] === 'id') + return { table: insert[1]!, where: [{ column: 'id', value: values[0] }] }; const where = /\bWHERE\b([\s\S]*)$/i.exec(statement); if (!where) return undefined; const predicate = /\bid\s*=\s*\$(\d+)/i.exec(where[1]!); if (!predicate) return undefined; const index = Number(predicate[1]) - 1; const update = /^\s*UPDATE\s+([a-z_]+)\s+SET/i.exec(statement); - if (update) return { table: update[1]!, column: 'id', value: values[index] }; + if (update) return { table: update[1]!, where: [{ column: 'id', value: values[index] }] }; const remove = /^\s*DELETE\s+FROM\s+([a-z_]+)/i.exec(statement); - if (remove) return { table: remove[1]!, column: 'id', value: values[index], deleted: true }; + if (remove) + return { + table: remove[1]!, + where: [{ column: 'id', value: values[index] }], + deleted: true, + }; return undefined; } diff --git a/scripts/integration-smoke.ts b/scripts/integration-smoke.ts index eb05460..df700af 100644 --- a/scripts/integration-smoke.ts +++ b/scripts/integration-smoke.ts @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process'; +import { isDeepStrictEqual } from 'node:util'; import { databaseReady, openDatabase, @@ -242,7 +243,6 @@ async function main(): Promise { (await primary.syncStatus(owner.id)).cursor === cursorBeforeRollback, 'failed transaction advanced the change cursor', ); - const failedReceiptId = uuidv7(); const failedReceiptTaskId = uuidv7(); await expectDomainError( @@ -667,6 +667,43 @@ async function main(): Promise { 'v2 delete was not persisted for a reloaded store', ); + // UPDATE ... RETURNING paths without an `id` predicate: settings (keyed by + // owner), notes (keyed by task) and the multi-row placement tombstone. + const returningTask = await restartStore.withMutation(() => + restartStore.createTask(owner.id, { + projectId: project.id, + category: 'FEATURE', + title: 'returning shapes', + priority: 'NONE', + }), + ); + assert( + ( + await restartStore.withMutation(() => + restartStore.updateNote(owner.id, returningTask.id, 'note v2', 1), + ) + ).contentMarkdown === 'note v2', + 'note update did not read its own row back', + ); + assert( + (await restartStore.updateSettings(owner.id, { timezone: 'UTC' }, 1)).version === 2, + 'settings update did not read its own row back', + ); + const returningEvent = await restartStore.createEvent(owner.id, 'returning event'); + await restartStore.withMutation(() => + restartStore.addPlacement(owner.id, returningTask.id, returningEvent.id), + ); + const cursorBeforeEventDelete = (await restartStore.syncStatus(owner.id)).cursor; + await restartStore.deleteTimePoint(owner.id, returningEvent.id, returningEvent.version); + assert( + BigInt((await restartStore.syncStatus(owner.id)).cursor) > BigInt(cursorBeforeEventDelete), + 'event deletion did not append placement tombstones', + ); + assert( + (await restartStore.getTask(owner.id, returningTask.id)).id === returningTask.id, + 'deleting an event also removed its task', + ); + console.log( `PASS: ${engine} migration, row-level transaction, owner isolation, two-process concurrency, idempotency, rollback, restart, multi-placement invariants, and SqlTreeStore v2 delete preview/tree deletion; cursor ${afterRestart.cursor}`, ); @@ -696,4 +733,3 @@ void main().catch((error: unknown) => { console.error(error); process.exitCode = 1; }); -import { isDeepStrictEqual } from 'node:util';