diff --git a/packages/safe-bash/docs/CSPLIT.md b/packages/safe-bash/docs/CSPLIT.md index f053a4ebb5..a3f14331cd 100644 --- a/packages/safe-bash/docs/CSPLIT.md +++ b/packages/safe-bash/docs/CSPLIT.md @@ -102,6 +102,27 @@ The default portable regex provider has an additional 65536-byte subject limit, so a larger `maxLineBytes` alone does not permit regex matching on longer lines. Provider-level limits and the command's limits both apply. +`maxBufferedBytes` includes twice the admitted input payload, 128 accounting +units for each retained line or partial-line reference, 256 units plus twice the +path's character count for each retained output receipt, and the active output +buffer (at most 65536 bytes). Finishing an output releases its buffer; cleanup +releases its receipt reservation. Only the immediate parent snapshot remains +retained after output creation. Admission checks run before payload allocation. +These limits apply together, so reaching one configured maximum does not promise +that every other maximum can be reached simultaneously; fragmented input and +retained outputs also consume the allowance. +Non-streaming reads additionally reserve the complete returned file array before +requesting it, pass its admitted size as the read bound, and retain that separate +charge until cleanup. Streaming avoids this extra retained-array allowance. + +Output uses a bounded 65536-byte buffer. A full buffer and a completed output +flush before byte counts are printed. Ordinary input or pattern errors flush +pending bytes before the `-k` retention decision. Cancellation discards pending +bytes; `-k` retains only the already committed prefix. This follows GNU's stdio +error-versus-signal policy, but does not promise GNU's platform-dependent buffer +threshold (the GNU 9.11 macOS comparison used 8192 bytes). A refused atomic flush +commits no prefix and prints no byte count for that output. + ## Environment and compatibility boundaries `LC_ALL`, `LC_CTYPE`, `LC_COLLATE`, and `LANG` select the supported C/POSIX or @@ -115,6 +136,15 @@ UTF-8, and command arguments cannot contain NUL. Input/output alias checks and bounded resource refusals intentionally protect the virtual input rather than copying unsafe or unbounded native effects. Filesystem identity and cleanup guarantees remain limited by the injected provider's declared capabilities. +Output requires the `atomicFileMutation` capability and both +`writeFileConditional` and `removeFileConditional`. Directories require stable, +scoped device/inode identity; output entries additionally require a revision. +Opening, appending, and removing an output atomically compare the captured +parent and file snapshots. Completed writes return their original committed +snapshot even if cancellation arrives before the response. Cleanup cannot adopt +or remove a replacement entry. Providers without these methods refuse output +mutation; checking a path before an ordinary write is not a fallback. + Output directories and entries require stable, scoped device/inode identity; symlink output parents and existing symlink outputs are refused. Existing outputs are also refused when known input metadata cannot establish its identity. A diff --git a/packages/safe-bash/src/commands/archive/unzip.ts b/packages/safe-bash/src/commands/archive/unzip.ts index 7e73d1952b..26b92bfdab 100644 --- a/packages/safe-bash/src/commands/archive/unzip.ts +++ b/packages/safe-bash/src/commands/archive/unzip.ts @@ -1,4 +1,4 @@ -import { collectBytes, readBytes, resolvePath, writeBytes, type CommandContext, type CommandDefinition, type FileStat } from "../../contracts/index.js"; +import { dirname, collectBytes, readBytes, resolvePath, writeBytes, type CommandContext, type CommandDefinition, type FileStat } from "../../contracts/index.js"; import { writeFileOutput } from "../../contracts/filesystem-output.js"; import { publicDiagnosticMessage } from "../../diagnostics.js"; import { Budget, bounded, checkPath, display, fail, settings, text, vfsPath, type ArchiveCommandsOptions } from "./internal.js"; @@ -97,8 +97,8 @@ export function createUnzipCommand(options: ArchiveCommandsOptions = {}): Comman let selected = 0; let total = 0; let actualTotal = 0; - const links: { path: string; shown: string; target: string; existing: FileStat | undefined; entry: ZipEntry }[] = []; - const directories: { path: string; entry: ZipEntry; identity: FileStat }[] = []; + const links: { path: string; shown: string; target: string; existing: FileStat | undefined; parent: FileStat; entry: ZipEntry }[] = []; + const directories: { path: string; entry: ZipEntry; identity: FileStat; parent: FileStat }[] = []; for (const entry of zip.entries) { await budget.member(entry.size); checkPath(entry.name, limits); @@ -120,17 +120,19 @@ export function createUnzipCommand(options: ArchiveCommandsOptions = {}): Comman for await (const chunk of readBytes(decodeZipEntry(entry, limits, context.signal), context.signal)) { await writeFileOutput(context, chunk, async () => { if (chunk.length) fail("directory has nonempty payload"); }); } + const parent = await extraction.operation(() => context.fs.lstat(dirname(path), { signal: context.signal })); const existing = await extraction.stat(path); + let identity = existing; if (existing && existing.type !== "directory") fail("directory destination is not a directory"); if (!existing) { - await extraction.operation(() => context.fs.mkdir(path, { signal: context.signal, mode: 0o755 })); + identity = await extraction.createDirectory(path, parent); await budget.output(` creating: ${filtered(shown)}\n`); } - const identity = await extraction.stat(path); if (!identity || identity.type !== "directory") fail("directory changed during creation"); - directories.push({ path, entry, identity }); + directories.push({ path, entry, identity, parent }); continue; } + let parent = await extraction.operation(() => context.fs.lstat(dirname(path), { signal: context.signal })); let existing = await extraction.destination(path, archivePath, archiveStat); let skip = false; let prompting = 0; @@ -158,6 +160,7 @@ export function createUnzipCommand(options: ArchiveCommandsOptions = {}): Comman path = extraction.member(root, renamed); shown = parsed.destination === undefined ? renamed : `${parsed.destination.endsWith("/") ? parsed.destination : `${parsed.destination}/`}${renamed}`; await extraction.parents(root, path, true); + parent = await extraction.operation(() => context.fs.lstat(dirname(path), { signal: context.signal })); existing = await extraction.destination(path, archivePath, archiveStat); continue; } @@ -177,10 +180,10 @@ export function createUnzipCommand(options: ArchiveCommandsOptions = {}): Comman const target = text(Buffer.concat(chunks)); await extraction.target(root, path, target); if (!context.fs.symlink || context.fs.capabilities.symlinks === false) fail("filesystem does not support symlinks"); - links.push({ path, shown, target, existing, entry }); + links.push({ path, shown, target, existing, parent, entry }); await budget.output(` linking: ${padded(filtered(shown))} -> ${filtered(target)} \n`); } else { - await extraction.publish(root, path, chunks, existing, entry.mode, entry.modified); + await extraction.publish(root, path, chunks, existing, parent, entry.mode, entry.modified); await budget.output(`${entry.method === 0 ? " extracting" : " inflating"}: ${padded(filtered(shown))} \n`); } } @@ -191,11 +194,11 @@ export function createUnzipCommand(options: ArchiveCommandsOptions = {}): Comman await extraction.parents(root, link.path, false); await extraction.target(root, link.path, link.target); await extraction.destination(link.path, archivePath, archiveStat); - await extraction.publish(root, link.path, [], link.existing, link.entry.mode, link.entry.modified, link.target); + await extraction.publish(root, link.path, [], link.existing, link.parent, link.entry.mode, link.entry.modified, link.target); await budget.output(` ${padded(filtered(link.shown))} -> ${filtered(link.target)}\n`); } - for (const { path, entry, identity } of directories.reverse()) { - await extraction.metadata(root, path, identity, entry.mode, entry.modified); + for (const { path, entry, identity, parent } of directories.reverse()) { + await extraction.metadata(root, path, identity, parent, entry.mode, entry.modified); } for (let index = 0; index < parsed.patterns.length; index++) if (!selection.matched.has(index)) { await budget.output(`caution: filename not matched: ${filtered(parsed.patterns[index]!)}\n`, true); diff --git a/packages/safe-bash/src/commands/archive/unzip/safety.ts b/packages/safe-bash/src/commands/archive/unzip/safety.ts index e79afa6933..36b968b86d 100644 --- a/packages/safe-bash/src/commands/archive/unzip/safety.ts +++ b/packages/safe-bash/src/commands/archive/unzip/safety.ts @@ -1,4 +1,4 @@ -import { dirname, isPathWithin, readBytes, resolvePath, type ByteSource, type CommandContext, type FileStat } from "../../../contracts/index.js"; +import { dirname, isPathWithin, readBytes, resolvePath, type ByteSource, type CommandContext, type FileStat, type FileStaging } from "../../../contracts/index.js"; import { retainFileSystemCleanup } from "poe-code/safe-fs/core"; import { checkPath, display, fail, hasIdentity, sameIdentity, type ArchiveLimits } from "../internal.js"; @@ -74,13 +74,23 @@ export class Extraction { const { fs, signal } = this.context; for (const component of raw.split("/")) { if (!component || component === ".") continue; + const parent = await this.operation(() => fs.lstat(current, { signal })); current = resolvePath(current, component); const stat = await this.stat(current); - if (!stat && create) await this.operation(() => fs.mkdir(current, { signal, mode: 0o755 })); + if (!stat && create) await this.createDirectory(current, parent); else if (!stat || stat.type !== "directory") fail(`unsafe non-directory or symlink ancestor: ${display(current)}`); } return current; } + async createDirectory(path: string, parent: FileStat): Promise { + const { fs, signal } = this.context; + const capabilities = await this.operation(async () => await fs.capabilitiesFor?.(path, { signal, create: true }) ?? fs.capabilities); + if (capabilities.atomicDirectoryMetadata !== true || !fs.prepareDirectory) fail("extraction requires atomic directory creation"); + return this.operation(() => fs.prepareDirectory!(path, { + signal, parent, expected: null, + ...(capabilities.permissions === false ? {} : { mode: 0o755 }), + })); + } async parents(root: string, path: string, create: boolean): Promise { if (!isPathWithin(root, path)) fail("extraction path escapes root"); await this.directory(dirname(path), create); @@ -143,75 +153,61 @@ export class Extraction { const current = await this.stat(path); if (!current || current.type !== expected.type || !sameIdentity(current, expected)) fail("extraction entry changed before mutation"); } - async metadata(root: string, path: string, identity: FileStat, mode: number, modified: Date): Promise { + async metadata(root: string, path: string, identity: FileStat, parent: FileStat, mode: number, modified: Date): Promise { const { fs, signal } = this.context; - if (fs.chmod && fs.capabilities.permissions !== false) { - await this.checked(root, path, identity); - await this.operation(() => fs.chmod!(path, mode & 0o777, { signal })); - } - if (fs.utimes && fs.capabilities.timestamps !== false) { - await this.checked(root, path, identity); - await this.operation(() => fs.utimes!(path, modified.getTime(), modified.getTime(), { signal })); - } + await this.parents(root, path, false); + const capabilities = await this.operation(async () => await fs.capabilitiesFor?.(path, { signal }) ?? fs.capabilities); + if (capabilities.permissions === false && capabilities.timestamps === false) return; + if (capabilities.atomicDirectoryMetadata !== true || !fs.prepareDirectory) fail("extraction metadata requires atomic entry conditions"); + await this.operation(() => fs.prepareDirectory!(path, { + signal, expected: identity, parent, + ...(capabilities.permissions === false ? {} : { mode: mode & 0o777 }), + ...(capabilities.timestamps === false ? {} : { atimeMs: modified.getTime(), mtimeMs: modified.getTime() }), + })); } - publish(root: string, path: string, chunks: readonly Uint8Array[], expected: FileStat | undefined, mode: number, modified: Date, target?: string): Promise { - const publication = this.stage(root, path, chunks, expected, mode, modified, target); + publish(root: string, path: string, chunks: readonly Uint8Array[], expected: FileStat | undefined, parent: FileStat, mode: number, modified: Date, target?: string): Promise { + const publication = this.stage(root, path, chunks, expected, parent, mode, modified, target); this.publications.add(publication); return publication.finally(() => { this.publications.delete(publication); }); } - private async stage(root: string, path: string, chunks: readonly Uint8Array[], expected: FileStat | undefined, mode: number, modified: Date, target: string | undefined): Promise { + private async stage(root: string, path: string, chunks: readonly Uint8Array[], expected: FileStat | undefined, parent: FileStat, mode: number, modified: Date, target: string | undefined): Promise { const { fs, signal } = this.context; - let temporary = ""; - let identity: FileStat | undefined; - let created = false; + const capabilities = await this.operation(async () => await fs.capabilitiesFor?.(path, { signal, create: true }) ?? fs.capabilities); + if (capabilities.atomicFileStaging !== true || !fs.createStagedFile || !fs.publishStagedFile || !fs.removeStagedFile) fail("extraction requires atomic owned file staging"); + let staging: FileStaging | undefined; let failure: { reason: unknown } | undefined; const cleanup = retainFileSystemCleanup(fs, async view => { - if (!created || !identity) return; - try { - let parent = "/"; - for (const component of dirname(temporary).split("/").filter(Boolean)) { - parent = resolvePath(parent, component); - if ((await view.lstat(parent)).type !== "directory") return; - } - const current = await view.lstat(temporary); - if (current.type === identity.type && sameIdentity(current, identity)) await view.rm(temporary); - } catch (error) { - if (typeof error !== "object" || error === null || !("code" in error) || error.code !== "ENOENT") throw error; + if (staging) { + if (!view.removeStagedFile) fail("extraction atomic cleanup unavailable"); + await view.removeStagedFile(staging); } }, { maxOperations: Math.min(4096, this.limits.maxDepth + 3) }); try { - await this.parents(root, path, true); + await this.parents(root, path, false); + const content = target === undefined ? { type: "file" as const, data: Buffer.concat(chunks) } : { type: "symlink" as const, target }; for (let attempt = 0; attempt < this.limits.maxMembers; attempt++) { - temporary = resolvePath(dirname(path), `.unzip-${++this.serial}`); + const temporary = resolvePath(dirname(path), `.unzip-${++this.serial}`); checkPath(temporary, this.limits); - if (temporary === path || await this.stat(temporary)) continue; - await this.operation(async () => { - if (target === undefined) await fs.writeFile(temporary, new Uint8Array(), { signal, flag: "wx", mode: 0o600 }); - else await fs.symlink!(target, temporary, { signal }); - created = true; - identity = await fs.lstat(temporary); - }); - if (!identity || !hasIdentity(identity)) fail("temporary file backing identity unavailable"); - break; - } - if (!created) fail("temporary file attempt limit exceeded"); - if (target === undefined) { - for (const chunk of chunks) { - await this.checked(root, temporary, identity!); - await this.operation(() => fs.appendFile(temporary, chunk, { signal })); + checkPath(`${temporary}/entry`, this.limits); + if (temporary === path) continue; + try { + await this.operation(async () => { + staging = await fs.createStagedFile!(temporary, "entry", content, { + signal, parent, + ...(target !== undefined || capabilities.permissions === false ? {} : { mode: mode & 0o777 }), + ...(target !== undefined || capabilities.timestamps === false ? {} : { atimeMs: modified.getTime(), mtimeMs: modified.getTime() }), + }); + }); + break; + } catch (error) { + signal.throwIfAborted(); + if (typeof error !== "object" || error === null || !("code" in error) || error.code !== "EEXIST") throw error; } - await this.metadata(root, temporary, identity!, mode, modified); } + if (!staging) fail("temporary file attempt limit exceeded"); await this.parents(root, path, false); - const current = await this.stat(path); - if (current ? !expected || !sameIdentity(current, expected) || current.type !== "file" : expected !== undefined) fail("destination changed before publication"); - const staging = await this.stat(temporary); - if (!staging || !identity || !sameIdentity(staging, identity) || staging.type !== (target === undefined ? "file" : "symlink")) fail("temporary file changed before publication"); - await this.operation(() => fs.rename(temporary, path, { signal })); - created = false; - } catch (error) { - failure = { reason: error }; - } + await this.operation(() => fs.publishStagedFile!(staging!, path, { signal, parent, destination: expected ?? null })); + } catch (error) { failure = { reason: error }; } try { await cleanup(); } catch (error) { if (failure) throw new AggregateError([failure.reason, error], "unzip publication and cleanup failed"); diff --git a/packages/safe-bash/src/commands/archive/zip.ts b/packages/safe-bash/src/commands/archive/zip.ts index e1b85114cf..45dee8b3fa 100644 --- a/packages/safe-bash/src/commands/archive/zip.ts +++ b/packages/safe-bash/src/commands/archive/zip.ts @@ -87,6 +87,7 @@ async function prepare(scope: ZipScope, parsed: ZipOptions, budget: Budget) { const outputName = vfsPath(context.cwd, parsed.archive); const parentName = dirname(outputName); const parent = await scope.operation(() => context.fs.realpath(parentName, { signal: context.signal })); + const parentStat = await scope.operation(() => context.fs.lstat(parent, { signal: context.signal })); const output = `${parent === "/" ? "" : parent}/${outputName.slice(outputName.lastIndexOf("/") + 1)}`; checkPath(output, limits); const existing = await scope.stat(output); @@ -197,7 +198,7 @@ async function prepare(scope: ZipScope, parsed: ZipOptions, budget: Budget) { } for (const { entry } of selected.values()) { entries.push(entry); append(entry, false); } const bytes = await writeZipArchive({ entries, comment: archive.comment }, limits, context.signal); - return { output, parentName, parent, existing, bytes, progress }; + return { output, parentName, parent, parentStat, existing, bytes, progress }; } export function createZipCommand(options: ArchiveCommandsOptions = {}): CommandDefinition { diff --git a/packages/safe-bash/src/commands/archive/zip/safety.ts b/packages/safe-bash/src/commands/archive/zip/safety.ts index 1b9317618d..c278b724ed 100644 --- a/packages/safe-bash/src/commands/archive/zip/safety.ts +++ b/packages/safe-bash/src/commands/archive/zip/safety.ts @@ -1,6 +1,6 @@ -import { readBytes, type ByteSource, type CommandContext, type FileStat } from "../../../contracts/index.js"; +import { readBytes, type ByteSource, type CommandContext, type FileStat, type FileStaging } from "../../../contracts/index.js"; import { retainFileSystemCleanup } from "poe-code/safe-fs/core"; -import { checkPath, fail, hasIdentity, sameIdentity, type ArchiveLimits } from "../internal.js"; +import { checkPath, fail, type ArchiveLimits } from "../internal.js"; export class ZipScope { readonly context: CommandContext; @@ -84,50 +84,33 @@ export interface ZipPublication { readonly parent: string; readonly parentName: string; readonly existing: FileStat | undefined; + readonly parentStat: FileStat; readonly bytes: Uint8Array; } export async function publishZip(scope: ZipScope, prepared: ZipPublication): Promise { const { fs, signal } = scope.context; - const capabilities = await scope.operation(() => fs.capabilitiesFor?.(prepared.output, { signal }) ?? fs.capabilities); - if (capabilities.atomicRename !== true || capabilities.exclusiveCreate !== true || capabilities.explicitDirectories !== true - || capabilities.permissions !== true || !fs.rmdir || (prepared.existing ? !fs.chmod : capabilities.atomicRenameNoReplace !== true)) fail("ZIP publication requires atomic rename, exclusive creation, private directories and mode preservation"); - let directory: { path: string; stat: FileStat } | undefined; - let temporary = ""; - let identity: FileStat | undefined; - let published = false; - let collision = false; + const capabilities = await scope.operation(() => fs.capabilitiesFor?.(prepared.output, { signal, create: true }) ?? fs.capabilities); + if (capabilities.atomicFileStaging !== true || !fs.createStagedFile || !fs.publishStagedFile || !fs.removeStagedFile) fail("ZIP publication requires atomic owned file staging"); + let staging: FileStaging | undefined; let failure: { reason: unknown } | undefined; - const checkOwner = async (): Promise => { - const parent = await scope.operation(() => fs.realpath(prepared.parentName, { signal })); - const owner = directory && await scope.stat(directory.path); - if (parent !== prepared.parent || !owner || owner.type !== "directory" || !sameIdentity(owner, directory!.stat)) fail("ZIP temporary ownership changed before mutation"); - }; const close = retainFileSystemCleanup(fs, async cleanup => { - if (!directory || !hasIdentity(directory.stat)) return; - const parent = await cleanup.realpath(prepared.parentName); - const owner = await cleanup.lstat(directory.path); - if (parent !== prepared.parent || owner.type !== "directory" || !sameIdentity(owner, directory.stat)) fail("ZIP temporary ownership changed before cleanup"); - if (temporary && !published && !collision) { - let current: FileStat | undefined; - try { current = await cleanup.lstat(temporary); } - catch (error) { if (typeof error !== "object" || error === null || !("code" in error) || error.code !== "ENOENT") throw error; } - if (current) { - if (!identity || !sameIdentity(current, identity) || current.type !== "file" || current.nlink !== 1) fail("ZIP temporary file changed before cleanup"); - await cleanup.rm(temporary); - } + if (staging) { + if (!cleanup.removeStagedFile) fail("ZIP atomic cleanup unavailable"); + await cleanup.removeStagedFile(staging); } - await cleanup.rmdir!(directory.path); }, { maxOperations: 16 }); try { for (let attempt = 0; attempt < Math.min(64, scope.limits.maxMembers); attempt++) { const path = `${prepared.parent === "/" ? "" : prepared.parent}/.zip-${attempt + 1}`; checkPath(path, scope.limits); + checkPath(`${path}/archive.zip`, scope.limits); if (path === prepared.output) continue; try { await scope.operation(async () => { - await fs.mkdir(path, { signal, mode: 0o700 }); - directory = { path, stat: await fs.lstat(path) }; + staging = await fs.createStagedFile!(path, "archive.zip", { type: "file", data: prepared.bytes }, { + signal, parent: prepared.parentStat, ...(prepared.existing ? { mode: prepared.existing.mode & 0o7777 } : {}), + }); }); break; } catch (error) { @@ -135,43 +118,12 @@ export async function publishZip(scope: ZipScope, prepared: ZipPublication): Pro if (typeof error !== "object" || error === null || !("code" in error) || error.code !== "EEXIST") throw error; } } - if (!directory) fail("ZIP temporary directory attempt limit exceeded"); - if (directory.stat.type !== "directory" || !hasIdentity(directory.stat) || (directory.stat.mode & 0o777) !== 0o700) fail("ZIP temporary directory ownership unavailable"); - temporary = `${directory.path}/archive.zip`; - checkPath(temporary, scope.limits); - await checkOwner(); - await scope.operation(async () => { - let writeFailure: { reason: unknown } | undefined; - try { await fs.writeFile(temporary, prepared.bytes, { signal, flag: "wx", ...(prepared.existing ? { mode: prepared.existing.mode & 0o7777 } : {}) }); } - catch (error) { - collision = typeof error === "object" && error !== null && "code" in error && error.code === "EEXIST"; - writeFailure = { reason: error }; - } - if (!collision) { - await checkOwner(); - try { identity = await fs.lstat(temporary); } - catch (error) { if (typeof error !== "object" || error === null || !("code" in error) || error.code !== "ENOENT") throw error; } - } - if (writeFailure) throw writeFailure.reason; - }); - if (!identity || !hasIdentity(identity) || identity.type !== "file" || identity.nlink !== 1 || identity.size !== prepared.bytes.length) fail("ZIP staged archive identity or size changed"); - if (prepared.existing) { - await checkOwner(); - await scope.operation(() => fs.chmod!(temporary, prepared.existing!.mode & 0o7777, { signal })); - } + if (!staging) fail("ZIP temporary directory attempt limit exceeded"); const parent = await scope.operation(() => fs.realpath(prepared.parentName, { signal })); - const current = await scope.stat(prepared.output); - if (parent !== prepared.parent || (prepared.existing ? !current || !sameIdentity(prepared.existing, current) - || current.type !== "file" || current.nlink !== 1 || current.size !== prepared.existing.size - || current.mode !== prepared.existing.mode || current.mtimeMs !== prepared.existing.mtimeMs || current.ctimeMs !== prepared.existing.ctimeMs : current !== undefined)) fail("archive backing entry changed before publication"); - const staging = await scope.stat(temporary); - const owner = await scope.stat(directory.path); - if (!staging || !sameIdentity(staging, identity) || staging.type !== "file" || staging.nlink !== 1 || staging.size !== prepared.bytes.length - || !owner || !sameIdentity(owner, directory.stat) || owner.type !== "directory") fail("ZIP temporary ownership changed before publication"); - await scope.operation(async () => { - await fs.rename(temporary, prepared.output, { signal, ...(!prepared.existing ? { noReplace: true } : {}) }); - published = true; - }); + if (parent !== prepared.parent) fail("archive parent changed before publication"); + await scope.operation(() => fs.publishStagedFile!(staging!, prepared.output, { + signal, parent: prepared.parentStat, destination: prepared.existing ?? null, + })); } catch (error) { failure = { reason: error }; } try { await close(); } catch (error) { diff --git a/packages/safe-bash/src/commands/csplit/command.ts b/packages/safe-bash/src/commands/csplit/command.ts index 98d6cab098..af81588b87 100644 --- a/packages/safe-bash/src/commands/csplit/command.ts +++ b/packages/safe-bash/src/commands/csplit/command.ts @@ -12,7 +12,9 @@ export type { CsplitCommandsOptions, CsplitLimits } from "./internal.js"; export function createCsplitCommandWithExecutor(executor: RegexExecutor, options: CsplitCommandsOptions = {}): CommandDefinition { const limits = settings(options); - return { name: "csplit", description: "Split files at bounded line and BRE boundaries", async execute(context) { + return { name: "csplit", description: "Split files at bounded line and BRE boundaries", + filesystemRequirements: [{ id: "split", description: "Atomically mutate owned VFS outputs", capabilities: ["atomicFileMutation"], mutates: true }], + async execute(context) { return withRegexSession(context, executor, async session => { const budget = new Budget(context, limits); const lifecycle = new Lifecycle(budget); diff --git a/packages/safe-bash/src/commands/csplit/internal.ts b/packages/safe-bash/src/commands/csplit/internal.ts index 435a1d8cbd..202f2ec26c 100644 --- a/packages/safe-bash/src/commands/csplit/internal.ts +++ b/packages/safe-bash/src/commands/csplit/internal.ts @@ -84,11 +84,19 @@ export function missing(error: unknown): boolean { } export class Budget { + private readonly buffers = new Map(); private work = 0; private checkpoint = 0; private diagnostics = 0; private output = 0; constructor(readonly context: CommandContext, readonly limits: CsplitLimits) {} + reserveBuffered(owner: object, bytes: number): void { + let total = bytes; + for (const [key, value] of this.buffers) if (key !== owner) total += value; + this.check(total, this.limits.maxBufferedBytes, "buffered bytes"); + if (bytes) this.buffers.set(owner, bytes); + else this.buffers.delete(owner); + } quote(value: string): string { const locale = this.context.env.LC_ALL || this.context.env.LC_CTYPE || this.context.env.LANG || "C"; this.charge(value.length); diff --git a/packages/safe-bash/src/commands/csplit/io.ts b/packages/safe-bash/src/commands/csplit/io.ts index c90925aa77..80a5fb3b08 100644 --- a/packages/safe-bash/src/commands/csplit/io.ts +++ b/packages/safe-bash/src/commands/csplit/io.ts @@ -4,6 +4,10 @@ import { retainFileSystemCleanup } from "poe-code/safe-fs/core"; import { Budget, CsplitError, fsDetail, missing, pathText } from "./internal.js"; import type { Options } from "./options.js"; +const retainedLineUnits = 128; +const retainedOutputUnits = 256; +const outputBufferBytes = 65_536; + export function sameIdentity(first: FileStat, second: FileStat): boolean { return ((typeof first.identityScope === "object" && first.identityScope !== null) || typeof first.identityScope === "symbol") && first.identityScope === second.identityScope && Number.isSafeInteger(first.dev) && Number.isSafeInteger(first.ino) @@ -12,12 +16,14 @@ export function sameIdentity(first: FileStat, second: FileStat): boolean { export class Lifecycle { private readonly pending = new Set>(); - private readonly cleanups: (() => Promise)[] = []; + private readonly cleanups = new Set<() => Promise>(); private closing: Promise | undefined; constructor(readonly budget: Budget) { budget.context.registerCleanup?.(() => this.close()); } - cleanup(action: () => Promise): void { + cleanup(action: () => Promise): () => Promise { this.assertOpen(); - this.cleanups.push(action); + const owned = async () => { try { await action(); } finally { this.cleanups.delete(owned); } }; + this.cleanups.add(owned); + return owned; } assertOpen(): void { this.budget.context.signal.throwIfAborted(); @@ -46,6 +52,7 @@ export class Lifecycle { } export class Lines { + private readonly fallbackAllocation = {}; private readonly lines: Uint8Array[] = []; private partial: Uint8Array[] = []; private partialBytes = 0; @@ -56,7 +63,17 @@ export class Lines { private path: string | undefined; identity: FileStat | undefined; constructor(readonly lifecycle: Lifecycle) { - lifecycle.cleanup(async () => { if (!this.ended) await this.iterator?.return?.(); }); + lifecycle.cleanup(async () => { + try { if (!this.ended) await this.iterator?.return?.(); } + finally { + this.lines.length = 0; this.partial = []; + lifecycle.budget.reserveBuffered(this, 0); + lifecycle.budget.reserveBuffered(this.fallbackAllocation, 0); + } + }); + } + private reserve(extraLines = 0, extraParts = 0): void { + this.lifecycle.budget.reserveBuffered(this, this.total * 2 + (this.lines.length + extraLines + this.partial.length + extraParts) * retainedLineUnits); } async open(name: string): Promise { if (name === "-") { this.identity = this.lifecycle.budget.context.stdinInput?.stat; return; } @@ -77,18 +94,24 @@ export class Lines { const capabilities = await this.lifecycle.operation(async () => await fs.capabilitiesFor?.(this.path!, { signal }) ?? fs.capabilities); if (fs.readStream && capabilities.streamingRead !== false) return fs.readStream(this.path, { signal, chunkSize: 65_536 }); if (this.identity!.size > Math.min(limits.maxInputBytes, limits.maxBufferedBytes / 2)) throw new CsplitError("buffered input bytes limit exceeded"); - const value = await this.lifecycle.operation(() => fs.readFile(this.path!, { signal, maxBytes: limits.maxInputBytes })); + const admittedBytes = this.identity!.size; + this.lifecycle.budget.reserveBuffered(this.fallbackAllocation, admittedBytes); + this.lifecycle.budget.reserveBuffered(this, admittedBytes * 2); + const value = await this.lifecycle.operation(() => fs.readFile(this.path!, { signal, maxBytes: admittedBytes })); + this.lifecycle.budget.check(value.length, admittedBytes, "input bytes"); return { async *[Symbol.asyncIterator]() { yield value; } }; } private appendLine(): void { const { budget } = this.lifecycle; budget.check(this.lines.length + 1, budget.limits.maxLines, "line count"); + this.reserve(1); const line = new Uint8Array(this.partialBytes); let offset = 0; for (const part of this.partial) { line.set(part, offset); offset += part.length; } this.lines.push(line); this.partial = []; this.partialBytes = 0; + this.reserve(); } async get(number: number): Promise { const { budget } = this.lifecycle; @@ -107,7 +130,7 @@ export class Lines { if (!(chunk instanceof Uint8Array)) throw new CsplitError("input produced a non-byte chunk"); this.total += chunk.length; budget.check(this.total, budget.limits.maxInputBytes, "input bytes"); - budget.check(this.total * 2, budget.limits.maxBufferedBytes, "buffered bytes"); + this.reserve(); if (!chunk.length) budget.check(++this.emptyChunks, budget.limits.maxEmptyChunks, "empty input chunks"); budget.charge(chunk.length); const owned = Uint8Array.from(chunk); @@ -116,6 +139,7 @@ export class Lines { if (owned[offset] === 10) { this.partialBytes += offset + 1 - start; budget.check(this.partialBytes, budget.limits.maxLineBytes, "line bytes"); + this.reserve(0, 1); this.partial.push(owned.subarray(start, offset + 1)); this.appendLine(); start = offset + 1; } @@ -127,6 +151,7 @@ export class Lines { if (start < owned.length) { this.partialBytes += owned.length - start; budget.check(this.partialBytes, budget.limits.maxLineBytes, "line bytes"); + this.reserve(0, 1); this.partial.push(owned.subarray(start)); } await budget.checkpointWork(); @@ -137,14 +162,16 @@ export class Lines { interface Output { readonly path: string; - readonly name: string; - readonly parents: readonly { path: string; identity: FileStat }[]; + readonly parent: FileStat; + readonly metadataBytes: number; identity?: FileStat; created: boolean; removed: boolean; elided: boolean; failed: boolean; size: number; + buffer?: Uint8Array; + buffered: number; cleanup(): Promise; } @@ -164,19 +191,13 @@ export class Outputs { let current = "/"; for (const component of ["", ...components]) { if (component) current = resolvePath(current, component); - const identity = await this.lifecycle.operation(() => context.fs.lstat(current, { signal: context.signal })); + const identity = Object.freeze({ ...await this.lifecycle.operation(() => context.fs.lstat(current, { signal: context.signal })) }); if (identity.type !== "directory" || !sameIdentity(identity, identity)) throw new CsplitError("unsafe output directory identity"); parents.push({ path: current, identity }); } return parents; } - private async checked(output: Output): Promise { - await this.checkedParents(output.parents); - const { context } = this.lifecycle.budget; - const current = await this.lifecycle.operation(() => context.fs.lstat(output.path, { signal: context.signal })); - if (!output.identity || current.type !== "file" || !sameIdentity(current, output.identity)) throw new CsplitError("output changed before mutation"); - } - private async checkedParents(parents: Output["parents"]): Promise { + private async checkedParents(parents: readonly { path: string; identity: FileStat }[]): Promise { const { context } = this.lifecycle.budget; for (const parent of parents) { const current = await this.lifecycle.operation(() => context.fs.lstat(parent.path, { signal: context.signal })); @@ -192,42 +213,37 @@ export class Outputs { budget.check(name.length, limits.maxPathBytes, "output filename bytes"); const path = resolvePath(context.cwd, pathText(name)); try { + const capabilities = await this.lifecycle.operation(async () => await context.fs.capabilitiesFor?.(path, { signal: context.signal, create: true }) ?? context.fs.capabilities); + if (capabilities.atomicFileMutation !== true || !context.fs.writeFileConditional || !context.fs.removeFileConditional) throw new CsplitError("atomic output mutations are not supported"); const parents = await this.parents(path); let existing: FileStat | undefined; - try { existing = await this.lifecycle.operation(() => context.fs.lstat(path, { signal: context.signal })); } + try { existing = Object.freeze({ ...await this.lifecycle.operation(() => context.fs.lstat(path, { signal: context.signal })) }); } catch (error) { context.signal.throwIfAborted(); if (!missing(error)) throw error; } if (existing && (existing.type !== "file" || !sameIdentity(existing, existing))) throw new CsplitError(`unsafe output file ${budget.quote(name)}`); this.input.assertOutput(path, existing); - const output: Output = { path, name, parents, created: false, removed: false, elided: false, failed: false, size: 0, cleanup: async () => {} }; - output.cleanup = retainFileSystemCleanup(context.fs, async view => { + const metadataBytes = retainedOutputUnits + path.length * 2; + const output: Output = { path, parent: parents.at(-1)!.identity, metadataBytes, created: false, removed: false, elided: false, failed: false, size: 0, buffered: 0, cleanup: async () => {} }; + budget.reserveBuffered(output, metadataBytes); + output.cleanup = this.lifecycle.cleanup(retainFileSystemCleanup(context.fs, async view => { + delete output.buffer; + output.buffered = 0; + budget.reserveBuffered(output, 0); if (!output.created || !output.identity || !output.elided && (this.succeeded || this.options.keep || this.preserve)) return; try { - for (const parent of parents) { - const current = await view.lstat(parent.path); - if (current.type !== "directory" || !sameIdentity(current, parent.identity)) return; - } - const current = await view.lstat(path); - if (current.type === "file" && sameIdentity(current, output.identity)) { - await view.rm(path); - output.removed = true; - } - } catch (error) { if (!missing(error)) throw error; } - }, { maxOperations: limits.maxPathDepth + 3 }); - this.lifecycle.cleanup(output.cleanup); - await this.checkedParents(parents); - let admitted: FileStat | undefined; - try { admitted = await this.lifecycle.operation(() => context.fs.lstat(path, { signal: context.signal })); } - catch (error) { context.signal.throwIfAborted(); if (!missing(error)) throw error; } - if (admitted ? !existing || admitted.type !== "file" || !sameIdentity(admitted, existing) : existing !== undefined) throw new CsplitError("output changed before creation"); - await this.checkedParents(parents); - await this.lifecycle.operation(async () => { - await context.fs.writeFile(path, new Uint8Array(), { signal: context.signal, flag: existing ? "w" : "wx" }); - output.created = true; - if (existing) output.identity = existing; - const identity = await context.fs.lstat(path, { signal: context.signal }); - if (identity.type !== "file" || !sameIdentity(identity, identity) || existing && !sameIdentity(existing, identity)) throw new CsplitError("output creation identity unavailable"); - output.identity = identity; - }); + if (!view.removeFileConditional) throw new FsError("ENOTSUP"); + await view.removeFileConditional(path, { expected: output.identity, parent: output.parent }); + output.removed = true; + } catch (error) { if (!missing(error) && !(error instanceof FsError && error.code === "EAGAIN")) throw error; } + }, { maxOperations: 1 })); + try { + await this.checkedParents(parents); + await this.lifecycle.operation(async () => { + output.identity = Object.freeze({ ...await context.fs.writeFileConditional!(path, new Uint8Array(), { + signal: context.signal, expected: existing ?? null, parent: output.parent, + }) }); + output.created = true; + }); + } finally { parents.length = 0; } this.index++; this.current = output; } catch (error) { @@ -235,24 +251,47 @@ export class Outputs { throw error; } } + private async flush(output: Output): Promise { + if (!output.buffered) return; + const { context } = this.lifecycle.budget; + try { + await this.lifecycle.operation(() => writeFileOutput(context, output.buffer!.subarray(0, output.buffered), async chunk => { + output.identity = Object.freeze({ ...await context.fs.writeFileConditional!(output.path, chunk, { + signal: context.signal, expected: output.identity!, parent: output.parent, append: true, + }) }); + output.buffered = 0; + })); + } catch (error) { output.failed = true; throw error; } + } async write(value: Uint8Array): Promise { const output = this.current; if (!output) throw new CsplitError("output is not open"); const { budget } = this.lifecycle; budget.fileBytes(value.length); - try { - await this.checked(output); - await this.lifecycle.operation(() => writeFileOutput(budget.context, value, chunk => budget.context.fs.appendFile(output.path, chunk, { signal: budget.context.signal }))); - output.size += value.length; - } catch (error) { output.failed = true; throw error; } + if (!output.buffer) { + const capacity = Math.min(outputBufferBytes, budget.limits.maxBufferedBytes); + budget.reserveBuffered(output, output.metadataBytes + capacity); + output.buffer = new Uint8Array(capacity); + } + for (let offset = 0; offset < value.length;) { + this.lifecycle.assertOpen(); + const count = Math.min(value.length - offset, output.buffer.length - output.buffered); + output.buffer.set(value.subarray(offset, offset + count), output.buffered); + output.buffered += count; + output.size += count; + offset += count; + if (output.buffered === output.buffer.length) await this.flush(output); + } } async finish(): Promise { const output = this.current; if (!output) return; this.current = undefined; if (output.failed) return; + await this.flush(output); + delete output.buffer; + this.lifecycle.budget.reserveBuffered(output, output.metadataBytes); if (!output.size && this.options.elide) { - await this.checked(output); output.elided = true; await output.cleanup(); if (!output.removed) throw new CsplitError("output changed before empty-file removal"); diff --git a/packages/safe-bash/src/contracts/filesystem.ts b/packages/safe-bash/src/contracts/filesystem.ts index f307880264..53c871cd95 100644 --- a/packages/safe-bash/src/contracts/filesystem.ts +++ b/packages/safe-bash/src/contracts/filesystem.ts @@ -1,2 +1,2 @@ export { ACCESS_MODES } from "poe-code/safe-fs/core"; -export type { FileType, EntryComparison, FileStat, DirectoryEntry, FileSystemCapabilities, FsOptions, CapabilityQueryOptions, RenameOptions, FileReadHandle, OpenReadFileOptions, FileResizeHandle, FileResizeOperation, FileResizeOptions, OpenResizeFileOptions, ReadFileOptions, ReadDirectoryOptions, WriteFileOptions, AppendFileOptions, MkdirOptions, RemoveOptions, CopyFileOptions, ReadStreamOptions, FileSystem, FileSystemFactory } from "poe-code/safe-fs/core"; +export type { ConditionalWriteFileOptions, ConditionalRemoveFileOptions, FileStaging, FileStagingEntry, StagedFileContent, CreateStagedFileOptions, PublishStagedFileOptions, PrepareDirectoryOptions, FileType, EntryComparison, FileStat, DirectoryEntry, FileSystemCapabilities, FsOptions, CapabilityQueryOptions, RenameOptions, FileReadHandle, OpenReadFileOptions, FileResizeHandle, FileResizeOperation, FileResizeOptions, OpenResizeFileOptions, ReadFileOptions, ReadDirectoryOptions, WriteFileOptions, AppendFileOptions, MkdirOptions, RemoveOptions, CopyFileOptions, ReadStreamOptions, FileSystem, FileSystemFactory } from "poe-code/safe-fs/core"; diff --git a/packages/safe-bash/tests/commands/csplit-review/lifecycle.test.ts b/packages/safe-bash/tests/commands/csplit-review/lifecycle.test.ts index 2263c5fa57..04aba21831 100644 --- a/packages/safe-bash/tests/commands/csplit-review/lifecycle.test.ts +++ b/packages/safe-bash/tests/commands/csplit-review/lifecycle.test.ts @@ -51,15 +51,15 @@ for (const phase of ["input next", "second output append"] as const) { }; } }); } else { - const appendFile = fs.appendFile.bind(fs); - fs.appendFile = async (path, bytes, options) => { - if (path !== "/work/xx01") return appendFile(path, bytes, options); + const writeFileConditional = fs.writeFileConditional.bind(fs); + fs.writeFileConditional = async (path, bytes, options) => { + if (path !== "/work/xx01" || !options.append) return writeFileConditional(path, bytes, options); operationSignal = options?.signal; admitted.resolve(); try { await gate.promise; options?.signal?.throwIfAborted(); - await appendFile(path, bytes, options); + return await writeFileConditional(path, bytes, options); } finally { completed = true; finished.resolve(); } }; } @@ -95,12 +95,11 @@ for (const phase of ["input next", "second output append"] as const) { }); } -for (const keep of [false, true]) test(`partial second append after first publication, keep=${keep}`, async () => { +for (const keep of [false, true]) test(`atomic second append refusal after first publication, keep=${keep}`, async () => { const fs = await fixture(); - const appendFile = fs.appendFile.bind(fs); - fs.appendFile = async (path, bytes, options) => { - if (path !== "/work/xx01") return appendFile(path, bytes, options); - await appendFile(path, bytes.subarray(0, 1), options); + const writeFileConditional = fs.writeFileConditional.bind(fs); + fs.writeFileConditional = async (path, bytes, options) => { + if (path !== "/work/xx01" || !options.append) return writeFileConditional(path, bytes, options); throw new FsError("ENOSPC"); }; const shell = new Shell({ fs, cwd: "/work" }).use(csplitCommands()); @@ -112,7 +111,7 @@ for (const keep of [false, true]) test(`partial second append after first public assert.deepEqual(await fs.readFile("/work/xx99"), Uint8Array.from(Buffer.from("foreign"))); if (keep) { assert.deepEqual(await fs.readFile("/work/xx00"), Uint8Array.from(Buffer.from("aa\n"))); - assert.deepEqual(await fs.readFile("/work/xx01"), Uint8Array.from(Buffer.from("b"))); + assert.deepEqual(await fs.readFile("/work/xx01"), new Uint8Array()); } else assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name).sort(), ["input", "xx99"]); } finally { await shell.dispose(); } }); @@ -120,15 +119,14 @@ for (const keep of [false, true]) test(`partial second append after first public test("cleanup failure for the first output does not abandon later owned cleanup", async () => { const fs = await fixture(); const failures: unknown[] = []; - const appendFile = fs.appendFile.bind(fs), remove = fs.rm.bind(fs); + const writeFileConditional = fs.writeFileConditional.bind(fs), remove = fs.removeFileConditional.bind(fs); const primary = new FsError("ENOSPC"); const cleanup = new Error("first output cleanup failed"); - fs.appendFile = async (path, bytes, options) => { - if (path !== "/work/xx01") return appendFile(path, bytes, options); - await appendFile(path, bytes.subarray(0, 1), options); + fs.writeFileConditional = async (path, bytes, options) => { + if (path !== "/work/xx01" || !options.append) return writeFileConditional(path, bytes, options); throw primary; }; - fs.rm = async (path, options) => { + fs.removeFileConditional = async (path, options) => { if (path === "/work/xx00") throw cleanup; await remove(path, options); }; @@ -150,8 +148,8 @@ test("input symlink backing identity is excluded before truncating the output", await fs.writeFile("/work/xx00", Buffer.from("aa\nbb\n")); await fs.symlink("xx00", "/work/input"); let writes = 0; - const writeFile = fs.writeFile.bind(fs); - fs.writeFile = async (...args) => { writes++; return writeFile(...args); }; + const writeFile = fs.writeFileConditional.bind(fs); + fs.writeFileConditional = async (...args) => { writes++; return writeFile(...args); }; const shell = new Shell({ fs, cwd: "/work" }).use(csplitCommands()); try { assert.equal((await shell.exec("csplit input 2")).exitCode, 1); @@ -166,14 +164,14 @@ for (const source of ["xx00", "./xx00", "source-link"]) test(`source/output alia await fs.mkdir("/work"); await fs.writeFile("/work/xx00", Buffer.from("aa\nbb\n")); if (source === "source-link") await fs.symlink("xx00", "/work/source-link"); - const stat = fs.stat.bind(fs), writeFile = fs.writeFile.bind(fs); + const stat = fs.stat.bind(fs), writeFile = fs.writeFileConditional.bind(fs); let writes = 0; fs.stat = async (path, options) => { const value = await stat(path, options); const { identityScope: ignoredScope, ino: ignoredInode, dev: ignoredDevice, ...unknown } = value; return unknown; }; - fs.writeFile = async (...args) => { writes++; return writeFile(...args); }; + fs.writeFileConditional = async (...args) => { writes++; return writeFile(...args); }; const shell = new Shell({ fs, cwd: "/work" }).use(csplitCommands()); try { assert.equal((await shell.exec(`csplit ${source} 2`)).exitCode, 1); diff --git a/packages/safe-bash/tests/commands/csplit/api.test.ts b/packages/safe-bash/tests/commands/csplit/api.test.ts index 00f961f03b..c3bed40bbb 100644 --- a/packages/safe-bash/tests/commands/csplit/api.test.ts +++ b/packages/safe-bash/tests/commands/csplit/api.test.ts @@ -4,6 +4,7 @@ import * as entry from "../../../src/commands/csplit/index.js"; import { createCsplitCommandWithExecutor } from "../../../src/commands/csplit/command.js"; import { RegexExecutor } from "../../../src/commands/regex-execution/portable.js"; import { createBoundedRegexProvider } from "../../../src/commands/regex-execution/bounded-provider.js"; +import { evaluateCommandSupport } from "../../../src/contracts/command-requirements.js"; test("public csplit entry exposes only the requested three factories", () => { assert.deepEqual(Object.keys(entry).sort(), ["createCsplitCommand", "createCsplitCommands", "csplitCommands"]); @@ -14,3 +15,10 @@ test("internal executor factory remains available for preset composition", async try { assert.equal(createCsplitCommandWithExecutor(executor).name, "csplit"); } finally { await executor.dispose(); } }); + +test("csplit declares atomic output support and refuses readonly capability metadata", () => { + const command = entry.createCsplitCommand(); + assert.equal(evaluateCommandSupport(command, { atomicFileMutation: true }).status, "supported"); + assert.equal(evaluateCommandSupport(command, { atomicFileMutation: false }).status, "unsupported"); + assert.equal(evaluateCommandSupport(command, { atomicFileMutation: true, readOnly: true }).status, "unsupported"); +}); diff --git a/packages/safe-bash/tests/commands/csplit/safety.test.ts b/packages/safe-bash/tests/commands/csplit/safety.test.ts index a4f7f1bfd7..40d7819995 100644 --- a/packages/safe-bash/tests/commands/csplit/safety.test.ts +++ b/packages/safe-bash/tests/commands/csplit/safety.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { MemoryFileSystem } from "../../../src/fs/memory/index.js"; import { Shell } from "../../../src/shell/shell.js"; -import { toByteSource, type CommandContext, type FileSystem } from "../../../src/contracts/index.js"; +import { FsError, toByteSource, type CommandContext, type FileStat, type FileSystem } from "../../../src/contracts/index.js"; import { bindFileOutputBudget } from "../../../src/contracts/filesystem-output.js"; import { createCsplitCommand, csplitCommands, type CsplitCommandsOptions } from "../../../src/commands/csplit/index.js"; @@ -12,6 +12,65 @@ async function fixture() { return fs; } +test("new split outputs request atomic mutation capabilities with create intent", async () => { + const fs = await fixture(); + const creations = new Set(); + const view = wrap(fs, { + async capabilitiesFor(path, options) { + try { await fs.lstat(path); } + catch (error) { + if (!(error instanceof FsError) || error.code !== "ENOENT") throw error; + if (!options?.create) return { ...fs.capabilities, atomicFileMutation: false }; + creations.add(path); + } + return fs.capabilities; + }, + }); + assert.deepEqual(await run(view, ["-", "2"]), { exitCode: 0, stdout: "2\n4\n", stderr: "" }); + assert.deepEqual(await fs.readFile("/work/xx00"), new TextEncoder().encode("a\n")); + assert.deepEqual(await fs.readFile("/work/xx01"), new TextEncoder().encode("b\nc\n")); + assert.deepEqual([...creations].sort(), ["/work/xx00", "/work/xx01"]); +}); + +for (const boundary of ["writeFile", "appendFile", "rm"] as const) test(`replacement at the ${boundary} boundary preserves foreign output bytes`, async () => { + const fs = await fixture(); + const path = "/work/xx00"; + const foreign = new TextEncoder().encode("foreign replacement"); + if (boundary === "writeFile") await fs.writeFile(path, new TextEncoder().encode("previous")); + let replaced = false; + const replace = async (candidate: string) => { + if (candidate !== path || replaced) return; + replaced = true; + await fs.rename(path, "/work/displaced"); + await fs.writeFile(path, foreign); + }; + const view = wrap(fs, { + async writeFileConditional(candidate, bytes, options) { + if (boundary === "writeFile" && !options.append || boundary === "appendFile" && options.append) await replace(candidate); + return fs.writeFileConditional(candidate, bytes, options); + }, + async removeFileConditional(candidate, options) { + if (boundary === "rm") await replace(candidate); + await fs.removeFileConditional(candidate, options); + }, + async writeFile(candidate, bytes, options) { + if (boundary === "writeFile") await replace(candidate); + await fs.writeFile(candidate, bytes, options); + }, + async appendFile(candidate, bytes, options) { + if (boundary === "appendFile") await replace(candidate); + await fs.appendFile(candidate, bytes, options); + }, + async rm(candidate, options) { + if (boundary === "rm") await replace(candidate); + await fs.rm(candidate, options); + }, + }); + await run(view, ["-", boundary === "rm" ? "20" : "2"]); + assert.equal(replaced, true, "the replacement must occur at the actual mutation boundary"); + assert.deepEqual(await fs.readFile(path), foreign); +}); + async function run(fs: FileSystem, args: string[], input = "a\nb\nc\n", options: CsplitCommandsOptions = {}, overrides: Partial = {}) { let stdout = "", stderr = ""; const result = await createCsplitCommand(options).execute({ @@ -31,6 +90,137 @@ function wrap(fs: FileSystem, overrides: Partial): FileSystem { } }); } +test("buffered tiny lines preserve bytes and counts with writes proportional to bytes", async () => { + const fs = await fixture(); + const input = "a\n".repeat(131_072); + let appends = 0; + const view = wrap(fs, { async writeFileConditional(path, value, options) { + if (options.append) { appends++; assert.ok(value.length <= 65_536); } + return fs.writeFileConditional(path, value, options); + } }); + const result = await run(view, ["-", "2"], input, { limits: { + maxInputBytes: 10 * 1024 * 1024, maxBufferedBytes: 36 * 1024 * 1024 + 65_536 + 128, maxLines: 131_072, + } }); + assert.deepEqual(result, { exitCode: 0, stdout: "2\n262142\n", stderr: "" }); + assert.equal(appends, 5); + assert.deepEqual(await fs.readFile("/work/xx01"), new TextEncoder().encode(input.slice(2))); +}); + +for (const keep of [false, true]) test(`buffered interruption retains only committed bytes, keep=${keep}`, async () => { + const fs = await fixture(); + const controller = new AbortController(); + const bytes = new TextEncoder().encode("a\n".repeat(35_000)); + const stdin = { async *[Symbol.asyncIterator]() { yield bytes; controller.abort(false); } }; + await assert.rejects(run(fs, [...(keep ? ["-k"] : []), "-", "999999"], "", {}, { signal: controller.signal, stdin }), error => error === false); + if (keep) assert.deepEqual(await fs.readFile("/work/xx00"), bytes.subarray(0, 65_536)); + else assert.deepEqual(await fs.readdir("/work"), []); +}); + +test("failed buffered flush does not print uncommitted byte counts", async () => { + const fs = await fixture(); + const view = wrap(fs, { async writeFileConditional(path, value, options) { + if (options.append) throw new FsError("ENOSPC"); + return fs.writeFileConditional(path, value, options); + } }); + const result = await run(view, ["-k", "-", "2"]); + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.equal((await fs.readFile("/work/xx00")).length, 0); +}); + +test("a provider that loses its committed receipt cannot authorize stale cleanup", async () => { + const fs = await fixture(); + const view = wrap(fs, { async writeFileConditional(path, value, options) { + const receipt = await fs.writeFileConditional(path, value, options); + if (options.append) throw false; + return receipt; + } }); + const result = await run(view, ["-", "2"]); + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.deepEqual(await fs.readFile("/work/xx00"), new TextEncoder().encode("a\n")); +}); + +test("retained receipts stay stable when a provider reuses its result object", async () => { + const fs = await fixture(); + let scratch: FileStat | undefined; + const view = wrap(fs, { async writeFileConditional(path, value, options) { + const receipt = await fs.writeFileConditional(path, value, options); + scratch = Object.assign(scratch ?? {}, receipt) as FileStat; + return scratch; + } }); + const result = await run(view, ["-", "2", "20"]); + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, "2\n4\n"); + assert.deepEqual(await fs.readdir("/work"), []); +}); + +test("buffered empty-file suppression reuses the name and prints only completed bytes", async () => { + const fs = await fixture(); + assert.deepEqual(await run(fs, ["-z", "-", "1"]), { exitCode: 0, stdout: "6\n", stderr: "" }); + assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name), ["xx00"]); + assert.deepEqual(await fs.readFile("/work/xx00"), new TextEncoder().encode("a\nb\nc\n")); +}); + +test("buffered payload and retained line units reject before excess allocation", async () => { + const fs = await fixture(); + const result = await run(fs, ["-", "2"], "a\nb\n", { limits: { maxBufferedBytes: 600 } }); + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /buffered bytes limit exceeded/); + assert.deepEqual(await fs.readdir("/work"), []); +}); + +test("fragmented input charges retained references before creating more fragments", async () => { + const fs = await fixture(); + let reads = 0; + const stdin = { async *[Symbol.asyncIterator]() { + for (let index = 0; index < 10; index++) { reads++; yield Uint8Array.of(97); } + } }; + const result = await run(fs, ["-", "2"], "", { limits: { maxBufferedBytes: 600 } }, { stdin }); + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /buffered bytes limit exceeded/); + assert.equal(reads, 3); + assert.deepEqual(await fs.readdir("/work"), []); +}); + +test("non-streaming input reserves its payload before requesting a file allocation", async () => { + const fs = await fixture(); + await fs.writeFile("/work/source", new Uint8Array(400).fill(97)); + let reads = 0; + const view = wrap(fs, { + async capabilitiesFor() { return { ...fs.capabilities, streamingRead: false }; }, + async readFile(path, options) { reads++; return fs.readFile(path, options); }, + }); + const result = await run(view, ["source", "2"], "", { limits: { maxBufferedBytes: 1000 } }); + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /buffered bytes limit exceeded/); + assert.equal(reads, 0); + assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name), ["source"]); +}); + +test("non-streaming reads pass the admitted bound and reject an oversized provider result", async () => { + const fs = await fixture(); + await fs.writeFile("/work/source", new TextEncoder().encode("a\nb\n")); + let admitted: number | undefined; + const view = wrap(fs, { + async capabilitiesFor() { return { ...fs.capabilities, streamingRead: false }; }, + async readFile(_path, options) { admitted = options?.maxBytes; return new Uint8Array(5); }, + }); + const result = await run(view, ["source", "2"]); + assert.equal(admitted, 4); + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /input bytes limit exceeded/); + assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name), ["source"]); +}); + +test("retained output receipts remain charged until cleanup releases them", async () => { + const fs = await fixture(); + const result = await run(fs, ["-k", "-s", "-", "1", "{*}"], "a\n".repeat(50), { limits: { maxBufferedBytes: 78_000 } }); + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /buffered bytes limit exceeded/); + assert.ok((await fs.readdir("/work")).length < 50); +}); + test("default error deletes preexisting outputs after truncation, not rollback", async () => { const fs = await fixture(); await fs.writeFile("/work/xx00", new TextEncoder().encode("previous")); @@ -129,9 +319,12 @@ test("unknown input identity permits distinct newly created outputs", async () = assert.deepEqual(await fs.readFile("/work/input"), new TextEncoder().encode("a\nb\n")); }); -for (const reason of [undefined, null, false, 0, ""]) test(`known output cleans after falsey append failure ${String(reason)}`, async () => { +for (const reason of [undefined, null, false, 0, ""]) test(`known output cleans after falsey atomic append refusal ${String(reason)}`, async () => { const fs = await fixture(); - const view = wrap(fs, { async appendFile(path, value, options) { await fs.appendFile(path, value, options); throw reason; } }); + const view = wrap(fs, { async writeFileConditional(path, value, options) { + if (options.append) throw reason; + return fs.writeFileConditional(path, value, options); + } }); assert.equal((await run(view, ["-", "2"])).exitCode, 1); assert.deepEqual(await fs.readdir("/work"), []); }); @@ -139,14 +332,18 @@ for (const reason of [undefined, null, false, 0, ""]) test(`known output cleans for (const reason of [null, false, 0, ""]) test(`known output cleans after cancellation ${String(reason)}`, async () => { const fs = await fixture(); const controller = new AbortController(); - const view = wrap(fs, { async appendFile(path, value, options) { await fs.appendFile(path, value, options); controller.abort(reason); } }); + const view = wrap(fs, { async writeFileConditional(path, value, options) { + const receipt = await fs.writeFileConditional(path, value, options); + if (options.append) controller.abort(reason); + return receipt; + } }); await assert.rejects(run(view, ["-", "2"], undefined, {}, { signal: controller.signal }), error => error === reason); assert.deepEqual(await fs.readdir("/work"), []); }); test("cleanup failure retains the primary failure in an aggregate", async () => { const fs = await fixture(); - const view = wrap(fs, { async rm() { throw false; } }); + const view = wrap(fs, { async removeFileConditional() { throw false; } }); await assert.rejects(run(view, ["-", "20"]), error => error instanceof AggregateError && error.errors.some((reason: unknown) => reason instanceof Error && reason.message.includes("line number out of range")) && error.errors.includes(false)); @@ -154,15 +351,17 @@ test("cleanup failure retains the primary failure in an aggregate", async () => test("elision cannot reuse a slot replaced during retained cleanup", async () => { const fs = await fixture(); - let calls = 0; - const view = wrap(fs, { async lstat(path, options) { - if (path === "/work/xx00" && ++calls === 5) { + let replaced = false; + const view = wrap(fs, { async removeFileConditional(path, options) { + if (path === "/work/xx00" && !replaced) { + replaced = true; await fs.rename(path, "/work/held"); await fs.writeFile(path, new TextEncoder().encode("foreign")); } - return fs.lstat(path, options); + await fs.removeFileConditional(path, options); } }); assert.equal((await run(view, ["-z", "-", "1"])).exitCode, 1); + assert.equal(replaced, true); assert.deepEqual(await fs.readFile("/work/xx00"), new TextEncoder().encode("foreign")); }); @@ -186,21 +385,25 @@ test("replacement after append cannot receive subsequent writes or cleanup", asy const fs = await fixture(); await fs.writeFile("/outside", new TextEncoder().encode("foreign")); let replaced = false; - const view = wrap(fs, { async appendFile(path, value, options) { - await fs.appendFile(path, value, options); - if (!replaced) { replaced = true; await fs.rename(path, "/work/held"); await fs.symlink("/outside", path); } + const view = wrap(fs, { async writeFileConditional(path, value, options) { + const receipt = await fs.writeFileConditional(path, value, options); + if (options.append && !replaced) { replaced = true; await fs.rename(path, "/work/held"); await fs.symlink("/outside", path); } + return receipt; } }); - assert.equal((await run(view, ["-", "3"])).exitCode, 1); + assert.equal((await run(view, ["-", "2"], "a".repeat(70_000) + "\nb\n")).exitCode, 1); + assert.equal(replaced, true); assert.deepEqual(await fs.readFile("/outside"), new TextEncoder().encode("foreign")); assert.equal((await fs.lstat("/work/xx00")).type, "symlink"); }); -test("unknown creation identity after cancellation remains a fail-closed artifact", async () => { +test("committed creation receipt permits owned cleanup after cancellation", async () => { const fs = await fixture(); const controller = new AbortController(); - const view = wrap(fs, { async writeFile(path, value, options) { await fs.writeFile(path, value, options); controller.abort(false); } }); + const view = wrap(fs, { async writeFileConditional(path, value, options) { + const receipt = await fs.writeFileConditional(path, value, options); controller.abort(false); return receipt; + } }); await assert.rejects(run(view, ["-", "2"], undefined, {}, { signal: controller.signal }), error => error === false); - assert.equal((await fs.readFile("/work/xx00")).length, 0); + assert.deepEqual(await fs.readdir("/work"), []); }); test("actual shell output cancellation drains known output cleanup and recovers", async () => { diff --git a/packages/safe-bash/tests/commands/unzip.test.ts b/packages/safe-bash/tests/commands/unzip.test.ts index 93e5e5ba25..39bdb1904e 100644 --- a/packages/safe-bash/tests/commands/unzip.test.ts +++ b/packages/safe-bash/tests/commands/unzip.test.ts @@ -236,9 +236,9 @@ function wrapped(fs: FileSystem, overrides: Partial): FileSystem { } }); } -for (const reason of [undefined, null, false, 0, ""]) test(`unzip cleans staging after falsey append failure ${String(reason)}`, async () => { +for (const reason of [undefined, null, false, 0, ""]) test(`unzip cleans staging after falsey publication failure ${String(reason)}`, async () => { const fs = await fixture([{ name: "file", body: "hello" }]); await fs.writeFile("/work/file", Buffer.from("keep")); - const faulty = wrapped(fs, { async appendFile() { throw reason; } }); + const faulty = wrapped(fs, { async publishStagedFile() { throw reason; } }); assert.equal((await run(faulty, ["-o", "sample.zip"])).exitCode, 2); assert.equal(Buffer.from(await fs.readFile("/work/file")).toString(), "keep"); assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name).sort(), ["file", "sample.zip"]); @@ -247,9 +247,10 @@ for (const reason of [undefined, null, false, 0, ""]) test(`unzip cleans staging test("unzip waits for staging cleanup after abort during exclusive acquisition", async () => { const fs = await fixture([{ name: "file", body: "hello" }]); const controller = new AbortController(); - const faulty = wrapped(fs, { async writeFile(path, bytes, options) { - await fs.writeFile(path, bytes, options); - if (path.includes(".unzip-")) controller.abort(false); + const faulty = wrapped(fs, { async createStagedFile(path, name, content, options) { + const receipt = await fs.createStagedFile!(path, name, content, options); + controller.abort(false); + return receipt; } }); await assert.rejects(run(faulty, ["sample.zip"], "", {}, { signal: controller.signal }), reason => reason === false); assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name), ["sample.zip"]); @@ -327,9 +328,10 @@ test("unzip registered cleanup waits for admitted publication and removes stagin const enteredPromise = new Promise(resolve => { entered = resolve; }); const blocked = new Promise(resolve => { unblock = resolve; }); let cleanup: (() => void | Promise) | undefined; - const delayed = wrapped(fs, { async writeFile(path, bytes, options) { - await fs.writeFile(path, bytes, options); - if (path.includes(".unzip-")) { entered(); await blocked; } + const delayed = wrapped(fs, { async createStagedFile(path, name, content, options) { + const receipt = await fs.createStagedFile!(path, name, content, options); + entered(); await blocked; + return receipt; } }); const result = run(delayed, ["sample.zip"], "", {}, { registerCleanup(handler) { cleanup = handler; } }); await enteredPromise; @@ -342,11 +344,12 @@ test("unzip registered cleanup waits for admitted publication and removes stagin test("unzip never removes a replacement at its failed staging name", async () => { const fs = await fixture([{ name: "file", body: "hello" }]); - const faulty = wrapped(fs, { async appendFile(path) { + const faulty = wrapped(fs, { async publishStagedFile(staging) { + const path = staging.file.path; await fs.rm(path); await fs.writeFile(path, Buffer.from("not ours")); throw false; } }); assert.equal((await run(faulty, ["sample.zip"])).exitCode, 2); - assert.equal(Buffer.from(await fs.readFile("/work/.unzip-1")).toString(), "not ours"); + assert.equal(Buffer.from(await fs.readFile("/work/.unzip-1/entry")).toString(), "not ours"); }); for (const streaming of [false, true]) test(`unzip cleanup waits for admitted ${streaming ? "stream" : "buffered"} archive reads`, async () => { @@ -382,7 +385,7 @@ test("unzip bounds archive-stream pulls, not only nonempty bytes", async () => { assert.ok(pulls <= 51, String(pulls)); }); -for (const after of ["append", "chmod"]) test(`unzip rejects staging symlink replacement after ${after} before metadata mutation`, async () => { +for (const after of ["creation", "publication"]) test(`unzip rejects staging symlink replacement at ${after}`, async () => { const fs = await fixture([{ name: "file", body: "hello", mode: 0o100640 }]); await fs.writeFile("/outside", Buffer.from("keep"), { mode: 0o604 }); await fs.utimes!("/outside", 946684800000, 946684800000); @@ -392,13 +395,14 @@ for (const after of ["append", "chmod"]) test(`unzip rejects staging symlink rep await fs.symlink!("/outside", path); }; const faulty = wrapped(fs, { - async appendFile(path, bytes, options) { - await fs.appendFile(path, bytes, options); - if (after === "append") await replace(path); + async createStagedFile(path, name, content, options) { + const receipt = await fs.createStagedFile!(path, name, content, options); + if (after === "creation") await replace(receipt.file.path); + return receipt; }, - async chmod(path, mode, options) { - await fs.chmod!(path, mode, options); - if (after === "chmod") await replace(path); + async publishStagedFile(staging, destination, options) { + if (after === "publication") await replace(staging.file.path); + return fs.publishStagedFile!(staging, destination, options); }, }); assert.equal((await run(faulty, ["sample.zip"])).exitCode, 2); @@ -406,7 +410,7 @@ for (const after of ["append", "chmod"]) test(`unzip rejects staging symlink rep assert.equal(outside.mode, before.mode); assert.equal(outside.mtimeMs, before.mtimeMs); assert.equal(Buffer.from(await fs.readFile("/outside")).toString(), "keep"); - assert.equal(await fs.readlink!("/work/.unzip-1"), "/outside"); + assert.equal(await fs.readlink!("/work/.unzip-1/entry"), "/outside"); }); test("unzip checks staging parents before later mutation and cleanup", async () => { @@ -414,10 +418,11 @@ test("unzip checks staging parents before later mutation and cleanup", async () await fs.mkdir("/outside"); await fs.writeFile("/outside/.unzip-1", Buffer.from("keep"), { mode: 0o604 }); const before = await fs.stat("/outside/.unzip-1"); - const faulty = wrapped(fs, { async appendFile(path, bytes, options) { - await fs.appendFile(path, bytes, options); + const faulty = wrapped(fs, { async createStagedFile(path, name, content, options) { + const receipt = await fs.createStagedFile!(path, name, content, options); await fs.rename("/work/dir", "/work/held-dir"); await fs.symlink!("/outside", "/work/dir"); + return receipt; } }); assert.equal((await run(faulty, ["sample.zip"])).exitCode, 2); const outside = await fs.stat("/outside/.unzip-1"); @@ -426,13 +431,13 @@ test("unzip checks staging parents before later mutation and cleanup", async () assert.equal(Buffer.from(await fs.readFile("/outside/.unzip-1")).toString(), "keep"); }); -test("unzip checks directory identity between chmod and timestamp restoration", async () => { +test("unzip checks directory identity atomically before metadata restoration", async () => { const fs = await fixture([{ name: "dir/", mode: 0o40750 }]); await fs.mkdir("/outside"); await fs.utimes!("/outside", 946684800000, 946684800000); const before = await fs.stat("/outside"); - const faulty = wrapped(fs, { async chmod(path, mode, options) { - await fs.chmod!(path, mode, options); - if (path === "/work/dir") { await fs.rename(path, "/work/held-dir"); await fs.symlink!("/outside", path); } + const faulty = wrapped(fs, { async prepareDirectory(path, options) { + if (path === "/work/dir" && options.expected) { await fs.rename(path, "/work/held-dir"); await fs.symlink!("/outside", path); } + return fs.prepareDirectory!(path, options); } }); assert.equal((await run(faulty, ["sample.zip"])).exitCode, 2); assert.equal((await fs.stat("/outside")).mtimeMs, before.mtimeMs); @@ -459,14 +464,15 @@ for (const operand of ["archive", "selection", "destination"]) test(`unzip Shell } finally { await shell.dispose(); } }); -test("unzip actual Shell drains retained staging cleanup after append abort", async () => { +test("unzip actual Shell drains retained staging cleanup after creation abort", async () => { const fs = await fixture([{ name: "file", body: "x".repeat(4096), method: 8 }]); await fs.writeFile("/work/file", Buffer.from("keep")); const archive = await fs.readFile("/work/sample.zip"); const controller = new AbortController(); - const dynamic = wrapped(fs, { async appendFile(path, bytes, options) { - await fs.appendFile(path, bytes, options); - if (path.startsWith("/work/.unzip-")) controller.abort(false); + const dynamic = wrapped(fs, { async createStagedFile(path, name, content, options) { + const receipt = await fs.createStagedFile!(path, name, content, options); + controller.abort(false); + return receipt; } }); const shell = new Shell({ fs: dynamic, cwd: "/work" }); shell.use({ name: "test-unzip", setup(host) { host.commands.register(createUnzipCommand()); } }); @@ -484,8 +490,8 @@ test("unzip preserves both publication and falsey retained-cleanup failures", as const original = new Error("publication failed"); const observed: unknown[] = []; const faulty = wrapped(fs, { - async appendFile() { throw original; }, - async rm() { throw false; }, + async publishStagedFile() { throw original; }, + async removeStagedFile() { throw false; }, }); const result = await run(faulty, ["sample.zip"], "", {}, { onInternalError(error) { observed.push(error); } }); assert.equal(result.exitCode, 2); @@ -498,13 +504,13 @@ test("unzip handled missing staging file does not replace the publication failur const fs = await fixture([{ name: "file", body: "hello" }]); const original = new Error("publication failed after removal"); const observed: unknown[] = []; - const faulty = wrapped(fs, { async appendFile(path) { await fs.rm(path); throw original; } }); + const faulty = wrapped(fs, { async publishStagedFile(staging) { await fs.rm(staging.file.path); throw original; } }); assert.equal((await run(faulty, ["sample.zip"], "", {}, { onInternalError(error) { observed.push(error); } })).exitCode, 2); assert.deepEqual(observed, [original]); assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name), ["sample.zip"]); }); -for (const replacement of [false, true]) test(`unzip acquisition-abort limitation: ${replacement ? "refuses foreign replacement cleanup" : "unverified temporary may remain"}`, async () => { +for (const replacement of [false, true]) test(`unzip atomic acquisition abort: ${replacement ? "refuses foreign replacement cleanup" : "removes owned staging"}`, async () => { const fs = await fixture([{ name: "file", body: "hello" }]); await fs.writeFile("/work/file", Buffer.from("keep")); await fs.writeFile("/outside", Buffer.from("outside sentinel"), { mode: 0o604 }); @@ -513,31 +519,33 @@ for (const replacement of [false, true]) test(`unzip acquisition-abort limitatio const outside = await fs.stat("/outside"); const controller = new AbortController(); let allocation: { path: string; stat: FileStat } | undefined; - const dynamic = wrapped(fs, { async writeFile(path, bytes, options) { - await fs.writeFile(path, bytes, options); - if (path.startsWith("/work/.unzip-")) { - allocation = { path, stat: await fs.lstat(path) }; + const dynamic = wrapped(fs, { async createStagedFile(directory, name, content, options) { + const receipt = await fs.createStagedFile!(directory, name, content, options); + const path = receipt.file.path; + if (directory.startsWith("/work/.unzip-")) { + allocation = { path, stat: receipt.file.stat }; if (replacement) { await fs.rename(path, "/work/held-stage"); await fs.symlink!("/outside", path); } controller.abort(false); } + return receipt; } }); const shell = new Shell({ fs: dynamic, cwd: "/work" }); shell.use({ name: "test-unzip", setup(host) { host.commands.register(createUnzipCommand()); } }); try { await assert.rejects(shell.exec("unzip -o sample.zip", { signal: controller.signal }), reason => reason === false); assert.ok(allocation); - const retained = await fs.lstat(replacement ? "/work/held-stage" : allocation.path); - assert.equal(retained.type, "file"); - assert.equal(retained.ino, allocation.stat.ino); - assert.equal(retained.mode & 0o777, 0o600); - assert.equal(retained.size, 0); if (replacement) { + const retained = await fs.lstat("/work/held-stage"); + assert.equal(retained.type, "file"); + assert.equal(retained.ino, allocation.stat.ino); + assert.equal(retained.mode, allocation.stat.mode); + assert.equal(retained.size, 5); assert.equal((await fs.lstat(allocation.path)).type, "symlink"); assert.equal(await fs.readlink!(allocation.path), "/outside"); - } + } else await assert.rejects(fs.lstat(allocation.path)); assert.equal(Buffer.from(await fs.readFile("/work/file")).toString(), "keep"); assert.deepEqual(await fs.readFile("/work/sample.zip"), archive); assert.equal(Buffer.from(await fs.readFile("/outside")).toString(), "outside sentinel"); diff --git a/packages/safe-bash/tests/commands/zip-atomic-ownership.test.ts b/packages/safe-bash/tests/commands/zip-atomic-ownership.test.ts new file mode 100644 index 0000000000..d42f345f33 --- /dev/null +++ b/packages/safe-bash/tests/commands/zip-atomic-ownership.test.ts @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { FsError, Shell, archiveCommands, createMemoryFileSystem, type FileSystem } from "../../src/index.js"; + +function intercept(fs: FileSystem, before: (method: PropertyKey, args: unknown[]) => Promise): FileSystem { + return new Proxy(fs, { get(target, method) { + const value: unknown = Reflect.get(target, method); + if (typeof value !== "function") return value; + return async (...args: unknown[]) => { await before(method, args); return Reflect.apply(value, target, args); }; + } }); +} + +test("archive creation and extraction request capabilities for missing outputs with create intent", async context => { + const fs = createMemoryFileSystem(); + await fs.mkdir("/work/source", { recursive: true }); + await fs.writeFile("/work/source/input", Buffer.from("payload")); + const creations = new Set(); + const wrapped = new Proxy(fs, { get(target, method) { + if (method === "capabilitiesFor") return (async (path, options) => { + try { await fs.lstat(path); } + catch (error) { + if (!(error instanceof FsError) || error.code !== "ENOENT") throw error; + if (!options?.create) return { ...fs.capabilities, atomicFileStaging: false, atomicDirectoryMetadata: false }; + creations.add(path); + } + return fs.capabilities; + }) satisfies NonNullable; + const value: unknown = Reflect.get(target, method); + return typeof value === "function" ? value.bind(target) : value; + } }); + const shell = new Shell({ fs: wrapped, cwd: "/work" }).use(archiveCommands()); + context.after(() => shell.dispose()); + const archive = await shell.exec("zip -r archive.zip source"); + assert.equal(archive.exitCode, 0, archive.stderr); + const extraction = await shell.exec("unzip archive.zip -d recovered"); + assert.equal(extraction.exitCode, 0, extraction.stderr); + assert.equal(Buffer.from(await fs.readFile("/work/recovered/source/input")).toString(), "payload"); + for (const path of ["/work/archive.zip", "/work/recovered", "/work/recovered/source/input"]) assert.ok(creations.has(path), path); +}); + +for (const command of ["zip", "unzip"]) { + test(`${command} refuses a destination replaced inside publication and preserves its bytes`, async context => { + const fs = createMemoryFileSystem(); + await fs.mkdir("/work"); + await fs.writeFile("/work/input", Buffer.from("original payload")); + const setup = new Shell({ fs, cwd: "/work" }).use(archiveCommands()); + context.after(() => setup.dispose()); + assert.equal((await setup.exec("zip archive.zip input")).exitCode, 0); + const destination = command === "zip" ? "/work/archive.zip" : "/work/input"; + let injected = false; + const wrapped = intercept(fs, async (method, args) => { + if (!injected && (method === "rename" || method === "publishStagedFile") && args[1] === destination) { + injected = true; + await fs.rm(destination); + await fs.writeFile(destination, Buffer.from("replacement owned by another operation")); + } + }); + const shell = new Shell({ fs: wrapped, cwd: "/work" }).use(archiveCommands()); + context.after(() => shell.dispose()); + const result = await shell.exec(command === "zip" ? "zip archive.zip input" : "unzip -o archive.zip"); + assert.equal(injected, true); + assert.notEqual(result.exitCode, 0); + assert.equal(Buffer.from(await fs.readFile(destination)).toString(), "replacement owned by another operation"); + }); + + test(`${command} cleanup never removes a replacement inserted inside its mutation`, async context => { + const fs = createMemoryFileSystem(); + await fs.mkdir("/work"); + await fs.writeFile("/work/input", Buffer.from("original payload")); + const setup = new Shell({ fs, cwd: "/work" }).use(archiveCommands()); + context.after(() => setup.dispose()); + assert.equal((await setup.exec("zip archive.zip input")).exitCode, 0); + let temporary = ""; + let injected = false; + const wrapped = intercept(fs, async (method, args) => { + if (method === "rename" || method === "publishStagedFile") { + temporary = typeof args[0] === "string" ? args[0] : (args[0] as { file: { path: string } }).file.path; + throw new FsError("EIO"); + } + if (!injected && temporary && ((method === "rm" && args[0] === temporary) || method === "removeStagedFile")) { + injected = true; + await fs.rm(temporary); + await fs.writeFile(temporary, Buffer.from("replacement must survive cleanup")); + } + }); + const shell = new Shell({ fs: wrapped, cwd: "/work" }).use(archiveCommands()); + context.after(() => shell.dispose()); + const result = await shell.exec(command === "zip" ? "zip archive.zip input" : "unzip -o archive.zip"); + assert.equal(injected, true); + assert.notEqual(result.exitCode, 0); + assert.equal(Buffer.from(await fs.readFile(temporary)).toString(), "replacement must survive cleanup"); + }); +} + +test("ZIP never claims a replacement inserted after staging creation as its own", async context => { + const fs = createMemoryFileSystem(); + await fs.mkdir("/work"); + await fs.writeFile("/work/input", Buffer.from("payload")); + let injected = false; + let replacement = ""; + const wrapped = new Proxy(fs, { get(target, method) { + const value: unknown = Reflect.get(target, method); + if (typeof value !== "function") return value; + return async (...args: unknown[]) => { + const result: unknown = await Reflect.apply(value, target, args); + const path = method === "createStagedFile" ? (result as { file: { path: string } }).file.path : args[0]; + if (!injected && ((method === "writeFile" && typeof path === "string" && path.includes("/.zip-")) || method === "createStagedFile")) { + injected = true; + replacement = path as string; + const bytes = await fs.readFile(replacement); + await fs.rm(replacement); + await fs.writeFile(replacement, bytes); + } + return result; + }; + } }); + const shell = new Shell({ fs: wrapped, cwd: "/work" }).use(archiveCommands()); + context.after(() => shell.dispose()); + const result = await shell.exec("zip archive.zip input"); + assert.equal(injected, true); + assert.notEqual(result.exitCode, 0); + await assert.rejects(fs.lstat("/work/archive.zip"), { code: "ENOENT" }); + assert.equal((await fs.lstat(replacement)).type, "file"); +}); + +test("unzip omits unsupported metadata during atomic directory and file creation", async () => { + const fs = createMemoryFileSystem(); + await fs.mkdir("/work/source", { recursive: true }); + await fs.writeFile("/work/source/input", Buffer.from("payload")); + const setup = new Shell({ fs, cwd: "/work" }).use(archiveCommands()); + assert.equal((await setup.exec("zip archive.zip source/input")).exitCode, 0); + await setup.dispose(); + let creations = 0; + const wrapped = new Proxy(fs, { get(target, method) { + if (method === "capabilities") return { ...target.capabilities, permissions: false, timestamps: false }; + const value: unknown = Reflect.get(target, method); + if (typeof value !== "function") return value; + return async (...args: unknown[]) => { + if (method === "prepareDirectory" || method === "createStagedFile") { + creations++; + const options = args.at(-1) as { mode?: number; atimeMs?: number; mtimeMs?: number }; + assert.equal(options.mode, undefined); + assert.equal(options.atimeMs, undefined); + assert.equal(options.mtimeMs, undefined); + } + return Reflect.apply(value, target, args); + }; + } }); + const shell = new Shell({ fs: wrapped, cwd: "/work" }).use(archiveCommands()); + try { + const result = await shell.exec("unzip -d output archive.zip"); + assert.equal(result.exitCode, 0, result.stderr); + assert.equal(creations, 3); + assert.equal(Buffer.from(await fs.readFile("/work/output/source/input")).toString(), "payload"); + } finally { await shell.dispose(); } +}); diff --git a/packages/safe-bash/tests/commands/zip-review.test.ts b/packages/safe-bash/tests/commands/zip-review.test.ts index 12ef07dca1..bfba399d1f 100644 --- a/packages/safe-bash/tests/commands/zip-review.test.ts +++ b/packages/safe-bash/tests/commands/zip-review.test.ts @@ -122,13 +122,15 @@ test("unzip review: replaced staging symlink cannot change outside metadata befo await fs.utimes!("/outside", 946684800000, 946684800000); const before = await fs.stat("/outside"); let staging = ""; - const dynamic = wrapped(fs, { async appendFile(path, bytes, options) { - await fs.appendFile(path, bytes, options); + const dynamic = wrapped(fs, { async createStagedFile(directory, name, content, options) { + const receipt = await fs.createStagedFile!(directory, name, content, options); + const path = receipt.file.path; if (path.startsWith("/work/.unzip-")) { staging = path; await fs.rename(path, "/work/held-stage"); await fs.symlink!("/outside", path); } + return receipt; } }); const result = await run(dynamic, "unzip", ["-o", "sample.zip"]); assert.notEqual(staging, ""); @@ -203,17 +205,19 @@ test("zip review: DOS headers use local wall time while UT extras retain the ins } }); -test("unzip review: falsey cancellation after a staged append waits for cleanup", async () => { +test("unzip review: falsey cancellation after staging creation waits for cleanup", async () => { const original = archive(Buffer.alloc(4096, 65), 8); const fs = await fixture(original); const controller = new AbortController(); let appended = false; - const dynamic = wrapped(fs, { async appendFile(path, bytes, options) { - await fs.appendFile(path, bytes, options); + const dynamic = wrapped(fs, { async createStagedFile(directory, name, content, options) { + const receipt = await fs.createStagedFile!(directory, name, content, options); + const path = receipt.file.path; if (path.startsWith("/work/.unzip-")) { appended = true; controller.abort(false); } + return receipt; } }); await assert.rejects(run(dynamic, "unzip", ["-o", "sample.zip"], { signal: controller.signal }), reason => reason === false); assert.equal(appended, true); @@ -254,12 +258,12 @@ for (const stop of ["caller", "dispose"]) test(`zip review: actual Shell ${stop} let settled = false; let disposed = false; let disposal: Promise | undefined; - const dynamic = wrapped(fs, { async writeFile(path, bytes, options) { + const dynamic = wrapped(fs, { async createStagedFile(path, name, content, options) { entered = true; writeSignal = options?.signal; try { await gate; - await fs.writeFile(path, bytes, options); + return await fs.createStagedFile!(path, name, content, options); } finally { completed = true; finishHost(); } } }); const shell = new Shell({ fs: dynamic, cwd: "/work" }); @@ -293,20 +297,19 @@ for (const stop of ["caller", "dispose"]) test(`zip review: actual Shell ${stop} } }); -test("zip review: actual Shell partial ENOSPC before publication preserves the previous archive", async () => { +test("zip review: actual Shell atomic allocation ENOSPC before publication preserves the previous archive", async () => { const original = archive(); const fs = await fixture(original); let failedPath: string | undefined; - const dynamic = wrapped(fs, { async writeFile(path, bytes, options) { - await fs.writeFile(path, bytes.subarray(0, Math.min(17, bytes.length)), options); + const dynamic = wrapped(fs, { async createStagedFile(path) { failedPath = path; - throw new FsError("ENOSPC", { path, syscall: "writeFile" }); + throw new FsError("ENOSPC", { path, syscall: "createStagedFile" }); } }); const shell = new Shell({ fs: dynamic, cwd: "/work" }); shell.commands.register(createZipCommand()); try { const result = await shell.exec("zip sample.zip file"); - assert.notEqual(failedPath, undefined, "the backend must partially write before reporting ENOSPC"); + assert.notEqual(failedPath, undefined, "the backend must attempt atomic allocation before reporting ENOSPC"); assert.equal(result.exitCode, 2); assert.match(result.stderr, /ENOSPC/u); assert.deepEqual(await fs.readFile("/work/sample.zip"), Uint8Array.from(original), "failed prepublication write must not destroy the existing ZIP"); @@ -445,19 +448,21 @@ for (const phase of ["metadata", "buffered input"]) test(`zip review: actual She } }); -test("unzip review: actual Shell abort after staged append removes its temporary without outside effects", async () => { +test("unzip review: actual Shell abort after staging creation removes its temporary without outside effects", async () => { const original = archive(Buffer.alloc(4096, 65), 8); const fs = await fixture(original); await fs.writeFile("/outside", Buffer.from("outside sentinel"), { mode: 0o604 }); await fs.utimes!("/outside", 946684800000, 946684800000); const controller = new AbortController(); let appended = false; - const dynamic = wrapped(fs, { async appendFile(path, bytes, options) { - await fs.appendFile(path, bytes, options); + const dynamic = wrapped(fs, { async createStagedFile(directory, name, content, options) { + const receipt = await fs.createStagedFile!(directory, name, content, options); + const path = receipt.file.path; if (path.startsWith("/work/.unzip-")) { appended = true; controller.abort(false); } + return receipt; } }); const shell = new Shell({ fs: dynamic, cwd: "/work" }); shell.commands.register(createUnzipCommand()); @@ -478,6 +483,8 @@ test("zip review: new archives honor a 0644-default VFS while updates preserve e const fs = await fixture(); const defaults = wrapped(fs, { async writeFile(path, bytes, options) { await fs.writeFile(path, bytes, { ...options, mode: options?.mode ?? 0o644 }); + }, async createStagedFile(path, name, content, options) { + return fs.createStagedFile!(path, name, content, { ...options, mode: options.mode ?? 0o644 }); } }); await defaults.writeFile("/work/default-file", Buffer.from("mode control")); const defaultMode = (await fs.stat("/work/default-file")).mode & 0o777; @@ -508,16 +515,18 @@ test("zip review: replaced staging directory cannot redirect archive mode restor let replaced = false; let stagingDirectory = ""; let outsideBytes: Uint8Array | undefined; - const dynamic = wrapped(fs, { async writeFile(path, bytes, options) { - await fs.writeFile(path, bytes, options); - if (!path.startsWith("/work/.zip-") || !path.endsWith("/archive.zip")) return; + const dynamic = wrapped(fs, { async createStagedFile(path, name, content, options) { + const receipt = await fs.createStagedFile!(path, name, content, options); + assert.equal(content.type, "file"); + const bytes = content.type === "file" ? content.data : new Uint8Array(); outsideBytes = Uint8Array.from(bytes); await fs.writeFile("/outside/archive.zip", bytes, { mode: 0o600 }); await fs.utimes!("/outside/archive.zip", 946684800000, 946684800000); - stagingDirectory = path.slice(0, path.lastIndexOf("/")); + stagingDirectory = path; await fs.rename(stagingDirectory, "/work/held-stage"); await fs.symlink!("/outside", stagingDirectory); replaced = true; + return receipt; } }); const shell = new Shell({ fs: dynamic, cwd: "/work" }); shell.commands.register(createZipCommand()); @@ -531,6 +540,6 @@ test("zip review: replaced staging directory cannot redirect archive mode restor assert.equal(await fs.readlink!(stagingDirectory), "/outside"); const outside = await fs.stat("/outside/archive.zip"); assert.deepEqual({ mode: outside.mode & 0o777, mtimeMs: outside.mtimeMs }, { mode: 0o600, mtimeMs: 946684800000 }, - "stage-parent proof must be refreshed after the awaited write and before chmod"); + "atomic publication must reject the original staging parent after replacement"); } finally { await shell.dispose(); } }); diff --git a/packages/safe-bash/tests/commands/zip.test.ts b/packages/safe-bash/tests/commands/zip.test.ts index 04afbe1310..11ad1dfc9d 100644 --- a/packages/safe-bash/tests/commands/zip.test.ts +++ b/packages/safe-bash/tests/commands/zip.test.ts @@ -236,10 +236,10 @@ test("zip copies reusable read fragments and prepares all files before publicati completed = true; } }; }, - async writeFile(path, bytes, options) { + async createStagedFile(path, name, content, options) { assert.equal(completed, true); writes++; - return fs.writeFile(path, bytes, options); + return fs.createStagedFile!(path, name, content, options); }, }); assert.equal((await run(observed, ["bundle", "file", "tree/child"])).exitCode, 0); @@ -364,12 +364,12 @@ for (const stop of [false, 0, "", null, "dispose"]) test(`zip drains an admitted let disposed = false; let writeSignal: AbortSignal | undefined; let disposal: Promise | undefined; - const observed = wrapped(fs, { async writeFile(path, bytes, options) { + const observed = wrapped(fs, { async createStagedFile(path, name, content, options) { entered = true; writeSignal = options?.signal; try { await held; - await fs.writeFile(path, bytes, options); + return await fs.createStagedFile!(path, name, content, options); } finally { completed = true; finish(); } } }); const shell = new Shell({ fs: observed, cwd: "/work" }); @@ -421,7 +421,7 @@ test("zip registers cleanup before filesystem admission and rejects new publicat } return fs.capabilities; }, - async writeFile(path, bytes, options) { writes++; return fs.writeFile(path, bytes, options); }, + async createStagedFile(path, name, content, options) { writes++; return fs.createStagedFile!(path, name, content, options); }, }); await assert.rejects(run(observed, ["bundle", "file"], {}, { registerCleanup(callback) { cleanup = callback; } }), reason => Object.is(reason, closingSignal?.reason)); assert.equal(closure, repeatedClosure); @@ -486,10 +486,9 @@ test("zip stages failed writes without destroying the old archive or leaving tem assert.equal((await run(fs, ["bundle", "file"])).exitCode, 0); const before = await fs.readFile("/work/bundle.zip"); let failedPath: string | undefined; - const broken = wrapped(fs, { async writeFile(path, bytes, options) { - await fs.writeFile(path, bytes.subarray(0, 17), options); + const broken = wrapped(fs, { async createStagedFile(path) { failedPath = path; - throw new FsError("ENOSPC", { path, syscall: "writeFile" }); + throw new FsError("ENOSPC", { path, syscall: "createStagedFile" }); } }); const result = await run(broken, ["bundle", "tree/child"]); assert.equal(result.exitCode, 2); @@ -506,7 +505,7 @@ test("zip staged replacement preserves output mode and leaves temporary-name col const collision = new TextEncoder().encode("not owned by zip"); await fs.writeFile("/work/.zip-1", collision); let source: string | undefined; - const observed = wrapped(fs, { async rename(from, to, options) { source = from; return fs.rename(from, to, options); } }); + const observed = wrapped(fs, { async publishStagedFile(staging, to, options) { source = staging.file.path; return fs.publishStagedFile!(staging, to, options); } }); assert.equal((await run(observed, ["bundle", "tree/child"])).exitCode, 0); assert.notEqual(source, undefined); assert.equal((await fs.stat("/work/bundle.zip")).mode & 0o7777, 0o640); @@ -518,6 +517,8 @@ test("zip new archives honor provider modes while updates preserve existing mode const fs = await fixture(); const defaults = wrapped(fs, { async writeFile(path, bytes, options) { await fs.writeFile(path, bytes, { ...options, mode: options?.mode ?? 0o644 }); + }, async createStagedFile(path, name, content, options) { + return fs.createStagedFile!(path, name, content, { ...options, mode: options.mode ?? 0o644 }); } }); await defaults.writeFile("/work/control", Buffer.from("control")); const shell = new Shell({ fs: defaults, cwd: "/work" }); @@ -540,13 +541,16 @@ test("zip checks staging ownership after writes before restoring archive mode", await fs.mkdir("/outside"); let outside: Uint8Array | undefined; let staging = ""; - const replaced = wrapped(fs, { async writeFile(path, bytes, options) { - await fs.writeFile(path, bytes, options); + const replaced = wrapped(fs, { async createStagedFile(path, name, content, options) { + const receipt = await fs.createStagedFile!(path, name, content, options); + assert.equal(content.type, "file"); + const bytes = content.type === "file" ? content.data : new Uint8Array(); outside = Uint8Array.from(bytes); await fs.writeFile("/outside/archive.zip", bytes, { mode: 0o600 }); - staging = path.slice(0, path.lastIndexOf("/")); + staging = path; await fs.rename(staging, "/work/held-stage"); await fs.symlink!("/outside", staging); + return receipt; } }); const shell = new Shell({ fs: replaced, cwd: "/work" }); shell.commands.register(createZipCommand()); @@ -559,7 +563,7 @@ test("zip checks staging ownership after writes before restoring archive mode", } finally { await shell.dispose(); } }); -for (const replacement of [false, true]) test(`zip unknown-identity acquisition limitation preserves ${replacement ? "foreign replacements" : "unverified directories"} after draining cancellation`, async () => { +for (const replacement of [false, true]) test(`zip atomic acquisition preserves ${replacement ? "foreign replacements" : "owned cleanup"} after draining cancellation`, async () => { const fs = await fixture(); assert.equal((await run(fs, ["bundle", "file"])).exitCode, 0); await fs.chmod!("/work/bundle.zip", 0o640); @@ -576,9 +580,8 @@ for (const replacement of [false, true]) test(`zip unknown-identity acquisition let settled = false; let staging = ""; let acquisitionSignal: AbortSignal | undefined; - const acquired = wrapped(fs, { async mkdir(path, options) { - await fs.mkdir(path, options); - if (!path.startsWith("/work/.zip-")) return; + const acquired = wrapped(fs, { async createStagedFile(path, name, content, options) { + const receipt = await fs.createStagedFile!(path, name, content, options); staging = path; acquisitionSignal = options?.signal; if (replacement) { @@ -587,7 +590,7 @@ for (const replacement of [false, true]) test(`zip unknown-identity acquisition } entered = true; controller.abort(false); - try { await held; } + try { await held; return receipt; } finally { completed = true; } } }); const shell = new Shell({ fs: acquired, cwd: "/work" }); @@ -611,11 +614,13 @@ for (const replacement of [false, true]) test(`zip unknown-identity acquisition assert.equal((await fs.stat("/outside/archive.zip")).mode & 0o777, 0o600); assert.equal((await fs.stat("/outside")).mode & 0o777, 0o701); if (replacement) assert.equal(await fs.readlink!(staging), "/outside"); - const retained = replacement ? "/work/held-stage" : staging; - assert.equal((await fs.lstat(retained)).type, "directory"); - assert.equal((await fs.lstat(retained)).mode & 0o777, 0o700); - assert.deepEqual(await fs.readdir(retained), []); - assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name), replacement ? [".zip-1", "bundle.zip", "file", "held-stage", "tree"] : [".zip-1", "bundle.zip", "file", "tree"]); + if (replacement) { + const retained = "/work/held-stage"; + assert.equal((await fs.lstat(retained)).type, "directory"); + assert.equal((await fs.lstat(retained)).mode & 0o777, 0o700); + assert.deepEqual((await fs.readdir(retained)).map(entry => entry.name), ["archive.zip"]); + } else await assert.rejects(fs.lstat(staging)); + assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name), replacement ? [".zip-1", "bundle.zip", "file", "held-stage", "tree"] : ["bundle.zip", "file", "tree"]); assert.equal((await shell.exec(":")).exitCode, 0); } finally { release(); diff --git a/packages/safe-bash/tests/fs/overlay/review-regressions.test.ts b/packages/safe-bash/tests/fs/overlay/review-regressions.test.ts index 79356f7ed7..a83d343479 100644 --- a/packages/safe-bash/tests/fs/overlay/review-regressions.test.ts +++ b/packages/safe-bash/tests/fs/overlay/review-regressions.test.ts @@ -21,6 +21,7 @@ class GetterStat implements FileStat { get mtimeMs() { return this.#values.mtimeMs; } get atimeMs() { return this.#values.atimeMs; } get ctimeMs() { return this.#values.ctimeMs; } + get revision() { return this.#values.revision; } get birthtimeMs() { return this.#values.birthtimeMs; } get identityScope() { return this.#values.identityScope; } get ino() { return this.#values.ino; } @@ -32,7 +33,7 @@ class GetterStat implements FileStat { const metadata: MutableStat = { type: "file", size: 5, allocatedBytes: 4096, preferredIoBlockSize: 4096, mode: 0o100640, mtimeMs: 101, atimeMs: 102, - ctimeMs: 103, birthtimeMs: 104, identityScope: Symbol(), ino: 105, dev: 0, nlink: 1, uid: 0, gid: 0, + ctimeMs: 103, revision: 7, birthtimeMs: 104, identityScope: Symbol(), ino: 105, dev: 0, nlink: 1, uid: 0, gid: 0, }; const shapes: Record FileStat> = { diff --git a/packages/safe-bash/tests/fs/readonly/metadata.test.ts b/packages/safe-bash/tests/fs/readonly/metadata.test.ts index 6a6e27e856..e523a202ba 100644 --- a/packages/safe-bash/tests/fs/readonly/metadata.test.ts +++ b/packages/safe-bash/tests/fs/readonly/metadata.test.ts @@ -42,7 +42,7 @@ function requiredStat(): Mutable { } function fullStat(): Mutable> { - return { ...requiredStat(), allocatedBytes: 4096, preferredIoBlockSize: 4096, birthtimeMs: 10, identityScope: Symbol(), ino: 21, dev: 22, nlink: 2, uid: 0, gid: 0 }; + return { ...requiredStat(), revision: 7, allocatedBytes: 4096, preferredIoBlockSize: 4096, birthtimeMs: 10, identityScope: Symbol(), ino: 21, dev: 22, nlink: 2, uid: 0, gid: 0 }; } for (const representation of ["prototype-accessors", "nonenumerable-own"] as const) { diff --git a/packages/safe-bash/tests/fs/readonly/readonly.test.ts b/packages/safe-bash/tests/fs/readonly/readonly.test.ts index 6ccc6e7b2c..8ee4bfae4b 100644 --- a/packages/safe-bash/tests/fs/readonly/readonly.test.ts +++ b/packages/safe-bash/tests/fs/readonly/readonly.test.ts @@ -326,6 +326,7 @@ test("capabilities are immutable, detached, conservative, and omit unknown exten const capabilities = { readOnly: false, append: true, symlinks: true, hardlinks: true, permissions: true, timestamps: true, atomicRename: true, atomicResize: true, streamingRead: true, streamingWrite: true, nativeExec: true, customWrites: true, + atomicFileMutation: true, atomicFileStaging: true, atomicDirectoryMetadata: true, }; const fixture = createFixture(true, capabilities); const filesystem = createReadOnlyFileSystem(fixture.filesystem); @@ -333,6 +334,7 @@ test("capabilities are immutable, detached, conservative, and omit unknown exten readOnly: true, append: false, symlinks: true, hardlinks: false, permissions: false, timestamps: false, atomicRename: false, atomicRenameNoReplace: false, streamingRead: true, streamingWrite: false, retainedRead: false, descriptorWriteStream: false, retainedResize: false, atomicResize: false, + atomicFileMutation: false, atomicFileStaging: false, atomicDirectoryMetadata: false, write: false, exclusiveCreate: false, mkdir: false, recursiveMkdir: false, remove: false, removeDirectory: false, recursiveRemove: false, rename: false, copy: false, exclusiveCopy: false, truncate: false, streamingAppend: false, randomAccessWrite: false, diff --git a/packages/safe-fs/src/contracts/filesystem.md b/packages/safe-fs/src/contracts/filesystem.md index 9f7a84300e..400f87bd7f 100644 --- a/packages/safe-fs/src/contracts/filesystem.md +++ b/packages/safe-fs/src/contracts/filesystem.md @@ -674,3 +674,75 @@ permitted. Replacing that historical generic row's explicit-mode/X_OK ENOTSUP expectations is an intentional profile delta, **not a source bug fix**. Preserve the red historical cohort, exact bytes, exclusivity, invalid-mode, chmod, authorization, cancellation and read-only assertions when revising backend tests. +# Conditional file mutations + +`atomicFileMutation` requires `writeFileConditional(path, data, options)` and +`removeFileConditional(path, options)`. Each operation atomically checks the +parent's scoped identity and directory type, and the file's scoped identity, +type, and revision. `expected: null` on a write requires absence and creates the +file, including with `append: true`; an expected existing file is never silently +recreated. Current permissions and ordinary provider write restrictions still +apply. `mode` affects creation only. + +A write replaces or appends the complete supplied byte array, or changes +nothing. Quota and allocation refusals happen before mutation. It returns the +original committed `FileStat`, not a subsequent path lookup. A pre-commit abort +refuses the operation; an abort arriving after commit must not suppress its +receipt. Removal only removes the exact expected file. Changed conditions fail +with `EAGAIN`; unavailable identity/revision guarantees fail with `ENOTSUP`. +Implementations must not substitute a read/check followed by an unconditional +mutation. Retained cleanup exposes only conditional removal and preserves its +existing operation budget and lifetime rules. + + +## Atomic owned staging + +`atomicFileStaging: true` requires `createStagedFile`, `publishStagedFile`, and +`removeStagedFile`. Creation atomically checks the supplied parent identity, +creates a private mode-0700 directory and an exclusive regular file or symlink, +and returns the original parent, directory, and file snapshots as a `FileStaging` +receipt. Once creation commits, the provider and forwarding views must return +the original receipt even if cancellation arrives before the promise settles; +the caller can then retain cleanup before honoring cancellation. It must not +derive ownership from a later path lookup. Allocation, +quota, permission, and unsupported-metadata failures must precede publication of +any staging entry. Implementations retain their existing file and aggregate +limits; this contract adds no larger byte allowance. + +Publication atomically verifies the original staging file, its private directory, +both parents, and the destination's supplied snapshot or explicit absence before +renaming. Parent and directory conditions compare stable identity and type; +child mutations may legitimately change their timestamps. File conditions also +compare `revision`, size, mode, link count, modification time, and change time. +`FileStat.revision`, when present, is a nonnegative safe integer that changes for +each content write or explicit metadata mutation, including same-tick same-size +writes. Access-time updates caused solely by reads need not change it. +Unknown identity or revision cannot satisfy a conditional file mutation. + +Cleanup atomically removes only the original staging file, if still present, +and its empty original private directory. A replacement entry, changed source, +or unexpected child must survive. An absent file after successful publication +is allowed. Cleanup never follows a replacement symlink or recursively removes +a directory. `retainFileSystemCleanup` exposes only this narrow cleanup method +in addition to its existing methods; its lifetime, operation limit, scope charge, +and pending-operation drain still apply after cancellation. + +`atomicDirectoryMetadata: true` requires `prepareDirectory`. With `expected: null` +it exclusively creates a directory after checking its parent's original +identity and returns the creation snapshot atomically. With an existing snapshot +it verifies the directory and parent identities before applying metadata and +returning the updated snapshot. Child writes do not invalidate this directory +identity condition. Unsupported timestamp fields must reject before mutation; +a caller must not infer timestamp support from this capability. + +These operations reject changed conditions with `EAGAIN` and unavailable +identity with `ENOTSUP`. A backend must withhold capabilities it cannot implement +atomically. Mount/device views preserve the original snapshots while translating +paths; cross-mount publication is refused. Read-only, quota, and overlay views +withhold unsupported owned staging rather than bypassing their policies. + +ZIP creation/update and unzip file extraction require `atomicFileStaging`. +Unzip directory creation and supported directory metadata restoration also +require `atomicDirectoryMetadata`. Listing an archive does not require mutation capabilities. Providers without these guarantees reject the +corresponding mutation; they must not fall back to check-then-rename or +check-then-delete operations. diff --git a/packages/safe-fs/src/contracts/filesystem.ts b/packages/safe-fs/src/contracts/filesystem.ts index 20b6b23480..8c77702e01 100644 --- a/packages/safe-fs/src/contracts/filesystem.ts +++ b/packages/safe-fs/src/contracts/filesystem.ts @@ -13,6 +13,7 @@ export interface FileStat { readonly atimeMs: number; readonly ctimeMs: number; readonly birthtimeMs?: number; + readonly revision?: number; readonly identityScope?: object | symbol; readonly ino?: number; readonly dev?: number; @@ -55,6 +56,9 @@ export interface FileSystemCapabilities { readonly permissions?: boolean; readonly timestamps?: boolean; readonly atomicRename?: boolean; + readonly atomicFileStaging?: boolean; + readonly atomicFileMutation?: boolean; + readonly atomicDirectoryMetadata?: boolean; readonly atomicRenameNoReplace?: boolean; readonly snapshotRmdir?: boolean; readonly streamingRead?: boolean; @@ -163,8 +167,61 @@ export interface ReadStreamOptions extends FsOptions { readonly chunkSize?: number; } +export interface ConditionalWriteFileOptions extends FsOptions { + readonly parent: FileStat; + readonly expected: FileStat | null; + readonly append?: boolean; + readonly mode?: number; +} + +export interface ConditionalRemoveFileOptions extends FsOptions { + readonly parent: FileStat; + readonly expected: FileStat; +} + +export interface FileStagingEntry { + readonly path: string; + readonly stat: FileStat; +} + +export interface FileStaging { + readonly parent: FileStagingEntry; + readonly directory: FileStagingEntry; + readonly file: FileStagingEntry; +} + +export type StagedFileContent = + | { readonly type: "file"; readonly data: Uint8Array } + | { readonly type: "symlink"; readonly target: string }; + +export interface CreateStagedFileOptions extends FsOptions { + readonly parent: FileStat; + readonly mode?: number; + readonly atimeMs?: number; + readonly mtimeMs?: number; +} + +export interface PublishStagedFileOptions extends FsOptions { + readonly parent: FileStat; + readonly destination: FileStat | null; +} + +export interface PrepareDirectoryOptions extends FsOptions { + readonly expected: FileStat | null; + readonly parent: FileStat; + readonly mode?: number; + readonly atimeMs?: number; + readonly mtimeMs?: number; +} + export interface FileSystem { + writeFileConditional?(path: string, data: Uint8Array, options: ConditionalWriteFileOptions): Promise; + removeFileConditional?(path: string, options: ConditionalRemoveFileOptions): Promise; readonly capabilities: FileSystemCapabilities; + prepareDirectory?(path: string, options: PrepareDirectoryOptions): Promise; + createStagedFile?(directoryPath: string, name: string, content: StagedFileContent, options: CreateStagedFileOptions): Promise; + publishStagedFile?(staging: FileStaging, destination: string, options: PublishStagedFileOptions): Promise; + removeStagedFile?(staging: FileStaging, options?: FsOptions): Promise; openReadFile?(path: string, options?: OpenReadFileOptions): Promise; openResizeFile?(path: string, options?: OpenResizeFileOptions): Promise; /** Missing targets reject ENOENT unless create is true; existing bytes survive refusals. */ diff --git a/packages/safe-fs/src/fs/capabilities.ts b/packages/safe-fs/src/fs/capabilities.ts index 85c7d4da5a..915d084c7b 100644 --- a/packages/safe-fs/src/fs/capabilities.ts +++ b/packages/safe-fs/src/fs/capabilities.ts @@ -1,4 +1,4 @@ -import type { FileReadHandle, FileResizeHandle, FileSystem, FileSystemCapabilities, OpenReadFileOptions, OpenResizeFileOptions } from "../contracts/filesystem.js"; +import type { FsOptions, FileReadHandle, FileResizeHandle, FileSystem, FileSystemCapabilities, OpenReadFileOptions, OpenResizeFileOptions } from "../contracts/filesystem.js"; import { FsError } from "../contracts/errors.js"; import { finishCleanup } from "../contracts/cleanup.js"; @@ -107,7 +107,7 @@ export function readOnlyCapabilities(capabilities: FileSystemCapabilities): File mkdir: false, recursiveMkdir: false, remove: false, removeDirectory: false, recursiveRemove: false, rename: false, copy: false, exclusiveCopy: false, truncate: false, streamingAppend: false, randomAccessWrite: false, hardlinks: false, permissions: false, timestamps: false, - descriptorWriteStream: false, retainedResize: false, atomicResize: false, + descriptorWriteStream: false, atomicResize: false, retainedResize: false, atomicFileMutation: false, atomicFileStaging: false, atomicDirectoryMetadata: false, atomicRename: false, atomicRenameNoReplace: false, streamingWrite: false, }); } @@ -116,8 +116,25 @@ export function quotaCapabilities(capabilities: FileSystemCapabilities): FileSys const streamingWrite = requireCapabilities(capabilities.write, capabilities.append, !capabilities.readOnly); const streamingAppend = requireCapabilities(capabilities.append, !capabilities.readOnly); const { streamingWrite: ignoredWrite, streamingAppend: ignoredAppend, ...rest } = capabilities; - return Object.freeze({ ...rest, descriptorWriteStream: false, atomicResize: false, + return Object.freeze({ ...rest, descriptorWriteStream: false, atomicResize: false, atomicFileMutation: false, atomicFileStaging: false, atomicDirectoryMetadata: false, ...(streamingWrite === undefined ? {} : { streamingWrite }), ...(streamingAppend === undefined ? {} : { streamingAppend }), }); } + +export function ownedMutationCapabilities(filesystem: FileSystem, capabilities = filesystem.capabilities): FileSystemCapabilities { + const unavailable: Record = {}; + if (capabilities.atomicFileMutation === true && (capabilities.readOnly === true || typeof filesystem.writeFileConditional !== "function" || typeof filesystem.removeFileConditional !== "function")) unavailable.atomicFileMutation = false; + if (capabilities.atomicFileStaging === true && (capabilities.readOnly === true + || typeof filesystem.createStagedFile !== "function" || typeof filesystem.publishStagedFile !== "function" || typeof filesystem.removeStagedFile !== "function")) unavailable.atomicFileStaging = false; + if (capabilities.atomicDirectoryMetadata === true && (capabilities.readOnly === true || typeof filesystem.prepareDirectory !== "function")) unavailable.atomicDirectoryMetadata = false; + return Object.keys(unavailable).length ? { ...capabilities, ...unavailable } : capabilities; +} + +export async function requireOwnedMutation(filesystem: FileSystem, path: string, + capability: "atomicFileMutation" | "atomicFileStaging" | "atomicDirectoryMetadata", options: FsOptions, create = false): Promise { + options.signal?.throwIfAborted(); + const capabilities = ownedMutationCapabilities(filesystem, await filesystem.capabilitiesFor?.(path, create ? { ...options, create: true } : options) ?? filesystem.capabilities); + options.signal?.throwIfAborted(); + if (capabilities[capability] !== true) throw new FsError("ENOTSUP", { path, syscall: capability }); +} diff --git a/packages/safe-fs/src/fs/devices/index.ts b/packages/safe-fs/src/fs/devices/index.ts index 2355a6382c..acb0abaceb 100644 --- a/packages/safe-fs/src/fs/devices/index.ts +++ b/packages/safe-fs/src/fs/devices/index.ts @@ -3,13 +3,14 @@ import type { AppendFileOptions, CapabilityQueryOptions, CopyFileOptions, DirectoryEntry, FileReadHandle, FileResizeHandle, FileResizeOperation, FileResizeOptions, FileStat, FileSystem, OpenReadFileOptions, OpenResizeFileOptions, FileSystemCapabilities, FsOptions, RenameOptions, MkdirOptions, ReadDirectoryOptions, ReadFileOptions, ReadStreamOptions, RemoveOptions, WriteFileOptions, + ConditionalWriteFileOptions, ConditionalRemoveFileOptions, CreateStagedFileOptions, FileStaging, PublishStagedFileOptions, PrepareDirectoryOptions, StagedFileContent, } from "../../contracts/filesystem.js"; import type { ByteSource } from "../../contracts/io.js"; import { admitDirectoryEntries, directoryEntryLimit } from "../directory-admission.js"; import { compareEntries, registerEntryAuthority, registerEntryView } from "../mount/comparison.js"; import { deviceDirectory, lexicalDevicePath, nullPath, resolveDevicePath } from "./path.js"; import { deviceReadStream, drainDeviceFile, drainDeviceInput } from "./stream.js"; -import { openRetainedReadFile, openRetainedResizeFile, retainedResizeCapabilities } from "../capabilities.js"; +import { openRetainedReadFile, openRetainedResizeFile, retainedResizeCapabilities, ownedMutationCapabilities, requireOwnedMutation } from "../capabilities.js"; import { pathNamespace } from "../path-namespace.js"; const views = new WeakMap(); @@ -20,12 +21,14 @@ const deviceCapabilities: FileSystemCapabilities = Object.freeze({ remove: false, removeDirectory: false, recursiveRemove: false, rename: false, mkdir: false, recursiveMkdir: false, symlinks: false, hardlinks: false, readlink: false, permissions: false, timestamps: false, truncate: false, randomAccessWrite: false, + atomicFileMutation: false, atomicFileStaging: false, atomicDirectoryMetadata: false, atomicRename: false, atomicRenameNoReplace: false, descriptorWriteStream: true, retainedResize: true, atomicResize: false, }); function globalCapabilities(filesystem: FileSystem): FileSystemCapabilities { const capabilities: Record = { readOnly: false }; const optional: Record = { + atomicFileMutation: ["writeFileConditional", "removeFileConditional"], atomicFileStaging: ["createStagedFile", "publishStagedFile", "removeStagedFile"], atomicDirectoryMetadata: ["prepareDirectory"], streamingRead: ["readStream"], streamingWrite: ["writeStream"], retainedRead: ["openReadFile"], streamingAppend: ["writeStream"], descriptorWriteStream: ["writeStream"], retainedResize: ["openResizeFile"], atomicResize: ["resizeFile"], symlinks: ["symlink", "readlink"], hardlinks: ["link"], permissions: ["chmod"], @@ -127,7 +130,7 @@ export class DeviceFileSystem implements FileSystem { options.signal?.throwIfAborted(); const selected = query === undefined || query === null ? undefined : await Reflect.apply(query, this.#filesystem, [path, options]); options.signal?.throwIfAborted(); - const observed = selected ?? this.#filesystem.capabilities; + const observed = ownedMutationCapabilities(this.#filesystem, selected ?? this.#filesystem.capabilities); options.signal?.throwIfAborted(); const capabilities = observed.retainedResize === true ? retainedResizeCapabilities(this.#filesystem, observed) : observed; const unavailable: Record = {}; @@ -331,6 +334,48 @@ export class DeviceFileSystem implements FileSystem { await this.#filesystem.rmdir(path, options); } + async writeFileConditional(path: string, data: Uint8Array, options: ConditionalWriteFileOptions): Promise { + await this.#mutable(path, options, false); + await requireOwnedMutation(this.#filesystem, path, "atomicFileMutation", options, options.expected === null); + if (!this.#filesystem.writeFileConditional) throw new FsError("ENOTSUP", { path }); + return this.#filesystem.writeFileConditional(path, data, options); + } + + async removeFileConditional(path: string, options: ConditionalRemoveFileOptions): Promise { + await this.#mutable(path, options, false); + await requireOwnedMutation(this.#filesystem, path, "atomicFileMutation", options); + if (!this.#filesystem.removeFileConditional) throw new FsError("ENOTSUP", { path }); + await this.#filesystem.removeFileConditional(path, options); + } + + async createStagedFile(path: string, name: string, content: StagedFileContent, options: CreateStagedFileOptions): Promise { + await this.#mutable(path, options, false); + await requireOwnedMutation(this.#filesystem, path, "atomicFileStaging", options, true); + if (!this.#filesystem.createStagedFile) throw new FsError("ENOTSUP", { path }); + return this.#filesystem.createStagedFile(path, name, content, options); + } + + async publishStagedFile(staging: FileStaging, destination: string, options: PublishStagedFileOptions): Promise { + for (const path of [staging.directory.path, staging.file.path, destination]) await this.#mutable(path, options, false); + await requireOwnedMutation(this.#filesystem, staging.directory.path, "atomicFileStaging", options); + if (!this.#filesystem.publishStagedFile) throw new FsError("ENOTSUP", { path: destination }); + await this.#filesystem.publishStagedFile(staging, destination, options); + } + + async removeStagedFile(staging: FileStaging, options: FsOptions = {}): Promise { + for (const path of [staging.directory.path, staging.file.path]) await this.#mutable(path, options, false); + await requireOwnedMutation(this.#filesystem, staging.directory.path, "atomicFileStaging", options); + if (!this.#filesystem.removeStagedFile) throw new FsError("ENOTSUP", { path: staging.directory.path }); + await this.#filesystem.removeStagedFile(staging, options); + } + + async prepareDirectory(path: string, options: PrepareDirectoryOptions): Promise { + await this.#mutable(path, options, false); + await requireOwnedMutation(this.#filesystem, path, "atomicDirectoryMetadata", options, options.expected === null); + if (!this.#filesystem.prepareDirectory) throw new FsError("ENOTSUP", { path }); + return this.#filesystem.prepareDirectory(path, options); + } + async rename(source: string, destination: string, options: RenameOptions = {}): Promise { await this.#mutable(source, options, false); await this.#mutable(destination, options, false); if (options.noReplace) { diff --git a/packages/safe-fs/src/fs/memory/index.ts b/packages/safe-fs/src/fs/memory/index.ts index cd9918a309..70aea0a0d4 100644 --- a/packages/safe-fs/src/fs/memory/index.ts +++ b/packages/safe-fs/src/fs/memory/index.ts @@ -4,6 +4,7 @@ import type { AppendFileOptions, CopyFileOptions, DirectoryEntry, EntryComparison, FileReadHandle, FileResizeHandle, FileStat, FileSystem, FsOptions, RenameOptions, MkdirOptions, ReadDirectoryOptions, ReadFileOptions, ReadStreamOptions, RemoveOptions, OpenReadFileOptions, OpenResizeFileOptions, WriteFileOptions, + ConditionalWriteFileOptions, ConditionalRemoveFileOptions, CreateStagedFileOptions, FileStaging, PublishStagedFileOptions, PrepareDirectoryOptions, StagedFileContent, } from "../../contracts/filesystem.js"; import type { ByteSource } from "../../contracts/io.js"; import { assertCallbackAuthorityAllowed, compareEntries, registerEntryAuthority } from "../mount/comparison.js"; @@ -18,6 +19,7 @@ import { normalizeMemoryFileSystemLimits, type MemoryFileSystemOptions } from ". export { defaultMemoryFileSystemLimits, type MemoryFileSystemLimits, type MemoryFileSystemOptions } from "./limits.js"; interface Metadata { + revision: number; mode: number; ino: number; nlink: number; @@ -120,6 +122,8 @@ export class MemoryFileSystem implements FileSystem { permissions: true, timestamps: true, atomicRename: true, + atomicFileStaging: true, atomicFileMutation: true, + atomicDirectoryMetadata: true, streamingRead: true, retainedRead: true, get retainedResize() { return stockRetainedResize(filesystem); }, @@ -171,7 +175,7 @@ export class MemoryFileSystem implements FileSystem { private metadata(mode: number): Metadata { const now = Date.now(); return { - mode, ino: this.nextInode++, nlink: 1, references: 0, + mode, ino: this.nextInode++, nlink: 1, references: 0, revision: 0, atimeMs: now, mtimeMs: now, ctimeMs: now, birthtimeMs: now, }; } @@ -238,6 +242,7 @@ export class MemoryFileSystem implements FileSystem { } private changed(node: MemoryNode): void { + node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, node.revision + 1); node.mtimeMs = node.ctimeMs = Date.now(); } @@ -311,6 +316,7 @@ export class MemoryFileSystem implements FileSystem { private snapshot(node: MemoryNode): FileStat { return { type: node.type, + ...(Number.isSafeInteger(node.revision) ? { revision: node.revision } : {}), preferredIoBlockSize: 4096, size: node.type === "file" ? node.data.byteLength : node.type === "symlink" ? new TextEncoder().encode(node.target).byteLength : 0, @@ -448,6 +454,153 @@ export class MemoryFileSystem implements FileSystem { return new Uint8Array(node.data); } + private expectEntry(node: MemoryNode | undefined, expected: FileStat | null, path: string, unchanged = true): void { + if (expected === null) { + if (node) this.fail("EAGAIN", "fileStaging", path); + return; + } + if (!node) this.fail("EAGAIN", "fileStaging", path); + const current = this.snapshot(node); + if (!expected.identityScope || expected.identityScope !== current.identityScope + || !Number.isSafeInteger(expected.ino) || !Number.isSafeInteger(expected.dev)) this.fail("ENOTSUP", "fileStaging", path); + if (current.ino !== expected.ino || current.dev !== expected.dev || current.type !== expected.type) this.fail("EAGAIN", "fileStaging", path); + if (unchanged) { + if (!Number.isSafeInteger(expected.revision) || !Number.isSafeInteger(current.revision)) this.fail("ENOTSUP", "fileStaging", path); + if (current.revision !== expected.revision || current.size !== expected.size || current.mode !== expected.mode + || current.nlink !== expected.nlink || current.mtimeMs !== expected.mtimeMs || current.ctimeMs !== expected.ctimeMs) this.fail("EAGAIN", "fileStaging", path); + } + } + + private stagingLocations(staging: FileStaging): { directory: Location; file: Location } { + const parent = this.entry(staging.parent.path, "fileStaging"); + this.expectEntry(parent.node, staging.parent.stat, staging.parent.path, false); + if (parent.node!.type !== "directory") this.fail("ENOTDIR", "fileStaging", staging.parent.path); + const directory = this.entry(staging.directory.path, "fileStaging"); + this.expectEntry(directory.node, staging.directory.stat, staging.directory.path, false); + if (directory.parent !== parent.node || directory.node!.type !== "directory" || (directory.node!.mode & 0o777) !== 0o700) this.fail("EAGAIN", "fileStaging", staging.directory.path); + const file = this.entry(staging.file.path, "fileStaging", true); + if (file.parent !== directory.node || !["file", "symlink"].includes(staging.file.stat.type)) this.fail("EAGAIN", "fileStaging", staging.file.path); + return { directory, file }; + } + + async createStagedFile(directoryPath: string, name: string, content: StagedFileContent, options: CreateStagedFileOptions): Promise { + options.signal?.throwIfAborted(); + if (!name || name === "." || name === ".." || name.includes("/") || name.includes("\0")) this.fail("EINVAL", "createStagedFile", directoryPath); + const location = this.entry(directoryPath, "createStagedFile", true); + if (location.node) this.fail("EEXIST", "createStagedFile", directoryPath); + this.expectEntry(location.parent, options.parent, directoryPath, false); + this.permission(location.parent, 3, "createStagedFile", directoryPath); + const mode = this.mode(options.mode, content.type === "file" ? 0o666 : 0o777, "createStagedFile", directoryPath); + for (const time of [options.atimeMs, options.mtimeMs]) if (time !== undefined && !Number.isFinite(time)) this.fail("EINVAL", "createStagedFile", directoryPath); + if (content.type === "file" && !(content.data instanceof Uint8Array)) throw new TypeError("Staged files require Uint8Array data"); + if (content.type === "symlink") this.validatePath(content.target, "createStagedFile"); + const retained = (location.name.length + name.length) * 2 + (content.type === "symlink" ? content.target.length * 2 : 0); + this.ledger.reserve(retained, 4, "createStagedFile", directoryPath); + let allocation: MemoryAllocation | undefined; + let directory: DirectoryNode; + let file: FileNode | SymlinkNode; + try { + if (content.type === "file") allocation = this.bytes(content.data, "createStagedFile", directoryPath); + directory = this.directory(0o700); + file = content.type === "file" + ? { ...this.metadata(typeModes.file | mode), type: "file", data: allocation!.data, allocation: allocation! } + : { ...this.metadata(typeModes.symlink | mode), type: "symlink", target: content.target }; + if (options.atimeMs !== undefined) file.atimeMs = options.atimeMs; + if (options.mtimeMs !== undefined) file.mtimeMs = options.mtimeMs; + directory.entries.set(name, file); + location.parent.entries.set(location.name, directory); + } catch (error) { allocation?.release(); this.ledger.release(retained, 4); throw error; } + this.changed(location.parent); + const parentPath = location.path.slice(0, location.path.lastIndexOf("/")) || "/"; + const receipt = (path: string, node: MemoryNode) => Object.freeze({ path, stat: Object.freeze(this.snapshot(node)) }); + return Object.freeze({ parent: receipt(parentPath, location.parent), directory: receipt(location.path, directory), file: receipt(`${location.path}/${name}`, file) }); + } + + async publishStagedFile(staging: FileStaging, destination: string, options: PublishStagedFileOptions): Promise { + options.signal?.throwIfAborted(); + const { directory, file } = this.stagingLocations(staging); + this.expectEntry(file.node, staging.file.stat, staging.file.path); + const target = this.entry(destination, "publishStagedFile", true); + this.expectEntry(target.parent, options.parent, destination, false); + this.expectEntry(target.node, options.destination, destination); + if (target.parent === directory.node || target.node === directory.node || target.node === file.node) this.fail("EINVAL", "publishStagedFile", destination); + if (target.node && (target.node.type !== "file" || target.node.nlink !== 1)) this.fail("EAGAIN", "publishStagedFile", destination); + return MemoryFileSystem.prototype.rename.call(this, staging.file.path, destination, { ...options, noReplace: options.destination === null }); + } + + async removeStagedFile(staging: FileStaging, options: FsOptions = {}): Promise { + options.signal?.throwIfAborted(); + const { directory, file } = this.stagingLocations(staging); + if (file.node) this.expectEntry(file.node, staging.file.stat, staging.file.path); + const node = directory.node as DirectoryNode; + if (node.entries.size !== (file.node ? 1 : 0)) this.fail("ENOTEMPTY", "removeStagedFile", staging.directory.path); + this.permission(directory.parent, 3, "removeStagedFile", staging.directory.path); + this.permission(node, 3, "removeStagedFile", staging.file.path); + directory.parent.entries.delete(directory.name); + if (file.node) { + node.entries.delete(file.name); + this.ledger.release(file.name.length * 2, 1); + file.node.nlink--; + file.node.ctimeMs = Date.now(); + file.node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, file.node.revision + 1); + this.releaseNode(file.node); + } + this.ledger.release(directory.name.length * 2, 1); + node.nlink = 0; + node.ctimeMs = Date.now(); + node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, node.revision + 1); + this.releaseNode(node); + this.changed(directory.parent); + } + + async prepareDirectory(path: string, options: PrepareDirectoryOptions): Promise { + options.signal?.throwIfAborted(); + const location = this.entry(path, "prepareDirectory", true); + this.expectEntry(location.parent, options.parent, path, false); + this.expectEntry(location.node, options.expected, path, false); + const mode = options.mode === undefined ? undefined : this.mode(options.mode, 0, "prepareDirectory", path); + for (const time of [options.atimeMs, options.mtimeMs]) if (time !== undefined && !Number.isFinite(time)) this.fail("EINVAL", "prepareDirectory", path); + let node = location.node; + if (!node) { + this.permission(location.parent, 3, "prepareDirectory", path); + node = this.addNode(location.parent, location.name, () => this.directory(mode ?? 0o777), "prepareDirectory", path); + } + if (node.type !== "directory") this.fail("ENOTDIR", "prepareDirectory", path); + if (mode !== undefined) node.mode = typeModes.directory | mode; + if (options.atimeMs !== undefined) node.atimeMs = options.atimeMs; + if (options.mtimeMs !== undefined) node.mtimeMs = options.mtimeMs; + node.ctimeMs = Date.now(); + node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, node.revision + 1); + return Object.freeze(this.snapshot(node)); + } + + async writeFileConditional(path: string, data: Uint8Array, options: ConditionalWriteFileOptions): Promise { + options.signal?.throwIfAborted(); + const location = this.entry(path, "writeFileConditional", true); + this.expectEntry(location.parent, options.parent, path, false); + this.expectEntry(location.node, options.expected, path); + if (location.node && location.node.type !== "file") this.fail("EINVAL", "writeFileConditional", path); + this.writeData(path, data, { ...(options.mode === undefined ? {} : { mode: options.mode }), flag: options.append ? "a" : "w" }, "writeFileConditional"); + return Object.freeze(this.snapshot(location.parent.entries.get(location.name)!)); + } + + async removeFileConditional(path: string, options: ConditionalRemoveFileOptions): Promise { + options.signal?.throwIfAborted(); + const location = this.entry(path, "removeFileConditional", true); + this.expectEntry(location.parent, options.parent, path, false); + this.expectEntry(location.node, options.expected, path); + const node = location.node!; + if (node.type !== "file") this.fail("EINVAL", "removeFileConditional", path); + this.permission(location.parent, 3, "removeFileConditional", path); + location.parent.entries.delete(location.name); + this.ledger.release(location.name.length * 2, 1); + node.nlink--; + node.ctimeMs = Date.now(); + node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, node.revision + 1); + this.releaseNode(node); + this.changed(location.parent); + } + async writeFile(path: string, data: Uint8Array, options: WriteFileOptions = {}): Promise { options.signal?.throwIfAborted(); this.writeData(path, data, options, "writeFile"); @@ -509,6 +662,7 @@ export class MemoryFileSystem implements FileSystem { this.ledger.release(location.name.length * 2, 1); node.nlink = 0; node.ctimeMs = Date.now(); + node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, node.revision + 1); this.releaseNode(node); this.changed(location.parent); } @@ -544,6 +698,7 @@ export class MemoryFileSystem implements FileSystem { } entry.nlink--; entry.ctimeMs = Date.now(); + entry.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, entry.revision + 1); this.releaseNode(entry); } this.changed(location.parent); @@ -577,11 +732,13 @@ export class MemoryFileSystem implements FileSystem { this.ledger.release(origin.name.length * 2, 1); target.node.nlink--; target.node.ctimeMs = Date.now(); + target.node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, target.node.revision + 1); this.releaseNode(target.node); } else if (nameGrowth < 0) this.ledger.release(-nameGrowth, 0); this.changed(origin.parent); this.changed(target.parent); node.ctimeMs = Date.now(); + node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, node.revision + 1); } catch (error) { if (error instanceof FsError) throw new FsError(error.code, { syscall: "rename", path: source, dest: destination, cause: error }); throw error; @@ -647,6 +804,7 @@ export class MemoryFileSystem implements FileSystem { catch (error) { this.ledger.release(target.name.length * 2, 1); throw error; } node.nlink++; node.ctimeMs = Date.now(); + node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, node.revision + 1); this.changed(target.parent); } @@ -656,6 +814,7 @@ export class MemoryFileSystem implements FileSystem { const node = this.resolve(path, "chmod").node!; node.mode = typeModes[node.type] | permissions; node.ctimeMs = Date.now(); + node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, node.revision + 1); } async utimes(path: string, atimeMs: number, mtimeMs: number, options: FsOptions = {}): Promise { @@ -665,6 +824,7 @@ export class MemoryFileSystem implements FileSystem { node.atimeMs = atimeMs; node.mtimeMs = mtimeMs; node.ctimeMs = Date.now(); + node.revision = Math.min(Number.MAX_SAFE_INTEGER + 1, node.revision + 1); } async truncate(path: string, length = 0, options: FsOptions = {}): Promise { diff --git a/packages/safe-fs/src/fs/mount/index.ts b/packages/safe-fs/src/fs/mount/index.ts index bc23f2e603..5362e9903c 100644 --- a/packages/safe-fs/src/fs/mount/index.ts +++ b/packages/safe-fs/src/fs/mount/index.ts @@ -4,11 +4,12 @@ import type { AppendFileOptions, CapabilityQueryOptions, CopyFileOptions, DirectoryEntry, FileReadHandle, FileResizeHandle, FileResizeOperation, FileResizeOptions, FileStat, FileSystem, OpenReadFileOptions, OpenResizeFileOptions, FileSystemCapabilities, FsOptions, RenameOptions, MkdirOptions, ReadDirectoryOptions, ReadFileOptions, ReadStreamOptions, RemoveOptions, WriteFileOptions, + ConditionalWriteFileOptions, ConditionalRemoveFileOptions, CreateStagedFileOptions, FileStaging, FileStagingEntry, PublishStagedFileOptions, PrepareDirectoryOptions, StagedFileContent, } from "../../contracts/filesystem.js"; import type { ByteSource } from "../../contracts/io.js"; import { readBytes } from "../../contracts/io.js"; import { finishCleanup } from "../../contracts/cleanup.js"; -import { openRetainedReadFile, openRetainedResizeFile, readOnlyCapabilities, retainedReadCapabilities, retainedResizeCapabilities } from "../capabilities.js"; +import { requireOwnedMutation, ownedMutationCapabilities, openRetainedReadFile, openRetainedResizeFile, readOnlyCapabilities, retainedReadCapabilities, retainedResizeCapabilities } from "../capabilities.js"; import { admitDirectoryEntries, directoryEntryLimit } from "../directory-admission.js"; import { normalizePath, validatePath } from "../../contracts/virtual-path.js"; import { pathNamespace } from "../path-namespace.js"; @@ -65,9 +66,10 @@ function fail(code: ErrnoCode): never { } function snapshotStat(stat: FileStat): FileStat { - const { type, size, allocatedBytes, preferredIoBlockSize, mode, mtimeMs, atimeMs, ctimeMs, birthtimeMs, identityScope, ino, dev, nlink, uid, gid } = stat; + const { type, size, allocatedBytes, preferredIoBlockSize, mode, mtimeMs, atimeMs, ctimeMs, birthtimeMs, revision, identityScope, ino, dev, nlink, uid, gid } = stat; return { type, size, mode, mtimeMs, atimeMs, ctimeMs, + ...(revision === undefined ? {} : { revision }), ...(allocatedBytes === undefined ? {} : { allocatedBytes }), ...(preferredIoBlockSize === undefined ? {} : { preferredIoBlockSize }), ...(birthtimeMs === undefined ? {} : { birthtimeMs }), @@ -135,6 +137,7 @@ export class MountFileSystem implements FileSystem { : mounts.every(({ backend }) => backend.capabilities.append === false) ? false : undefined; const common = (capability: string): boolean | undefined => { const optional: Record = { + atomicFileMutation: ["writeFileConditional", "removeFileConditional"], atomicFileStaging: ["createStagedFile", "publishStagedFile", "removeStagedFile"], atomicDirectoryMetadata: ["prepareDirectory"], symlinks: ["symlink", "readlink"], hardlinks: ["link"], permissions: ["chmod"], timestamps: ["utimes"], readlink: ["readlink"], descriptorWriteStream: ["writeStream"], retainedResize: ["openResizeFile"], atomicResize: ["resizeFile"], }; @@ -149,7 +152,7 @@ export class MountFileSystem implements FileSystem { return values.every(value => value === true) ? true : values.every(value => value === false) ? false : undefined; }; const semantics = Object.fromEntries([ - "read", "stat", "readdir", "realpath", "access", + "atomicFileMutation", "atomicFileStaging", "atomicDirectoryMetadata", "read", "stat", "readdir", "realpath", "access", "write", "append", "exclusiveCreate", "explicitDirectories", "implicitDirectories", "mkdir", "recursiveMkdir", "remove", "removeDirectory", "recursiveRemove", "rename", "atomicRenameNoReplace", "copy", "exclusiveCopy", "readlink", "truncate", "streamingAppend", "randomAccessWrite", "descriptorWriteStream", "retainedResize", "atomicResize", "symlinks", "hardlinks", "permissions", "timestamps", @@ -176,8 +179,8 @@ export class MountFileSystem implements FileSystem { allowMissing: options.create ?? true, ...(options.create === undefined ? {} : { resizeCreate: options.create }), }); - const observed = await location.mount.backend.capabilitiesFor?.(location.local, options) - ?? location.mount.backend.capabilities; + const observed = ownedMutationCapabilities(location.mount.backend, await location.mount.backend.capabilitiesFor?.(location.local, options) + ?? location.mount.backend.capabilities); const declared = observed.descriptorWriteStream === true && (typeof location.mount.backend.writeStream !== "function" || observed.readOnly === true || observed.streamingWrite === false) ? { ...observed, descriptorWriteStream: false } : observed; @@ -258,12 +261,12 @@ export class MountFileSystem implements FileSystem { private async operation( syscall: string, path: string, options: FsOptions, - action: () => Promise, dest?: string, + action: () => Promise, dest?: string, preserveReceipt = false, ): Promise { try { options.signal?.throwIfAborted(); const result = await action(); - options.signal?.throwIfAborted(); + if (!preserveReceipt) options.signal?.throwIfAborted(); return result; } catch (error) { throw this.error(error, syscall, path, options, dest); @@ -533,6 +536,94 @@ export class MountFileSystem implements FileSystem { }); } + private async localStaging(staging: FileStaging, options: FsOptions): Promise<{ mount: Mount; staging: FileStaging }> { + const parent = await this.resolve(staging.parent.path, options, { followFinal: false, entry: true }); + const directory = await this.resolve(staging.directory.path, options, { followFinal: false, entry: true }); + const file = await this.resolve(staging.file.path, options, { followFinal: false, entry: true, allowMissing: true }); + if (parent.mount !== directory.mount || directory.mount !== file.mount) fail("EXDEV"); + if (this.protected(directory.path) || this.protected(file.path)) fail("EBUSY"); + for (const location of [directory, file]) this.mutable(location); + return { mount: directory.mount, staging: { + parent: { path: parent.local, stat: staging.parent.stat }, + directory: { path: directory.local, stat: staging.directory.stat }, + file: { path: file.local, stat: staging.file.stat }, + } }; + } + + writeFileConditional(path: string, data: Uint8Array, options: ConditionalWriteFileOptions): Promise { + return this.operation("writeFileConditional", path, options, async () => { + const location = await this.resolve(path, options, { followFinal: false, entry: true, allowMissing: true }); + if (this.protected(location.path)) fail("EBUSY"); + this.mutable(location); + const backend = location.mount.backend; + await requireOwnedMutation(backend, location.local, "atomicFileMutation", options, options.expected === null); + if (!backend.writeFileConditional) fail("ENOTSUP"); + return snapshotStat(await backend.writeFileConditional(location.local, data, options)); + }, undefined, true); + } + + removeFileConditional(path: string, options: ConditionalRemoveFileOptions): Promise { + return this.operation("removeFileConditional", path, options, async () => { + const location = await this.resolve(path, options, { followFinal: false, entry: true, allowMissing: true }); + if (this.protected(location.path)) fail("EBUSY"); + this.mutable(location); + const backend = location.mount.backend; + await requireOwnedMutation(backend, location.local, "atomicFileMutation", options); + if (!backend.removeFileConditional) fail("ENOTSUP"); + await backend.removeFileConditional(location.local, options); + }); + } + + createStagedFile(path: string, name: string, content: StagedFileContent, options: CreateStagedFileOptions): Promise { + return this.operation("createStagedFile", path, options, async () => { + const location = await this.resolve(path, options, { followFinal: false, entry: true, allowMissing: true, missingDirectory: true }); + if (this.protected(location.path)) fail("EBUSY"); + this.mutable(location); + const backend = location.mount.backend; + await requireOwnedMutation(backend, location.local, "atomicFileStaging", options, true); + if (!backend.createStagedFile) fail("ENOTSUP"); + const receipt = await backend.createStagedFile(location.local, name, content, options); + const map = (entry: FileStagingEntry): FileStagingEntry => Object.freeze({ + path: location.mount.path === "/" ? entry.path : `${location.mount.path}${entry.path === "/" ? "" : entry.path}`, + stat: Object.freeze(snapshotStat(entry.stat)), + }); + return Object.freeze({ parent: map(receipt.parent), directory: map(receipt.directory), file: map(receipt.file) }); + }, undefined, true); + } + + publishStagedFile(staging: FileStaging, destination: string, options: PublishStagedFileOptions): Promise { + return this.operation("publishStagedFile", staging.file.path, options, async () => { + const local = await this.localStaging(staging, options); + const target = await this.resolve(destination, options, { followFinal: false, entry: true, allowMissing: true }); + if (this.protected(target.path)) fail("EBUSY"); + if (local.mount !== target.mount) fail("EXDEV"); + this.mutable(target); + await requireOwnedMutation(local.mount.backend, local.staging.directory.path, "atomicFileStaging", options); + if (!local.mount.backend.publishStagedFile) fail("ENOTSUP"); + await local.mount.backend.publishStagedFile(local.staging, target.local, options); + }, destination); + } + + removeStagedFile(staging: FileStaging, options: FsOptions = {}): Promise { + return this.operation("removeStagedFile", staging.directory.path, options, async () => { + const local = await this.localStaging(staging, options); + await requireOwnedMutation(local.mount.backend, local.staging.directory.path, "atomicFileStaging", options); + if (!local.mount.backend.removeStagedFile) fail("ENOTSUP"); + await local.mount.backend.removeStagedFile(local.staging, options); + }); + } + + prepareDirectory(path: string, options: PrepareDirectoryOptions): Promise { + return this.operation("prepareDirectory", path, options, async () => { + const location = await this.resolve(path, options, { followFinal: false, entry: true, allowMissing: true, missingDirectory: true }); + if (this.protected(location.path)) fail("EBUSY"); + this.mutable(location); + await requireOwnedMutation(location.mount.backend, location.local, "atomicDirectoryMetadata", options, options.expected === null); + if (!location.mount.backend.prepareDirectory) fail("ENOTSUP"); + return snapshotStat(await location.mount.backend.prepareDirectory(location.local, options)); + }, undefined, true); + } + rename(source: string, destination: string, options: RenameOptions = {}): Promise { return this.operation("rename", source, options, async () => { const origin = await this.resolve(source, options, { followFinal: false, entry: true }); diff --git a/packages/safe-fs/src/fs/overlay/index.ts b/packages/safe-fs/src/fs/overlay/index.ts index ccc235c5e5..179b05485e 100644 --- a/packages/safe-fs/src/fs/overlay/index.ts +++ b/packages/safe-fs/src/fs/overlay/index.ts @@ -54,12 +54,13 @@ interface LinkOrigin { type LinkMetadata = Pick; function snapshotStat(stat: FileStat): FileStat { - const { type, size, allocatedBytes, preferredIoBlockSize, mode, mtimeMs, atimeMs, ctimeMs, birthtimeMs, identityScope, ino, dev, nlink, uid, gid } = stat; + const { type, size, allocatedBytes, preferredIoBlockSize, mode, mtimeMs, atimeMs, ctimeMs, birthtimeMs, revision, identityScope, ino, dev, nlink, uid, gid } = stat; return { type, size, mode, mtimeMs, atimeMs, ctimeMs, ...(allocatedBytes === undefined ? {} : { allocatedBytes }), ...(preferredIoBlockSize === undefined ? {} : { preferredIoBlockSize }), ...(birthtimeMs === undefined ? {} : { birthtimeMs }), + ...(revision === undefined ? {} : { revision }), ...(identityScope === undefined ? {} : { identityScope }), ...(ino === undefined ? {} : { ino }), ...(dev === undefined ? {} : { dev }), @@ -164,6 +165,7 @@ export class OverlayFileSystem implements FileSystem { ].filter(([, value]) => value !== undefined)); this.capabilities = Object.freeze({ ...semantics, + atomicFileMutation: false, atomicFileStaging: false, atomicDirectoryMetadata: false, implicitDirectories: false, readlink: upper.readlink === true && this.#lower.capabilities.readlink === true ? true : upper.readlink === false && this.#lower.capabilities.readlink === false ? false : undefined, diff --git a/packages/safe-fs/src/fs/quota/index.ts b/packages/safe-fs/src/fs/quota/index.ts index 9b1ce9bd65..df7cb485e0 100644 --- a/packages/safe-fs/src/fs/quota/index.ts +++ b/packages/safe-fs/src/fs/quota/index.ts @@ -294,7 +294,7 @@ export function withFileSystemQuota(fs: FileSystem, options: FileSystemQuotaOpti // capabilities and methods without violating invariants on own properties. return new Proxy(Object.create(fs) as FileSystem, { get(_target, property) { - if (property === "canonicalizeMissingTarget" || property === "resizeFile") return undefined; + if (property === "writeFileConditional" || property === "removeFileConditional" || property === "resizeFile" || property === "canonicalizeMissingTarget" || property === "createStagedFile" || property === "publishStagedFile" || property === "removeStagedFile" || property === "prepareDirectory") return undefined; if (property === "capabilities") return quotaCapabilities(retainedResizeCapabilities(fs, retainedReadCapabilities(fs))); if (property === "capabilitiesFor") return async (path: string, fsOptions?: FsOptions) => { const capabilities = await fs.capabilitiesFor?.(path, fsOptions) ?? fs.capabilities; diff --git a/packages/safe-fs/src/fs/readonly/index.ts b/packages/safe-fs/src/fs/readonly/index.ts index 93ebc384a7..d69b335779 100644 --- a/packages/safe-fs/src/fs/readonly/index.ts +++ b/packages/safe-fs/src/fs/readonly/index.ts @@ -17,9 +17,10 @@ function readOnly(syscall: string, path: string, dest?: string): never { } function snapshotStat(stat: FileStat): FileStat { - const { type, size, allocatedBytes, preferredIoBlockSize, mode, mtimeMs, atimeMs, ctimeMs, birthtimeMs, identityScope, ino, dev, nlink, uid, gid } = stat; + const { type, size, allocatedBytes, preferredIoBlockSize, mode, mtimeMs, atimeMs, ctimeMs, birthtimeMs, revision, identityScope, ino, dev, nlink, uid, gid } = stat; return { type, size, mode, mtimeMs, atimeMs, ctimeMs, + ...(revision === undefined ? {} : { revision }), ...(allocatedBytes === undefined ? {} : { allocatedBytes }), ...(preferredIoBlockSize === undefined ? {} : { preferredIoBlockSize }), ...(birthtimeMs === undefined ? {} : { birthtimeMs }), diff --git a/packages/safe-fs/src/fs/scoped.ts b/packages/safe-fs/src/fs/scoped.ts index f32ee75325..7283287fe2 100644 --- a/packages/safe-fs/src/fs/scoped.ts +++ b/packages/safe-fs/src/fs/scoped.ts @@ -1,12 +1,13 @@ -import type { FileReadHandle, FileResizeHandle, FileSystem, FsOptions, OpenResizeFileOptions, RenameOptions } from "../contracts/filesystem.js"; +import type { FileStaging, FileReadHandle, FileResizeHandle, FileSystem, FsOptions, OpenResizeFileOptions, RenameOptions } from "../contracts/filesystem.js"; import { FsError } from "../contracts/errors.js"; import type { ByteSource } from "../contracts/io.js"; import { finishCleanup } from "../contracts/cleanup.js"; import { registerEntryView } from "./mount/comparison.js"; -import { openRetainedResizeFile, retainedResizeCapabilities } from "./capabilities.js"; +import { openRetainedResizeFile, retainedResizeCapabilities, ownedMutationCapabilities, requireOwnedMutation } from "./capabilities.js"; const originals = new WeakMap void }>(); const operations = new Set([ + "writeFileConditional", "removeFileConditional", "createStagedFile", "publishStagedFile", "removeStagedFile", "prepareDirectory", "access", "appendFile", "canonicalizeMissingTarget", "capabilitiesFor", "chmod", "compareEntry", "copyFile", "link", "lstat", "mkdir", "openReadFile", "openResizeFile", "readFile", "readStream", "readdir", "readlink", "realpath", "rename", "resizeFile", "rm", "rmdir", "stat", "symlink", "truncate", "utimes", @@ -127,7 +128,7 @@ export function scopeFileSystem(filesystem: FileSystem, charge: () => void, sign return Reflect.set(original, property, value, original); }, get(_target, property) { - if (property === "capabilities") return retainedResizeCapabilities(original); + if (property === "capabilities") return ownedMutationCapabilities(original, retainedResizeCapabilities(original)); const method: unknown = Reflect.get(original, property, original); if (typeof method !== "function") return method; const cached = methods.get(property); @@ -137,6 +138,14 @@ export function scopeFileSystem(filesystem: FileSystem, charge: () => void, sign const options = args.at(-1); admit(options && typeof options === "object" && "signal" in options ? options as FsOptions : undefined); } + if (["writeFileConditional", "removeFileConditional", "createStagedFile", "publishStagedFile", "removeStagedFile", "prepareDirectory"].includes(String(property))) return (async () => { + const path = typeof args[0] === "string" ? args[0] : (args[0] as FileStaging).directory.path; + const options = args[property === "createStagedFile" ? 3 : property === "publishStagedFile" || property === "writeFileConditional" ? 2 : 1] as FsOptions | undefined; + const create = property === "createStagedFile" || (property === "writeFileConditional" || property === "prepareDirectory") && options !== undefined && "expected" in options && options.expected === null; + await requireOwnedMutation(original, path, property === "prepareDirectory" ? "atomicDirectoryMetadata" : property === "writeFileConditional" || property === "removeFileConditional" ? "atomicFileMutation" : "atomicFileStaging", options ?? {}, create); + assertOpen(options); + return Reflect.apply(method, original, args); + })(); if (property === "compareEntry") { const peer = args[1] as FileSystem; args[1] = originals.get(peer)?.filesystem ?? peer; @@ -159,7 +168,7 @@ export function scopeFileSystem(filesystem: FileSystem, charge: () => void, sign return wrapResizeHandle(await openRetainedResizeFile(original, path, resizeOptions(options))); } : property === "capabilitiesFor" - ? async (...args: unknown[]) => retainedResizeCapabilities(original, await dispatch(...args) as FileSystem["capabilities"]) + ? async (...args: unknown[]) => ownedMutationCapabilities(original, retainedResizeCapabilities(original, await dispatch(...args) as FileSystem["capabilities"])) : property === "openReadFile" ? async (...args: unknown[]) => wrapHandle(await dispatch(...args) as FileReadHandle) : property === "readStream" @@ -180,6 +189,8 @@ export function scopeFileSystem(filesystem: FileSystem, charge: () => void, sign } export interface RetainedFileSystemCleanupView { + readonly removeFileConditional?: NonNullable; + readonly removeStagedFile?: NonNullable; readonly lstat: FileSystem["lstat"]; readonly realpath: FileSystem["realpath"]; readonly rm: (path: string, options?: FsOptions) => Promise; @@ -205,7 +216,8 @@ export function retainFileSystemCleanup( let operations = 0; let active = false; let closing: Promise | undefined; - const invoke = (method: keyof RetainedFileSystemCleanupView, path: string, settings?: FsOptions): Promise => { + const invoke = (method: keyof RetainedFileSystemCleanupView, input: string | FileStaging, settings?: FsOptions): Promise => { + const path = typeof input === "string" ? input : input.directory.path; if (!active) { const rejected = Promise.reject(new FsError("EBADF", { syscall: method, path, message: "cleanup callback is not active" })); void rejected.catch(() => {}); @@ -221,7 +233,9 @@ export function retainFileSystemCleanup( const operation = backing[method]; if (typeof operation !== "function") throw new FsError("ENOTSUP", { syscall: method, path }); const parameters = method === "rm" ? { ...settings, recursive: false, force: false } : settings; - result = Promise.resolve(Reflect.apply(operation, backing, [path, parameters])); + result = method === "removeStagedFile" || method === "removeFileConditional" + ? requireOwnedMutation(backing, path, method === "removeFileConditional" ? "atomicFileMutation" : "atomicFileStaging", settings ?? {}).then(() => Reflect.apply(operation, backing, [input, parameters])) + : Promise.resolve(Reflect.apply(operation, backing, [input, parameters])); } catch (error) { result = Promise.reject(error); void result.catch(() => {}); @@ -235,6 +249,8 @@ export function retainFileSystemCleanup( return result; }; const view: RetainedFileSystemCleanupView = Object.freeze(Object.assign(Object.create(null) as RetainedFileSystemCleanupView, { + ...(typeof backing.removeFileConditional === "function" ? { removeFileConditional: invoke.bind(undefined, "removeFileConditional") as NonNullable } : {}), + ...(typeof backing.removeStagedFile === "function" ? { removeStagedFile: invoke.bind(undefined, "removeStagedFile") as NonNullable } : {}), lstat: invoke.bind(undefined, "lstat") as RetainedFileSystemCleanupView["lstat"], realpath: invoke.bind(undefined, "realpath") as RetainedFileSystemCleanupView["realpath"], rm: invoke.bind(undefined, "rm") as RetainedFileSystemCleanupView["rm"], diff --git a/packages/safe-fs/tests/conditional-mutations.test.ts b/packages/safe-fs/tests/conditional-mutations.test.ts new file mode 100644 index 0000000000..9373ea9e29 --- /dev/null +++ b/packages/safe-fs/tests/conditional-mutations.test.ts @@ -0,0 +1,165 @@ +import assert from "node:assert/strict"; +import { test } from "vitest"; +import { MemoryFileSystem } from "../src/fs/memory/index.js"; +import { MountFileSystem } from "../src/fs/mount/index.js"; +import { DeviceFileSystem } from "../src/fs/devices/index.js"; +import { ReadOnlyFileSystem } from "../src/fs/readonly/index.js"; +import { withFileSystemQuota } from "../src/fs/quota/index.js"; +import { retainFileSystemCleanup, scopeFileSystem } from "../src/fs/scoped.js"; +import type { FileSystem } from "../src/contracts/filesystem.js"; +import { FsError } from "../src/contracts/errors.js"; + +for (const kind of ["scope", "mount", "devices"] as const) test(`${kind} preserves create intent for owned mutations without marking deletion as creation`, async () => { + const memory = new MemoryFileSystem(); + const queries: { path: string; create: boolean }[] = []; + const backend = new Proxy(memory, { get(target, property) { + if (property === "capabilitiesFor") return (async (path, options) => { + queries.push({ path, create: options?.create === true }); + try { await memory.lstat(path); } + catch (error) { + if (!(error instanceof FsError) || error.code !== "ENOENT") throw error; + if (!options?.create) return { ...memory.capabilities, atomicFileMutation: false, atomicFileStaging: false, atomicDirectoryMetadata: false }; + } + return memory.capabilities; + }) satisfies NonNullable; + const value: unknown = Reflect.get(target, property); + return typeof value === "function" ? value.bind(target) : value; + } }); + const fs = kind === "scope" ? scopeFileSystem(backend, () => {}, new AbortController().signal) + : kind === "mount" ? new MountFileSystem({ root: backend }) : new DeviceFileSystem(backend); + const parent = await fs.lstat("/"); + const file = await fs.writeFileConditional!("/file", Uint8Array.of(1), { parent, expected: null }); + await fs.prepareDirectory!("/directory", { parent, expected: null }); + const staging = await fs.createStagedFile!("/.stage", "payload", { type: "file", data: Uint8Array.of(2) }, { parent }); + await fs.publishStagedFile!(staging, "/published", { parent, destination: null }); + assert.deepEqual(await memory.readFile("/published"), Uint8Array.of(2)); + for (const path of ["/file", "/directory", "/.stage"]) assert.ok(queries.some(query => query.path === path && query.create), path); + queries.length = 0; + await fs.removeFileConditional!("/file", { parent, expected: file }); + await fs.removeStagedFile!(staging); + assert.ok(queries.length > 0); + assert.ok(queries.every(query => !query.create)); + await assert.rejects(memory.lstat("/file"), { code: "ENOENT" }); +}); + +for (const kind of ["memory", "mount", "devices"] as const) test(`conditional raw writes and removal preserve committed receipts through ${kind}`, async () => { + const memory = new MemoryFileSystem(); + const fs: FileSystem = kind === "memory" ? memory : kind === "devices" ? new DeviceFileSystem(memory) + : new MountFileSystem({ root: new MemoryFileSystem(), mounts: { "/work": memory } }); + const directory = kind === "mount" ? "/work" : "/"; + const path = kind === "mount" ? "/work/file" : "/file"; + const parent = await fs.lstat(directory); + const first = await fs.writeFileConditional!(path, Uint8Array.of(0xc3), { parent, expected: null, append: true, mode: 0o600 }); + await assert.rejects(fs.writeFileConditional!(path, Uint8Array.of(9), { parent, expected: null }), { code: "EAGAIN" }); + const second = await fs.writeFileConditional!(path, Uint8Array.of(0, 255), { parent, expected: first, append: true }); + assert.ok(second.revision! > first.revision!); + assert.equal(second.mode & 0o777, 0o600); + assert.deepEqual(await memory.readFile("/file"), Uint8Array.of(0xc3, 0, 255)); + await assert.rejects(fs.removeFileConditional!(path, { parent, expected: first }), { code: "EAGAIN" }); + await fs.removeFileConditional!(path, { parent, expected: second }); + await assert.rejects(memory.lstat("/file"), { code: "ENOENT" }); +}); + +test("conditional writes reject stale revision even with matching size and timestamps", async () => { + const fs = new MemoryFileSystem(); + const parent = await fs.lstat("/"); + const before = await fs.writeFileConditional("/file", Uint8Array.of(1), { parent, expected: null }); + await fs.writeFile("/file", Uint8Array.of(2)); + const current = await fs.lstat("/file"); + const stale = { ...current, revision: before.revision! }; + await assert.rejects(fs.writeFileConditional("/file", Uint8Array.of(3), { parent, expected: stale }), { code: "EAGAIN" }); + assert.deepEqual(await fs.readFile("/file"), Uint8Array.of(2)); +}); + +test("conditional output cannot follow a replaced parent or adopt a replacement file", async () => { + const fs = new MemoryFileSystem(); + await fs.mkdir("/work"); + const parent = await fs.lstat("/work"); + const before = await fs.writeFileConditional("/work/file", Uint8Array.of(1), { parent, expected: null }); + await fs.rename("/work", "/held"); + await fs.mkdir("/work"); + await fs.writeFile("/work/file", Uint8Array.of(9)); + await assert.rejects(fs.writeFileConditional("/work/file", Uint8Array.of(2), { parent, expected: before, append: true }), { code: "EAGAIN" }); + await assert.rejects(fs.removeFileConditional("/work/file", { parent, expected: before }), { code: "EAGAIN" }); + assert.deepEqual(await fs.readFile("/work/file"), Uint8Array.of(9)); + assert.deepEqual(await fs.readFile("/held/file"), Uint8Array.of(1)); +}); + +test("conditional file-size failure preserves original bytes and revision", async () => { + const fs = new MemoryFileSystem({ maxFileBytes: 2 }); + const parent = await fs.lstat("/"); + const expected = await fs.writeFileConditional("/file", Uint8Array.of(1, 2), { parent, expected: null }); + await assert.rejects(fs.writeFileConditional("/file", Uint8Array.of(3), { parent, expected, append: true }), { code: "EFBIG" }); + assert.equal((await fs.lstat("/file")).revision, expected.revision); + assert.deepEqual(await fs.readFile("/file"), Uint8Array.of(1, 2)); +}); + +test("unknown file revisions and parent identities refuse before mutation", async () => { + const fs = new MemoryFileSystem(); + const parent = await fs.lstat("/"); + const expected = await fs.writeFileConditional("/file", Uint8Array.of(1), { parent, expected: null }); + const unknownRevision = { ...expected }; + delete unknownRevision.revision; + await assert.rejects(fs.writeFileConditional("/file", Uint8Array.of(2), { parent, expected: unknownRevision }), { code: "ENOTSUP" }); + const unknownParent = { ...parent }; + delete unknownParent.identityScope; + await assert.rejects(fs.removeFileConditional("/file", { parent: unknownParent, expected }), { code: "ENOTSUP" }); + assert.deepEqual(await fs.readFile("/file"), Uint8Array.of(1)); +}); + +test("conditional cleanup budget refuses without deleting a file", async () => { + const fs = new MemoryFileSystem(); + const parent = await fs.lstat("/"); + const expected = await fs.writeFileConditional("/file", Uint8Array.of(1), { parent, expected: null }); + const cleanup = retainFileSystemCleanup(fs, async view => { + await view.removeFileConditional!("/file", { parent, expected }); + }, { maxOperations: 0 }); + await assert.rejects(cleanup(), { code: "EFBIG" }); + assert.deepEqual(await fs.readFile("/file"), Uint8Array.of(1)); +}); + +test("readonly and quota views do not advertise or expose conditional mutation bypasses", async () => { + const fs = new MemoryFileSystem(); + for (const view of [new ReadOnlyFileSystem(fs), withFileSystemQuota(fs, { maxBytes: 10 })] as FileSystem[]) { + assert.equal(view.capabilities.atomicFileMutation, false); + assert.equal((await view.capabilitiesFor?.("/file") ?? view.capabilities).atomicFileMutation, false); + assert.equal(view.writeFileConditional, undefined); + assert.equal(view.removeFileConditional, undefined); + } +}); + +test("composed views refuse an atomic capability whose writer is missing", async () => { + const memory = new MemoryFileSystem(); + const incomplete = new Proxy(memory, { get(target, property) { + if (property === "writeFileConditional") return undefined; + const value: unknown = Reflect.get(target, property); + return typeof value === "function" ? value.bind(target) : value; + } }); + const signal = new AbortController().signal; + for (const view of [scopeFileSystem(incomplete, () => {}, signal), new DeviceFileSystem(incomplete), new MountFileSystem({ root: incomplete })]) { + assert.equal(view.capabilities.atomicFileMutation, false); + assert.equal((await view.capabilitiesFor?.("/file") ?? view.capabilities).atomicFileMutation, false); + } +}); + +test("scoped committed receipt survives cancellation and retained conditional cleanup is charged", async () => { + const fs = new MemoryFileSystem(); + const controller = new AbortController(); + const original = fs.writeFileConditional.bind(fs); + fs.writeFileConditional = async (...args) => { + const receipt = await original(...args); + controller.abort(false); + return receipt; + }; + let charges = 0; + const scoped = scopeFileSystem(fs, () => {}, controller.signal, () => { charges++; }); + const parent = await scoped.lstat("/"); + const cleanup = retainFileSystemCleanup(scoped, async view => { + await view.removeFileConditional!("/file", { parent, expected: receipt }); + }, { maxOperations: 1 }); + const receipt = await scoped.writeFileConditional!("/file", Uint8Array.of(1), { parent, expected: null }); + assert.equal(controller.signal.aborted, true); + await cleanup(); + assert.equal(charges, 1); + await assert.rejects(fs.lstat("/file"), { code: "ENOENT" }); +}); diff --git a/packages/safe-fs/tests/file-staging.test.ts b/packages/safe-fs/tests/file-staging.test.ts new file mode 100644 index 0000000000..5e31bd5665 --- /dev/null +++ b/packages/safe-fs/tests/file-staging.test.ts @@ -0,0 +1,243 @@ +import assert from "node:assert/strict"; +import { afterEach, test, vi } from "vitest"; + +afterEach(() => vi.restoreAllMocks()); +import { createMemoryFileSystem } from "../src/fs/memory/index.js"; +import { createDeviceFileSystem } from "../src/fs/devices/index.js"; +import { createMountFileSystem } from "../src/fs/mount/index.js"; +import { scopeFileSystem, retainFileSystemCleanup } from "../src/fs/scoped.js"; +import { withFileSystemQuota } from "../src/fs/quota/index.js"; + +async function fixture() { + const fs = createMemoryFileSystem(); + await fs.mkdir("/work"); + const parent = await fs.lstat("/work"); + const staged = await fs.createStagedFile("/work/.stage", "file", { type: "file", data: new Uint8Array([1, 2, 3]) }, { parent }); + return { fs, parent, staged }; +} + +test("owned staging publishes exact bytes and cleans only its private directory", async () => { + const { fs, parent, staged } = await fixture(); + assert.ok(Object.isFrozen(staged) && Object.isFrozen(staged.file.stat)); + await fs.publishStagedFile(staged, "/work/output", { parent, destination: null }); + await fs.removeStagedFile(staged); + assert.deepEqual(await fs.readFile("/work/output"), new Uint8Array([1, 2, 3])); + assert.deepEqual((await fs.readdir("/work")).map(entry => entry.name), ["output"]); +}); + +for (const target of ["file", "directory", "parent"] as const) { + test(`owned staging refuses a replaced ${target} during publication and cleanup`, async () => { + const { fs, parent, staged } = await fixture(); + const path = staged[target].path; + await fs.rename(path, `${path}-old`); + if (target === "file") await fs.writeFile(path, new Uint8Array([9])); + else await fs.mkdir(path); + await assert.rejects(fs.publishStagedFile(staged, "/work/output", { parent, destination: null })); + await assert.rejects(fs.removeStagedFile(staged)); + assert.equal((await fs.lstat(path)).type, target === "file" ? "file" : "directory"); + }); +} + +test("same-tick same-size writes invalidate the original staging revision", async () => { + vi.spyOn(Date, "now").mockReturnValue(1234); + const { fs, parent, staged } = await fixture(); + await fs.writeFile(staged.file.path, new Uint8Array([4, 5, 6])); + assert.equal((await fs.lstat(staged.file.path)).mtimeMs, staged.file.stat.mtimeMs); + await assert.rejects(fs.publishStagedFile(staged, "/work/output", { parent, destination: null }), { code: "EAGAIN" }); + await assert.rejects(fs.removeStagedFile(staged), { code: "EAGAIN" }); + assert.deepEqual(await fs.readFile(staged.file.path), new Uint8Array([4, 5, 6])); +}); + +test("failed staging allocation leaves no directory and releases its metadata charge", async () => { + const fs = createMemoryFileSystem({ maxFileBytes: 1, maxMetadataUnits: 7 }); + await fs.mkdir("/work"); + const parent = await fs.lstat("/work"); + await assert.rejects(fs.createStagedFile("/work/.stage", "file", { type: "file", data: new Uint8Array(2) }, { parent }), { code: "EFBIG" }); + assert.deepEqual(await fs.readdir("/work"), []); + const staged = await fs.createStagedFile("/work/.stage", "file", { type: "file", data: new Uint8Array(1) }, { parent }); + await fs.removeStagedFile(staged); +}); + +test("retained atomic cleanup charges once after cancellation and refuses an exhausted budget", async () => { + const { fs, staged } = await fixture(); + const controller = new AbortController(); + let cleanupCharges = 0; + const scoped = scopeFileSystem(fs, () => {}, controller.signal, () => { cleanupCharges++; }); + let escapedCleanup: typeof fs.removeStagedFile | undefined; + const close = retainFileSystemCleanup(scoped, view => { + escapedCleanup = view.removeStagedFile!; + return escapedCleanup(staged); + }, { maxOperations: 1 }); + const denied = retainFileSystemCleanup(scoped, view => view.removeStagedFile!(staged), { maxOperations: 0 }); + controller.abort(); + await assert.rejects(denied(), { code: "EFBIG" }); + await close(); + await close(); + await assert.rejects(escapedCleanup!(staged), { code: "EBADF" }); + assert.equal(cleanupCharges, 1); + assert.deepEqual(await fs.readdir("/work"), []); +}); + +test("mount staging receipts preserve original identity while translating paths", async () => { + const root = createMemoryFileSystem(); + const backend = createMemoryFileSystem(); + const fs = createMountFileSystem({ root, mounts: { "/mounted": backend } }); + const parent = await fs.lstat("/mounted"); + const staged = await fs.createStagedFile("/mounted/.stage", "file", { type: "file", data: new Uint8Array([7]) }, { parent }); + assert.equal(staged.file.path, "/mounted/.stage/file"); + await fs.publishStagedFile(staged, "/mounted/output", { parent, destination: null }); + await fs.removeStagedFile(staged); + assert.deepEqual(await backend.readFile("/output"), new Uint8Array([7])); +}); + +test("quota wrappers withhold atomic staging instead of bypassing byte accounting", () => { + const fs = withFileSystemQuota(createMemoryFileSystem(), { maxBytes: 1 }); + assert.equal(fs.capabilities.atomicFileStaging, false); + assert.equal(fs.createStagedFile, undefined); + assert.equal(fs.publishStagedFile, undefined); + assert.equal(fs.removeStagedFile, undefined); +}); + +test("directory receipts reject a replacement but allow legitimate child mutations before metadata", async () => { + const fs = createMemoryFileSystem(); + await fs.mkdir("/work"); + const parent = await fs.lstat("/work"); + const expected = await fs.prepareDirectory("/work/extracted", { parent, expected: null, mode: 0o755 }); + await fs.writeFile("/work/extracted/child", new Uint8Array([1])); + const updated = await fs.prepareDirectory("/work/extracted", { parent, expected, mode: 0o700, mtimeMs: 1234 }); + assert.equal(updated.mtimeMs, 1234); + await fs.rename("/work/extracted", "/work/original"); + await fs.mkdir("/work/extracted", { mode: 0o755 }); + await assert.rejects(fs.prepareDirectory("/work/extracted", { parent, expected: updated, mode: 0o000 }), { code: "EAGAIN" }); + assert.equal((await fs.lstat("/work/extracted")).mode & 0o777, 0o755); +}); + +test("directory creation rejects an exchanged parent before creating any child", async () => { + const fs = createMemoryFileSystem(); + await fs.mkdir("/work"); + const parent = await fs.lstat("/work"); + await fs.rename("/work", "/old"); + await fs.mkdir("/work"); + await assert.rejects(fs.prepareDirectory("/work/child", { parent, expected: null }), { code: "EAGAIN" }); + assert.deepEqual(await fs.readdir("/work"), []); +}); + +test("cleanup preserves every entry when an unexpected child occupies the owned directory", async () => { + const { fs, staged } = await fixture(); + await fs.writeFile(`${staged.directory.path}/foreign`, new Uint8Array([9])); + await assert.rejects(fs.removeStagedFile(staged), { code: "ENOTEMPTY" }); + assert.deepEqual(await fs.readFile(staged.file.path), new Uint8Array([1, 2, 3])); + assert.deepEqual(await fs.readFile(`${staged.directory.path}/foreign`), new Uint8Array([9])); +}); + +test("same-tick destination changes cannot be replaced using an older receipt", async () => { + vi.spyOn(Date, "now").mockReturnValue(1234); + const { fs, parent, staged } = await fixture(); + await fs.writeFile("/work/output", new Uint8Array([7, 8, 9])); + const destination = await fs.lstat("/work/output"); + await fs.writeFile("/work/output", new Uint8Array([9, 8, 7])); + await assert.rejects(fs.publishStagedFile(staged, "/work/output", { parent, destination }), { code: "EAGAIN" }); + await fs.removeStagedFile(staged); + assert.deepEqual(await fs.readFile("/work/output"), new Uint8Array([9, 8, 7])); +}); + +test("mount publication refuses crossing into another backend without changing either side", async () => { + const root = createMemoryFileSystem(); + const backend = createMemoryFileSystem(); + const fs = createMountFileSystem({ root, mounts: { "/mounted": backend } }); + const parent = await fs.lstat("/mounted"); + const staged = await fs.createStagedFile("/mounted/.stage", "file", { type: "file", data: new Uint8Array([7]) }, { parent }); + await assert.rejects(fs.publishStagedFile(staged, "/output", { parent: await fs.lstat("/"), destination: null }), { code: "EXDEV" }); + assert.deepEqual(await backend.readFile("/.stage/file"), new Uint8Array([7])); + await assert.rejects(root.lstat("/output"), { code: "ENOENT" }); + await fs.removeStagedFile(staged); +}); + +test("mounted and scoped capability queries reject incomplete staging providers", async () => { + const memory = createMemoryFileSystem(); + const incomplete = new Proxy(memory, { get(target, key) { + if (key === "publishStagedFile") return undefined; + const value: unknown = Reflect.get(target, key); + return typeof value === "function" ? value.bind(target) : value; + } }); + const mounted = createMountFileSystem({ root: incomplete }); + assert.equal(mounted.capabilities.atomicFileStaging, false); + assert.equal((await mounted.capabilitiesFor("/")).atomicFileStaging, false); + const scoped = scopeFileSystem(incomplete, () => {}, new AbortController().signal); + assert.equal(scoped.capabilities.atomicFileStaging, false); + assert.equal((await scoped.capabilitiesFor?.("/") ?? scoped.capabilities).atomicFileStaging, false); +}); + +test("mount creation returns its original receipt when cancellation follows backend commit", async () => { + const memory = createMemoryFileSystem(); + const controller = new AbortController(); + const backend = new Proxy(memory, { get(target, key) { + if (key === "createStagedFile") return async (...args: Parameters) => { + const receipt = await memory.createStagedFile(...args); + controller.abort(false); + return receipt; + }; + const value: unknown = Reflect.get(target, key); + return typeof value === "function" ? value.bind(target) : value; + } }); + const fs = createMountFileSystem({ root: backend }); + const staged = await fs.createStagedFile("/.stage", "file", { type: "file", data: new Uint8Array([7]) }, { parent: await fs.lstat("/"), signal: controller.signal }); + assert.equal(controller.signal.aborted, true); + await fs.removeStagedFile(staged); + assert.deepEqual(await memory.readdir("/"), []); +}); + +test("retained file metadata revisions advance when a directory entry is removed", async () => { + const fs = createMemoryFileSystem(); + await fs.writeFile("/file", new Uint8Array([1])); + const handle = await fs.openReadFile("/file"); + try { + const before = await handle.stat(); + await fs.rm("/file"); + const after = await handle.stat(); + assert.equal(after.nlink, 0); + assert.ok(after.revision! > before.revision!); + } finally { await handle.close(); } +}); + +for (const kind of ["mount", "device", "scope"] as const) test(`${kind} refuses atomic methods when the backend withholds its guarantee`, async () => { + const memory = createMemoryFileSystem(); + const backend = new Proxy(memory, { get(target, key) { + if (key === "capabilities") return { ...target.capabilities, atomicFileStaging: false, atomicDirectoryMetadata: false }; + const value: unknown = Reflect.get(target, key); + return typeof value === "function" ? value.bind(target) : value; + } }); + const fs = kind === "mount" ? createMountFileSystem({ root: backend }) : kind === "device" ? createDeviceFileSystem(backend) : scopeFileSystem(backend, () => {}, new AbortController().signal); + const parent = await memory.lstat("/"); + await assert.rejects(fs.createStagedFile!("/.stage", "file", { type: "file", data: new Uint8Array([1]) }, { parent }), { code: "ENOTSUP" }); + await assert.rejects(fs.prepareDirectory!("/directory", { parent, expected: null }), { code: "ENOTSUP" }); + assert.deepEqual(await memory.readdir("/"), []); +}); + +test("scoped staging cannot start after cancellation during its capability query", async () => { + const memory = createMemoryFileSystem(); + const controller = new AbortController(); + const backend = new Proxy(memory, { get(target, key) { + if (key === "capabilitiesFor") return async () => { + controller.abort(false); + return target.capabilities; + }; + const value: unknown = Reflect.get(target, key); + return typeof value === "function" ? value.bind(target) : value; + } }); + const fs = scopeFileSystem(backend, () => {}, controller.signal); + await assert.rejects(fs.createStagedFile!("/.stage", "file", { type: "file", data: new Uint8Array([1]) }, { parent: await memory.lstat("/") }), reason => reason === false); + assert.deepEqual(await memory.readdir("/"), []); +}); + +test("retained cleanup refuses a backend that withholds atomic staging", async () => { + const { fs, staged } = await fixture(); + const backend = new Proxy(fs, { get(target, key) { + if (key === "capabilities") return { ...target.capabilities, atomicFileStaging: false }; + const value: unknown = Reflect.get(target, key); + return typeof value === "function" ? value.bind(target) : value; + } }); + const close = retainFileSystemCleanup(backend, view => view.removeStagedFile!(staged), { maxOperations: 1 }); + await assert.rejects(close(), { code: "ENOTSUP" }); + assert.deepEqual(await fs.readFile(staged.file.path), new Uint8Array([1, 2, 3])); +}); diff --git a/packages/safe-fs/tests/scoped-cleanup.test.ts b/packages/safe-fs/tests/scoped-cleanup.test.ts index 308306296d..04baad56a9 100644 --- a/packages/safe-fs/tests/scoped-cleanup.test.ts +++ b/packages/safe-fs/tests/scoped-cleanup.test.ts @@ -69,7 +69,7 @@ test("retained cleanup has one shared close promise and a frozen minimal view", escaped = view; assert.ok(Object.isFrozen(view)); assert.equal(Object.getPrototypeOf(view), null); - assert.deepEqual(Object.keys(view).sort(), ["lstat", "realpath", "rm", "rmdir"]); + assert.deepEqual(Object.keys(view).sort(), ["lstat", "realpath", "removeFileConditional", "removeStagedFile", "rm", "rmdir"]); assert.equal("writeFile" in view, false); assert.equal("capabilities" in view, false); started.resolve(); @@ -77,7 +77,7 @@ test("retained cleanup has one shared close promise and a frozen minimal view", }); const first = close(); assert.equal(close(), first); - await started.promise; + await Promise.race([started.promise, first]); gate.resolve(); await first; assert.equal(close(), first); diff --git a/scripts/fixtures/safe-packages-browser.mjs b/scripts/fixtures/safe-packages-browser.mjs index 4fd2c6fa90..ebf20c10ab 100644 --- a/scripts/fixtures/safe-packages-browser.mjs +++ b/scripts/fixtures/safe-packages-browser.mjs @@ -25,7 +25,7 @@ if (JSON.stringify(commandNames) !== JSON.stringify(expectedAgentCommandNames)) throw new Error(`Default browser command inventory differs: ${JSON.stringify(commandNames)}`); } const declaredCommands = [ - "[", "basename", "cat", "cmp", "cp", "cut", "dirname", "echo", "false", "fmt", "grep", "head", "ln", "ls", + "[", "basename", "cat", "cmp", "cp", "csplit", "cut", "dirname", "echo", "false", "fmt", "grep", "head", "ln", "ls", "mkdir", "mv", "numfmt", "printf", "pwd", "readlink", "realpath", "rg", "rm", "rmdir", "sed", "shuf", "sort", "tail", "tee", "test", "touch", "tr", "true", "truncate", "uniq", "wc", ]; @@ -35,7 +35,7 @@ for (const definition of definitions.filter(definition => !Object.hasOwn(definit const support = evaluateCommandSupport(definition, { readOnly: true }); if (support.declared || support.status !== "partial" || support.modes.length) throw new Error(`Undeclared browser command support became optimistic: ${definition.name}`); } -for (const [name, expected] of [["printf", "supported"], ["mkdir", "unsupported"], ["tee", "partial"], ["truncate", "unsupported"]]) { +for (const [name, expected] of [["printf", "supported"], ["mkdir", "unsupported"], ["csplit", "unsupported"], ["tee", "partial"], ["truncate", "unsupported"]]) { const definition = definitions.find(command => command.name === name); if (!definition || evaluateCommandSupport(definition, { readOnly: true }).status !== expected) { throw new Error(`Browser filesystem capability evaluation failed: ${name}`);