From 0e015dd4c53dfd8555375ea911843bdf9d87c269 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:41:33 +0000 Subject: [PATCH] feat: prevent compose output string overflow Co-authored-by: neilime <314088+neilime@users.noreply.github.com> Signed-off-by: Emilien Escalle --- README.md | 16 ++ docs/api.md | 24 +++ src/index.ts | 124 +++++++++++++-- test/compose.test.ts | 348 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 500 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 11a9dcd0..86041a3f 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,22 @@ compose.upAll({ cwd: path.join(__dirname), log: true }).then( ) ``` +For very chatty commands, use `callback` to process output chunks as they arrive and `maxOutputLength` to limit the UTF-16 code units retained per stream (the same units as JavaScript's `string.length`). It defaults to Node.js's `buffer.constants.MAX_STRING_LENGTH`. Stdout keeps the beginning of its output; stderr keeps the end so recent errors remain available. The result's `truncated.out` and `truncated.err` flags indicate whether output was dropped. Commands that parse stdout reject with an error mentioning `maxOutputLength` if stdout was truncated. + +Set `maxOutputLength: 0` to disable buffering completely while still receiving every chunk through `callback`. Both `result.out` and `result.err` will be empty strings: + +```javascript +await compose.logs('web', { + cwd: path.join(__dirname), + follow: true, + maxOutputLength: 0, + callback: (chunk, streamSource) => { + const stream = streamSource === 'stderr' ? process.stderr : process.stdout + stream.write(chunk) + } +}) +``` + Start specific services using `compose.upMany`: ```javascript diff --git a/docs/api.md b/docs/api.md index 44a1fef5..17ce4f68 100644 --- a/docs/api.md +++ b/docs/api.md @@ -41,9 +41,12 @@ All commands return a `Promise({object})` with stdout and stderr strings and an out: 'stdout contents', err: 'stderr contents', exitCode: 0, // !== 0 in case of an error + truncated: { out: false, err: false }, } ``` +`truncated.out` and `truncated.err` indicate whether any output was dropped from the corresponding stream, including when buffering is disabled. These flags are also present on results rejected for a nonzero exit code and on typed results with a `data` field. Commands that parse stdout (`config`, `configServices`, `configVolumes`, `ps`, `images`, `port`, `version`, and `stats`) reject with an error mentioning `maxOutputLength` if stdout was truncated. Truncating only stderr does not prevent parsing stdout. + ## Progress Callback Although the return type is a `Promise`, it is still possible to get the process progress before the `Promise` resolves, by passing a callback function to the optional `callback` parameter. @@ -60,6 +63,26 @@ compose.upAll({ ) ``` +### Output buffering + +Use `maxOutputLength` to limit how much output is retained independently for stdout and stderr. The limit counts UTF-16 code units, as measured by JavaScript's `string.length`, and defaults to Node.js's `buffer.constants.MAX_STRING_LENGTH`. Stdout retains the beginning of the output; stderr retains a trailing window so the most recent error messages remain available. Reaching the limit does not stop the command, and `callback` and `log` still receive all output. + +Set `maxOutputLength: 0` to disable buffering completely and process output only as it arrives. `out` and `err` remain empty strings, and each truncation flag becomes `true` if its stream emits nonempty output: + +```typescript +await compose.logs('web', { + cwd: path.join(__dirname), + follow: true, + maxOutputLength: 0, + callback: (chunk, streamSource) => { + const stream = streamSource === 'stderr' ? process.stderr : process.stdout + stream.write(chunk) + } +}) +``` + +Finite limits are rounded down and clamped between `0` and `buffer.constants.MAX_STRING_LENGTH`. Non-finite values use the default limit. + ## Options `docker-compose` accepts these params: @@ -74,6 +97,7 @@ compose.upAll({ | `log` | `boolean` | Enable console logging | | `composeOptions` | `string[] \| Array` | Options for all commands (e.g., `--verbose`) | | `commandOptions` | `string[] \| Array` | Options for specific command | +| `maxOutputLength` | `number` | Maximum UTF-16 code units retained per stream; defaults to `buffer.constants.MAX_STRING_LENGTH`. `0` disables buffering. | | `callback` | `(chunk: Buffer, sourceStream?: 'stdout' \| 'stderr') => void` | Progress callback | ### Example with options diff --git a/src/index.ts b/src/index.ts index 08edc538..ea9d940e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ import childProcess from 'child_process' +import { constants as bufferConstants } from 'buffer' import yaml from 'yaml' import mapPorts from './map-ports' export type { ComposeSpecification } from './compose-spec' @@ -26,6 +27,8 @@ export interface IDockerComposeOptions { composeOptions?: string[] | (string | string[])[] commandOptions?: string[] | (string | string[])[] env?: NodeJS.ProcessEnv + /** Maximum UTF-16 code units retained per stream; 0 disables buffering. */ + maxOutputLength?: number callback?: (chunk: Buffer, streamSource?: 'stdout' | 'stderr') => void } @@ -83,17 +86,93 @@ export interface IDockerComposeResult { exitCode: number | null out: string err: string + truncated: { + out: boolean + err: boolean + } } -export type TypedDockerComposeResult = { - exitCode: number | null - out: string - err: string +export type TypedDockerComposeResult = IDockerComposeResult & { data: T } const nonEmptyString = (v: string) => v !== '' +const MAX_CAPTURED_OUTPUT_LENGTH = bufferConstants.MAX_STRING_LENGTH + +interface BufferedOutput { + chunks: string[] + head: number + length: number + truncated: boolean +} + +const createBufferedOutput = (): BufferedOutput => ({ + chunks: [], + head: 0, + length: 0, + truncated: false +}) + +const appendBufferedOutput = ( + output: BufferedOutput, + stream: 'out' | 'err', + chunk: Buffer, + maxLength: number +): void => { + if (chunk.length === 0) { + return + } + + if (maxLength === 0 || (stream === 'out' && output.length >= maxLength)) { + output.truncated = true + return + } + + const chunkString = chunk.toString() + const remainingLength = maxLength - output.length + if (chunkString.length > remainingLength) { + output.truncated = true + } + + const retainedChunk = + stream === 'err' + ? chunkString.slice(-maxLength) + : chunkString.slice(0, remainingLength) + output.chunks.push(retainedChunk) + output.length += retainedChunk.length + + // Evict individual chunks without copying the retained window on each write. + while (output.length > maxLength) { + const first = output.chunks[output.head] + const excessLength = output.length - maxLength + if (first.length <= excessLength) { + output.length -= first.length + output.chunks[output.head++] = '' + } else { + output.chunks[output.head] = first.slice(excessLength) + output.length -= excessLength + } + } + + // Release discarded entries without shifting the array on every eviction. + if (output.head > 0 && output.head * 2 >= output.chunks.length) { + output.chunks = output.chunks.slice(output.head) + output.head = 0 + } +} + +const assertCompleteOutput = ( + result: IDockerComposeResult, + command: string +): void => { + if (result.truncated.out) { + throw new Error( + `Cannot parse docker compose ${command} output because stdout was truncated by maxOutputLength. Increase maxOutputLength or omit it.` + ) + } +} + export type DockerComposePsResultService = { name: string command: string @@ -310,6 +389,17 @@ export const execCompose = ( const cwd = options.cwd const env = options.env || undefined const executable = options.executable + const requestedMaxOutputLength = + options.maxOutputLength ?? MAX_CAPTURED_OUTPUT_LENGTH + const maxOutputLength = Number.isFinite(requestedMaxOutputLength) + ? Math.max( + 0, + Math.min( + Math.floor(requestedMaxOutputLength), + MAX_CAPTURED_OUTPUT_LENGTH + ) + ) + : MAX_CAPTURED_OUTPUT_LENGTH let executablePath: string let executableArgs: string[] = [] @@ -335,25 +425,27 @@ export const execCompose = ( reject(err) }) - const result: IDockerComposeResult = { - exitCode: null, - err: '', - out: '' - } + const stdout = createBufferedOutput() + const stderr = createBufferedOutput() childProc.stdout.on('data', (chunk): void => { - result.out += chunk.toString() + appendBufferedOutput(stdout, 'out', chunk, maxOutputLength) options.callback?.(chunk, 'stdout') }) childProc.stderr.on('data', (chunk): void => { - result.err += chunk.toString() + appendBufferedOutput(stderr, 'err', chunk, maxOutputLength) options.callback?.(chunk, 'stderr') }) childProc.on('exit', (exitCode): void => { - result.exitCode = exitCode setTimeout(() => { + const result: IDockerComposeResult = { + exitCode, + out: stdout.chunks.join(''), + err: stderr.chunks.join(''), + truncated: { out: stdout.truncated, err: stderr.truncated } + } if (exitCode === 0) { resolve(result) } else { @@ -583,6 +675,7 @@ export const config = async function ( ): Promise> { try { const result = await execCompose('config', [], options) + assertCompleteOutput(result, 'config') const config = yaml.parse(result.out) return { ...result, @@ -598,6 +691,7 @@ export const configServices = async function ( ): Promise> { try { const result = await execCompose('config', ['--services'], options) + assertCompleteOutput(result, 'config --services') const services = result.out.split('\n').filter(nonEmptyString) return { ...result, @@ -613,6 +707,7 @@ export const configVolumes = async function ( ): Promise> { try { const result = await execCompose('config', ['--volumes'], options) + assertCompleteOutput(result, 'config --volumes') const volumes = result.out.split('\n').filter(nonEmptyString) return { ...result, @@ -628,6 +723,7 @@ export const ps = async function ( ): Promise> { try { const result = await execCompose('ps', [], options) + assertCompleteOutput(result, 'ps') const data = mapPsOutput(result.out, options) return { ...result, @@ -648,6 +744,7 @@ export const images = async function ( commandOptions: [...(options?.commandOptions || []), ['--format', 'json']] } const result = await execCompose('images', [], jsonOptions) + assertCompleteOutput(result, 'images') const data = mapImListOutput(result.out, jsonOptions) return { ...result, @@ -713,6 +810,7 @@ export const port = async function ( try { const result = await execCompose('port', args, options) + assertCompleteOutput(result, 'port') const [address, port] = result.out.split(':') return { ...result, @@ -731,6 +829,7 @@ export const version = async function ( ): Promise> { try { const result = await execCompose('version', ['--short'], options) + assertCompleteOutput(result, 'version') const version = result.out.replace('\n', '').trim() return { ...result, @@ -749,6 +848,7 @@ export const stats = async function ( try { const result = await execCompose('stats', args, options) + assertCompleteOutput(result, 'stats') // Remove first and last quote from output, as well as newline. const output = result.out.replace('\n', '').trim().slice(1, -1) return JSON.parse(output) diff --git a/test/compose.test.ts b/test/compose.test.ts index 99a957aa..4f47f27e 100644 --- a/test/compose.test.ts +++ b/test/compose.test.ts @@ -8,6 +8,7 @@ import { beforeAll, MockInstance } from 'vitest' +import { EventEmitter } from 'events' import Docker, { ContainerInfo } from 'dockerode' import * as compose from '../src' import childProcess from 'child_process' @@ -1297,3 +1298,350 @@ describe('executable path resolution', (): void => { ) }) }) + +describe('output buffering limits', (): void => { + let spawnSpy: MockInstance + let mockProc: any + + beforeEach((): void => { + vi.useFakeTimers() + + const stdout = new EventEmitter() as EventEmitter & { + pipe: ReturnType + } + stdout.pipe = vi.fn() + const stderr = new EventEmitter() as EventEmitter & { + pipe: ReturnType + } + stderr.pipe = vi.fn() + + mockProc = new EventEmitter() + mockProc.stdout = stdout + mockProc.stderr = stderr + mockProc.stdin = { write: vi.fn(), end: vi.fn() } + + spawnSpy = vi + .spyOn(childProcess, 'spawn') + .mockReturnValue(mockProc as childProcess.ChildProcess) + }) + + afterEach((): void => { + spawnSpy.mockRestore() + vi.useRealTimers() + }) + + it('caps stdout and stderr buffering independently', async (): Promise => { + const callback = vi.fn() + const promise = compose.execCompose('up', [], { + callback, + maxOutputLength: 5 + }) + + mockProc.stdout.emit('data', Buffer.from('123')) + mockProc.stdout.emit('data', Buffer.from('456')) + mockProc.stderr.emit('data', Buffer.from('abcdef')) + mockProc.stderr.emit('data', Buffer.from('gh')) + mockProc.stdout.emit('data', Buffer.from('789')) + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + const result = await promise + + expect(result.out).toBe('12345') + expect(result.err).toBe('defgh') + expect(result.truncated).toEqual({ out: true, err: true }) + expect(callback.mock.calls).toEqual([ + [Buffer.from('123'), 'stdout'], + [Buffer.from('456'), 'stdout'], + [Buffer.from('abcdef'), 'stderr'], + [Buffer.from('gh'), 'stderr'], + [Buffer.from('789'), 'stdout'] + ]) + }) + + it.each([ + { limit: 5, chunks: ['ab', 'cd', 'ef', 'ghi', 'jk'] }, + { limit: 5, chunks: ['ab', 'cde', 'fghij'] }, + { limit: 5, chunks: ['ab', 'cd', '0123456789', 'xy'] }, + { limit: 1, chunks: ['ab', 'c', 'de', 'f'] }, + { limit: 3, chunks: ['é', '😀', 'x', '😀'] } + ])( + 'retains a stderr tail of $limit code units across chunks $chunks', + async ({ limit, chunks }): Promise => { + const promise = compose.execCompose('logs', [], { + maxOutputLength: limit + }) + + for (const chunk of chunks) { + mockProc.stderr.emit('data', Buffer.from(chunk)) + } + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + expect(await promise).toEqual({ + exitCode: 0, + out: '', + err: chunks.join('').slice(-limit), + truncated: { out: false, err: true } + }) + } + ) + + it.each([127, 1_000_000])( + 'retains the stdout prefix and stderr tail with many small chunks and a limit of %i', + async (maxOutputLength): Promise => { + const promise = compose.execCompose('logs', [], { maxOutputLength }) + const chunks = Array.from( + { length: 20000 }, + (_, index) => `${index.toString().padStart(6, '0')} ${'x'.repeat(92)}\n` + ) + + for (const chunk of chunks) { + mockProc.stdout.emit('data', Buffer.from(chunk)) + mockProc.stderr.emit('data', Buffer.from(chunk)) + } + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + const completeOutput = chunks.join('') + expect(await promise).toEqual({ + exitCode: 0, + out: completeOutput.slice(0, maxOutputLength), + err: completeOutput.slice(-maxOutputLength), + truncated: { out: true, err: true } + }) + } + ) + + it.each([0, 1])( + 'includes output received after exit code %i before the result settles', + async (exitCode): Promise => { + const promise = compose.execCompose('logs', [], { maxOutputLength: 5 }) + const expected = { + exitCode, + out: 'abcde', + err: 'defgh', + truncated: { out: true, err: true } + } + const settled = + exitCode === 0 + ? expect(promise).resolves.toEqual(expected) + : expect(promise).rejects.toEqual(expected) + + mockProc.stdout.emit('data', Buffer.from('abc')) + mockProc.stderr.emit('data', Buffer.from('abc')) + mockProc.emit('exit', exitCode) + await vi.advanceTimersByTimeAsync(250) + mockProc.stdout.emit('data', Buffer.from('defgh')) + mockProc.stderr.emit('data', Buffer.from('defgh')) + + await vi.advanceTimersByTimeAsync(250) + await settled + } + ) + + it('disables buffering at zero while forwarding every chunk', async (): Promise => { + const callback = vi.fn() + const promise = compose.execCompose('logs', [], { + maxOutputLength: 0, + callback, + log: true + }) + + mockProc.stdout.emit('data', Buffer.from('first output')) + mockProc.stderr.emit('data', Buffer.from('an error')) + mockProc.stdout.emit('data', Buffer.from('more output')) + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + expect(await promise).toEqual({ + exitCode: 0, + out: '', + err: '', + truncated: { out: true, err: true } + }) + expect(callback.mock.calls).toEqual([ + [Buffer.from('first output'), 'stdout'], + [Buffer.from('an error'), 'stderr'], + [Buffer.from('more output'), 'stdout'] + ]) + expect(mockProc.stdout.pipe).toHaveBeenCalledWith(process.stdout) + expect(mockProc.stderr.pipe).toHaveBeenCalledWith(process.stderr) + }) + + it('retains complete output by default', async (): Promise => { + const promise = compose.execCompose('up', []) + const output = 'stdout é😀\n'.repeat(10000) + const error = 'stderr é😀\n'.repeat(10000) + + mockProc.stdout.emit('data', Buffer.from(output)) + mockProc.stderr.emit('data', Buffer.from(error)) + mockProc.stdout.emit('data', Buffer.from('done')) + mockProc.stderr.emit('data', Buffer.from('finished')) + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + expect(await promise).toEqual({ + exitCode: 0, + out: output + 'done', + err: error + 'finished', + truncated: { out: false, err: false } + }) + }) + + it.each([0, 5])( + 'does not report truncation when output exactly fits a limit of %i', + async (maxOutputLength): Promise => { + const promise = compose.execCompose('up', [], { maxOutputLength }) + const output = 'x'.repeat(maxOutputLength) + + for (const stream of [mockProc.stdout, mockProc.stderr]) { + stream.emit('data', Buffer.from(output)) + stream.emit('data', Buffer.alloc(0)) + } + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + expect(await promise).toEqual({ + exitCode: 0, + out: output, + err: output, + truncated: { out: false, err: false } + }) + } + ) + + it('counts UTF-16 code units for each stream', async (): Promise => { + const promise = compose.execCompose('up', [], { maxOutputLength: 3 }) + + mockProc.stdout.emit('data', Buffer.from('é😀x')) + mockProc.stderr.emit('data', Buffer.from('é😀x')) + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + expect(await promise).toMatchObject({ + out: 'é😀', + err: '😀x', + truncated: { out: true, err: true } + }) + }) + + it.each([ + { limit: -1, out: '', err: '', truncated: true }, + { limit: 2.9, out: 'ab', err: 'ef', truncated: true }, + { limit: NaN, out: 'abcdef', err: 'abcdef', truncated: false }, + { limit: Infinity, out: 'abcdef', err: 'abcdef', truncated: false }, + { limit: -Infinity, out: 'abcdef', err: 'abcdef', truncated: false } + ])( + 'normalizes a maxOutputLength of $limit', + async ({ limit, out, err, truncated }): Promise => { + const promise = compose.execCompose('up', [], { maxOutputLength: limit }) + + mockProc.stdout.emit('data', Buffer.from('abcdef')) + mockProc.stderr.emit('data', Buffer.from('abcdef')) + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + expect(await promise).toMatchObject({ + out, + err, + truncated: { out: truncated, err: truncated } + }) + } + ) + + it('preserves the final error and truncation flags on rejection', async (): Promise => { + const promise = compose.execCompose('up', [], { maxOutputLength: 12 }) + const rejection = expect(promise).rejects.toEqual({ + exitCode: 1, + out: 'ok', + err: 'fatal: error', + truncated: { out: false, err: true } + }) + + mockProc.stdout.emit('data', Buffer.from('ok')) + mockProc.stderr.emit('data', Buffer.from('progress '.repeat(10))) + mockProc.stderr.emit('data', Buffer.from('fatal: error')) + mockProc.emit('exit', 1) + + await vi.advanceTimersByTimeAsync(500) + await rejection + }) + + const parsedCommands = [ + { name: 'config', run: compose.config, output: 'services: {}\n' }, + { + name: 'configServices', + run: compose.configServices, + output: 'web\ndb\n' + }, + { + name: 'configVolumes', + run: compose.configVolumes, + output: 'data\nlogs\n' + }, + { + name: 'ps', + run: (options: compose.IDockerComposeOptions) => + compose.ps({ ...options, commandOptions: ['--services'] }), + output: 'web\ndb\n' + }, + { name: 'images', run: compose.images, output: '[]\n' }, + { + name: 'port', + run: (options: compose.IDockerComposeOptions) => + compose.port('web', 80, options), + output: '127.0.0.1:8080\n' + }, + { name: 'version', run: compose.version, output: '2.0.0\n' }, + { + name: 'stats', + run: (options: compose.IDockerComposeOptions) => + compose.stats('web', options), + output: '"{ "CPUPerc": "0.00%" }"\n' + } + ] + + it.each(parsedCommands)( + 'rejects truncated stdout before parsing $name output', + async ({ run, output }): Promise => { + // For YAML and line-based output, truncation can still look valid. + const promise = run({ maxOutputLength: output.length - 2 }) + const rejection = expect(promise).rejects.toThrow( + 'stdout was truncated by maxOutputLength' + ) + + mockProc.stdout.emit('data', Buffer.from(output)) + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + await rejection + } + ) + + it.each(parsedCommands)( + 'parses complete $name output when only stderr is truncated', + async ({ name, run, output }): Promise => { + const promise = run({ maxOutputLength: output.length }) + + mockProc.stdout.emit('data', Buffer.from(output)) + mockProc.stderr.emit( + 'data', + Buffer.from('warning '.repeat(output.length)) + ) + mockProc.emit('exit', 0) + + await vi.advanceTimersByTimeAsync(500) + const result = await promise + if (name === 'stats') { + expect(result).toEqual({ CPUPerc: '0.00%' }) + } else { + expect(result).toMatchObject({ + out: output, + truncated: { out: false, err: true }, + data: expect.any(Object) + }) + } + } + ) +})