diff --git a/src/__tests__/MockResponsesServer.test.ts b/src/__tests__/MockResponsesServer.test.ts new file mode 100644 index 00000000..25a1b1c6 --- /dev/null +++ b/src/__tests__/MockResponsesServer.test.ts @@ -0,0 +1,129 @@ +import {afterEach, describe, expect, it, vi} from "vitest"; +import { + execCommandResponse, functionCallResponse, startMockResponsesServer, type MockResponsesServer, +} from "./integration/mock-responses-server"; + +async function response(server: MockResponsesServer, body: unknown, path = "/responses") { + const result = await fetch(`${server.baseUrl}${path}`, {method: "POST", body: JSON.stringify(body)}); + return {status: result.status, body: await result.text()}; +} + +function functionItem(stream: string) { + return stream.split("\n").filter(line => line.startsWith("data: ")) + .map(line => JSON.parse(line.slice(6))) + .find(event => event.type === "response.output_item.done")?.item; +} + +describe("mock Responses API", () => { + let server: MockResponsesServer | undefined; + afterEach(async () => { + await server?.close(); + server = undefined; + vi.restoreAllMocks(); + }); + + it("keeps FIFO replies when no request handler is installed", async () => { + server = await startMockResponsesServer(); + server.enqueue("First queued reply", "Second queued reply"); + const replies = [await response(server, {kind: "first"})]; + server.setErrorResponse({status: 429, body: {error: "Scripted rate limit"}}); + replies.push(await response(server, {kind: "limited"})); + server.setErrorResponse(null); + replies.push(await response(server, {kind: "second"})); + replies.push(await response(server, {kind: "unscripted"})); + await expect(JSON.stringify({ + replies, requests: server.requests, unexpectedRequests: server.unexpectedRequests, + }, null, 2)).toMatchFileSnapshot("data/mock-responses-queue.json"); + }); + + it("routes concurrent requests by body even when they complete out of order", async () => { + server = await startMockResponsesServer(); + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + server.setResponseHandler(async request => { + if (request !== null && typeof request === "object" && "kind" in request && request.kind === "slow") { + await held; + return "Slow reply"; + } + return "Fast reply"; + }); + let slowCompleted = false; + const slow = response(server, {kind: "slow"}).then(result => { + slowCompleted = true; + return result; + }); + void slow.catch(() => {}); + try { + await vi.waitFor(() => expect(server!.requests).toEqual([{kind: "slow"}])); + const fast = await response(server, {kind: "fast"}); + expect(slowCompleted).toBe(false); + release(); + const replies = {fast, slow: await slow, requests: server.requests, unexpectedRequests: server.unexpectedRequests}; + await expect(JSON.stringify(replies, null, 2)).toMatchFileSnapshot("data/mock-responses-concurrent.json"); + } finally { + release(); + } + }); + + it("keeps error overrides and unmatched requests ahead of request handlers", async () => { + server = await startMockResponsesServer(); + const handler = vi.fn(() => "Selected reply"); + server.setResponseHandler(handler); + server.setErrorResponse({status: 503, body: {error: "Scripted outage"}}); + const unavailable = await response(server, {kind: "limited"}); + const unmatched = await response(server, {kind: "wrong-path"}, "/not-responses"); + expect(handler).not.toHaveBeenCalled(); + server.setErrorResponse(null); + const selected = await response(server, {kind: "selected"}); + expect(handler).toHaveBeenCalledExactlyOnceWith({kind: "selected"}); + await expect(JSON.stringify({ + unavailable, unmatched, selected, unexpectedRequests: server.unexpectedRequests, + }, null, 2)).toMatchFileSnapshot("data/mock-responses-precedence.json"); + }); + + it("does not fall back to queued replies when a request handler returns undefined", async () => { + server = await startMockResponsesServer(); + server.enqueue("Not a fallback"); + server.setResponseHandler(() => undefined); + expect(await response(server, {kind: "unscripted"})).toEqual({status: 400, body: "mock: unexpected request #1"}); + expect(server.unexpectedRequests).toBe(1); + expect(server.requests).toEqual([{kind: "unscripted"}]); + }); + + it.each([false, true])("keeps diagnostics out of HTTP responses (async: %s)", async asynchronous => { + const error = new Error("Private handler diagnostic"); + error.stack = "Error: Private handler diagnostic\n at handler (/private/internal.ts:42:1)"; + const failure = asynchronous ? error.stack : error; + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + server = await startMockResponsesServer(); + server.setResponseHandler(asynchronous + ? async () => { throw failure; } + : () => { throw failure; }); + + const failed = await response(server, {}); + expect(failed.body).not.toContain("Private handler diagnostic"); + expect(failed.body).not.toContain("/private/internal.ts"); + expect(log).toHaveBeenCalledWith("Mock Responses API handler failed", failure); + + server.setResponseHandler(() => "Next scripted response"); + const next = await response(server, {}); + await expect(JSON.stringify({ + status: failed.status, + body: failed.body, + nextStatus: next.status, + nextBody: next.body, + }, null, 2)).toMatchFileSnapshot("data/mock-responses-handler-failure.json"); + }); + + it("scripts named and namespaced function calls without changing shell-call payloads", async () => { + const shellArgs = {cmd: "echo hello"}; + const shell = functionItem(execCommandResponse("shell", shellArgs)); + expect(shell).toEqual(functionItem(functionCallResponse("shell", "exec_command", shellArgs))); + const calls = [ + shell, + functionItem(functionCallResponse("lookup", "lookup", {query: "example"})), + functionItem(functionCallResponse("lookup-namespaced", "lookup", {query: "example"}, "helpers")), + ]; + await expect(JSON.stringify(calls, null, 2)).toMatchFileSnapshot("data/mock-responses-function-calls.json"); + }); +}); diff --git a/src/__tests__/data/mock-responses-concurrent.json b/src/__tests__/data/mock-responses-concurrent.json new file mode 100644 index 00000000..c97baa9a --- /dev/null +++ b/src/__tests__/data/mock-responses-concurrent.json @@ -0,0 +1,19 @@ +{ + "fast": { + "status": 200, + "body": "Fast reply" + }, + "slow": { + "status": 200, + "body": "Slow reply" + }, + "requests": [ + { + "kind": "slow" + }, + { + "kind": "fast" + } + ], + "unexpectedRequests": 0 +} \ No newline at end of file diff --git a/src/__tests__/data/mock-responses-function-calls.json b/src/__tests__/data/mock-responses-function-calls.json new file mode 100644 index 00000000..73f2dd1f --- /dev/null +++ b/src/__tests__/data/mock-responses-function-calls.json @@ -0,0 +1,24 @@ +[ + { + "type": "function_call", + "id": "fc_shell", + "call_id": "shell", + "name": "exec_command", + "arguments": "{\"cmd\":\"echo hello\"}" + }, + { + "type": "function_call", + "id": "fc_lookup", + "call_id": "lookup", + "name": "lookup", + "arguments": "{\"query\":\"example\"}" + }, + { + "type": "function_call", + "id": "fc_lookup-namespaced", + "call_id": "lookup-namespaced", + "name": "lookup", + "namespace": "helpers", + "arguments": "{\"query\":\"example\"}" + } +] \ No newline at end of file diff --git a/src/__tests__/data/mock-responses-handler-failure.json b/src/__tests__/data/mock-responses-handler-failure.json new file mode 100644 index 00000000..ff73db95 --- /dev/null +++ b/src/__tests__/data/mock-responses-handler-failure.json @@ -0,0 +1,6 @@ +{ + "status": 500, + "body": "mock: response handler failed", + "nextStatus": 200, + "nextBody": "Next scripted response" +} \ No newline at end of file diff --git a/src/__tests__/data/mock-responses-precedence.json b/src/__tests__/data/mock-responses-precedence.json new file mode 100644 index 00000000..0db1ee0b --- /dev/null +++ b/src/__tests__/data/mock-responses-precedence.json @@ -0,0 +1,15 @@ +{ + "unavailable": { + "status": 503, + "body": "{\"error\":\"Scripted outage\"}" + }, + "unmatched": { + "status": 404, + "body": "mock: unexpected request #2" + }, + "selected": { + "status": 200, + "body": "Selected reply" + }, + "unexpectedRequests": 1 +} \ No newline at end of file diff --git a/src/__tests__/data/mock-responses-queue.json b/src/__tests__/data/mock-responses-queue.json new file mode 100644 index 00000000..74d02e3d --- /dev/null +++ b/src/__tests__/data/mock-responses-queue.json @@ -0,0 +1,35 @@ +{ + "replies": [ + { + "status": 200, + "body": "First queued reply" + }, + { + "status": 429, + "body": "{\"error\":\"Scripted rate limit\"}" + }, + { + "status": 200, + "body": "Second queued reply" + }, + { + "status": 400, + "body": "mock: unexpected request #4" + } + ], + "requests": [ + { + "kind": "first" + }, + { + "kind": "limited" + }, + { + "kind": "second" + }, + { + "kind": "unscripted" + } + ], + "unexpectedRequests": 1 +} \ No newline at end of file diff --git a/src/__tests__/integration/mock-responses-server.ts b/src/__tests__/integration/mock-responses-server.ts index cfc2a960..51920061 100644 --- a/src/__tests__/integration/mock-responses-server.ts +++ b/src/__tests__/integration/mock-responses-server.ts @@ -2,9 +2,10 @@ import http from "node:http"; import type {AddressInfo} from "node:net"; type MockErrorResponse = {status: number; body: unknown}; +type ResponseHandler = (request: unknown) => string | undefined | Promise; /** - * Local stand-in for the OpenAI Responses API. Each `POST /v1/responses` from Codex gets the next queued reply; + * Local stand-in for the OpenAI Responses API. By default, each `POST /v1/responses` gets the next queued reply; * a tool-call turn needs two: the call, then the answer after Codex sends the tool output. */ export interface MockResponsesServer { @@ -12,12 +13,14 @@ export interface MockResponsesServer { readonly baseUrl: string; /** Bodies of every request Codex sent, in order; shows what Codex passed back to the model. */ readonly requests: unknown[]; - /** Requests with no queued reply (answered 400/404): Codex did something the test didn't script. */ + /** Requests with no scripted reply (answered 400/404): Codex did something the test didn't script. */ readonly unexpectedRequests: number; /** Queues replies built with `textResponse`, `execCommandResponse` or `applyPatchResponse`. */ enqueue(...responses: string[]): void; /** Keeps returning the HTTP error across runtime retries, until cleared with null. */ setErrorResponse(response: MockErrorResponse | null): void; + /** Overrides queued replies; return undefined to flag an unexpected request. */ + setResponseHandler(handler: ResponseHandler): void; close(): Promise; } @@ -33,6 +36,7 @@ function parseBody(body: string): unknown { export async function startMockResponsesServer(): Promise { const queue: string[] = []; let errorResponse: MockErrorResponse | null = null; + let responseHandler: ResponseHandler | undefined; const requests: unknown[] = []; let unexpectedRequests = 0; let requestCount = 0; @@ -40,11 +44,22 @@ export async function startMockResponsesServer(): Promise { const server = http.createServer((req, res) => { let body = ""; req.on("data", (chunk) => body += chunk); - req.on("end", () => { + req.on("end", async () => { requestCount++; - requests.push(parseBody(body)); + const parsed = parseBody(body); + requests.push(parsed); const isResponsesCall = req.method === "POST" && (req.url ?? "").split("?")[0]!.endsWith("/responses"); - const next = isResponsesCall ? errorResponse ?? queue.shift() : undefined; + let next: string | MockErrorResponse | undefined; + try { + next = isResponsesCall ? errorResponse ?? (responseHandler ? await responseHandler(parsed) : queue.shift()) : undefined; + } catch (error) { + console.error("Mock Responses API handler failed", error); + if (res.destroyed) return; + res.writeHead(500); + res.end("mock: response handler failed"); + return; + } + if (res.destroyed) return; if (next === undefined) { unexpectedRequests++; res.writeHead(isResponsesCall ? 400 : 404); @@ -76,6 +91,7 @@ export async function startMockResponsesServer(): Promise { queue.push(...responses); }, setErrorResponse: response => { errorResponse = response; }, + setResponseHandler: handler => { responseHandler = handler; }, close: async () => { server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); @@ -121,8 +137,14 @@ export function textResponse(...chunks: string[]): string { /** The model calls the shell tool `exec_command` with `args` (e.g. `{cmd: "echo hi"}`). */ export function execCommandResponse(callId: string, args: object): string { + return functionCallResponse(callId, "exec_command", args); +} + +/** The model calls a named function, optionally within a tool namespace. */ +export function functionCallResponse(callId: string, name: string, args: object, namespace?: string): string { return wrap(nextId(), [ - {type: "response.output_item.done", output_index: 0, item: {type: "function_call", id: `fc_${callId}`, call_id: callId, name: "exec_command", arguments: JSON.stringify(args)}}, + {type: "response.output_item.done", output_index: 0, item: {type: "function_call", id: `fc_${callId}`, call_id: callId, name, + ...(namespace ? {namespace} : {}), arguments: JSON.stringify(args)}}, ]); }