diff --git a/docs/Testing.md b/docs/Testing.md index a6c383aca..f45218878 100644 --- a/docs/Testing.md +++ b/docs/Testing.md @@ -19,6 +19,8 @@ The workhorse. Every command gets unit tests. They should cover every code branc Run against real Ably services with real credentials (via env vars). These cover the entire journey — the CLI's interaction with the actual Ably service, end to end. Every command must have an E2E test. E2E tests should cover the happy path and major sad paths (e.g., invalid capabilities, nonexistent resources). They are slow and can incur costs, so use them deliberately. +Ably counts every distinct client ID it sees towards the account's MAU, so E2E tests never mint random client IDs: use `getTestClientId(prefix)` from `test/helpers/e2e-test-helper.ts`, which draws from a small fixed pool per prefix (successive calls return distinct IDs), and CLI subprocesses that don't pass `--client-id` act as the fixed `ABLY_CLIENT_ID` set for the `e2e` project in `vitest.config.ts`. + ### TTY tests Use `node-pty` to create a real pseudo-terminal. This is the only way to test readline SIGINT handling, which doesn't work with piped stdio. Local only — cannot run in CI (no TTY in GitHub Actions runners). Rebuild `node-pty` with `pnpm rebuild node-pty` if it fails to load. diff --git a/test/e2e/auth/auth-tokens-e2e.test.ts b/test/e2e/auth/auth-tokens-e2e.test.ts index 3424d00e0..4a6087294 100644 --- a/test/e2e/auth/auth-tokens-e2e.test.ts +++ b/test/e2e/auth/auth-tokens-e2e.test.ts @@ -9,6 +9,7 @@ import { cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, + getTestClientId, } from "../../helpers/e2e-test-helper.js"; import { parseNdjsonLines } from "../../helpers/ndjson.js"; @@ -177,7 +178,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Auth Tokens E2E Tests", () => { { "x-ably-appId": appId, "x-ably-capability": { "*": ["*"] }, - "x-ably-clientId": `e2e-revoke-key-client-${randomUUID().slice(0, 8)}`, + "x-ably-clientId": getTestClientId("e2e-revoke-key-client"), "x-ably-revocation-key": revocationKey, // The CLI always connects as a server, which token auth allows // only with this claim. diff --git a/test/e2e/channels/channel-presence-subscribe-e2e.test.ts b/test/e2e/channels/channel-presence-subscribe-e2e.test.ts index a9dee0f31..3df33c189 100644 --- a/test/e2e/channels/channel-presence-subscribe-e2e.test.ts +++ b/test/e2e/channels/channel-presence-subscribe-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -38,8 +38,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Channel Presence Subscribe E2E Tests", () => { ); const channel = getUniqueChannelName("pres-sub"); - const subClientId = getUniqueClientId("sub-client"); - const enterClientId = getUniqueClientId("enter-client"); + const subClientId = getTestClientId("sub-client"); + const enterClientId = getTestClientId("enter-client"); // Start presence subscriber const subscriber = await startSubscribeCommand( diff --git a/test/e2e/connections/connections.test.ts b/test/e2e/connections/connections.test.ts index 41a754081..17d0182db 100644 --- a/test/e2e/connections/connections.test.ts +++ b/test/e2e/connections/connections.test.ts @@ -147,6 +147,9 @@ describe("Connections E2E Tests", () => { const cliPath = join(process.cwd(), "bin", "run.js"); const testChannelName = `test-live-connections-${Date.now()}`; + // Deliberately unique rather than pooled: the test finds its own + // connection in the lifecycle log by client ID, and a pooled ID could + // match a concurrent run's connection and hide a failure. const testClientId = `test-client-${Date.now()}`; // Step 1: Start live connection log monitoring diff --git a/test/e2e/rooms/rooms-e2e.test.ts b/test/e2e/rooms/rooms-e2e.test.ts index 5560eb00d..f2cc1bd9f 100644 --- a/test/e2e/rooms/rooms-e2e.test.ts +++ b/test/e2e/rooms/rooms-e2e.test.ts @@ -2,7 +2,7 @@ import { describe, it, beforeEach, afterEach, expect } from "vitest"; import { SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, testOutputFiles, testCommands, @@ -30,8 +30,8 @@ describe("Rooms E2E Tests", () => { testCommands.length = 0; testRoom = getUniqueChannelName("room"); - client1Id = getUniqueClientId("client1"); - client2Id = getUniqueClientId("client2"); + client1Id = getTestClientId("client1"); + client2Id = getTestClientId("client2"); }); afterEach(async () => { diff --git a/test/e2e/rooms/rooms-messages-e2e.test.ts b/test/e2e/rooms/rooms-messages-e2e.test.ts index f142b8c19..c6ba7ea0d 100644 --- a/test/e2e/rooms/rooms-messages-e2e.test.ts +++ b/test/e2e/rooms/rooms-messages-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -18,7 +18,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Messages E2E Tests", () => { beforeEach(() => { resetTestTracking(); testRoom = getUniqueChannelName("room-msg"); - clientId = getUniqueClientId("msg-client"); + clientId = getTestClientId("msg-client"); }); afterEach(async () => { diff --git a/test/e2e/rooms/rooms-messages-reactions-e2e.test.ts b/test/e2e/rooms/rooms-messages-reactions-e2e.test.ts index 620850be4..7861065d3 100644 --- a/test/e2e/rooms/rooms-messages-reactions-e2e.test.ts +++ b/test/e2e/rooms/rooms-messages-reactions-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -25,8 +25,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Message Reactions E2E Tests", () => { beforeEach(() => { resetTestTracking(); testRoom = getUniqueChannelName("room-msgreact"); - client1Id = getUniqueClientId("msgreact-sub"); - client2Id = getUniqueClientId("msgreact-send"); + client1Id = getTestClientId("msgreact-sub"); + client2Id = getTestClientId("msgreact-send"); }); afterEach(async () => { diff --git a/test/e2e/rooms/rooms-messages-subscribe-e2e.test.ts b/test/e2e/rooms/rooms-messages-subscribe-e2e.test.ts index d2d309652..685875cbc 100644 --- a/test/e2e/rooms/rooms-messages-subscribe-e2e.test.ts +++ b/test/e2e/rooms/rooms-messages-subscribe-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -24,8 +24,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Messages Subscribe E2E Tests", () => { beforeEach(() => { resetTestTracking(); testRoom = getUniqueChannelName("room-sub"); - subscriberId = getUniqueClientId("subscriber"); - senderId = getUniqueClientId("sender"); + subscriberId = getTestClientId("subscriber"); + senderId = getTestClientId("sender"); }); afterEach(async () => { diff --git a/test/e2e/rooms/rooms-occupancy-e2e.test.ts b/test/e2e/rooms/rooms-occupancy-e2e.test.ts index 649e11054..e79970ddf 100644 --- a/test/e2e/rooms/rooms-occupancy-e2e.test.ts +++ b/test/e2e/rooms/rooms-occupancy-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -25,7 +25,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Occupancy E2E Tests", () => { beforeEach(() => { resetTestTracking(); testRoom = getUniqueChannelName("room-occ"); - clientId = getUniqueClientId("occ-client"); + clientId = getTestClientId("occ-client"); }); afterEach(async () => { @@ -91,7 +91,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Occupancy E2E Tests", () => { await new Promise((resolve) => setTimeout(resolve, 2000)); // Enter presence to trigger an occupancy change - const enterClientId = getUniqueClientId("occ-enter"); + const enterClientId = getTestClientId("occ-enter"); enterer = await startPresenceCommand( [ "rooms", diff --git a/test/e2e/rooms/rooms-presence-e2e.test.ts b/test/e2e/rooms/rooms-presence-e2e.test.ts index 1cdbdeed5..9a3fc178c 100644 --- a/test/e2e/rooms/rooms-presence-e2e.test.ts +++ b/test/e2e/rooms/rooms-presence-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -23,7 +23,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Presence E2E Tests", () => { beforeEach(() => { resetTestTracking(); testRoom = getUniqueChannelName("room-pres"); - clientId = getUniqueClientId("pres-client"); + clientId = getTestClientId("pres-client"); }); afterEach(async () => { diff --git a/test/e2e/rooms/rooms-presence-subscribe-e2e.test.ts b/test/e2e/rooms/rooms-presence-subscribe-e2e.test.ts index 56c6f5b6e..03d3b53c2 100644 --- a/test/e2e/rooms/rooms-presence-subscribe-e2e.test.ts +++ b/test/e2e/rooms/rooms-presence-subscribe-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -35,8 +35,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Presence Subscribe E2E Tests", () => { ); const testRoom = getUniqueChannelName("room-pres-sub"); - const subClientId = getUniqueClientId("pres-sub"); - const enterClientId = getUniqueClientId("pres-enter"); + const subClientId = getTestClientId("pres-sub"); + const enterClientId = getTestClientId("pres-enter"); // Start presence subscriber const subscriber = await startSubscribeCommand( diff --git a/test/e2e/rooms/rooms-reactions-e2e.test.ts b/test/e2e/rooms/rooms-reactions-e2e.test.ts index 8fa04e029..ab861802b 100644 --- a/test/e2e/rooms/rooms-reactions-e2e.test.ts +++ b/test/e2e/rooms/rooms-reactions-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -24,8 +24,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Reactions E2E Tests", () => { beforeEach(() => { resetTestTracking(); testRoom = getUniqueChannelName("room-react"); - client1Id = getUniqueClientId("react-sub"); - client2Id = getUniqueClientId("react-send"); + client1Id = getTestClientId("react-sub"); + client2Id = getTestClientId("react-send"); }); afterEach(async () => { diff --git a/test/e2e/rooms/rooms-typing-e2e.test.ts b/test/e2e/rooms/rooms-typing-e2e.test.ts index 5cdbabbfa..e40a4eca6 100644 --- a/test/e2e/rooms/rooms-typing-e2e.test.ts +++ b/test/e2e/rooms/rooms-typing-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -24,8 +24,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Typing E2E Tests", () => { beforeEach(() => { resetTestTracking(); testRoom = getUniqueChannelName("room-typing"); - subscriberId = getUniqueClientId("type-sub"); - typerId = getUniqueClientId("typer"); + subscriberId = getTestClientId("type-sub"); + typerId = getTestClientId("typer"); }); afterEach(async () => { diff --git a/test/e2e/spaces/spaces-crud-e2e.test.ts b/test/e2e/spaces/spaces-crud-e2e.test.ts index 69ded644e..258ba179e 100644 --- a/test/e2e/spaces/spaces-crud-e2e.test.ts +++ b/test/e2e/spaces/spaces-crud-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -23,7 +23,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Spaces CRUD E2E Tests", () => { beforeEach(() => { resetTestTracking(); spaceName = getUniqueChannelName("space"); - clientId = getUniqueClientId("space-client"); + clientId = getTestClientId("space-client"); }); afterEach(async () => { diff --git a/test/e2e/spaces/spaces-e2e.test.ts b/test/e2e/spaces/spaces-e2e.test.ts index 742f9f208..8ad896ae4 100644 --- a/test/e2e/spaces/spaces-e2e.test.ts +++ b/test/e2e/spaces/spaces-e2e.test.ts @@ -2,7 +2,7 @@ import { describe, it, beforeEach, afterEach, expect } from "vitest"; import { SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, createTempOutputFile, runLongRunningBackgroundProcess, readProcessOutput, @@ -29,8 +29,8 @@ describe("Spaces E2E Tests", () => { testCommands.length = 0; testSpaceId = getUniqueChannelName("space"); - client1Id = getUniqueClientId("client1"); - client2Id = getUniqueClientId("client2"); + client1Id = getTestClientId("client1"); + client2Id = getTestClientId("client2"); }); afterEach(async () => { diff --git a/test/e2e/spaces/spaces-locations-e2e.test.ts b/test/e2e/spaces/spaces-locations-e2e.test.ts index 21609d54c..6636b2c7c 100644 --- a/test/e2e/spaces/spaces-locations-e2e.test.ts +++ b/test/e2e/spaces/spaces-locations-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -26,7 +26,7 @@ describe.skipIf(SHOULD_SKIP_E2E)( beforeEach(() => { resetTestTracking(); spaceName = getUniqueChannelName("space-loc"); - clientId = getUniqueClientId("loc-client"); + clientId = getTestClientId("loc-client"); }); afterEach(async () => { diff --git a/test/e2e/spaces/spaces-occupancy-e2e.test.ts b/test/e2e/spaces/spaces-occupancy-e2e.test.ts index 7e245b0e4..451347aca 100644 --- a/test/e2e/spaces/spaces-occupancy-e2e.test.ts +++ b/test/e2e/spaces/spaces-occupancy-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -23,7 +23,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Spaces Occupancy E2E Tests", () => { beforeEach(() => { resetTestTracking(); spaceName = getUniqueChannelName("space-occ"); - clientId = getUniqueClientId("occ-client"); + clientId = getTestClientId("occ-client"); }); afterEach(async () => { diff --git a/test/e2e/spaces/spaces-subscribe-e2e.test.ts b/test/e2e/spaces/spaces-subscribe-e2e.test.ts index c488075c8..57a7620df 100644 --- a/test/e2e/spaces/spaces-subscribe-e2e.test.ts +++ b/test/e2e/spaces/spaces-subscribe-e2e.test.ts @@ -3,7 +3,7 @@ import { E2E_API_KEY, SHOULD_SKIP_E2E, getUniqueChannelName, - getUniqueClientId, + getTestClientId, cleanupTrackedResources, setupTestFailureHandler, resetTestTracking, @@ -35,8 +35,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Spaces Subscribe E2E Tests", () => { ); const spaceName = getUniqueChannelName("space-sub"); - const subClientId = getUniqueClientId("space-sub-client"); - const enterClientId = getUniqueClientId("space-enter-client"); + const subClientId = getTestClientId("space-sub-client"); + const enterClientId = getTestClientId("space-enter-client"); // Start the space subscriber const subscriber = await startSubscribeCommand( diff --git a/test/helpers/e2e-test-helper.ts b/test/helpers/e2e-test-helper.ts index 01d1df474..b49662d7d 100644 --- a/test/helpers/e2e-test-helper.ts +++ b/test/helpers/e2e-test-helper.ts @@ -59,10 +59,23 @@ export function getUniqueChannelName(prefix: string): string { } /** - * Create a unique client ID for testing + * Number of client IDs each prefix cycles through. Ably counts every distinct + * client ID it sees towards the account's MAU, so E2E runs draw from a fixed + * pool rather than minting a new ID per test on every PR. */ -export function getUniqueClientId(prefix = "cli-e2e-test"): string { - return `${prefix}-${randomUUID()}`; +const CLIENT_ID_POOL_SIZE = 8; + +const clientIdPoolCursors = new Map(); + +/** + * A client ID from a bounded pool. Successive calls with the same prefix + * return different IDs (up to the pool size), so a test that needs two + * distinct clients gets them; across runs the same IDs are reused. + */ +export function getTestClientId(prefix = "cli-e2e-test"): string { + const cursor = clientIdPoolCursors.get(prefix) ?? 0; + clientIdPoolCursors.set(prefix, cursor + 1); + return `${prefix}-${cursor % CLIENT_ID_POOL_SIZE}`; } /** @@ -84,7 +97,7 @@ export function createAblyClient(): Ably.PubSubHttpClient { // throw new Error('Structurally invalid E2E_ABLY_API_KEY detected'); } - const clientId = getUniqueClientId(); + const clientId = getTestClientId(); const _keyPrefix = E2E_API_KEY.split(":")[0]?.split(".")[0] || "unknown-app"; const _keyId = E2E_API_KEY.split(":")[0]?.split(".")[1]?.slice(0, 4) || "unknown-key"; @@ -118,7 +131,7 @@ export function createAblyRealtimeClient(): Ably.PubSubRealtimeClient { // throw new Error('Structurally invalid E2E_ABLY_API_KEY detected'); } - const clientId = getUniqueClientId(); + const clientId = getTestClientId(); const _keyPrefix = E2E_API_KEY.split(":")[0]?.split(".")[0] || "unknown-app"; const _keyId = E2E_API_KEY.split(":")[0]?.split(".")[1]?.slice(0, 4) || "unknown-key"; diff --git a/vitest.config.ts b/vitest.config.ts index 4e5ca5897..02bc4c1fb 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -71,6 +71,9 @@ export default defineConfig({ setupFiles: ["./test/setup.ts", "./test/e2e/setup.ts"], env: { ABLY_API_KEY: undefined, + // One fixed identity for CLI subprocesses that don't pass + // --client-id, so runs don't mint a new MAU per fresh config dir. + ABLY_CLIENT_ID: "cli-e2e-default", }, testTimeout: 20000, // Allow 20s per test for plenty of time on actions hookTimeout: 60000, // 60 seconds for hooks