Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/Testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ The workhorse. Every command gets unit tests. They should cover every code branc

Run against real Ably services with real credentials (via env vars). These cover the entire journey — the CLI's interaction with the actual Ably service, end to end. Every command must have an E2E test. E2E tests should cover the happy path and major sad paths (e.g., invalid capabilities, nonexistent resources). They are slow and can incur costs, so use them deliberately.

Ably counts every distinct client ID it sees towards the account's MAU, so E2E tests never mint random client IDs: use `getTestClientId(prefix)` from `test/helpers/e2e-test-helper.ts`, which draws from a small fixed pool per prefix (successive calls return distinct IDs), and CLI subprocesses that don't pass `--client-id` act as the fixed `ABLY_CLIENT_ID` set for the `e2e` project in `vitest.config.ts`.

### TTY tests

Use `node-pty` to create a real pseudo-terminal. This is the only way to test readline SIGINT handling, which doesn't work with piped stdio. Local only — cannot run in CI (no TTY in GitHub Actions runners). Rebuild `node-pty` with `pnpm rebuild node-pty` if it fails to load.
Expand Down
3 changes: 2 additions & 1 deletion test/e2e/auth/auth-tokens-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
getTestClientId,
} from "../../helpers/e2e-test-helper.js";
import { parseNdjsonLines } from "../../helpers/ndjson.js";

Expand Down Expand Up @@ -177,7 +178,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Auth Tokens E2E Tests", () => {
{
"x-ably-appId": appId,
"x-ably-capability": { "*": ["*"] },
"x-ably-clientId": `e2e-revoke-key-client-${randomUUID().slice(0, 8)}`,
"x-ably-clientId": getTestClientId("e2e-revoke-key-client"),
"x-ably-revocation-key": revocationKey,
// The CLI always connects as a server, which token auth allows
// only with this claim.
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/channels/channel-presence-subscribe-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand Down Expand Up @@ -38,8 +38,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Channel Presence Subscribe E2E Tests", () => {
);

const channel = getUniqueChannelName("pres-sub");
const subClientId = getUniqueClientId("sub-client");
const enterClientId = getUniqueClientId("enter-client");
const subClientId = getTestClientId("sub-client");
const enterClientId = getTestClientId("enter-client");

// Start presence subscriber
const subscriber = await startSubscribeCommand(
Expand Down
3 changes: 3 additions & 0 deletions test/e2e/connections/connections.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ describe("Connections E2E Tests", () => {

const cliPath = join(process.cwd(), "bin", "run.js");
const testChannelName = `test-live-connections-${Date.now()}`;
// Deliberately unique rather than pooled: the test finds its own
// connection in the lifecycle log by client ID, and a pooled ID could
// match a concurrent run's connection and hide a failure.
const testClientId = `test-client-${Date.now()}`;

// Step 1: Start live connection log monitoring
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/rooms/rooms-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { describe, it, beforeEach, afterEach, expect } from "vitest";
import {
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
testOutputFiles,
testCommands,
Expand Down Expand Up @@ -30,8 +30,8 @@ describe("Rooms E2E Tests", () => {
testCommands.length = 0;

testRoom = getUniqueChannelName("room");
client1Id = getUniqueClientId("client1");
client2Id = getUniqueClientId("client2");
client1Id = getTestClientId("client1");
client2Id = getTestClientId("client2");
});

afterEach(async () => {
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/rooms/rooms-messages-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -18,7 +18,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Messages E2E Tests", () => {
beforeEach(() => {
resetTestTracking();
testRoom = getUniqueChannelName("room-msg");
clientId = getUniqueClientId("msg-client");
clientId = getTestClientId("msg-client");
});

afterEach(async () => {
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/rooms/rooms-messages-reactions-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -25,8 +25,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Message Reactions E2E Tests", () => {
beforeEach(() => {
resetTestTracking();
testRoom = getUniqueChannelName("room-msgreact");
client1Id = getUniqueClientId("msgreact-sub");
client2Id = getUniqueClientId("msgreact-send");
client1Id = getTestClientId("msgreact-sub");
client2Id = getTestClientId("msgreact-send");
});

afterEach(async () => {
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/rooms/rooms-messages-subscribe-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -24,8 +24,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Messages Subscribe E2E Tests", () => {
beforeEach(() => {
resetTestTracking();
testRoom = getUniqueChannelName("room-sub");
subscriberId = getUniqueClientId("subscriber");
senderId = getUniqueClientId("sender");
subscriberId = getTestClientId("subscriber");
senderId = getTestClientId("sender");
});

afterEach(async () => {
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/rooms/rooms-occupancy-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -25,7 +25,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Occupancy E2E Tests", () => {
beforeEach(() => {
resetTestTracking();
testRoom = getUniqueChannelName("room-occ");
clientId = getUniqueClientId("occ-client");
clientId = getTestClientId("occ-client");
});

afterEach(async () => {
Expand Down Expand Up @@ -91,7 +91,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Occupancy E2E Tests", () => {
await new Promise((resolve) => setTimeout(resolve, 2000));

// Enter presence to trigger an occupancy change
const enterClientId = getUniqueClientId("occ-enter");
const enterClientId = getTestClientId("occ-enter");
enterer = await startPresenceCommand(
[
"rooms",
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/rooms/rooms-presence-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -23,7 +23,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Presence E2E Tests", () => {
beforeEach(() => {
resetTestTracking();
testRoom = getUniqueChannelName("room-pres");
clientId = getUniqueClientId("pres-client");
clientId = getTestClientId("pres-client");
});

afterEach(async () => {
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/rooms/rooms-presence-subscribe-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand Down Expand Up @@ -35,8 +35,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Presence Subscribe E2E Tests", () => {
);

const testRoom = getUniqueChannelName("room-pres-sub");
const subClientId = getUniqueClientId("pres-sub");
const enterClientId = getUniqueClientId("pres-enter");
const subClientId = getTestClientId("pres-sub");
const enterClientId = getTestClientId("pres-enter");

// Start presence subscriber
const subscriber = await startSubscribeCommand(
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/rooms/rooms-reactions-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -24,8 +24,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Reactions E2E Tests", () => {
beforeEach(() => {
resetTestTracking();
testRoom = getUniqueChannelName("room-react");
client1Id = getUniqueClientId("react-sub");
client2Id = getUniqueClientId("react-send");
client1Id = getTestClientId("react-sub");
client2Id = getTestClientId("react-send");
});

afterEach(async () => {
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/rooms/rooms-typing-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -24,8 +24,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Rooms Typing E2E Tests", () => {
beforeEach(() => {
resetTestTracking();
testRoom = getUniqueChannelName("room-typing");
subscriberId = getUniqueClientId("type-sub");
typerId = getUniqueClientId("typer");
subscriberId = getTestClientId("type-sub");
typerId = getTestClientId("typer");
});

afterEach(async () => {
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/spaces/spaces-crud-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -23,7 +23,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Spaces CRUD E2E Tests", () => {
beforeEach(() => {
resetTestTracking();
spaceName = getUniqueChannelName("space");
clientId = getUniqueClientId("space-client");
clientId = getTestClientId("space-client");
});

afterEach(async () => {
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/spaces/spaces-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { describe, it, beforeEach, afterEach, expect } from "vitest";
import {
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
createTempOutputFile,
runLongRunningBackgroundProcess,
readProcessOutput,
Expand All @@ -29,8 +29,8 @@ describe("Spaces E2E Tests", () => {
testCommands.length = 0;

testSpaceId = getUniqueChannelName("space");
client1Id = getUniqueClientId("client1");
client2Id = getUniqueClientId("client2");
client1Id = getTestClientId("client1");
client2Id = getTestClientId("client2");
});

afterEach(async () => {
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/spaces/spaces-locations-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -26,7 +26,7 @@ describe.skipIf(SHOULD_SKIP_E2E)(
beforeEach(() => {
resetTestTracking();
spaceName = getUniqueChannelName("space-loc");
clientId = getUniqueClientId("loc-client");
clientId = getTestClientId("loc-client");
});

afterEach(async () => {
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/spaces/spaces-occupancy-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand All @@ -23,7 +23,7 @@ describe.skipIf(SHOULD_SKIP_E2E)("Spaces Occupancy E2E Tests", () => {
beforeEach(() => {
resetTestTracking();
spaceName = getUniqueChannelName("space-occ");
clientId = getUniqueClientId("occ-client");
clientId = getTestClientId("occ-client");
});

afterEach(async () => {
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/spaces/spaces-subscribe-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
E2E_API_KEY,
SHOULD_SKIP_E2E,
getUniqueChannelName,
getUniqueClientId,
getTestClientId,
cleanupTrackedResources,
setupTestFailureHandler,
resetTestTracking,
Expand Down Expand Up @@ -35,8 +35,8 @@ describe.skipIf(SHOULD_SKIP_E2E)("Spaces Subscribe E2E Tests", () => {
);

const spaceName = getUniqueChannelName("space-sub");
const subClientId = getUniqueClientId("space-sub-client");
const enterClientId = getUniqueClientId("space-enter-client");
const subClientId = getTestClientId("space-sub-client");
const enterClientId = getTestClientId("space-enter-client");

// Start the space subscriber
const subscriber = await startSubscribeCommand(
Expand Down
23 changes: 18 additions & 5 deletions test/helpers/e2e-test-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,23 @@ export function getUniqueChannelName(prefix: string): string {
}

/**
* Create a unique client ID for testing
* Number of client IDs each prefix cycles through. Ably counts every distinct
* client ID it sees towards the account's MAU, so E2E runs draw from a fixed
* pool rather than minting a new ID per test on every PR.
*/
export function getUniqueClientId(prefix = "cli-e2e-test"): string {
return `${prefix}-${randomUUID()}`;
const CLIENT_ID_POOL_SIZE = 8;

const clientIdPoolCursors = new Map<string, number>();

/**
* A client ID from a bounded pool. Successive calls with the same prefix
* return different IDs (up to the pool size), so a test that needs two
* distinct clients gets them; across runs the same IDs are reused.
*/
export function getTestClientId(prefix = "cli-e2e-test"): string {
const cursor = clientIdPoolCursors.get(prefix) ?? 0;
clientIdPoolCursors.set(prefix, cursor + 1);
return `${prefix}-${cursor % CLIENT_ID_POOL_SIZE}`;
}

/**
Expand All @@ -84,7 +97,7 @@ export function createAblyClient(): Ably.PubSubHttpClient {
// throw new Error('Structurally invalid E2E_ABLY_API_KEY detected');
}

const clientId = getUniqueClientId();
const clientId = getTestClientId();
const _keyPrefix = E2E_API_KEY.split(":")[0]?.split(".")[0] || "unknown-app";
const _keyId =
E2E_API_KEY.split(":")[0]?.split(".")[1]?.slice(0, 4) || "unknown-key";
Expand Down Expand Up @@ -118,7 +131,7 @@ export function createAblyRealtimeClient(): Ably.PubSubRealtimeClient {
// throw new Error('Structurally invalid E2E_ABLY_API_KEY detected');
}

const clientId = getUniqueClientId();
const clientId = getTestClientId();
const _keyPrefix = E2E_API_KEY.split(":")[0]?.split(".")[0] || "unknown-app";
const _keyId =
E2E_API_KEY.split(":")[0]?.split(".")[1]?.slice(0, 4) || "unknown-key";
Expand Down
3 changes: 3 additions & 0 deletions vitest.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ export default defineConfig({
setupFiles: ["./test/setup.ts", "./test/e2e/setup.ts"],
env: {
ABLY_API_KEY: undefined,
// One fixed identity for CLI subprocesses that don't pass
// --client-id, so runs don't mint a new MAU per fresh config dir.
ABLY_CLIENT_ID: "cli-e2e-default",
},
testTimeout: 20000, // Allow 20s per test for plenty of time on actions
hookTimeout: 60000, // 60 seconds for hooks
Expand Down