From 674d8a1ba03e2a92ec36c6270bbf801a66b82bd1 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Sat, 3 Oct 2026 00:57:10 -0700 Subject: [PATCH] feat: identify every gitops API request with a User-Agent Only `npm run sim` and `npm run check` identified themselves. Setup, pull, push, apply, promote, cleanup, rollback, call and audit sent Node's default `node` User-Agent, about a sixth of all api.vapi.ai traffic, so gitops usage beyond simulations couldn't be counted. - src/user-agent.ts: `vapi-gitops-/`, plus ` (ci)` when CI or GITHUB_ACTIONS is set. The command is the npm script that started the process (so `npm run apply` labels the pull and push it runs as apply, and a promotion's applies as promote), else the entry script's name. sim and check keep their fixed labels, which analytics already counts simulation runs by. - Every fetch to the Vapi API sends it. tests/user-agent-coverage.test.ts fails on a fetch without it and checks api.ts against a local server. - cleanup-safety and new-file-gate tests sent about a dozen requests to the real api.vapi.ai per `npm test` (fake key, 401s), which the new User-Agent made visible in the request logs, and which would have counted every fork's CI run as usage. They now point at a dead local address. - how-it-works.md says what the API sees; AGENTS.md says every request sends the header. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 + docs/guides/how-it-works.md | 9 ++++ src/api.ts | 4 ++ src/call.ts | 6 ++- src/cleanup.ts | 11 ++++- src/interactive.ts | 6 ++- src/push.ts | 2 + src/rollback-cmd.ts | 2 + src/setup.ts | 14 +++++-- src/user-agent.ts | 59 +++++++++++++++++++++++--- tests/cleanup-safety.test.ts | 7 +++- tests/new-file-gate.test.ts | 12 ++++-- tests/user-agent-coverage.test.ts | 60 +++++++++++++++++++++++++++ tests/user-agent.test.ts | 69 +++++++++++++++++++++++++++---- 14 files changed, 239 insertions(+), 24 deletions(-) create mode 100644 tests/user-agent-coverage.test.ts diff --git a/AGENTS.md b/AGENTS.md index 044cda4..18feb75 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -319,6 +319,8 @@ For changes under `src/`, `tests/` or `.github/`: table in the same change. - Changing an example under `examples/`? Doc snippets that start with `# examples/` must match the file exactly (`npm test` checks). +- Every request to the Vapi API sends `"User-Agent": userAgentGet()` from + `src/user-agent.ts`; `npm test` fails on a `fetch` without it. - Commit messages follow Conventional Commits (`fix(pull): …`, `docs: …`). - When you hit engine friction ("this should be better"), add or update an entry in `improvements.md` in the same change. Upstream's log collects diff --git a/docs/guides/how-it-works.md b/docs/guides/how-it-works.md index 6646e5c..92ef1f6 100644 --- a/docs/guides/how-it-works.md +++ b/docs/guides/how-it-works.md @@ -145,6 +145,15 @@ Tracks resource ID ↔ Vapi UUID mappings per org: Every resource type has a section. Keys are sorted, so diffs stay readable. +## What Vapi sees + +Every API request uses the org's private key and identifies the tool with a +User-Agent: `vapi-gitops-/`, plus ` (ci)` when the `CI` or +`GITHUB_ACTIONS` variable is set. For example, `npm run apply` in a GitHub +workflow sends `vapi-gitops-apply/1.0.0 (ci)`. Vapi uses it to count how +the tool is used. Nothing else is sent beyond the requests themselves; there +is no separate telemetry. + ## Where things live | Path | What it is | diff --git a/src/api.ts b/src/api.ts index 5fe790c..c7f0e88 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,5 +1,6 @@ import { DRY_RUN, VAPI_BASE_URL, VAPI_TOKEN } from "./config.ts"; import type { VapiResponse } from "./types.ts"; +import { userAgentGet } from "./user-agent.ts"; import { INITIAL_DELAY_MS, MAX_RETRIES, @@ -97,6 +98,7 @@ export async function vapiRequest( headers: { "Content-Type": "application/json", Authorization: `Bearer ${VAPI_TOKEN}`, + "User-Agent": userAgentGet(), }, body: JSON.stringify(body), }); @@ -140,6 +142,7 @@ export async function vapiGet(endpoint: string): Promise { method: "GET", headers: { Authorization: `Bearer ${VAPI_TOKEN}`, + "User-Agent": userAgentGet(), }, }); @@ -188,6 +191,7 @@ export async function vapiDelete(endpoint: string): Promise { method: "DELETE", headers: { Authorization: `Bearer ${VAPI_TOKEN}`, + "User-Agent": userAgentGet(), }, }); diff --git a/src/call.ts b/src/call.ts index 9e569c1..fb3b364 100644 --- a/src/call.ts +++ b/src/call.ts @@ -6,6 +6,7 @@ import { dirname, join, resolve } from "path"; import * as readline from "readline"; import { fileURLToPath } from "url"; import type { Environment, StateFile } from "./types.ts"; +import { userAgentGet } from "./user-agent.ts"; const require = createRequire(import.meta.url); @@ -362,6 +363,7 @@ async function createCall( headers: { "Content-Type": "application/json", Authorization: `Bearer ${config.token}`, + "User-Agent": userAgentGet(), }, body: JSON.stringify(body), }); @@ -1046,7 +1048,9 @@ function createMicrophoneStream(onData: (data: Buffer) => void): { } catch (error) { const msg = error instanceof Error ? error.message : String(error); if (msg.includes("Cannot find module")) { - console.warn("⚠️ 'mic' module not installed. Microphone input disabled."); + console.warn( + "⚠️ 'mic' module not installed. Microphone input disabled.", + ); console.warn(" Install with: npm install mic"); } else if (msg.includes("sox") || msg.includes("rec")) { console.warn("⚠️ sox/rec not found. Required for microphone input."); diff --git a/src/cleanup.ts b/src/cleanup.ts index 13723c7..4ded1d5 100644 --- a/src/cleanup.ts +++ b/src/cleanup.ts @@ -11,6 +11,7 @@ import { FOLDER_MAP } from "./resource-parse.ts"; import { slugify } from "./slug-utils.ts"; import { loadState } from "./state.ts"; import type { ResourceType } from "./types.ts"; +import { userAgentGet } from "./user-agent.ts"; // ───────────────────────────────────────────────────────────────────────────── // Dangerous Sync - Delete everything NOT in state file @@ -29,7 +30,10 @@ function isRecord(value: unknown): value is Record { async function vapiGet(endpoint: string, debug = false): Promise { await sleep(REQUEST_DELAY_MS); const response = await fetch(`${VAPI_BASE_URL}${endpoint}`, { - headers: { Authorization: `Bearer ${VAPI_TOKEN}` }, + headers: { + Authorization: `Bearer ${VAPI_TOKEN}`, + "User-Agent": userAgentGet(), + }, }); if (!response.ok) { throw new Error(`GET ${endpoint} failed: ${response.status}`); @@ -67,7 +71,10 @@ async function vapiDelete(endpoint: string): Promise { await sleep(REQUEST_DELAY_MS); const response = await fetch(`${VAPI_BASE_URL}${endpoint}`, { method: "DELETE", - headers: { Authorization: `Bearer ${VAPI_TOKEN}` }, + headers: { + Authorization: `Bearer ${VAPI_TOKEN}`, + "User-Agent": userAgentGet(), + }, }); if (!response.ok && response.status !== 404) { throw new Error(`DELETE ${endpoint} failed: ${response.status}`); diff --git a/src/interactive.ts b/src/interactive.ts index 9e9534a..6348df1 100644 --- a/src/interactive.ts +++ b/src/interactive.ts @@ -9,6 +9,7 @@ import searchableCheckbox, { BACK_SENTINEL } from "./searchableCheckbox.js"; // the launcher, which runs before any org/token is selected. import { isBackupCopyFile } from "./slug-utils.ts"; import type { StateFile } from "./types.ts"; +import { userAgentGet } from "./user-agent.ts"; // ───────────────────────────────────────────────────────────────────────────── // Constants @@ -223,7 +224,10 @@ async function apiGet( ): Promise { const response = await fetch(`${baseUrl}${endpoint}`, { method: "GET", - headers: { Authorization: `Bearer ${token}` }, + headers: { + Authorization: `Bearer ${token}`, + "User-Agent": userAgentGet(), + }, }); if (!response.ok) { const text = await response.text(); diff --git a/src/push.ts b/src/push.ts index 5bac449..ea80668 100644 --- a/src/push.ts +++ b/src/push.ts @@ -34,6 +34,7 @@ import { import { reconcileStateKeyForResource } from "./reconcile-state-key.ts"; import { writeSnapshot } from "./snapshot.ts"; import { mergeScoped } from "./state-merge.ts"; +import { userAgentGet } from "./user-agent.ts"; import { summarizeFindings, validateNoIgnoredReferences, @@ -303,6 +304,7 @@ async function upsertResourceWithStateRecovery(options: { method: "GET", headers: { Authorization: `Bearer ${process.env.VAPI_TOKEN}`, + "User-Agent": userAgentGet(), }, }, ); diff --git a/src/rollback-cmd.ts b/src/rollback-cmd.ts index ee3e0f2..6908bfb 100644 --- a/src/rollback-cmd.ts +++ b/src/rollback-cmd.ts @@ -13,6 +13,7 @@ import { existsSync, readFileSync } from "fs"; import { dirname, join } from "path"; import { fileURLToPath } from "url"; import { listSnapshotTimestamps, loadSnapshot } from "./snapshot.ts"; +import { userAgentGet } from "./user-agent.ts"; const __dirname = dirname(fileURLToPath(import.meta.url)); const BASE_DIR = join(__dirname, ".."); @@ -186,6 +187,7 @@ async function main(): Promise { headers: { Authorization: `Bearer ${cfg.token}`, "Content-Type": "application/json", + "User-Agent": userAgentGet(), }, body: JSON.stringify(entry.payload.platform), }); diff --git a/src/setup.ts b/src/setup.ts index e87d66a..4255e3e 100644 --- a/src/setup.ts +++ b/src/setup.ts @@ -20,6 +20,7 @@ import { SETUP_USAGE, } from "./setup-args.ts"; import { slugify } from "./slug-utils.ts"; +import { userAgentGet } from "./user-agent.ts"; // ───────────────────────────────────────────────────────────────────────────── // Constants @@ -94,7 +95,10 @@ const c = { async function apiGet(token: string, endpoint: string): Promise { const response = await fetch(`${vapiBaseUrl}${endpoint}`, { method: "GET", - headers: { Authorization: `Bearer ${token}` }, + headers: { + Authorization: `Bearer ${token}`, + "User-Agent": userAgentGet(), + }, }); if (!response.ok) { @@ -367,7 +371,9 @@ async function runDirectSetup(options: DirectSetupOptions): Promise { const resourceDir = join(BASE_DIR, "resources", slug); const stateFile = join(BASE_DIR, `.vapi-state.${slug}.json`); - console.log(c.bold(`\n Vapi GitOps — non-interactive setup for "${slug}"\n`)); + console.log( + c.bold(`\n Vapi GitOps — non-interactive setup for "${slug}"\n`), + ); // Never clobber an org that already has local state. Re-running setup is // a destructive operation in the wizard (it deletes and re-pulls), and an @@ -517,7 +523,9 @@ async function main(): Promise { // so explain the non-interactive path instead. if (!process.stdin.isTTY) { console.error( - c.red("\n ✗ The setup wizard needs an interactive terminal (stdin is not a TTY).\n"), + c.red( + "\n ✗ The setup wizard needs an interactive terminal (stdin is not a TTY).\n", + ), ); console.error(SETUP_USAGE); process.exit(1); diff --git a/src/user-agent.ts b/src/user-agent.ts index 39a383b..866f305 100644 --- a/src/user-agent.ts +++ b/src/user-agent.ts @@ -1,11 +1,19 @@ -// User-Agent for the API requests this tool makes, so simulation runs started -// from gitops can be told apart in the platform's analytics. +// User-Agent for every API request this tool makes, so gitops traffic can be +// told apart in the platform's request logs and analytics: +// +// vapi-gitops-/[ (ci)] +// +// `` is the npm script that started the process (`npm run apply` +// labels the pull and push it runs as `apply`), or the entry script's name +// when it was run directly, as the PR check workflow does. The `sim` and +// `check` labels are fixed by their callers, because analytics already counts +// simulation runs by those prefixes; keep them stable. // // Config-free on purpose (like api-key.ts): importing config.ts would parse // argv and exit, which breaks importing this from sim.ts and tests. import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; const PACKAGE_JSON_PATH = join( @@ -14,6 +22,12 @@ const PACKAGE_JSON_PATH = join( "package.json", ); +export interface UserAgentContext { + env: NodeJS.ProcessEnv; + // The entry script, process.argv[1]. + scriptPath?: string; +} + function packageVersionRead(): string { try { const parsed: unknown = JSON.parse( @@ -34,6 +48,41 @@ function packageVersionRead(): string { return "unknown"; } -export function userAgentGet(product: "sim" | "check"): string { - return `vapi-gitops-${product}/${packageVersionRead()}`; +const PACKAGE_VERSION = packageVersionRead(); + +// A User-Agent product token allows few characters; keep to a safe subset. +function tokenClean(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, ""); +} + +function commandNameGet(context: UserAgentContext): string { + const npmScript = context.env.npm_lifecycle_event; + if (npmScript && tokenClean(npmScript)) return tokenClean(npmScript); + const script = context.scriptPath + ? basename(context.scriptPath).replace(/\.[cm]?[jt]s$/, "") + : ""; + return tokenClean(script.replace(/-cmd$/, "")) || "cli"; +} + +function ciRun(env: NodeJS.ProcessEnv): boolean { + const ci = env.CI?.toLowerCase(); + return ( + env.GITHUB_ACTIONS === "true" || + (ci !== undefined && ci !== "" && ci !== "false" && ci !== "0") + ); +} + +export function userAgentGet( + product?: "sim" | "check", + context: UserAgentContext = { + env: process.env, + scriptPath: process.argv[1], + }, +): string { + const command = product ?? commandNameGet(context); + const ci = ciRun(context.env) ? " (ci)" : ""; + return `vapi-gitops-${command}/${PACKAGE_VERSION}${ci}`; } diff --git a/tests/cleanup-safety.test.ts b/tests/cleanup-safety.test.ts index 0575aba..08749c9 100644 --- a/tests/cleanup-safety.test.ts +++ b/tests/cleanup-safety.test.ts @@ -92,7 +92,12 @@ function runCleanup( ["--import", "tsx", "src/cleanup.ts", "test-cleanup-org", ...args], { cwd, - env: { ...process.env, VAPI_TOKEN: "fake-token-not-used" }, + env: { + ...process.env, + VAPI_TOKEN: "fake-token-not-used", + // Nothing listens here: tests must never reach the real API. + VAPI_BASE_URL: "http://127.0.0.1:9", + }, encoding: "utf-8", timeout: 20_000, }, diff --git a/tests/new-file-gate.test.ts b/tests/new-file-gate.test.ts index 4ffb6d1..5860f2c 100644 --- a/tests/new-file-gate.test.ts +++ b/tests/new-file-gate.test.ts @@ -19,9 +19,8 @@ import { fileURLToPath } from "node:url"; process.argv = ["node", "test", "test-fixture-org"]; process.env.VAPI_TOKEN = process.env.VAPI_TOKEN || "test-token-not-used"; -const { detectOrphanYamls, formatGateMessage } = await import( - "../src/new-file-gate.ts" -); +const { detectOrphanYamls, formatGateMessage } = + await import("../src/new-file-gate.ts"); import type { OrphanReport } from "../src/new-file-gate.ts"; import type { ResourceState, ResourceType, StateFile } from "../src/types.ts"; @@ -459,7 +458,12 @@ function runPush( ["--import", "tsx", "src/push.ts", fx.env, ...extraArgs], { cwd: fx.dir, - env: { ...process.env, VAPI_TOKEN: "fake-token-not-used" }, + env: { + ...process.env, + VAPI_TOKEN: "fake-token-not-used", + // Nothing listens here: tests must never reach the real API. + VAPI_BASE_URL: "http://127.0.0.1:9", + }, encoding: "utf-8", timeout: 30_000, }, diff --git a/tests/user-agent-coverage.test.ts b/tests/user-agent-coverage.test.ts new file mode 100644 index 0000000..7da62db --- /dev/null +++ b/tests/user-agent-coverage.test.ts @@ -0,0 +1,60 @@ +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +// Every request gitops makes to the Vapi API must carry the gitops +// User-Agent, or that traffic is indistinguishable from any other Node +// script ("node"). api.ts carries push, pull, apply and promote, so it's +// checked against a real server; the other call sites are checked by +// reading them. + +const SRC = fileURLToPath(new URL("../src", import.meta.url)); + +// GitHub's API, not Vapi's: the commit status client sets its own. +const NOT_VAPI = new Set(["check-status.ts"]); + +test("every fetch to the Vapi API in src/ sets the User-Agent", () => { + const missing: string[] = []; + for (const file of readdirSync(SRC).filter((f) => f.endsWith(".ts"))) { + if (NOT_VAPI.has(file)) continue; + const lines = readFileSync(join(SRC, file), "utf8").split("\n"); + lines.forEach((line, index) => { + if (!/\bfetch\(/.test(line)) return; + // The options object follows within a few lines. + const call = lines.slice(index, index + 12).join("\n"); + if (!call.includes('"User-Agent"')) missing.push(`${file}:${index + 1}`); + }); + } + assert.deepEqual(missing, []); +}); + +test("api.ts requests carry the command's User-Agent", async () => { + const seen: Array = []; + const server = createServer((req, res) => { + seen.push(req.headers["user-agent"]); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end("[]"); + }); + await new Promise((resolve) => server.listen(0, resolve)); + const { port } = server.address() as AddressInfo; + // config.ts reads these at import. + process.argv = ["node", "src/push.ts", "ua-test-org"]; + process.env.VAPI_TOKEN = "test-token-not-used"; + process.env.VAPI_BASE_URL = `http://127.0.0.1:${port}`; + process.env.npm_lifecycle_event = "apply"; + delete process.env.CI; + delete process.env.GITHUB_ACTIONS; + try { + const { vapiGet } = await import("../src/api.ts"); + const { userAgentGet } = await import("../src/user-agent.ts"); + await vapiGet("/assistant"); + assert.deepEqual(seen, [userAgentGet()]); + assert.match(seen[0] ?? "", /^vapi-gitops-apply\/[^ ]+$/); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/tests/user-agent.test.ts b/tests/user-agent.test.ts index 5ba18f8..efc4ae5 100644 --- a/tests/user-agent.test.ts +++ b/tests/user-agent.test.ts @@ -6,18 +6,73 @@ import test from "node:test"; import { runSimulation } from "../src/sim.ts"; import { userAgentGet } from "../src/user-agent.ts"; -// The User-Agent is how gitops-started simulation runs are counted in the -// platform's analytics (`user_agent` on the run-started event), so its -// format is a contract worth pinning. +// The User-Agent is how gitops traffic is counted in the platform's request +// logs and analytics (simulation runs by `user_agent` on the run-started +// event), so its format is a contract worth pinning. const packageJsonPath = new URL("../package.json", import.meta.url); const packageVersion = ( JSON.parse(readFileSync(packageJsonPath, "utf-8")) as { version: string } ).version; -test("userAgentGet: names the product and the package version", () => { - assert.equal(userAgentGet("sim"), `vapi-gitops-sim/${packageVersion}`); - assert.equal(userAgentGet("check"), `vapi-gitops-check/${packageVersion}`); +const at = ( + env: NodeJS.ProcessEnv, + scriptPath?: string, + product?: "sim" | "check", +) => userAgentGet(product, { env, scriptPath }); + +test("userAgentGet: sim and check keep their fixed labels", () => { + assert.deepEqual( + [ + at({}, "/repo/src/sim-cmd.ts", "sim"), + at( + { npm_lifecycle_event: "promote" }, + "/repo/src/promote-cmd.ts", + "check", + ), + ], + [ + `vapi-gitops-sim/${packageVersion}`, + `vapi-gitops-check/${packageVersion}`, + ], + ); +}); + +test("userAgentGet: names the npm script, else the entry script", () => { + assert.deepEqual( + [ + // `npm run apply` runs pull.ts and push.ts as children: still apply. + at({ npm_lifecycle_event: "apply" }, "/repo/src/push.ts"), + at({}, "/repo/src/check-cmd.ts"), + at({}, "/repo/src/pull.ts"), + at({ npm_lifecycle_event: "check:All" }), + at({}), + ], + [ + `vapi-gitops-apply/${packageVersion}`, + `vapi-gitops-check/${packageVersion}`, + `vapi-gitops-pull/${packageVersion}`, + `vapi-gitops-check-all/${packageVersion}`, + `vapi-gitops-cli/${packageVersion}`, + ], + ); +}); + +test("userAgentGet: marks runs in CI", () => { + const marked = (env: NodeJS.ProcessEnv) => + at({ npm_lifecycle_event: "push", ...env }).endsWith(" (ci)"); + assert.deepEqual( + [ + { GITHUB_ACTIONS: "true" }, + { CI: "true" }, + { CI: "1" }, + { CI: "false" }, + { CI: "0" }, + { CI: "" }, + {}, + ].map(marked), + [true, true, true, false, false, false, false], + ); }); test("runSimulation: sends the sim User-Agent on run create", async () => { @@ -53,5 +108,5 @@ test("runSimulation: sends the sim User-Agent on run create", async () => { } assert.equal(seen.method, "POST"); assert.equal(seen.url, "/eval/simulation/run"); - assert.equal(seen.userAgent, `vapi-gitops-sim/${packageVersion}`); + assert.equal(seen.userAgent, userAgentGet("sim")); });