From 92656e0b4dc4baf5c05737d65ce5a69581619908 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Fri, 2 Oct 2026 23:41:12 -0700 Subject: [PATCH] docs: prepare for public release: scrub internal names, fix examples, add community files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove customer, product and person names, a customer incident's details, internal tool names and internal ticket IDs from docs, agent instructions, code comments and test fixtures. Lessons are kept in neutral terms; fixture renames don't change what tests check. - Fix README examples a new user would copy and break on: personalities need an assistant and scenarios need instructions and evaluations (the old examples used fields the API rejects), state files store {"uuid": …} entries, and squads hand off through handoff tools. - Add examples/starter, a complete small org, and build the README's File Formats section from its files. tests/examples.test.ts checks that every example org passes validate and has the fields the API requires, that the starter's PR check builds cleanly, and that every doc snippet naming an example file matches it exactly. - Add CONTRIBUTING.md, SECURITY.md (GitHub private vulnerability reporting), and issue forms; update package.json's description. Co-Authored-By: Claude Opus 5.5 --- .conductor/setup.sh | 2 +- .github/ISSUE_TEMPLATE/bug_report.yml | 34 ++++ .github/ISSUE_TEMPLATE/config.yml | 8 + .github/ISSUE_TEMPLATE/feature_request.yml | 15 ++ .gitignore | 2 +- CLAUDE.md | 5 +- CONTRIBUTING.md | 47 +++++ README.md | 189 +++++++++++------- SECURITY.md | 26 +++ docs/learnings/simulations.md | 2 +- docs/learnings/voicemail-detection.md | 4 +- examples/starter/.vapi-state.starter.json | 1 + examples/starter/README.md | 21 ++ .../starter/assistants/receptionist.md | 31 +++ .../resources/starter/assistants/scheduler.md | 18 ++ .../simulations/personalities/calm-caller.yml | 10 + .../simulations/scenarios/books-cleaning.yml | 15 ++ .../starter/simulations/suites/core.yml | 3 + .../simulations/tests/books-cleaning-calm.yml | 3 + .../resources/starter/squads/front-desk.yml | 4 + .../structuredOutputs/booking-confirmed.yml | 5 + .../structuredOutputs/call-summary.yml | 10 + .../starter/tools/book-appointment.yml | 16 ++ .../starter/tools/handoff-to-scheduler.yml | 7 + .../starter/tools/lookup-patient.yml | 14 ++ examples/starter/vapi-checks.yml | 6 + improvements.md | 54 ++--- package.json | 2 +- src/audit.ts | 4 +- src/check-status.ts | 2 +- src/credentials.ts | 2 +- src/new-file-gate.ts | 2 +- src/pull.ts | 2 +- src/push.ts | 2 +- src/types.ts | 2 +- tests/audit.test.ts | 28 +-- tests/credentials.test.ts | 4 +- tests/drift.test.ts | 2 +- tests/examples.test.ts | 169 ++++++++++++++++ tests/pull-same-name-clobber.test.ts | 2 +- tests/recanonicalize.test.ts | 8 +- tests/slug-utils.test.ts | 4 +- 42 files changed, 652 insertions(+), 135 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md create mode 100644 examples/starter/.vapi-state.starter.json create mode 100644 examples/starter/README.md create mode 100644 examples/starter/resources/starter/assistants/receptionist.md create mode 100644 examples/starter/resources/starter/assistants/scheduler.md create mode 100644 examples/starter/resources/starter/simulations/personalities/calm-caller.yml create mode 100644 examples/starter/resources/starter/simulations/scenarios/books-cleaning.yml create mode 100644 examples/starter/resources/starter/simulations/suites/core.yml create mode 100644 examples/starter/resources/starter/simulations/tests/books-cleaning-calm.yml create mode 100644 examples/starter/resources/starter/squads/front-desk.yml create mode 100644 examples/starter/resources/starter/structuredOutputs/booking-confirmed.yml create mode 100644 examples/starter/resources/starter/structuredOutputs/call-summary.yml create mode 100644 examples/starter/resources/starter/tools/book-appointment.yml create mode 100644 examples/starter/resources/starter/tools/handoff-to-scheduler.yml create mode 100644 examples/starter/resources/starter/tools/lookup-patient.yml create mode 100644 examples/starter/vapi-checks.yml create mode 100644 tests/examples.test.ts diff --git a/.conductor/setup.sh b/.conductor/setup.sh index b4fd020..ffc2a84 100755 --- a/.conductor/setup.sh +++ b/.conductor/setup.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Conductor setup script for gitops-mudflap. +# Conductor setup script for this repository. # # Wire-up: this script is dispatched from `conductor.json` at the repo root: # {"scripts": {"setup": "bash .conductor/setup.sh"}} diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..52b5b67 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,34 @@ +name: Bug report +description: Something in the sync engine, CLI or workflows didn't work as documented. +labels: [bug] +body: + - type: textarea + id: what-happened + attributes: + label: What happened? + description: What you expected, and what happened instead. + validations: + required: true + - type: textarea + id: command + attributes: + label: Command and output + description: The exact command you ran and its output. Remove API keys and any customer data. + render: shell + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: The commit or release of this repo you're on (`git log -1 --oneline`). + - type: input + id: node + attributes: + label: Node version + description: Output of `node --version`. + - type: dropdown + id: region + attributes: + label: Vapi region + options: [US, EU] diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..7992db1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Report a security vulnerability + url: https://github.com/VapiAI/gitops/security/advisories/new + about: Please report security issues privately, not as public issues. + - name: Vapi platform docs and support + url: https://docs.vapi.ai + about: Questions about the Vapi platform itself rather than this repository. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..336ee25 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,15 @@ +name: Feature request +description: Suggest a change or new capability. +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: What problem would this solve? + description: Describe what you're trying to do and what gets in the way today. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: What would you like to happen? diff --git a/.gitignore b/.gitignore index 9520d34..7e16615 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ node_modules/ pnpm-lock.yaml # Environment files (secrets - never commit these!) -# Covers dev/stg/prod and any org slug (e.g. .env.roofr-production) +# Covers dev/stg/prod and any org slug (e.g. .env.acme-production) .env .env.* !.env.example diff --git a/CLAUDE.md b/CLAUDE.md index 85f964e..3c64d2e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ When both files exist, follow both. If guidance overlaps, treat `AGENTS.md` as t **The Vapi PATCH API does NOT deep-merge nested objects. When you PATCH a nested object (`model`, `voice`, `transcriber`, `messagePlan`, `analysisPlan`, `artifactPlan`, `voicemailDetection`, `startSpeakingPlan`, `stopSpeakingPlan`) with a partial body, the API REPLACES the entire object — wiping every field you didn't include.** -This wiped three live-production assistants' system prompts on 2026-05-13 (gitops-mudflap iForm barge fleet). The PATCH was `{"model": {"model": "gpt-4.1", "provider": "openai", "maxTokens": 260, "temperature": 0.3, "toolIds": [...]}}` — looked complete, but did NOT include `model.messages`. Result: prompts gone, live calls ran with empty system prompt until the operator forced a restore. +This wiped three live-production assistants' system prompts in a customer fork on 2026-05-13. The PATCH was `{"model": {"model": "gpt-4.1", "provider": "openai", "maxTokens": 260, "temperature": 0.3, "toolIds": [...]}}` — looked complete, but did NOT include `model.messages`. Result: prompts gone, live calls ran with empty system prompt until the operator forced a restore. **Mandatory workflow for any direct API PATCH against a nested object:** @@ -88,8 +88,7 @@ file paths with line numbers so future readers can verify your claims. When a fix lands, mark the entry `[RESOLVED YYYY-MM-DD] (#)` at the top — don't delete it. The history is the point. -Customer-fork logs (`gitops-mudflap/improvements.md`, -`gitops-amazon3p/improvements.md`) feed upstream: when an entry there is +Customer forks' own `improvements.md` logs feed upstream: when an entry there is generic enough to apply across customers, surface it here in the same revision. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d8be3ad --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,47 @@ +# Contributing + +Thanks for helping improve Vapi GitOps. Bug reports, docs fixes and new +learnings are all welcome. + +## Reporting a problem + +- **Bugs and feature requests:** open a [GitHub issue](https://github.com/VapiAI/gitops/issues/new/choose). + For bugs, include the command you ran, its output (with API keys removed), + your Node version, and whether the org is in the US or EU region. +- **Security issues:** don't open a public issue. Follow [SECURITY.md](SECURITY.md). +- **Questions about the Vapi platform itself** (not this repo) belong with + [Vapi support](https://docs.vapi.ai). + +## Making a change + +1. Fork the repository and create a branch. +2. Install and check that everything passes before you start: + + ```bash + nvm use + npm ci + npm run build # type-checks src/ and tests/ + npm test + ``` + +3. Make your change, with tests for any behaviour change. Tests use + `node:test` and live in `tests/`; they never call the real Vapi API. +4. Run `npm run build` and `npm test` again, then open a pull request. + +Commit messages and PR titles follow [Conventional Commits](https://www.conventionalcommits.org/) +(`fix(pull): …`, `feat(check): …`, `docs: …`). + +## Where things go + +| You're adding… | Put it in | +| --- | --- | +| A Vapi platform gotcha, recipe or troubleshooting guide | `docs/learnings/.md`, plus a row in [`docs/learnings/README.md`](docs/learnings/README.md) and the table in `AGENTS.md` for a new file | +| A sync-engine pain point and its fix (pull, push, state, cleanup) | `improvements.md`, in its Problem → Current behavior → Risk → Current mitigation → Possible fix → Status format | +| Setup or orientation for new users | `README.md` (keep it short; link to a guide for depth) | +| An example users can copy | `examples/`. Snippets in the docs that start with `# examples/` must match the file exactly; `npm test` checks this. | + +Don't edit `docs/changelog.md` here. It's a template for your own +deployment's change log once you fork the repo. + +Coding agents (Claude Code, Cursor, Codex) read [`AGENTS.md`](AGENTS.md) and +[`CLAUDE.md`](CLAUDE.md); keep them in step when a convention changes. diff --git a/README.md b/README.md index 195397f..3e36f03 100644 --- a/README.md +++ b/README.md @@ -750,134 +750,188 @@ Squad push ## File Formats -### Assistants with System Prompts (`.md`) +Every snippet below is a file from [`examples/starter/`](examples/starter/), a +small dental-clinic front desk with two assistants, tools, a handoff and a +simulation suite. CI checks that each snippet matches its file and that the +example passes `validate`, so you can copy from here safely. -Markdown with YAML frontmatter — the system prompt is readable Markdown below the config: +A resource's ID is its path under the type folder, without the extension +(`tools/lookup-patient.yml` is `lookup-patient`). Reference other resources +by that ID, never by UUID: the engine resolves IDs to UUIDs per org. + +### Assistants (`.md` or `.yml`) + +Markdown with YAML frontmatter: the frontmatter is the assistant config and the +body is its system prompt. ```markdown + --- -name: My Assistant -voice: - provider: 11labs - voiceId: abc123 +name: Receptionist +firstMessage: Thanks for calling Bright Smile Dental. How can I help? model: - model: gpt-4.1 provider: openai + model: gpt-4.1 + temperature: 0.3 toolIds: - - my-tool -firstMessage: Hello! How can I help you? + - lookup-patient + - handoff-to-scheduler + tools: + - type: endCall +voice: + provider: 11labs + voiceId: sarah +artifactPlan: + structuredOutputIds: + - call-summary --- -# Identity & Purpose - -You are a helpful assistant for the business you represent. +# Identity -# Conversation Flow +You are the receptionist for Bright Smile Dental, 123 Main St. The clinic is +open Monday to Friday, 8am to 5pm. -1. Greet the user -2. Ask how you can help -3. Resolve their issue +# Flow -# Rules - -- Always be polite -- Never make up information +1. Ask for the caller's phone number and call `lookup_patient` with it. +2. If they want to book, change or check an appointment, hand off to the + Scheduler with `handoff_to_scheduler`. Don't book anything yourself. +3. Answer general questions (hours, address) briefly yourself. ``` ### Tools (`.yml`) ```yaml +# examples/starter/resources/starter/tools/lookup-patient.yml type: function function: - name: get_weather - description: Get the current weather for a location + name: lookup_patient + description: Look up the caller's patient record by phone number. parameters: type: object properties: - location: + phone: type: string - description: The city name + description: The caller's phone number required: - - location + - phone server: - url: https://my-api.com/weather + url: https://example.com/vapi/lookup-patient +``` + +Handoffs between assistants are tools too. Give each one an explicit +`function.name` if your prompts mention it by name: + +```yaml +# examples/starter/resources/starter/tools/handoff-to-scheduler.yml +type: handoff +function: + name: handoff_to_scheduler +destinations: + - type: assistant + assistantId: scheduler + description: Books, changes and checks appointments. ``` ### Structured Outputs (`.yml`) ```yaml -name: Call Summary +# examples/starter/resources/starter/structuredOutputs/call-summary.yml +name: call-summary type: ai -description: Summarizes the key points of a call +description: Summarizes the call for the front-desk log. schema: type: object properties: summary: type: string - sentiment: - type: string - enum: [positive, neutral, negative] -assistant_ids: - - my-assistant + booked: + type: boolean ``` ### Squads (`.yml`) ```yaml -name: Support Squad +# examples/starter/resources/starter/squads/front-desk.yml +name: Front Desk members: - - assistantId: intake-agent - assistantDestinations: - - type: assistant - assistantId: specialist-agent - message: Transferring you to a specialist. - - assistantId: specialist-agent + - assistantId: receptionist + - assistantId: scheduler ``` +Members hand off to each other through handoff tools on the assistants, as +above. Prefer them over the legacy `assistantDestinations` field. + ### Evals (`.yml`) -```yaml -name: Booking Happy Path -type: eval -# (eval config as per Vapi API) -``` +An eval file is the body of the [Evals API](https://docs.vapi.ai/api-reference/evals) +create request, written as YAML. ### Simulations -**Personality** (`simulations/personalities/`): +**Personality** (`simulations/personalities/`): the simulated caller, as an +assistant config. ```yaml -name: Skeptical Sam -description: A doubtful caller who questions everything -prompt: You are skeptical and need convincing before trusting information. +# examples/starter/resources/starter/simulations/personalities/calm-caller.yml +name: Calm caller +assistant: + model: + provider: openai + model: gpt-4.1-mini + messages: + - role: system + content: > + You are a patient calling a dental clinic. Follow your scenario, + answer questions briefly, and don't invent details. ``` -**Scenario** (`simulations/scenarios/`): +**Scenario** (`simulations/scenarios/`): what the caller does, how the call is +judged (at least one evaluation), and mock results for the tools it calls. ```yaml -name: Happy Path - New Customer -description: New customer calling to schedule an appointment -prompt: | - You are a new customer calling to schedule your first appointment. +# examples/starter/resources/starter/simulations/scenarios/books-cleaning.yml +name: Books a cleaning +instructions: > + You are Jordan Lee, phone 206-555-0142, an existing patient. Book a teeth + cleaning for next Tuesday morning and accept the first slot offered. Once + the booking is confirmed, say thanks and goodbye. +evaluations: + - structuredOutputId: booking-confirmed + comparator: "=" + value: true + required: true +toolMocks: + - toolName: lookup_patient + result: '{"found": true, "patientId": "P-1001"}' + - toolName: book_appointment + result: '{"success": true, "date": "next Tuesday", "time": "09:00"}' ``` -**Simulation** (`simulations/tests/`): +**Simulation** (`simulations/tests/`): a personality paired with a scenario. ```yaml -name: Booking Test Case 1 -personalityId: skeptical-sam -scenarioId: happy-path-new-customer +# examples/starter/resources/starter/simulations/tests/books-cleaning-calm.yml +name: Books a cleaning (calm caller) +personalityId: calm-caller +scenarioId: books-cleaning ``` **Simulation Suite** (`simulations/suites/`): ```yaml -name: Booking Flow Tests +# examples/starter/resources/starter/simulations/suites/core.yml +name: Core simulationIds: - - booking-test-case-1 - - booking-test-case-2 + - books-cleaning-calm ``` +### TypeScript resources (`.ts`) + +Any resource can also be a `.ts` file whose default export is the resource +object, useful for generating config. It is executed when loaded, so treat +`.ts` resources like code in review. + --- ## How the Engine Works @@ -947,7 +1001,7 @@ server: credentialId: my-server-credential # State file (environment-specific) -# "my-server-credential": "2f6db611-ad08-4099-8bd8-74db37b0a07e" +# "credentials": { "my-server-credential": { "uuid": "2f6db611-ad08-4099-8bd8-74db37b0a07e" } } ``` ### State File @@ -956,14 +1010,15 @@ Tracks resource ID ↔ Vapi UUID mappings per org: ```json { - "credentials": { "my-cred": "uuid-0000" }, - "tools": { "my-tool": "uuid-1234" }, - "assistants": { "my-assistant": "uuid-5678" }, - "squads": { "my-squad": "uuid-abcd" }, - "evals": { "booking-happy-path": "uuid-efgh" } + "assistants": { "my-assistant": { "uuid": "9c0f3f42-…" } }, + "credentials": { "my-cred": { "uuid": "2f6db611-…" } }, + "squads": { "my-squad": { "uuid": "51a9e1c7-…" } }, + "tools": { "my-tool": { "uuid": "d4b8a2e0-…" } } } ``` +Every resource type has a section. Keys are sorted, so diffs stay readable. + --- ## Project Structure diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..bca35cc --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,26 @@ +# Security + +## Reporting a vulnerability + +Please report security issues privately through GitHub: +[**Report a vulnerability**](https://github.com/VapiAI/gitops/security/advisories/new) +(the repository's Security tab). Don't open a public issue or pull request +for a suspected vulnerability. + +Include what you found, how to reproduce it, and the impact you expect. We'll +acknowledge the report and keep you updated as we investigate. + +## Keeping your deployment safe + +- **Never commit API keys.** Keys live in `.env.` files, which are + gitignored, or in CI secrets. The tools never accept a key as a command-line + flag, so it can't leak into shell history. +- **Committed state contains IDs, not secrets.** `.vapi-state..json` maps + resource names to Vapi UUIDs. Credential secrets and phone-number + provisioning are never written to the repository. +- **`.ts` resource files run code.** Loading a TypeScript resource executes its + module, including in `validate`, `promote` plans and PR checks. Review them + like code, and only run them from trusted branches. +- **PR checks run with your key on same-repository branches.** Forked PRs get + a dry run with no secrets. See "Cost and safety" in the README's + PR Checks section for what a check still sends to real providers. diff --git a/docs/learnings/simulations.md b/docs/learnings/simulations.md index 5578c99..6541d1b 100644 --- a/docs/learnings/simulations.md +++ b/docs/learnings/simulations.md @@ -236,7 +236,7 @@ branch's files. Setup is in the README's "PR Checks" section; these are the behaviours worth knowing when a check surprises you. - **Inline matches stored, with two known differences.** A 2026-10-01 parity - run (TEST-141) scored inline and stored versions of the same squad 15/15 + run scored inline and stored versions of the same squad 15/15 each, with the same handoff and business-tool sequences. The differences: - **Generated handoff names:** `handoff_to_` inline vs `handoff_to_` stored. A prompt or judge that names the generated diff --git a/docs/learnings/voicemail-detection.md b/docs/learnings/voicemail-detection.md index 565ce90..a5ac53e 100644 --- a/docs/learnings/voicemail-detection.md +++ b/docs/learnings/voicemail-detection.md @@ -51,7 +51,7 @@ Outbound Call ### ⚠️ Platform `voicemailDetection` MUST be disabled on the gatekeeper (Two-Agent Relay) -This is the single most non-obvious failure mode in two-agent voicemail relay setups. If the gatekeeper assistant has Vapi's platform `voicemailDetection: { provider: vapi }` configured, the call will **silently break on every voicemail** — the bot never speaks, the message never lands, the call ends with `endedReason: voicemail` after ~10–22s of dead air. We hit this on mudflap-test's iform voicemail triage squad on 2026-05-14 and only diagnosed it after pulling full Axiom event timelines. +This is the single most non-obvious failure mode in two-agent voicemail relay setups. If the gatekeeper assistant has Vapi's platform `voicemailDetection: { provider: vapi }` configured, the call will **silently break on every voicemail** — the bot never speaks, the message never lands, the call ends with `endedReason: voicemail` after ~10–22s of dead air. We hit this on a production voicemail triage squad on 2026-05-14 and only diagnosed it after pulling the full call event timelines. **The mechanism (verified via call 019e2827 on 2026-05-14):** @@ -69,7 +69,7 @@ This is the single most non-obvious failure mode in two-agent voicemail relay se **Fix:** Disable platform `voicemailDetection` on the gatekeeper assistant. Either: -- **Recommended**: don't set the field at all on the gatekeeper. Multilingual triage classifiers that never had `voicemailDetection` configured (e.g. `iform-triage-classifier-multilingual-d98136d9`, `iform-triage-multilingual-classic-f6b53e27` on mudflap-test) work where same-shape squads with VMD-on classifiers failed. +- **Recommended**: don't set the field at all on the gatekeeper. Multilingual triage classifiers that never had `voicemailDetection` configured (two multilingual classifiers in the same test org) work where same-shape squads with VMD-on classifiers failed. - **If you can't modify the underlying assistant** (e.g. a customer is gatekeeping the base classifier UUID for another reason): fork the classifier and use the fork in the squad. `assistantOverrides.voicemailDetection: null` does **NOT** work — Vapi's API silently drops the field. Verified via direct PATCH test on 2026-05-14. - **Never** set `voicemailDetection` on the gatekeeper AND rely on the handoff path. They are mutually exclusive architectures. diff --git a/examples/starter/.vapi-state.starter.json b/examples/starter/.vapi-state.starter.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/examples/starter/.vapi-state.starter.json @@ -0,0 +1 @@ +{} diff --git a/examples/starter/README.md b/examples/starter/README.md new file mode 100644 index 0000000..3d83969 --- /dev/null +++ b/examples/starter/README.md @@ -0,0 +1,21 @@ +# Starter example + +A small dental-clinic front desk, as a complete gitops org: + +- two assistants (`receptionist`, `scheduler`) with a handoff between them; +- function tools, a structured output, and a squad (`front-desk`); +- a simulation suite (`core`) with a personality, a scenario that mocks its + tools, and a check in `vapi-checks.yml`. + +The README's File Formats section is built from these files, and CI checks +that every example here passes `validate` and that the PR check builds. To +try it: + +```bash +cp -R examples/starter/resources/starter resources/my-org +npm run validate -- my-org +npm run check -- core --dry-run # after adapting vapi-checks.yml to my-org +``` + +Replace the `example.com` server URLs with your own endpoints before you +deploy. Nothing under `examples/` is loaded by the engine. diff --git a/examples/starter/resources/starter/assistants/receptionist.md b/examples/starter/resources/starter/assistants/receptionist.md new file mode 100644 index 0000000..8fcd22b --- /dev/null +++ b/examples/starter/resources/starter/assistants/receptionist.md @@ -0,0 +1,31 @@ +--- +name: Receptionist +firstMessage: Thanks for calling Bright Smile Dental. How can I help? +model: + provider: openai + model: gpt-4.1 + temperature: 0.3 + toolIds: + - lookup-patient + - handoff-to-scheduler + tools: + - type: endCall +voice: + provider: 11labs + voiceId: sarah +artifactPlan: + structuredOutputIds: + - call-summary +--- + +# Identity + +You are the receptionist for Bright Smile Dental, 123 Main St. The clinic is +open Monday to Friday, 8am to 5pm. + +# Flow + +1. Ask for the caller's phone number and call `lookup_patient` with it. +2. If they want to book, change or check an appointment, hand off to the + Scheduler with `handoff_to_scheduler`. Don't book anything yourself. +3. Answer general questions (hours, address) briefly yourself. diff --git a/examples/starter/resources/starter/assistants/scheduler.md b/examples/starter/resources/starter/assistants/scheduler.md new file mode 100644 index 0000000..f725469 --- /dev/null +++ b/examples/starter/resources/starter/assistants/scheduler.md @@ -0,0 +1,18 @@ +--- +name: Scheduler +model: + provider: openai + model: gpt-4.1 + temperature: 0.3 + toolIds: + - book-appointment + tools: + - type: endCall +voice: + provider: 11labs + voiceId: sarah +--- + +You are the scheduler for Bright Smile Dental. When the caller picks a time, +call `book_appointment`, then confirm the booked date and time. Keep replies +short. diff --git a/examples/starter/resources/starter/simulations/personalities/calm-caller.yml b/examples/starter/resources/starter/simulations/personalities/calm-caller.yml new file mode 100644 index 0000000..0ef9c92 --- /dev/null +++ b/examples/starter/resources/starter/simulations/personalities/calm-caller.yml @@ -0,0 +1,10 @@ +name: Calm caller +assistant: + model: + provider: openai + model: gpt-4.1-mini + messages: + - role: system + content: > + You are a patient calling a dental clinic. Follow your scenario, + answer questions briefly, and don't invent details. diff --git a/examples/starter/resources/starter/simulations/scenarios/books-cleaning.yml b/examples/starter/resources/starter/simulations/scenarios/books-cleaning.yml new file mode 100644 index 0000000..3247abc --- /dev/null +++ b/examples/starter/resources/starter/simulations/scenarios/books-cleaning.yml @@ -0,0 +1,15 @@ +name: Books a cleaning +instructions: > + You are Jordan Lee, phone 206-555-0142, an existing patient. Book a teeth + cleaning for next Tuesday morning and accept the first slot offered. Once + the booking is confirmed, say thanks and goodbye. +evaluations: + - structuredOutputId: booking-confirmed + comparator: "=" + value: true + required: true +toolMocks: + - toolName: lookup_patient + result: '{"found": true, "patientId": "P-1001"}' + - toolName: book_appointment + result: '{"success": true, "date": "next Tuesday", "time": "09:00"}' diff --git a/examples/starter/resources/starter/simulations/suites/core.yml b/examples/starter/resources/starter/simulations/suites/core.yml new file mode 100644 index 0000000..80461f1 --- /dev/null +++ b/examples/starter/resources/starter/simulations/suites/core.yml @@ -0,0 +1,3 @@ +name: Core +simulationIds: + - books-cleaning-calm diff --git a/examples/starter/resources/starter/simulations/tests/books-cleaning-calm.yml b/examples/starter/resources/starter/simulations/tests/books-cleaning-calm.yml new file mode 100644 index 0000000..af1ba81 --- /dev/null +++ b/examples/starter/resources/starter/simulations/tests/books-cleaning-calm.yml @@ -0,0 +1,3 @@ +name: Books a cleaning (calm caller) +personalityId: calm-caller +scenarioId: books-cleaning diff --git a/examples/starter/resources/starter/squads/front-desk.yml b/examples/starter/resources/starter/squads/front-desk.yml new file mode 100644 index 0000000..e5e6785 --- /dev/null +++ b/examples/starter/resources/starter/squads/front-desk.yml @@ -0,0 +1,4 @@ +name: Front Desk +members: + - assistantId: receptionist + - assistantId: scheduler diff --git a/examples/starter/resources/starter/structuredOutputs/booking-confirmed.yml b/examples/starter/resources/starter/structuredOutputs/booking-confirmed.yml new file mode 100644 index 0000000..d0b845c --- /dev/null +++ b/examples/starter/resources/starter/structuredOutputs/booking-confirmed.yml @@ -0,0 +1,5 @@ +name: booking-confirmed +type: ai +schema: + type: boolean + description: Did the assistant confirm a booking date and time with the caller? diff --git a/examples/starter/resources/starter/structuredOutputs/call-summary.yml b/examples/starter/resources/starter/structuredOutputs/call-summary.yml new file mode 100644 index 0000000..702df63 --- /dev/null +++ b/examples/starter/resources/starter/structuredOutputs/call-summary.yml @@ -0,0 +1,10 @@ +name: call-summary +type: ai +description: Summarizes the call for the front-desk log. +schema: + type: object + properties: + summary: + type: string + booked: + type: boolean diff --git a/examples/starter/resources/starter/tools/book-appointment.yml b/examples/starter/resources/starter/tools/book-appointment.yml new file mode 100644 index 0000000..a7a851f --- /dev/null +++ b/examples/starter/resources/starter/tools/book-appointment.yml @@ -0,0 +1,16 @@ +type: function +function: + name: book_appointment + description: Book an appointment slot the caller has accepted. + parameters: + type: object + properties: + date: + type: string + time: + type: string + required: + - date + - time +server: + url: https://example.com/vapi/book-appointment diff --git a/examples/starter/resources/starter/tools/handoff-to-scheduler.yml b/examples/starter/resources/starter/tools/handoff-to-scheduler.yml new file mode 100644 index 0000000..5f7b2a8 --- /dev/null +++ b/examples/starter/resources/starter/tools/handoff-to-scheduler.yml @@ -0,0 +1,7 @@ +type: handoff +function: + name: handoff_to_scheduler +destinations: + - type: assistant + assistantId: scheduler + description: Books, changes and checks appointments. diff --git a/examples/starter/resources/starter/tools/lookup-patient.yml b/examples/starter/resources/starter/tools/lookup-patient.yml new file mode 100644 index 0000000..98506ee --- /dev/null +++ b/examples/starter/resources/starter/tools/lookup-patient.yml @@ -0,0 +1,14 @@ +type: function +function: + name: lookup_patient + description: Look up the caller's patient record by phone number. + parameters: + type: object + properties: + phone: + type: string + description: The caller's phone number + required: + - phone +server: + url: https://example.com/vapi/lookup-patient diff --git a/examples/starter/vapi-checks.yml b/examples/starter/vapi-checks.yml new file mode 100644 index 0000000..9847d0a --- /dev/null +++ b/examples/starter/vapi-checks.yml @@ -0,0 +1,6 @@ +version: 1 +checks: + core: + org: starter + targets: [squads/front-desk] + suites: [core] diff --git a/improvements.md b/improvements.md index 39810d2..b219832 100644 --- a/improvements.md +++ b/improvements.md @@ -94,7 +94,7 @@ you which stack PR closes the row.** ## 1. `push` has no drift detection — silently overwrites concurrent dashboard edits -**Discovered:** customer-fork log (Amazon3p `improvements.md` #1, 2026-04-17) +**Discovered:** a customer fork's `improvements.md` log (2026-04-17) ### Problem @@ -150,7 +150,7 @@ know nobody else touches the dashboard. ## 2. `apply` (pull → push) silently drops dashboard edits to files modified locally -**Discovered:** customer-fork log (Amazon3p #2, 2026-04-17) +**Discovered:** a customer fork's log (2026-04-17) ### Problem @@ -196,7 +196,7 @@ sibling `.platform.yml` for manual 3-way merge. ## 3. No rollback command — `git revert + push` inherits all of #1's problems -**Discovered:** customer-fork log (Amazon3p #3, 2026-04-17) +**Discovered:** a customer fork's log (2026-04-17) ### Problem @@ -240,7 +240,7 @@ the *current platform payload* to ## 4. State file is identity-only — no content snapshots -**Discovered:** customer-fork log (Amazon3p #4, 2026-04-17) +**Discovered:** a customer fork's log (2026-04-17) ### Problem @@ -288,7 +288,7 @@ G, H, I, J. ## 5. No `push --dry-run` / pre-push diff -**Discovered:** customer-fork log (Mudflap #6 + Amazon3p #5, 2026-04-17/28) +**Discovered:** two customer forks' logs (2026-04-17/28) ### Problem @@ -326,7 +326,7 @@ mitigates #1, #3, #6. ## 6. No optimistic concurrency at the API protocol level -**Discovered:** customer-fork log (Amazon3p #6, 2026-04-17) +**Discovered:** a customer fork's log (2026-04-17) ### Problem @@ -381,7 +381,7 @@ platform team to confirm support, then ship Stack I behind a flag. ## 7. Voice edits drop pronunciation-dictionary attachments (Cartesia + 11labs) -**Discovered:** customer-fork log (Amazon3p #7, 2026-04-19) +**Discovered:** a customer fork's log (2026-04-19) ### Problem @@ -451,7 +451,7 @@ warning covering both shapes. ## 8. Dashboard prompt edits can in-place duplicate the existing prompt -**Discovered:** customer-fork log (Amazon3p #8, 2026-04-19) +**Discovered:** a customer fork's log (2026-04-19) ### Problem @@ -500,7 +500,7 @@ is partial — duplicated prompts can also be authored deliberately). ## 9. Provider-specific voice fields nest differently — schema mismatch only surfaces at push time -**Discovered:** customer-fork log (Amazon3p #9, 2026-04-19) +**Discovered:** a customer fork's log (2026-04-19) ### Problem @@ -557,7 +557,7 @@ exact-key short-circuit and the create path. Adoption re-keys state to the canonical UUID, drops stale duplicate state keys (orphan-deletion guard), and routes through `applyTool` for the standard PATCH + drift-check flow. -**Discovered:** customer-fork log (Amazon3p #10, 2026-04-29) +**Discovered:** a customer fork's log (2026-04-29) ### Problem @@ -616,7 +616,7 @@ dedup is the second layer for the bootstrap-renamed case. ## 11. Bidirectional SO ↔ assistant attachment has no validation -**Discovered:** customer-fork log (Mudflap #3, 2026-04-28) +**Discovered:** a customer fork's log (2026-04-28) ### Problem @@ -659,7 +659,7 @@ Manual: grep both files when editing one side. Easy to miss. ## 12. State file accumulates UUIDs without source files (silent drift) -**Discovered:** customer-fork log (Mudflap #2, 2026-04-28) +**Discovered:** a customer fork's log (2026-04-28) ### Problem @@ -709,7 +709,7 @@ state-orphans-without-source remain. **[RESOLVED 2026-04-30] (Stack A)** -**Discovered:** customer-fork log (Mudflap #4, 2026-04-28) +**Discovered:** a customer fork's log (2026-04-28) ### Problem @@ -728,7 +728,7 @@ decisions. `.gitignore` extended with `.agent/`, `.agent/handoffs/`, `.claude/handoffs/` (the existing `.claude/` line covered the latter -already, but Mudflap's log explicitly called out `.agent/` which was +already, but one fork's log explicitly called out `.agent/` which was uncovered). Removed the legacy `requested improvements.md` line — that was a per-engineer convention superseded by adopting upstream `improvements.md`. @@ -739,7 +739,7 @@ was a per-engineer convention superseded by adopting upstream **[RESOLVED 2026-04-30] (Stack A)** -**Discovered:** customer-fork log (Mudflap #5, 2026-04-28) +**Discovered:** a customer fork's log (2026-04-28) ### Problem @@ -759,7 +759,7 @@ document multi-file push. Verified intentional in `src/config.ts:104-184` ## 15. Scoped push still rewrites the entire state file -**Discovered:** customer-fork log (Mudflap #7, 2026-04-28) +**Discovered:** a customer fork's log (2026-04-28) ### Problem @@ -794,7 +794,7 @@ distinguish "stale" from "just-not-touched." ## 16. No CLI runner for simulation suites (despite engine tracking them) -**Discovered:** customer-fork log (Mudflap #8, 2026-04-28) +**Discovered:** a customer fork's log (2026-04-28) ### Problem @@ -840,7 +840,7 @@ incompatible follow-up. ## 17. State file key-order churn produces noisy diffs -**Discovered:** customer-fork log (Mudflap #1, 2026-04-28) +**Discovered:** a customer fork's log (2026-04-28) ### Problem @@ -879,7 +879,7 @@ out in the PR description. ## 18. Structured-output evaluation `name` capped at 40 chars with no client-side validation -**Discovered:** customer-fork log (Mudflap #9, 2026-04-29) +**Discovered:** a customer fork's log (2026-04-29) ### Problem @@ -916,7 +916,7 @@ assistant `name` capped at 40 too). ## 19. No engine warning when `maxTokens` is too low for a tool-using assistant -**Discovered:** customer-fork log (Mudflap #10, 2026-04-29) +**Discovered:** a customer fork's log (2026-04-29) ### Problem @@ -949,7 +949,7 @@ If `model.maxTokens < floor`, warn (non-blocking). ## 20. Prompt vocabulary leaks into TTS -**Discovered:** customer-fork log (Mudflap #11, 2026-04-29) +**Discovered:** a customer fork's log (2026-04-29) ### Problem @@ -1050,7 +1050,7 @@ RESOLVED 2026-05-11 (#TBD — PR number updates when opened). **[RESOLVED 2026-06-03] (#TBD)** -**Discovered:** during a `vitali-org` pull after renaming an assistant in the +**Discovered:** during a test-org pull after renaming an assistant in the dashboard ("call-transfer-test" → "call-transfer-test-1"). Pull created a second file `call-transfer-test-1-c95f4c6b.md` next to the existing `call-transfer-test-c95f4c6b.md` — two files for one UUID. @@ -1174,7 +1174,7 @@ RESOLVED 2026-06-03 (#TBD — PR number updates when opened). ## 24. Bare `push` is too easy to use as the deploy path -**Problem.** PR #41 review (dhruva-reddy): operators shouldn't have to memorize +**Problem.** PR #41 review: operators shouldn't have to memorize "`validate` && `apply` && avoid `push`". Raw `push` skips apply's validate-then-pull safety yet reads like the natural deploy verb, so it keeps getting used as one. @@ -1207,7 +1207,7 @@ apply's built-in validate). ## 25. Interactive flows lack automated coverage -**Problem.** PR #41 review (dhruva-reddy): the interactive picker +**Problem.** PR #41 review: the interactive picker (`src/interactive.ts` — Back/Cancel/empty-selection states) and the local-wins-apply-stays-clean invariant have no automated tests, and these are exactly the paths that regress while unit tests for path parsing still pass. @@ -1728,7 +1728,7 @@ fields. **[RESOLVED 2026-10-01]** -**Discovered:** 2026-10-01, while designing simulation PR checks (TEST-141). +**Discovered:** 2026-10-01, while designing simulation PR checks. ### Problem @@ -1779,7 +1779,7 @@ None needed once the fix below lands. **[RESOLVED 2026-10-01]** -**Discovered:** 2026-10-01, TEST-141 (and PAL-608, where customers hand-maintain shell workflows for this). +**Discovered:** 2026-10-01, from customers hand-maintaining shell workflows for this. ### Problem @@ -1827,7 +1827,7 @@ None needed once the fix below lands. **[RESOLVED 2026-10-01]** -**Discovered:** 2026-10-01, while planning the promotion check gate (TEST-141). +**Discovered:** 2026-10-01, while planning the promotion check gate. ### Problem diff --git a/package.json b/package.json index 8c8415f..9d083a9 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "vapi-gitops", "version": "1.0.0", - "description": "GitOps management for Vapi resources (Assistants, Structured Outputs, Tools)", + "description": "Manage Vapi assistants, squads, tools, structured outputs, simulations and evals as code, with drift-safe sync, cross-org promotion and simulation PR checks", "type": "module", "private": true, "license": "Apache-2.0", diff --git a/src/audit.ts b/src/audit.ts index 9ca5a73..1de1992 100644 --- a/src/audit.ts +++ b/src/audit.ts @@ -156,8 +156,8 @@ function extractRemoteName(resource: VapiResource): string | undefined { // Build the candidate resourceId(s) that a dashboard-orphan UUID would map to, // so we can check them against `.vapi-ignore`. Two shapes are produced because // real customer .vapi-ignore patterns target either form: -// - the bare name-slug (e.g. `assistants/iform-triage-classifier`) -// - the `-` form pull.ts emits (`assistants/iform-...-d98136d9`) +// - the bare name-slug (e.g. `assistants/triage-classifier`) +// - the `-` form pull.ts emits (`assistants/triage-...-d98136d9`) function candidateResourceIdsForRemote(resource: VapiResource): string[] { const name = extractRemoteName(resource); const shortId = resource.id.slice(0, 8); diff --git a/src/check-status.ts b/src/check-status.ts index 9d61e65..87b6f75 100644 --- a/src/check-status.ts +++ b/src/check-status.ts @@ -2,7 +2,7 @@ // // Per check and target: `Vapi Evals / / `, pending with the // run link as soon as the run exists, then the verdict. The aggregate -// `Vapi Evals` (the name PAL-608 specifies) is what branch protection +// `Vapi Evals` (a stable, documented name) is what branch protection // requires: per-target statuses only exist on PRs that touch a check. export const AGGREGATE_CONTEXT = "Vapi Evals"; diff --git a/src/credentials.ts b/src/credentials.ts index c352deb..21e7a23 100644 --- a/src/credentials.ts +++ b/src/credentials.ts @@ -4,7 +4,7 @@ import type { StateFile } from "./types.ts"; // Credential Resolution — resolve org-specific credential UUIDs across environments // // Credentials are pulled from the API and stored in state (name-slug → UUID). -// Resource files store credential NAMES (e.g., "roofr-server-credential"). +// Resource files store credential NAMES (e.g., "acme-server-credential"). // Push resolves names → UUIDs. Pull resolves UUIDs → names. // // Replacement is scoped to `credentialId` / `credentialIds` fields only. diff --git a/src/new-file-gate.ts b/src/new-file-gate.ts index 654781d..f0fe0f5 100644 --- a/src/new-file-gate.ts +++ b/src/new-file-gate.ts @@ -10,7 +10,7 @@ // (c) MOVED file — file copied without the state entry being rekeyed // // Silently treating every orphan as case (a) is what produced the duplicate -// fleet we surfaced during the gitops-mudflap working session 2026-05-13. +// fleet we surfaced in a customer fork on 2026-05-13. // Flow F (`mv foo.md bar.md` + push), Flow G (dashboard rename → pull writes // new file but leaves stale YAML), and Flow M (`apply` compresses Flow G into // one click) all share this shape. diff --git a/src/pull.ts b/src/pull.ts index 4e71fea..2eca30a 100644 --- a/src/pull.ts +++ b/src/pull.ts @@ -851,7 +851,7 @@ export async function pullResourceType( // - `state[resourceType]` carries prior-pull claims loaded from // disk. Without this, if the dashboard returns the new same-name // twin BEFORE the tracked one, the new twin sees `newStateSection` - // empty and clobbers the tracked file. The customer's mudflap-prod + // empty and clobbers the tracked file. A customer's production // 5-Rileys investigation surfaced this ordering dependency. // - `newStateSection` carries intra-pull claims from earlier // iterations. Handles the converse (tracked-then-twin order). diff --git a/src/push.ts b/src/push.ts index 3270e9e..d5efc93 100644 --- a/src/push.ts +++ b/src/push.ts @@ -1674,7 +1674,7 @@ async function main(): Promise { // Orphan-YAML pre-flight gate. Runs ONCE for ALL resource types after // bootstrap (so state-recovery has a chance to rekey first) and BEFORE // any apply phase. Halts push when local files exist with no state entry - // — the duplicate-creation pattern we surfaced during the gitops-mudflap + // — the duplicate-creation pattern we surfaced in a customer fork's // working session 2026-05-13 (see src/new-file-gate.ts for context). // // Skipped during explicit `--bootstrap` runs: a bootstrap is supposed to diff --git a/src/types.ts b/src/types.ts index d285a43..f0c8ed4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -60,7 +60,7 @@ export type ResourceType = | "simulationSuites" | "evals"; -// Any slug-like string: "dev", "prod", "roofr-production", etc. +// Any slug-like string: "dev", "prod", "acme-production", etc. export type Environment = string; // Well-known names kept for backward-compatible npm scripts diff --git a/tests/audit.test.ts b/tests/audit.test.ts index d348bd2..72cac00 100644 --- a/tests/audit.test.ts +++ b/tests/audit.test.ts @@ -294,9 +294,9 @@ test("content-identical: 2 entries share hash, 1 entry has distinct hash → 1 f test("sibling-base-slug: bare + 2 suffixed entries cluster under same base → 1 finding, 3 slugs", async () => { const state = makeStateFile({ assistants: { - "iform-barge": makeStateEntry("uuid-1"), - "iform-barge-d98136d9": makeStateEntry("uuid-2"), - "iform-barge-f6b53e27": makeStateEntry("uuid-3"), + "intake-agent": makeStateEntry("uuid-1"), + "intake-agent-d98136d9": makeStateEntry("uuid-2"), + "intake-agent-f6b53e27": makeStateEntry("uuid-3"), }, }); const findings = await runAudit(baseOpts(state)); @@ -304,9 +304,9 @@ test("sibling-base-slug: bare + 2 suffixed entries cluster under same base → 1 assert.equal(siblings.length, 1); assert.equal(siblings[0]!.resourceIds.length, 3); assert.deepEqual(siblings[0]!.resourceIds, [ - "iform-barge", - "iform-barge-d98136d9", - "iform-barge-f6b53e27", + "intake-agent", + "intake-agent-d98136d9", + "intake-agent-f6b53e27", ]); // No content-identical overlap here → message does NOT contain cross-ref. assert.equal( @@ -318,8 +318,8 @@ test("sibling-base-slug: bare + 2 suffixed entries cluster under same base → 1 test("sibling-base-slug: siblings that share a hash get cross-reference to content-identical", async () => { const state = makeStateFile({ assistants: { - "iform-barge": makeStateEntry("uuid-1", "hash-shared"), - "iform-barge-d98136d9": makeStateEntry("uuid-2", "hash-shared"), + "intake-agent": makeStateEntry("uuid-1", "hash-shared"), + "intake-agent-d98136d9": makeStateEntry("uuid-2", "hash-shared"), }, }); const findings = await runAudit(baseOpts(state)); @@ -336,7 +336,7 @@ test("sibling-base-slug: siblings that share a hash get cross-reference to conte test("sibling-base-slug: only one entry (no siblings) → 0 findings", async () => { const state = makeStateFile({ assistants: { - "iform-barge": makeStateEntry("uuid-1"), + "intake-agent": makeStateEntry("uuid-1"), }, }); const findings = await runAudit(baseOpts(state)); @@ -504,12 +504,12 @@ test("integration: orphan-yaml + collision + content-identical(4) + sibling-base "riley-2": makeStateEntry("uuid-r2", "H1"), "riley-3": makeStateEntry("uuid-r3", "H1"), "riley-4": makeStateEntry("uuid-r4", "H1"), - // 3 slugs sharing base "iform-barge"; 2 of them share hash H2 so + // 3 slugs sharing base "intake-agent"; 2 of them share hash H2 so // sibling-base-slug message picks up the cross-ref AND we get one // extra content-identical finding for those 2. - "iform-barge": makeStateEntry("uuid-s1", "H2"), - "iform-barge-d98136d9": makeStateEntry("uuid-s2", "H2"), - "iform-barge-f6b53e27": makeStateEntry("uuid-s3"), + "intake-agent": makeStateEntry("uuid-s1", "H2"), + "intake-agent-d98136d9": makeStateEntry("uuid-s2", "H2"), + "intake-agent-f6b53e27": makeStateEntry("uuid-s3"), }, }); @@ -544,7 +544,7 @@ test("integration: orphan-yaml + collision + content-identical(4) + sibling-base ]); // The sibling finding must carry the cross-ref token because 2 of the 3 - // siblings (iform-barge, iform-barge-d98136d9) also appear in a + // siblings (intake-agent, intake-agent-d98136d9) also appear in a // content-identical cluster. const sibling = findings.find((f) => f.rule === "sibling-base-slug")!; assert.ok(sibling.message.includes("overlaps with content-identical")); diff --git a/tests/credentials.test.ts b/tests/credentials.test.ts index 09c85d7..2864cd4 100644 --- a/tests/credentials.test.ts +++ b/tests/credentials.test.ts @@ -38,12 +38,12 @@ function makeState(creds: Record): StateFile { test("replaceCredentialRefs swaps at credentialId keys", () => { const state = makeState({ - "roofr-server-credential": "11111111-1111-1111-1111-111111111111", + "acme-server-credential": "11111111-1111-1111-1111-111111111111", }); const input = { server: { url: "https://example.com", - credentialId: "roofr-server-credential", + credentialId: "acme-server-credential", }, }; const out = replaceCredentialRefs(input, forwardMap(state)); diff --git a/tests/drift.test.ts b/tests/drift.test.ts index 73fbf61..ae90704 100644 --- a/tests/drift.test.ts +++ b/tests/drift.test.ts @@ -786,7 +786,7 @@ test("canonicalizeForHash: strips server-managed fields (id, orgId, createdAt, u // short-circuit baseline preservation. // // Regression coverage for a bug introduced by the drift-direction-classifier -// PR (#38) and caught by the E2E both-diverged smoke test on mudflap-iform-test: +// PR (#38) and caught by the E2E both-diverged smoke test on a test org: // pull rebuilds each state section from EMPTY, and the classifier short-circuit // branches wrote back a bare `{ uuid }` — dropping the baseline, so the next // pull classified the resource as `no-baseline` and could never detect drift diff --git a/tests/examples.test.ts b/tests/examples.test.ts new file mode 100644 index 0000000..93e015d --- /dev/null +++ b/tests/examples.test.ts @@ -0,0 +1,169 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { checksConfigParse } from "../src/check-config.ts"; +import { checkPayloadBuild } from "../src/check-payload.ts"; +import { promotionStateParse } from "../src/promotion.ts"; +import { orgResourcesRead } from "../src/resource-parse.ts"; + +// The examples are what new users copy, so they must be valid: every org +// under examples/ passes `validate` and meets the API's minimum fields, the +// starter's PR check builds cleanly, and every doc snippet that names an +// example file is that file, byte for byte. + +const REPO = fileURLToPath(new URL("..", import.meta.url)); + +// Every examples//resources// directory. +function exampleOrgs(): Array<{ root: string; org: string }> { + const orgs: Array<{ root: string; org: string }> = []; + for (const example of readdirSync(join(REPO, "examples"))) { + const resources = join(REPO, "examples", example, "resources"); + if (!existsSync(resources)) continue; + for (const org of readdirSync(resources)) + orgs.push({ root: join(REPO, "examples", example), org }); + } + return orgs; +} + +function validateRun( + resourcesDir: string, + org: string, +): { code: number | null; output: string } { + const dir = mkdtempSync(join(tmpdir(), "vapi-examples-")); + try { + cpSync(join(REPO, "src"), join(dir, "src"), { recursive: true }); + cpSync(join(REPO, "package.json"), join(dir, "package.json")); + symlinkSync(join(REPO, "node_modules"), join(dir, "node_modules"), "dir"); + mkdirSync(join(dir, "resources")); + cpSync(resourcesDir, join(dir, "resources", org), { recursive: true }); + writeFileSync(join(dir, `.env.${org}`), "VAPI_TOKEN=fake-token-not-used\n"); + const result = spawnSync( + process.execPath, + ["--import", "tsx", "src/validate-cmd.ts", org], + { cwd: dir, encoding: "utf8", timeout: 30_000 }, + ); + return { code: result.status, output: `${result.stdout}${result.stderr}` }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +test("every example org passes validate", () => { + const results = exampleOrgs().map(({ root, org }) => { + const run = validateRun(join(root, "resources", org), org); + return [org, run.code, run.code === 0 ? "" : run.output]; + }); + assert.deepEqual( + results, + results.map(([org]) => [org, 0, ""]), + ); +}); + +test("every example simulation resource has the fields the API requires", async () => { + const problems: string[] = []; + for (const { root, org } of exampleOrgs()) { + const resources = await orgResourcesRead(root, org, { ignorePatterns: [] }); + for (const resource of resources.values()) { + const at = `${org}/${resource.type}/${resource.id}`; + const data = resource.data; + if ( + resource.type === "personalities" && + typeof data.assistant !== "object" + ) + problems.push(`${at}: needs an assistant`); + if (resource.type === "scenarios") { + if (typeof data.instructions !== "string" || data.instructions === "") + problems.push(`${at}: needs instructions`); + if (!Array.isArray(data.evaluations) || data.evaluations.length === 0) + problems.push(`${at}: needs at least one evaluation`); + } + if ( + resource.type === "simulations" && + (!data.personalityId || !data.scenarioId) + ) + problems.push(`${at}: needs personalityId and scenarioId`); + if ( + resource.type === "simulationSuites" && + !Array.isArray(data.simulationIds) + ) + problems.push(`${at}: needs simulationIds`); + } + } + assert.deepEqual(problems, []); +}); + +test("the starter example's PR check builds with no errors or warnings", async () => { + const root = join(REPO, "examples/starter"); + const check = checksConfigParse( + readFileSync(join(root, "vapi-checks.yml"), "utf8"), + ).checks.core!; + const state = promotionStateParse( + readFileSync(join(root, ".vapi-state.starter.json"), "utf8"), + ); + const result = checkPayloadBuild({ + check, + target: check.targets[0]!, + resources: await orgResourcesRead(root, "starter"), + sourceState: state, + runState: state, + }); + assert.deepEqual( + [result.errors, result.warnings, result.body?.simulations.length], + [[], [], 1], + ); +}); + +// A fenced block whose first line is `# examples/` (or +// `` in Markdown) must be that file exactly. +const SNIPPET_RE = + /```[a-z]*\n(?:# |)?\n([\s\S]*?)```/g; + +function markdownFiles(): string[] { + const files = ["README.md"]; + const walk = (dir: string) => { + for (const entry of readdirSync(join(REPO, dir), { withFileTypes: true })) { + const path = `${dir}/${entry.name}`; + if (entry.isDirectory()) walk(path); + else if (entry.name.endsWith(".md")) files.push(path); + } + }; + walk("docs"); + return files; +} + +test("doc snippets that name an example file match it exactly", () => { + const snippets: Array<[string, string]> = []; + const mismatched: string[] = []; + for (const file of markdownFiles()) { + for (const match of readFileSync(join(REPO, file), "utf8").matchAll( + SNIPPET_RE, + )) { + const [, path, body] = match; + snippets.push([file, path!]); + const target = join(REPO, path!); + if (!existsSync(target)) + mismatched.push(`${file}: ${path} does not exist`); + else if (readFileSync(target, "utf8") !== body) + mismatched.push(`${file}: snippet differs from ${path}`); + } + } + assert.ok( + snippets.length > 0, + "expected at least one example snippet in the docs", + ); + assert.deepEqual(mismatched, []); +}); diff --git a/tests/pull-same-name-clobber.test.ts b/tests/pull-same-name-clobber.test.ts index de1f852..5c20b79 100644 --- a/tests/pull-same-name-clobber.test.ts +++ b/tests/pull-same-name-clobber.test.ts @@ -30,7 +30,7 @@ import { Worker } from "node:worker_threads"; // Without the fix, B silently overwrites `riley.md` and the state mapping // for `riley` flips to B — orphaning A's UUID with no on-disk artifact. // -// Reproduces the mudflap "5 Rileys" customer scenario that will keep getting +// Reproduces the "5 Rileys" customer scenario that will keep getting // triggered as Vapi auto-seeds same-named twins for new orgs. // ───────────────────────────────────────────────────────────────────────────── diff --git a/tests/recanonicalize.test.ts b/tests/recanonicalize.test.ts index e9283e4..076b644 100644 --- a/tests/recanonicalize.test.ts +++ b/tests/recanonicalize.test.ts @@ -349,14 +349,14 @@ test("recanonicalize: handles UUID prefix match case-insensitively", () => { test("recanonicalize: collapses multi-dash base slug ('foo-vmd-') — the exact shape the orphan-gate pairing missed", () => { // From the live incident: state key was - // `iform-voicemail-triage-squad-llm-only-vmd-004c5108`. The orphan-gate's + // `voicemail-triage-squad-llm-only-vmd-004c5108`. The orphan-gate's // extractBaseSlug pairing failed because base = "...-vmd" not "...". // This pass operates on raw UUID-suffix shape, so it recanonicalizes // regardless of how many dash-segments precede the UUID8 — as long as // the canonical local file exists. const state = makeStateFile({ squads: { - "iform-voicemail-triage-squad-llm-only-vmd-004c5108": makeStateEntry( + "voicemail-triage-squad-llm-only-vmd-004c5108": makeStateEntry( "004c5108-aaaa-bbbb-cccc-dddddddddddd", ), }, @@ -364,13 +364,13 @@ test("recanonicalize: collapses multi-dash base slug ('foo-vmd-') — the const report = recanonicalizeStateKeys({ state, fileExists: makeFileExists( - new Set(["squads/iform-voicemail-triage-squad-llm-only-vmd.yml"]), + new Set(["squads/voicemail-triage-squad-llm-only-vmd.yml"]), ), }); assert.equal(report.rekeys.length, 1); assert.equal( report.rekeys[0]!.toKey, - "iform-voicemail-triage-squad-llm-only-vmd", + "voicemail-triage-squad-llm-only-vmd", ); assert.equal(report.conflicts.length, 0); }); diff --git a/tests/slug-utils.test.ts b/tests/slug-utils.test.ts index cd0dde0..1ca56f9 100644 --- a/tests/slug-utils.test.ts +++ b/tests/slug-utils.test.ts @@ -160,11 +160,11 @@ test("isEngineSuffixedSlug: strips UUID dashes defensively (malformed UUID with test("isEngineSuffixedSlug: handles multi-segment base", () => { const result = isEngineSuffixedSlug( - "iform-voicemail-triage-squad-llm-only-vmd-004c5108", + "voicemail-triage-squad-llm-only-vmd-004c5108", "004c5108-aaaa-bbbb-cccc-dddddddddddd", ); assert.deepEqual(result, { - base: "iform-voicemail-triage-squad-llm-only-vmd", + base: "voicemail-triage-squad-llm-only-vmd", suffix: "004c5108", }); });