diff --git a/.github/workflows/promotion.yml b/.github/workflows/promotion.yml index 5f679cd..888d1f4 100644 --- a/.github/workflows/promotion.yml +++ b/.github/workflows/promotion.yml @@ -56,6 +56,10 @@ jobs: - name: Reconcile configured promotions id: promotion + # On the step, not the job: a job timeout would cancel the + # always() commit step below, and a gate check (orgs..check) + # can run up to its timeoutMinutes per gated org. + timeout-minutes: 90 shell: bash run: | set -euo pipefail diff --git a/README.md b/README.md index 8d2e165..195397f 100644 --- a/README.md +++ b/README.md @@ -439,6 +439,30 @@ its cleaned state after downstream deletion completes. See [sync behavior](docs/learnings/sync-behavior.md#cross-org-promotion-deletions) for the exact lifecycle. +#### Check before promoting (optional) + +Gate an org on a [PR check](#pr-checks-simulations-against-your-branch): +nothing is promoted **out of** it unless the check passes there first. + +```yaml +# promotion.yml +orgs: + example-staging: + check: staging-core # a vapi-checks.yml check whose org (and runOrg) is example-staging +``` + +- Plans print `check would run staging-core in example-staging ( simulations × targets)` + and run nothing. +- On `--apply`, the check runs against `resources/example-staging/` at the + promoted commit, in example-staging, with that org's key from + `VAPI_PROMOTION_TOKENS`, before any file is written to the destination. A + failure, an incomplete run (timeout, billing) or a build error blocks the + transition with the run link; transitions that already applied are still + committed. +- A pass is reused for later transitions out of the same org in the same run, + until something is promoted into it. +- Transitions with no changes skip the check. + #### Rolling Back a Promotion Treat a promotion rollback as a new, auditable Git change: revert the source @@ -588,6 +612,12 @@ Require the **commit status `Vapi Evals`** in branch protection — not the status, so dispatch again after that. Running one named check by hand never changes `Vapi Evals`. +### Gate promotion on a check (optional) + +Multi-org repos can require a check to pass in an org before anything is +promoted out of it: set `orgs..check: ` in `promotion.yml` (see +[Check before promoting](#check-before-promoting-optional)). + ### Dedicated CI org (optional; recommended with `toolMocks: off`) 1. `npm run setup -- my-ci-org --resources none`. diff --git a/promotion.example.yml b/promotion.example.yml index 82bd78a..84165e0 100644 --- a/promotion.example.yml +++ b/promotion.example.yml @@ -8,6 +8,10 @@ orgs: baseUrl: https://api.vapi.ai example-staging: baseUrl: https://api.vapi.ai + # Optional gate: nothing is promoted out of example-staging unless this + # vapi-checks.yml check (whose org and runOrg are example-staging) passes + # there first. Plans print what it would run; --apply runs it. + # check: staging-core bindings: credentials: default: bind diff --git a/src/promote-cmd.ts b/src/promote-cmd.ts index 408046b..7600ffd 100644 --- a/src/promote-cmd.ts +++ b/src/promote-cmd.ts @@ -2,9 +2,16 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import type { CheckDefinition } from "./check-config.ts"; import type { OrgConnection } from "./org-connection.ts"; import { childRun, connectionLoad, tokensParse } from "./org-connection.ts"; import type { PromotionConfig, PromotionPipeline } from "./promotion.ts"; +import type { PromotionGateResult } from "./promotion-gate.ts"; +import { + promotionChecksLoad, + promotionGatePlanLine, + promotionGateRun, +} from "./promotion-gate.ts"; import { promotionConfigParse, promotionPlanApply, @@ -42,6 +49,17 @@ export const APPLIED_PATHS_FILE = "tmp/promotion-applied.txt"; export interface PromotionDeps { childRun: typeof orgScriptRun; + checkRun: ( + check: CheckDefinition, + connection: OrgConnection, + ) => Promise; +} + +// Gated orgs' checks, and the orgs whose check already passed in this run. +// A pass stays valid until a transition applies into that org. +interface PromotionGates { + checks: Map; + passed: Set; } function argumentsParse(args: string[]): PromotionArguments { @@ -215,6 +233,7 @@ async function transitionRun( tokens: Map, allowEmptySourceDeletion: boolean, deps: PromotionDeps, + gates: PromotionGates, ): Promise { const run = deps.childRun; if (apply) { @@ -247,7 +266,23 @@ async function transitionRun( for (const change of plan.changes) console.log(` ${change.kind.padEnd(6)} ${change.path}`); if (plan.changes.length === 0) console.log(" no changes"); - if (!apply || plan.changes.length === 0) return false; + if (plan.changes.length === 0) return false; + const check = gates.checks.get(transition.source); + if (check && !apply) + console.log(await promotionGatePlanLine(ROOT_DIR, check)); + if (!apply) return false; + if (check && !gates.passed.has(transition.source)) { + console.log(` check running ${check.name} in ${transition.source}…`); + const result = await deps.checkRun( + check, + orgConnection(config, transition.source, tokens), + ); + if (result.outcome !== "passed") + throw new Error( + `Promotion out of ${transition.source} blocked: check ${check.name} ${result.outcome} (${result.url ?? result.reason})`, + ); + gates.passed.add(transition.source); + } await promotionPlanApply(plan); const changedPaths = plan.changes.map( (change) => `resources/${transition.target}/${change.path}`, @@ -259,18 +294,30 @@ async function transitionRun( ["--force", "--allow-new-files", "--resolve=ours", ...changedPaths], ); appliedPathsRecord(transition.target); + // The target's files just changed, so an earlier pass no longer covers it. + gates.passed.delete(transition.target); return plan.changes.some((change) => change.kind === "delete"); } export async function promotionCommandRun( args = process.argv.slice(2), - deps: PromotionDeps = { childRun: orgScriptRun }, + overrides: Partial = {}, ): Promise { + const deps: PromotionDeps = { + childRun: orgScriptRun, + checkRun: (check, connection) => + promotionGateRun(ROOT_DIR, check, connection), + ...overrides, + }; const parsed = argumentsParse(args); const configPath = resolve(ROOT_DIR, "promotion.yml"); if (!existsSync(configPath)) throw new Error("promotion.yml is required at the repository root"); const config = promotionConfigParse(readFileSync(configPath, "utf8")); + const gates: PromotionGates = { + checks: promotionChecksLoad(ROOT_DIR, config), + passed: new Set(), + }; const tokens = parsed.apply ? tokensParse(TOKENS_ENV) : new Map(); @@ -292,6 +339,7 @@ export async function promotionCommandRun( tokens, deletionAuthorizedSources.has(sourceKey), deps, + gates, ); if (deleted) deletionAuthorizedSources.add( diff --git a/src/promotion-gate.ts b/src/promotion-gate.ts new file mode 100644 index 0000000..6e3cd49 --- /dev/null +++ b/src/promotion-gate.ts @@ -0,0 +1,107 @@ +// The promotion check gate: `orgs..check: ` in promotion.yml +// names a vapi-checks.yml check that must pass in that org before any +// transition promotes out of it. The gate runs the same check as the PR +// workflow, built from the source org's files at the promoted commit. + +import type { CheckDefinition } from "./check-config.ts"; +import { CHECKS_CONFIG_FILE, checksConfigLoad } from "./check-config.ts"; +import { checkJobsBuild } from "./check-build.ts"; +import type { CheckOutcome } from "./check-run.ts"; +import { checkRunAll, MIN_START_MS } from "./check-run.ts"; +import type { OrgConnection } from "./org-connection.ts"; +import type { PromotionConfig } from "./promotion.ts"; +import { userAgentGet } from "./user-agent.ts"; + +export interface PromotionGateResult { + outcome: CheckOutcome; + reason: string; + url?: string; +} + +const DEFAULT_BASE_URL = "https://api.vapi.ai"; +const OUTCOME_RANK: Record = { + passed: 0, + built: 0, + incomplete: 1, + failed: 2, + error: 3, +}; + +// The gated orgs' checks, validated up front so a typo fails before any +// transition applies. +export function promotionChecksLoad( + rootDir: string, + config: PromotionConfig, +): Map { + const gated = Object.entries(config.orgs).filter(([, org]) => org.check); + const checks = new Map(); + if (gated.length === 0) return checks; + const checksConfig = checksConfigLoad(rootDir); + if (!checksConfig) + throw new Error( + `promotion.yml gates ${gated.map(([slug]) => slug).join(", ")} on checks, but there is no ${CHECKS_CONFIG_FILE}`, + ); + for (const [slug, org] of gated) { + const check = checksConfig.checks[org.check!]; + if (!check) + throw new Error( + `orgs.${slug}.check: no check named ${org.check} in ${CHECKS_CONFIG_FILE}`, + ); + if (check.org !== slug || check.runOrg !== slug) + throw new Error( + `orgs.${slug}.check: check ${check.name} must read and run in ${slug} (it reads ${check.org} and runs in ${check.runOrg})`, + ); + checks.set(slug, check); + } + return checks; +} + +// The plan-only line: what the gate would run, built offline. +export async function promotionGatePlanLine( + rootDir: string, + check: CheckDefinition, +): Promise { + const jobs = await checkJobsBuild(rootDir, check); + const broken = jobs.find((job) => !job.result.body); + if (broken) + return ` check would run ${check.name} in ${check.org}, but its payload doesn't build: ${broken.result.errors[0]}`; + const simulations = jobs[0]?.result.body?.simulations.length ?? 0; + return ` check would run ${check.name} in ${check.org} (${simulations} simulation${simulations === 1 ? "" : "s"} × ${jobs.length} target${jobs.length === 1 ? "" : "s"})`; +} + +// Run the check live in the source org and reduce it to one result: the +// worst target wins, so anything short of every target passing blocks. +export async function promotionGateRun( + rootDir: string, + check: CheckDefinition, + connection: OrgConnection, +): Promise { + const jobs = await checkJobsBuild(rootDir, check); + const controller = new AbortController(); + const abort = () => controller.abort(); + process.once("SIGINT", abort); + process.once("SIGTERM", abort); + try { + const results = await checkRunAll({ + jobs, + connectionFor: () => ({ + token: connection.token, + baseUrl: check.baseUrl ?? connection.baseUrl ?? DEFAULT_BASE_URL, + userAgent: userAgentGet("check"), + }), + deadline: Date.now() + check.timeoutMinutes * 60_000 + MIN_START_MS, + signal: controller.signal, + }); + const worst = results.reduce((a, b) => + OUTCOME_RANK[b.outcome] > OUTCOME_RANK[a.outcome] ? b : a, + ); + for (const result of results) + console.log( + ` check ${result.job.label}: ${result.outcome} — ${result.reason}${result.url ? ` (${result.url})` : ""}`, + ); + return { outcome: worst.outcome, reason: worst.reason, url: worst.url }; + } finally { + process.off("SIGINT", abort); + process.off("SIGTERM", abort); + } +} diff --git a/src/promotion.ts b/src/promotion.ts index 7ece0a5..0582b6e 100644 --- a/src/promotion.ts +++ b/src/promotion.ts @@ -29,6 +29,9 @@ export interface PromotionBindings { export interface PromotionOrg { baseUrl?: string; bindings: PromotionBindings; + // A vapi-checks.yml check that must pass in this org before anything is + // promoted out of it. + check?: string; } export interface PromotionPipeline { @@ -152,9 +155,17 @@ export function promotionConfigParse(content: string): PromotionConfig { const org = object(value ?? {}, `org ${slug}`); if (org.baseUrl !== undefined && typeof org.baseUrl !== "string") throw new Error(`org ${slug}.baseUrl must be a string`); + if ( + org.check !== undefined && + (typeof org.check !== "string" || !SLUG_RE.test(org.check)) + ) + throw new Error( + `org ${slug}.check must be a check name from vapi-checks.yml`, + ); orgs[slug] = { baseUrl: typeof org.baseUrl === "string" ? org.baseUrl : undefined, bindings: promotionBindingsParse(org.bindings), + ...(typeof org.check === "string" ? { check: org.check } : {}), }; } const pipelinesRaw = object(raw.pipelines, "promotion.yml pipelines"); diff --git a/tests/promotion-gate.test.ts b/tests/promotion-gate.test.ts new file mode 100644 index 0000000..d4ec18b --- /dev/null +++ b/tests/promotion-gate.test.ts @@ -0,0 +1,333 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import type { CheckDefinition } from "../src/check-config.ts"; +import type { PromotionGateResult } from "../src/promotion-gate.ts"; + +// promote-cmd.ts binds its root at import: one fixture repo for the file, +// rebuilt per test. +const ROOT = mkdtempSync(join(tmpdir(), "promotion-gate-")); +process.env.VAPI_GITOPS_ROOT = ROOT; +const { promotionCommandRun } = await import("../src/promote-cmd.ts"); + +const CHECK_FILES: Record = { + "assistants/intake.yml": "name: Intake\nmodel:\n provider: openai\n", + "structuredOutputs/ok.yml": "name: ok\nschema:\n type: boolean\n", + "simulations/scenarios/s1.yml": + "name: S1\ninstructions: Hi.\nevaluations:\n - structuredOutputId: ok\n comparator: '='\n value: true\n required: true\n", + "simulations/personalities/calm.yml": "name: Calm\n", + "simulations/tests/t1.yml": "name: T1\npersonalityId: calm\nscenarioId: s1\n", +}; + +function write(path: string, content: string): void { + mkdirSync(dirname(join(ROOT, path)), { recursive: true }); + writeFileSync(join(ROOT, path), content); +} + +interface FixtureArgs { + orgs: Record; // org → check name + pipelines: Record; + files?: Record; // repo-relative + checks?: string; // vapi-checks.yml body after `checks:` +} + +function fixture(args: FixtureArgs): void { + rmSync(ROOT, { recursive: true, force: true }); + mkdirSync(ROOT, { recursive: true }); + const orgs = Object.entries(args.orgs) + .map( + ([org, check]) => ` ${org}:${check ? `\n check: ${check}` : " {}"}`, + ) + .join("\n"); + const pipelines = Object.entries(args.pipelines) + .map( + ([name, list]) => + ` ${name}:\n orgs: [${list.join(", ")}]\n resources: ['**/*']`, + ) + .join("\n"); + write( + "promotion.yml", + `version: 1\norgs:\n${orgs}\npipelines:\n${pipelines}\n`, + ); + if (args.checks !== undefined) + write("vapi-checks.yml", `version: 1\nchecks:\n${args.checks}`); + for (const org of Object.keys(args.orgs)) + write(`.vapi-state.${org}.json`, "{}\n"); + for (const [path, content] of Object.entries(args.files ?? {})) + write(path, content); + write(".gitignore", "tmp/\n.env.*\n"); + execFileSync("git", ["init", "-q", "-b", "main"], { cwd: ROOT }); +} + +const checkFor = (org: string, name = `${org}-core`) => + ` ${name}:\n org: ${org}\n targets: [assistants/intake]\n simulations: [t1]\n`; + +function filesFor(org: string): Record { + return Object.fromEntries( + Object.entries(CHECK_FILES).map(([path, content]) => [ + `resources/${org}/${path}`, + content, + ]), + ); +} + +interface Recorder { + applies: string[]; + checks: string[]; + output: string; +} + +async function promote( + args: string[], + gate: PromotionGateResult | ((check: CheckDefinition) => PromotionGateResult), +): Promise { + const recorder: Recorder = { applies: [], checks: [], output: "" }; + process.env.VAPI_PROMOTION_TOKENS = JSON.stringify({ + a: "t", + b: "t", + c: "t", + d: "t", + }); + const log = console.log; + console.log = (...parts: unknown[]) => { + recorder.output += `${parts.join(" ")}\n`; + }; + try { + await promotionCommandRun(args, { + childRun: (script, org) => { + if (script === "src/apply.ts") recorder.applies.push(org); + }, + checkRun: async (check) => { + recorder.checks.push(`${check.name}@${check.org}`); + return typeof gate === "function" ? gate(check) : gate; + }, + }); + return recorder; + } catch (error) { + return { ...recorder, error: (error as Error).message }; + } finally { + console.log = log; + } +} + +const PASS: PromotionGateResult = { + outcome: "passed", + reason: "1 of 1 simulations passed", + url: "https://run/pass", +}; +const FAIL: PromotionGateResult = { + outcome: "failed", + reason: "1 of 1 simulations failed", + url: "https://run/fail", +}; +const ONE_STEP = ["--pipeline", "release", "--from", "a", "--to", "b"]; + +test("a passing gate lets the transition apply", async () => { + fixture({ + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("a"), + }); + const result = await promote([...ONE_STEP, "--apply"], PASS); + assert.deepEqual( + [result.error, result.checks, result.applies], + [undefined, ["a-core@a"], ["b"]], + ); +}); + +test("a failed or incomplete gate blocks, naming the run, and leaves the target untouched", async () => { + const outcomes: Array<[PromotionGateResult, string]> = [ + [ + FAIL, + "Promotion out of a blocked: check a-core failed (https://run/fail)", + ], + [ + { outcome: "incomplete", reason: "timed out after 1200s; run canceled" }, + "Promotion out of a blocked: check a-core incomplete (timed out after 1200s; run canceled)", + ], + ]; + const seen = []; + for (const [gate] of outcomes) { + fixture({ + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("a"), + }); + const result = await promote([...ONE_STEP, "--apply"], gate); + seen.push([ + result.error, + result.applies, + existsSync(join(ROOT, "resources/b/assistants/intake.yml")), + ]); + } + assert.deepEqual( + seen, + outcomes.map(([, message]) => [message, [], false]), + ); +}); + +test("a plan-only run says what the gate would run and never runs it", async () => { + fixture({ + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("a"), + }); + const result = await promote(ONE_STEP, FAIL); + assert.deepEqual( + [ + result.error, + result.checks, + result.output.includes( + " check would run a-core in a (1 simulation × 1 target)", + ), + ], + [undefined, [], true], + ); +}); + +test("a transition with no changes skips the gate", async () => { + fixture({ + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: { ...filesFor("a"), ...filesFor("b") }, + checks: checkFor("a"), + }); + const result = await promote([...ONE_STEP, "--apply"], FAIL); + assert.deepEqual( + [result.error, result.checks, result.applies], + [undefined, [], []], + ); +}); + +test("gate configuration errors stop the run before anything applies", async () => { + const cases: Array<[FixtureArgs, string]> = [ + [ + { + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + }, + "promotion.yml gates a on checks, but there is no vapi-checks.yml", + ], + [ + { + orgs: { a: "nope", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("a"), + }, + "orgs.a.check: no check named nope in vapi-checks.yml", + ], + [ + { + orgs: { a: "b-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("b"), + }, + "orgs.a.check: check b-core must read and run in a (it reads b and runs in b)", + ], + ]; + const seen = []; + for (const [args] of cases) { + fixture(args); + const result = await promote([...ONE_STEP, "--apply"], PASS); + seen.push([result.error, result.applies]); + } + assert.deepEqual( + seen, + cases.map(([, message]) => [message, []]), + ); +}); + +test("a pass is reused for the same org until something applies into it", async () => { + // p1: b→c (gate on b), p2: a→b (changes b), p3: b→d (gate on b again). + fixture({ + orgs: { a: undefined, b: "b-core", c: undefined, d: undefined }, + pipelines: { p1: ["b", "c"], p2: ["a", "b"], p3: ["b", "d"] }, + files: { + ...filesFor("b"), + ...filesFor("a"), + "resources/a/assistants/extra.yml": "name: Extra\n", + }, + checks: checkFor("b"), + }); + const reused = await promote(["--all", "--apply"], PASS); + // Two pipelines out of a gated org with nothing applied into it in between. + fixture({ + orgs: { a: "a-core", b: undefined, c: undefined }, + pipelines: { p1: ["a", "b"], p2: ["a", "c"] }, + files: filesFor("a"), + checks: checkFor("a"), + }); + const cached = await promote(["--all", "--apply"], PASS); + assert.deepEqual( + [ + reused.error, + reused.checks, + reused.applies, + cached.checks, + cached.applies, + ], + [ + undefined, + ["b-core@b", "b-core@b"], + ["c", "b", "d"], + ["a-core@a"], + ["b", "c"], + ], + ); +}); + +test.after(() => rmSync(ROOT, { recursive: true, force: true })); + +// With no `check:` in promotion.yml the gate must be invisible: the same +// plan output as before gates existed, no check run, and vapi-checks.yml +// never read (here it is invalid, so reading it would throw). +const UNGATED_PLAN_OUTPUT = [ + "", + "release: a → b", + " create assistants/intake.yml", + " create simulations/personalities/calm.yml", + " create simulations/scenarios/s1.yml", + " create simulations/tests/t1.yml", + " create structuredOutputs/ok.yml", + "", +].join("\n"); + +test("with no gate configured, promotion is unchanged: same plan output, no check, checks config never read", async () => { + const ungated: FixtureArgs = { + orgs: { a: undefined, b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + }; + fixture(ungated); + write("vapi-checks.yml", "version: 999\nnot: [valid\n"); + const plan = await promote(ONE_STEP, FAIL); + fixture(ungated); + write("vapi-checks.yml", "version: 999\nnot: [valid\n"); + const applied = await promote([...ONE_STEP, "--apply"], FAIL); + assert.deepEqual( + [ + plan.error, + plan.output, + plan.checks, + applied.error, + applied.checks, + applied.applies, + ], + [undefined, UNGATED_PLAN_OUTPUT, [], undefined, [], ["b"]], + ); +}); diff --git a/tests/promotion.test.ts b/tests/promotion.test.ts index 08e418c..615efbb 100644 --- a/tests/promotion.test.ts +++ b/tests/promotion.test.ts @@ -449,3 +449,19 @@ test("promote CLI dry run uses the reviewed config without writing target files" await fx.cleanup(); } }); + +test("promotionConfigParse reads an org's optional check gate and rejects a non-slug", () => { + const config = (check: string) => + `version: 1\norgs:\n dev:\n check: ${check}\n prod: {}\npipelines:\n release:\n orgs: [dev, prod]\n resources: ['**/*']\n`; + let error = ""; + try { + promotionConfigParse(config("Not A Slug")); + } catch (caught) { + error = (caught as Error).message; + } + const parsed = promotionConfigParse(config("dev-core")); + assert.deepEqual( + [parsed.orgs.dev?.check, parsed.orgs.prod?.check, error], + ["dev-core", undefined, "org dev.check must be a check name from vapi-checks.yml"], + ); +});