diff --git a/.agents/skills/verify-dependency-pr/SKILL.md b/.agents/skills/verify-dependency-pr/SKILL.md new file mode 100644 index 000000000..6b4181900 --- /dev/null +++ b/.agents/skills/verify-dependency-pr/SKILL.md @@ -0,0 +1,150 @@ +--- +name: verify-dependency-pr +description: Verify a Dependabot (or other dependency-update) PR on ldcli to the standard of a diligent reviewer, and produce a verdict comment (safe to merge, needs a human decision, block, or incomplete). Use to verify, review, triage, or check a dependency update PR, given its number or branch. +--- + +# Verify a dependency PR + +You verify the PR to the standard of a diligent reviewer. After your run, a person must not have to verify anything again. A person only answers the decisions that your comment asks for. + +You do not approve, request changes, merge, or push to the PR branch. Verification writes files only. The caller runs `post-comment.sh` to post the comment. + +All paths are relative to the repository root. Output goes to `.verify-out/pr-/`, which git ignores. + +## Verdicts + +| Verdict | Meaning | Exit code | +|---|---|---| +| `block` | The PR must not merge as it is. The comment gives the fix. | 1 | +| `needs-human` | Verification is complete. A person must answer one or more specific questions. | 1 | +| `incomplete` | A required check, gate, or review did not run, or proved nothing. A rerun or more agent work closes it, not a person. | 2 | +| `safe-to-merge` | Every required check and review ran and passed. | 0 | + +A gate is a check that must pass, for example the build or the tests. A failure that also happens on base is "pre-existing" and does not count against the PR. Some checks list separate findings: actionlint, golangci-lint, prettier, and `npm ls`. If each finding of the PR also occurs on base, the failure is pre-existing. A pre-existing failure never satisfies a gate. + +## The diligent reviewer standard + +This table shows what must be true before the PR can be safe to merge. The tier comes from `result.json` (`.tier`). You can raise the tier, but you cannot lower it. + +| Tier | Typical updates | Required | +|---|---|---| +| low | A patch of a direct dependency, a dev dependency, or a transitive-only change | The baseline checks pass. | +| medium | A minor update of a runtime dependency, or any update in a risk area (CGO, LaunchDarkly SDKs, CLI libraries, UI framework, Docker base image) | The low items, plus an impact review of each direct update. If upstream behavior changes reach ldcli, at least one generated check must prove the change. | +| high | A major update (including a 0.x minor), a code generator, release tooling, a go directive change, or more than three direct updates | The medium items, plus each breaking change mapped to the ldcli code that it touches. | + +The baseline checks cover each ecosystem: build, tests, generated code, UI dist, smoke tests, transitive changes, licenses, upstream notes, and Actions interfaces. The playbooks list them. + +## Procedure + +### 1. Run the baseline + +```bash +scripts/dependency-pr/verify.sh --pr # or --branch +``` + +The script makes two worktrees. `work/base` is the tip of the base branch. `work/pr` is the PR merged into that tip. The script classifies the updates, runs the baseline checks, and writes `result.json`, `comment.md`, and `logs/`. If Docker is available and the update touches CGO, the go directive, or release tooling, add `--profile full`. + +Read these fields in `result.json`: + +- `.classification`: the updates, the tier, the risk tags, and the reasons for the tier. +- `.blocks`, `.decisions`, `.incomplete`: the items that set the verdict. +- `.checks[]`: the outcome of each check. The log of a check is in `logs//.log`. Its evidence is in `state/checks///`. +- `.fixes`: machine-applicable fix recipes. Do not apply them. A later step will. + +If the exit code is 2 because a tool is missing, install the tool and run the script again. + +### 2. Do the impact review (medium and high tier) + +Open the playbook for each ecosystem in `.classification.ecosystems`: + +| Ecosystem | Playbook | +|---|---| +| gomod | [playbooks/gomod.md](playbooks/gomod.md) | +| npm-ui (`internal/dev_server/ui`) | [playbooks/npm-ui.md](playbooks/npm-ui.md) | +| npm-wrapper (root `package.json`) | [playbooks/npm-wrapper.md](playbooks/npm-wrapper.md) | +| github-actions | [playbooks/github-actions.md](playbooks/github-actions.md) | +| docker | [playbooks/docker.md](playbooks/docker.md) | + +For each direct update, do these steps: + +1. Read the upstream notes. The `upstream-changes` check saves them in `state/checks/upstream-changes/pr/notes/`, and its details give the compare link. For an npm package in a monorepo, the notes come from the package `CHANGELOG.md`. For `golang.org/x/*` modules, the notes are the commit messages. The summary gives the coverage of each update. If the notes are partial or missing, read the upstream diff from the compare link for the rest of the range. Also read the PR body. If the `pr-disclosure` check names direct updates that the body does not name, review those updates the same as the others. Only Dependabot's own text counts as disclosure. The check ignores bot summaries and quoted release notes. An update that a named update requires is "forced", and it needs no decision. +2. Find breaking changes, security fixes, deprecations, new minimum versions (Go, Node, runner), new peer requirements, and license changes. +3. Find where ldcli uses the dependency. For Go, use `rg -l '"' --glob '*.go'` and `go list -deps ./... | rg `. For the UI, use `rg "from '" internal/dev_server/ui/src`. +4. Map each upstream change to the ldcli code that it reaches. A change that no ldcli code reaches is "not reachable". +5. Read the `transitive-changes` and `license-changes` details. Explain each new module, major transitive update, and license change. + +Write `agent/impact.json` in the output directory: + +```json +{ + "schema": 1, + "summary": "What changed upstream, and what in ldcli it reaches.", + "tier": "medium", + "tier_reasons": ["Only if you raise the tier: the reason"], + "changelog": [{"package": "github.com/mattn/go-sqlite3", "range": "1.14.28..1.14.52", "notes": "…", "breaking": false, "url": "https://…"}], + "usage": ["internal/dev_server/db/sqlite.go"], + "behavior_changes_reachable": true, + "no_local_proof": "Only if no local check can run the changed code: the reason", + "findings": [ + {"severity": "info", "text": "A fact for the reader.", "evidence": ["url or path"]}, + {"severity": "decide", "question": "Accept X for Y?", "text": "Why only a person can decide this.", "evidence": ["…"]}, + {"severity": "block", "text": "Concrete breakage, for example a removed API that ldcli calls.", "evidence": ["…"]} + ] +} +``` + +The verdict is `incomplete` if `changelog` does not name every direct update, or if `behavior_changes_reachable` is missing. + +Sometimes no local check can run the changed code. An example is an action that only release workflows use. In that case, set `no_local_proof` to the reason. The verdict then asks a person to accept the change without a local proof. Do not use `no_local_proof` to avoid a check that you can write. + +### 3. Write generated checks + +If `behavior_changes_reachable` is true, write at least one discriminating check. Follow [reference/generated-checks.md](reference/generated-checks.md). Put the checks in `generated/` in the output directory. The important rules: + +- A discriminating check must fail on the old version and pass on the new version. Only such a check counts as proof. +- A guard asserts behavior that must not change. It passes on both versions. A guard never counts as proof, but a guard that fails on the PR blocks the PR. +- Assert behavior, not version strings. Test code that the update reaches. + +### 4. Run the generated checks + +```bash +scripts/dependency-pr/verify.sh --pr --phase generated +``` + +This command runs each generated check on base and on the PR. It reads `agent/impact.json`, calculates the verdict again, and writes `comment.md` again. If you change only `impact.json`, use `--phase render`. If a generated check is `error`, `invalid`, or `fails-both`, fix it and run the phase again. + +### 5. Report + +Report the verdict, the blocks, the decisions, the incomplete items, and the path to `comment.md`. Give pre-existing problems on main separately. A generated check can find the same type of problem in future updates. In that case, recommend it for the baseline (see "Promotion" in the reference). + +To post, the caller runs: + +```bash +scripts/dependency-pr/post-comment.sh --out-dir .verify-out/pr- --dry-run # then without --dry-run +``` + +The script edits one comment in place. It does not post if the PR head moved after verification. + +## When to ask a person + +Ask a person only when the evidence is complete and the next step is a choice. Each decision is one question that a person can answer with yes or no. Put the evidence next to it. + +These items are decisions: + +- Accept a change that no automated check can run, for example "Accept actions/checkout v6 in release workflows that PR CI does not run?" +- Accept a new license, a license change, or a new npm install script. +- Accept a visible change to the CLI, for example different help text or flags. +- Choose between options, for example "Remove the unused react-window instead of updating it?" + +These items are not decisions: + +- A stale dist, untidy `go.mod`, or generated code that is out of date. These are blocks with a fix. +- A check that did not run because a tool is missing. This item is incomplete. +- A breaking change that you have not mapped yet. Do the mapping. + +## Rules + +- Do not fix the PR yourself. Each failing check records a fix recipe in `result.json` for a later step. +- Do not run ad hoc commands and report them as baseline results. Put them in generated checks, so that they run on both versions. +- Do not trust the local environment. `verify.sh` removes `LD_*` variables and uses private XDG directories, because local credentials make `go test` fail. Generated checks get the same environment. +- A check that fails because of a tool or environment problem must report `incomplete`, not `fail`. diff --git a/.agents/skills/verify-dependency-pr/playbooks/docker.md b/.agents/skills/verify-dependency-pr/playbooks/docker.md new file mode 100644 index 000000000..71c25c69d --- /dev/null +++ b/.agents/skills/verify-dependency-pr/playbooks/docker.md @@ -0,0 +1,24 @@ +# Playbook: Docker base image (`Dockerfile.goreleaser`) + +The published image is `FROM alpine:` plus the static `ldcli` binary from goreleaser. The release builds the images. PR CI does not build them. + +## Diligent reviewer standard + +1. The new tag exists. +2. The release image builds from the PR with a binary made as the release makes it: CGO for SQLite, musl, static link. +3. In the image, `ldcli --version` runs, HTTPS to LaunchDarkly works with the CA bundle of the image, and the dev server starts and serves its API. The dev server uses SQLite, so this step tests CGO on musl. +4. The changes between the two base image versions are known (musl, CA bundle, busybox, end of support). + +## What the baseline checks + +`docker-image` (gate) covers items 1 to 3. It needs Docker and `musl-gcc` (package `musl-tools`). If one of them is missing, or if a registry lookup or image pull fails, the check reports `incomplete`, not `fail`. `binary-smoke` also runs. + +## What the agent must do + +- Read the Alpine release notes for each minor version in the range. ldcli links statically, so it needs only the kernel interface and the certificates in `/etc/ssl`. +- Write down the support status. Alpine branches get security fixes for about two years. A move away from a branch that has no more support is a security improvement. +- If a scanner is available (`docker scout cves`, `trivy image`), compare the CVE counts of the old and new image. + +## When to ask a person + +If the image builds and runs and the impact review is complete, the PR can be safe to merge. Ask a person only if the new base image changes what users see, for example a removed shell or a different user. diff --git a/.agents/skills/verify-dependency-pr/playbooks/github-actions.md b/.agents/skills/verify-dependency-pr/playbooks/github-actions.md new file mode 100644 index 000000000..a1cf20419 --- /dev/null +++ b/.agents/skills/verify-dependency-pr/playbooks/github-actions.md @@ -0,0 +1,38 @@ +# Playbook: GitHub Actions (`.github/workflows`, `.github/actions`) + +## Diligent reviewer standard + +1. Each third-party action uses a full commit SHA with a version comment (SEC-7924, #668). +2. The workflows pass `actionlint`. +3. For each updated action, the inputs that the repository passes still exist, and no new required input is missing. The outputs that later steps read still exist. +4. Changed input defaults, the runtime (for example `node20` to `node24`), and the runner requirements are known. +5. The `permissions` blocks of the workflows did not change, or the change is approved. +6. PR CI runs each workflow that uses the action. For a workflow that runs only at release, a person accepts the risk. +7. The upstream release notes between the two versions are known. + +## What the baseline checks + +| Statement | Check | +|---|---| +| 1 | `actions-pinning`. Actions from `actions/`, `github/`, and `launchdarkly/` are exempt, as in the current repository. | +| 2 | `actionlint` (v1.7.7, installed through `go install` when it is missing) | +| 3, 4, 5 | `actions-coverage`. It reads the upstream `action.yml` at both refs and traces composite actions back to the workflows that call them. It also compares the `permissions` blocks on base and PR. If the interface does not match, the PR is blocked. If workflows pin the action at different old refs (v4 and v5), the check compares each old ref with the new ref. An output that no version declares is set at run time, so the check cannot compare it. The details name it. | +| 6 | `actions-coverage`. A breaking update that a workflow without a `pull_request` trigger uses becomes a decision. | +| 7 | `upstream-changes` | + +PR CI runs only `go.yml`, `dev-server-ui.yml`, `dependency-scan.yml`, and `lint-pr-title.yml`. `release-please.yml`, `manual-publish.yml`, `check-openapi-updates.yml`, and the `publish` and `publish-npm` composite actions do not run on a PR. + +## What the agent must do + +- Read the release notes of each major version in the range. Note new runner minimum versions, changed defaults, and changed credential or token behavior. +- Look at the steps that use the action in workflows that PR CI does not run. Make sure that their inputs, outputs, and side effects (for example `git push` after `actions/checkout`) still work. Put each check that you can write as a guard in `generated/`. +- Dependabot does not scan the composite actions in `.github/actions/`. Note any version difference that the update creates between a workflow and a composite action. +- For release-please majors, compare `release-please-config.json` and `.release-please-manifest.json` with the configuration schema of the new version, and the outputs that `release-please.yml` reads. + +## When to ask a person + +- A breaking update is used in workflows that PR CI does not run. Ask: "Accept in , which PR CI does not run?" Give the interface comparison, the runtime change, and the guard results as evidence. +- A workflow `permissions` block changed. +- A non-breaking update changes the default of an input that the repository does not set. + +No local check can run a release workflow. The comment can recommend a `manual-publish` dry run (`dry-run: true`) as follow-up. diff --git a/.agents/skills/verify-dependency-pr/playbooks/gomod.md b/.agents/skills/verify-dependency-pr/playbooks/gomod.md new file mode 100644 index 000000000..b07babdbc --- /dev/null +++ b/.agents/skills/verify-dependency-pr/playbooks/gomod.md @@ -0,0 +1,54 @@ +# Playbook: Go modules (`go.mod` and `go.sum`) + +## Diligent reviewer standard + +A diligent reviewer makes sure that these statements are true for a Go update: + +1. ldcli builds, passes `go vet`, and passes its tests with the new version. +2. `go mod tidy` and `go generate ./...` make no change. If a generator changed, the regenerated code builds. +3. The binary runs. All commands print help, and the dev server serves its UI and API. +4. Every release target compiles. The release uses CGO for SQLite on linux (musl, static), windows (mingw), and macOS (osxcross). +5. The upstream changes between the two versions are known. Each breaking change, security fix, and behavior change is mapped to ldcli code, or is shown to be not reachable. +6. Each new or changed module that goes into the binary is known, and its license is in the policy. +7. A go or toolchain directive change works with every tool that CI and the release use. +8. No new reachable vulnerability appears. + +## What the baseline checks + +| Statement | Check | +|---|---| +| 1 | `go-build-vet`, `go-test` (both are gates) | +| 2 | `go-mod-tidy`, `go-generate-drift` | +| 3 | `binary-smoke` (gate), `cli-help-diff` | +| 4 | `release-snapshot` (full profile, needs Docker). It is a gate for updates with the `cgo` or `go-directive` tag. If the private release image cannot be pulled, the check uses the public `goreleaser/goreleaser-cross:v1.24.2` image and musl.cc toolchains, both pinned. The summary names the image, and the details list the fidelity gaps. | +| 5 | `upstream-changes` collects the notes and the compare link. For `golang.org/x/*` modules, it uses the `github.com/golang` mirror, and the notes are the commit messages. The agent does the mapping. | +| 6 | `transitive-changes`, `license-changes` | +| 7 | `go-directive`, plus `golangci-lint` and `release-snapshot` as gates when the tag `go-directive` is set | +| 8 | `govulncheck`. Only a new reachable advisory fails. The details list the advisories that the PR fixes (reachable, in an imported package, or in a required module) and the reachable advisories that stay. | + +## What the agent must do + +- Read the notes in `state/checks/upstream-changes/pr/notes/`. If a module has no notes, read the compare diff. Look at the API changes in the packages that ldcli imports. +- Find the ldcli code that uses the module: `rg -l '"' --glob '*.go'`. For an indirect update, find the reason with `go mod why -m ` in `work/pr`. +- For each changed transitive module in the `transitive-changes` details, find what pulls it in, and say if it changes behavior that ldcli uses. +- If a directive change appears, read the release notes of the new Go version for changes that affect ldcli. + +## Risk areas + +| Tag or module | What to look at | Generated-check ideas | +|---|---|---| +| `codegen`: oapi-codegen, kin-openapi, strcase | The generator and its runtime library must change together (for example `oapi-codegen/runtime`). `go-generate-drift` must not become worse than on base. | Regenerate in `work/pr`, build, and run `go test ./internal/dev_server/api/...`. | +| `mocks`: go.uber.org/mock | The repository commits the mocks. A diff after regeneration means that the mock format changed. | Regenerate the mocks and run `go test ./internal/dev_server/...`. | +| `cgo`, `dev-server`: mattn/go-sqlite3 | The dev server stores its state in SQLite: `internal/dev_server/db`, `events_db`, `db/backup`. | A temporary test in the affected package. For example, the #829 check measures allocations per row in `events_db.QueryEvents`. | +| `ld-sdk`: go-server-sdk, go-sdk-common | The dev server forwards SDK streams and evaluations (`internal/dev_server/sdk`, `adapters`). | Start the dev server, import a project from a fixture, and evaluate a flag through the `/sdk` endpoints. | +| `setup`: sdk-meta | It supplies the SDK lists for `setup` and quickstart. | Compare the SDK list output on base and PR. Make sure that each SDK ID that ldcli uses still exists. | +| `cli-surface`: cobra, pflag, viper, mapstructure | Flag parsing, the order of `LD_*` variables and the configuration file, and the usage templates. `cli-help-diff` shows the help changes. | A test of the configuration order for `--base-uri`: the flag first, then `LD_*`, then the file. | +| `tui`: charmbracelet | The interactive flows have few tests. | Help output and output without a terminal for `setup` and quickstart. | + +## When to ask a person + +- `cli-help-diff` shows a change in help text or flags. Ask if the change is acceptable. +- A new module or a changed module has a license outside the policy, or its license changed. +- The impact review finds a behavior change that ldcli users will see. Ask if the change is acceptable, and state the change. + +A pre-existing `go-generate-drift` failure (from #720) is not caused by the PR. The comment lists it once. It does not block the PR. diff --git a/.agents/skills/verify-dependency-pr/playbooks/npm-ui.md b/.agents/skills/verify-dependency-pr/playbooks/npm-ui.md new file mode 100644 index 000000000..8b2e4441b --- /dev/null +++ b/.agents/skills/verify-dependency-pr/playbooks/npm-ui.md @@ -0,0 +1,55 @@ +# Playbook: dev-server UI (`internal/dev_server/ui`, npm) + +## Diligent reviewer standard + +A diligent reviewer makes sure that these statements are true for a UI update: + +1. `npm ci` resolves the tree with no peer conflict, and `npm ls --all` reports no new problem. +2. Lint, format, tests, and the production build pass. +3. The committed `dist/` is the same as a fresh build. The Go binary embeds `dist/`, so a stale `dist/` ships old code. +4. The binary with the new `dist/` serves the UI. +5. Each updated direct dependency is used. An unused dependency is a candidate for removal. +6. The upstream changes are known: breaking changes, new `engines` and peer requirements, and security fixes. +7. New and changed transitive packages are known. No new install script runs without approval. Licenses are in the policy. +8. No new advisory appears in the runtime dependencies. + +## What the baseline checks + +| Statement | Check | +|---|---| +| 1 | `ui-npm-ci` (gate), `ui-npm-ls` | +| 2 | `ui-lint`, `ui-prettier`, `ui-test` (gate), `ui-build-drift` | +| 3 | `ui-build-drift`. If it fails, `result.json` has the fix recipe `ui-dist-rebuild`. | +| 4 | `binary-smoke`. It runs after `ui-build-drift`, so it embeds the new build. | +| 5 | `ui-dep-usage`. Its details also list `engines`, peer dependencies, and deprecation notes. | +| 6 | `upstream-changes` collects the notes. For a monorepo package (react-router, `@launchpad-ui/*`), the notes come from the package `CHANGELOG.md`. The agent does the mapping. `pr-disclosure` names the direct updates that a grouped PR does not list in its description. | +| 7 | `transitive-changes`, `license-changes`. An install script that the package already had on base is not new. The license question groups a package with its `-*` platform packages. It also names dev-only build tools. | +| 8 | `ui-npm-audit`. It compares advisory IDs (GHSA). A package that npm audit flags only through another package has no advisory of its own. | + +## Known failure modes + +- A runtime dependency change rebuilds `dist/index.html`. Dependabot does not commit it, so the `dev-server UI` CI job fails (#639, #640). The verdict is `block`, with the rebuild as the fix. +- `@launchpad-ui/*`, React 18, and the `react-router-dom` v6 `overrides` conflict with each other (#638, #642, #723, #779). If `npm ci` fails, the verdict is `block`. A coordinated upgrade or a scoped `overrides` entry fixes it (#777 shows the pattern). +- No code imports `react-window` (#831). `ui-dep-usage` asks if the dependency can go. + +## What the agent must do + +- Read the notes for each direct update. For a major update, find the removed or renamed APIs and search for them in `src/`. +- Compare the `engines` and peer requirements in the `ui-dep-usage` details with the Node version that CI uses (`lts/*`) and with React 18. +- devDependencies do not ship, but they can change the format (prettier), the lint rules, or the built `dist/` (vite, typescript). Look at `ui-prettier` and `ui-build-drift`. +- For a security update, name the advisory that the update fixes (`ui-npm-audit` lists the fixed advisories). + +## Generated-check ideas + +| Update | Idea | +|---|---| +| A component library (`@launchpad-ui/*`) | A guard: a vitest render test of the screens that use the changed components, in a temporary `src/__verify__/` folder. | +| `launchdarkly-js-client-sdk` | Start the dev server, load `/ui/`, and make sure that the bundle connects to the `/sdk` endpoints of the dev server. | +| `vite`, plugins, typescript | A guard: `dist/index.html` is one file with no external scripts, has `
`, and its size is within 20% of base. | +| An unused dependency | A guard: the new build is the same, byte for byte, as the build on base. | + +## When to ask a person + +- An updated dependency is unused. Ask if it can go instead of the update. +- A new advisory appears in the runtime dependencies. Ask if it is acceptable, and name the advisory. +- A new install script or a license outside the policy appears. diff --git a/.agents/skills/verify-dependency-pr/playbooks/npm-wrapper.md b/.agents/skills/verify-dependency-pr/playbooks/npm-wrapper.md new file mode 100644 index 000000000..b2a914557 --- /dev/null +++ b/.agents/skills/verify-dependency-pr/playbooks/npm-wrapper.md @@ -0,0 +1,24 @@ +# Playbook: npm wrapper package (root `package.json`) + +The root package publishes `@launchdarkly/ldcli` to npm. Its only dependency is `@go-task/go-npm`. The `postinstall` script of go-npm downloads the GitHub release archive that `goBinary.url` names, for the version in `package.json`. PR CI does not test this path. + +## Diligent reviewer standard + +1. The package installs with the locked go-npm version, and the postinstall script installs a binary that runs and reports the package version. +2. The packed contents (`npm pack`) contain only the expected files. +3. The upstream changes of go-npm are known: URL templates, archive formats, platform names, and `engines`. +4. The license of go-npm is in the policy. + +## What the baseline checks + +`npm-wrapper-install` (gate) covers items 1 and 2. It installs into a scratch directory with scripts on, as users do, and with a scratch `npm_config_prefix`. Newer npm versions do not have `npm bin`, so go-npm copies the binary into `$npm_config_prefix/bin`. The check needs network access to npm and GitHub. `upstream-changes` collects the notes for item 3. `license-changes` covers item 4. + +## What the agent must do + +- npm does not publish the lockfile. Users get the go-npm version that the range in `package.json` allows (`^0.2.0`). A change to the lockfile only changes what CI and the check install. Only a range change in `package.json` reaches users. State which case applies. +- Make sure that the archive names from `.goreleaser.yaml` (`archives.name_template`) still match what go-npm expects. +- The release publishes through `scripts/publish-npm.sh` and `.github/actions/publish-npm`, with npm trusted publishing (npm 11.5.1 or later). Make sure that the update needs no change to the publish flags. + +## When to ask a person + +The tier is high by default, because every npm user runs this code at install time. If the evidence is complete and nothing failed, the PR can be safe to merge. Ask a person only if go-npm changes what users get, for example a new install location or a new platform mapping. diff --git a/.agents/skills/verify-dependency-pr/reference/generated-checks.md b/.agents/skills/verify-dependency-pr/reference/generated-checks.md new file mode 100644 index 000000000..9981aa5d4 --- /dev/null +++ b/.agents/skills/verify-dependency-pr/reference/generated-checks.md @@ -0,0 +1,95 @@ +# Generated checks: format and rules + +A generated check is a check that the agent writes for one PR. The runner runs it on the base version and on the PR version. + +## Layout + +``` +.verify-out/pr-/generated/ + checks.json manifest + .sh one script for each check +``` + +```json +{ + "schema": 1, + "checks": [ + { + "id": "events-query-cancellation-overhead", + "title": "Debug-events query adds no per-row allocations under a request context", + "kind": "discriminating", + "rationale": "go-sqlite3 1.14.51 stopped starting a goroutine and a channel for each row when ctx.Done() != nil. events_db.QueryEvents reads rows under the HTTP request context.", + "script": "events-query-cancellation-overhead.sh", + "timeout": 300 + } + ] +} +``` + +- `id`: kebab-case, unique. +- `kind`: `discriminating` (expected to fail on base and pass on the PR) or `guard` (expected to pass on both). +- `timeout`: seconds. The default is 600. + +## Script contract + +The runner runs each script two times: one time with `SIDE=base` and `WT=`, and one time with `SIDE=pr` and `WT=`. In both runs, the working directory is `$WT`, the `LD_*` variables are not set, and the XDG configuration, state, and data directories are private. Each script must obey these rules: + +1. Start with `source "$VERIFY_ROOT/lib/check.sh"`. +2. Work on `$WT` only. Write temporary files in `$ARTIFACTS`, not in the worktree. If the script changes the worktree, call `restore_tree` before it finishes. +3. Finish with one of `pass`, `fail`, `incomplete`, or `skip`. If a tool is missing or the network fails, use `incomplete`, not `fail`. If the script stops without one of these (a crash or a timeout), the runner records `error`. + +Helpers: `detail "markdown line"` (shown in the comment), `detail_block file [lines]`, `recommend "action"`, `run cmd…` (writes the command to the log), `build_ldcli `, `ensure_ui_deps`, `free_port`, `updates_for `, `$UI_DIR_REL`. + +For Go behavior, a temporary test file works best. Copy it into the package under test, run only that test, and then delete it. This is the check for go-sqlite3 1.14.52 (#829). It measured 2.03 extra allocations for each row on the old version and 0.03 on the new version, so it is proven: + +```bash +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +dst="$WT/internal/dev_server/events_db/zz_verify_alloc_test.go" +cp "$(dirname "$0")/events_query_alloc_test.go.txt" "$dst" +trap 'rm -f "$dst"' EXIT +out=$(cd "$WT" && go test -count=1 -run '^TestVerifyQueryEventsCancellationOverhead$' -v ./internal/dev_server/events_db/ 2>&1) +rc=$? +echo "$out" +measured=$(grep -m1 '^allocs:' <<<"$out") +[ -n "$measured" ] || fail "test did not run: $(tail -n1 <<<"$out")" +[ $rc -eq 0 ] && pass "no per-row allocations under a cancellable context ($measured)" +fail "allocates per row under a cancellable context ($measured)" +``` + +The test runs `QueryEvents` on 500 rows two times: one time with `context.Background()` and one time with a context that can be cancelled. It uses `testing.AllocsPerRun`, and it fails if the difference is more than 0.5 allocations for each row. Measure a difference against a control, not an absolute number. Then unrelated allocations cancel out. + +Give helper files the extension `.txt`, so that Go tools do not read them from the generated directory. + +For CLI behavior, build with `build_ldcli "$ARTIFACTS/ldcli"` and test the output. For the dev server, start it as `binary-smoke.sh` does: a free port, `--access-token verify-smoke-placeholder`, and `XDG_STATE_HOME` in `$ARTIFACTS`. + +## Rules + +1. A check must fail on the old version before it counts. The runner applies this rule. A discriminating check counts only with the outcome `proven`. If it passes on both versions, it does not count. Then the check probably does not reach the changed code. Change the check. +2. Test behavior, not versions. `sqlite_version() >= 3.50` or "package.json says 2.3.2" proves nothing about ldcli. Run code paths that ldcli uses. +3. Name the ldcli code path that the check covers in `rationale`. If the changed code is not reachable from ldcli, set `behavior_changes_reachable` to false in `impact.json`, and give the reason. +4. Make each check deterministic. Do not use sleeps to wait for events. Do not depend on the clock. Do not use the network, unless the summary says so. Two runs must give the same result. +5. Generated checks add to the baseline. They do not replace or skip baseline checks. +6. Fix a broken check. Do not delete it. The outcomes `error`, `invalid`, and `fails-both` make the verdict `incomplete`. + +## Outcomes + +| Kind | Base | PR | Outcome | Effect on the verdict | +|---|---|---|---|---| +| discriminating | fail | pass | proven | Counts as proof of a reachable change | +| discriminating | pass | pass | not-discriminating | Does not count | +| discriminating | pass | fail | regression | block | +| discriminating | fail | fail | fails-both | incomplete: fix the check or the analysis | +| guard | pass | pass | holds | Supports the review, but is not proof | +| guard | pass | fail | regression | block | +| guard | fail | any | invalid | incomplete: fix the check | +| any | error, skip, or incomplete | any | error or incomplete | incomplete | + +## Promotion into the baseline + +Promote a check if it can find the same type of problem in future updates. Examples are "the regenerated oapi code builds" and "the dev-server database works with the new SQLite". Do not promote a check that is about one version. + +1. Copy the script to `scripts/dependency-pr/checks/.sh`. Replace fixed versions and package names with values from `$CLASSIFICATION` (`updates_for gomod`, and so on). +2. Add an entry to `scripts/dependency-pr/checks/registry.json` with `when` (ecosystems), an optional `packages` regex (for example `"^github.com/mattn/go-sqlite3$"`), `compare_base: true`, `required`, and `gate` if the check must pass. +3. If a failure has a mechanical fix, record it with `fix_recipe`. +4. Open a normal PR. A person reviews the promotion. diff --git a/.claude/skills b/.claude/skills new file mode 120000 index 000000000..2b7a412b8 --- /dev/null +++ b/.claude/skills @@ -0,0 +1 @@ +../.agents/skills \ No newline at end of file diff --git a/.cursor/skills b/.cursor/skills new file mode 120000 index 000000000..2b7a412b8 --- /dev/null +++ b/.cursor/skills @@ -0,0 +1 @@ +../.agents/skills \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4dea585d6..91734d5ca 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dist/ node_modules/ devserver.db ldcli +.verify-out/ diff --git a/.npmignore b/.npmignore index 3d1d171a4..791b1a6b7 100644 --- a/.npmignore +++ b/.npmignore @@ -7,3 +7,8 @@ go.sum Makefile .goreleaser.yaml Dockerfile.goreleaser +.agents/ +.claude/ +.cursor/ +.verify-out/ +scripts/dependency-pr/ diff --git a/AGENTS.md b/AGENTS.md index 8e6787a2c..a35981c04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ make build # Build binary as ./ldcli make test # Run all tests (go test ./...) go test ./path/to/pkg # Run tests for a specific package make generate # Regenerate code from OpenAPI spec (go generate ./...) -make vendor # Tidy and vendor dependencies +make vendor # go mod tidy + go mod vendor (vendor/ is not committed; usually you only want go mod tidy) make install-hooks # Install git pre-commit hooks make openapi-spec-update # Download latest OpenAPI spec and regenerate code ``` @@ -122,13 +122,25 @@ npm run build # Production build (checked into repo) - Go tests use `testify` for assertions and `go.uber.org/mock` for mocking - Mock generation via `mockgen` - Test data in `cmd/resources/test_data/` and `cmd/config/testdata/` +- `LD_*` environment variables and `~/.config/ldcli/config.yml` leak into `cmd/` tests and make some of them fail (e.g. `cmd/setup`, `cmd/whoami`). Run tests with `LD_*` unset and `XDG_CONFIG_HOME` pointed at an empty directory. ## Pre-commit Hooks -Installed via `make install-hooks`. Checks: -- `go fmt` formatting -- `go.mod`/`go.sum` tidiness -- Dev server UI tests and build (requires npm) +Installed via `make install-hooks`. `.pre-commit-config.yaml` runs only: +- `golangci-lint` v1.63.4 (default linters, no `.golangci.yml`) +- `end-of-file-fixer` + +It does **not** check `go fmt`, `go mod tidy`, or the UI build. Run `go mod tidy` yourself; `scripts/dependency-pr/verify.sh` checks tidiness, codegen drift, and UI `dist/` drift. + +## Dependency PR Verification + +To verify a Dependabot PR, follow the `verify-dependency-pr` skill (`.agents/skills/verify-dependency-pr/SKILL.md`). The skill holds the verification to the standard of a diligent reviewer, so that a person only answers specific decisions. + +- Run `scripts/dependency-pr/verify.sh --pr `, or `make verify-dependency-pr ARGS="--pr "`. +- The script writes `.verify-out/pr-/result.json` and `comment.md`. It does not post, approve, or merge. +- The verdict is `safe-to-merge`, `needs-human` (with one question for each decision), `block` (with the fix), or `incomplete` (a check did not run). +- The caller runs `post-comment.sh` to post the comment. +- Unit tests: `scripts/dependency-pr/test/run.sh`. Replays of past PRs (needs network): `scripts/dependency-pr/test/replay.sh`. ## Linting diff --git a/Makefile b/Makefile index 59ba190ae..4b71a5cb4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build generate log test vendor +.PHONY: build generate log test vendor verify-dependency-pr build: go build -o ldcli @@ -28,3 +28,6 @@ test: vendor: go mod tidy && go mod vendor + +verify-dependency-pr: + scripts/dependency-pr/verify.sh $(ARGS) diff --git a/scripts/dependency-pr/checks/actionlint.sh b/scripts/dependency-pr/checks/actionlint.sh new file mode 100755 index 000000000..b38d001e4 --- /dev/null +++ b/scripts/dependency-pr/checks/actionlint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +bin=$(go_tool github.com/rhysd/actionlint/cmd/actionlint v1.7.7) || incomplete "could not install actionlint" +run "$bin" -oneline -no-color | tee "$ARTIFACTS/actionlint.out" +rc=${PIPESTATUS[0]} +if [ "$rc" -eq 1 ]; then + grep -E '^\.github/' "$ARTIFACTS/actionlint.out" | sed -E 's/:[0-9]+:[0-9]+:/:/' | sort -u >"$ARTIFACTS/findings" + findings_file "$ARTIFACTS/findings" + detail_block "$ARTIFACTS/actionlint.out" 25 + fail "actionlint reports $(wc -l <"$ARTIFACTS/findings") finding(s)" +elif [ "$rc" -ne 0 ]; then + incomplete "actionlint could not run (exit $rc); see log" +fi +pass "actionlint clean" diff --git a/scripts/dependency-pr/checks/actions-coverage.sh b/scripts/dependency-pr/checks/actions-coverage.sh new file mode 100755 index 000000000..8809954e4 --- /dev/null +++ b/scripts/dependency-pr/checks/actions-coverage.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# PR CI runs only the workflows that trigger on pull_request. Release +# workflows (release-please, manual-publish, the publish composite) never run +# on a PR. This check verifies what it can from the action metadata, and asks +# a person only to accept what no check can run. +source "$VERIFY_ROOT/lib/check.sh" + +python3 -c 'import yaml' 2>/dev/null || incomplete "python3 with PyYAML is required for workflow analysis" +updates_for github-actions | jq -s '.' >"$ARTIFACTS/updates.json" +if ! run python3 "$VERIFY_ROOT/lib/actions_analysis.py" "$WT" "$ARTIFACTS/updates.json" "$BASE_WT" >"$ARTIFACTS/report.json"; then + incomplete "workflow analysis failed; see log" +fi +cat "$ARTIFACTS/report.json" +R="$ARTIFACTS/report.json" + +detail "| Action | Change | Used in | Runs on PR CI? | Upstream action.yml |" +detail "|---|---|---|---|---|" +jq -r '.actions[] | + "| `\(.name)` | \(if (.replaced | length) > 1 then (.replaced | join(", ")) else (.from // "∅") end) → \(.to // "∅") | \([.usages[].file] | unique | join(", ")) | \( + if (.usages | length) == 0 then "n/a" + elif all(.usages[]; .runs_on_pr) then "yes" + elif any(.usages[]; .runs_on_pr) then "partly, not " + ([.usages[] | select(.runs_on_pr | not) | .workflows[]] | unique | join(", ")) + else "**no**" end) | \( + if .upstream == null then "not compared" + elif .upstream.status != "ok" then "could not fetch" + else ("inputs used: " + (if (.upstream.inputs_used | length) > 0 then (.upstream.inputs_used | join(", ")) else "none" end) + + "; outputs read: " + (if (.upstream.outputs_read | length) > 0 then (.upstream.outputs_read | join(", ")) else "none" end) + + "; runtime " + (.upstream.runs_using | join(" → ")) + + (if (.upstream.defaults_changed | length) > 0 then "; changed defaults: " + ([.upstream.defaults_changed[] | "\(.input) \(.from // "none") → \(.to // "none")"] | join(", ")) else "" end)) end) |"' "$R" >>"$ARTIFACTS/details.md" +jq -r '.actions[] | select(.upstream.status == "ok" and ((.upstream.outputs_undeclared // []) | length) > 0) + | "- `\(.name)` does not declare the outputs \(.upstream.outputs_undeclared | join(", ")) in either version. The action sets them at run time, so this check cannot compare them."' "$R" >>"$ARTIFACTS/details.md" +jq -r '.actions[] | select(.upstream.status == "ok" and ((.upstream.defaults_same_on_github_com // []) | length) > 0) + | "- `\(.name)`: the default of \([.upstream.defaults_same_on_github_com[].input] | join(", ")) changed only for GitHub Enterprise Server. It gives the same value on github.com."' "$R" >>"$ARTIFACTS/details.md" +jq -r '.permissions_changes[] | "- permissions changed in \(.where): `\(.from)` → `\(.to)`"' "$R" >>"$ARTIFACTS/details.md" + +broken=$(jq -r '[.actions[] | select(.upstream.status == "ok") | .name as $n | .upstream + | ((.removed_but_used | map("\($n) no longer has input \(.)")) + (.new_required | map("\($n) requires new input \(.)")) + (.outputs_missing | map("\($n) no longer has output \(.)")))[]] | join("; ")' "$R") +if [ -n "$broken" ]; then + fingerprint "$broken" + recommend "Update the workflow steps for the new action interface: $broken" + fail "Workflow steps do not match the new action interface: $broken" +fi + +unfetched=$(jq -r '[.actions[] | select(.breaking and .upstream.status != "ok") | .name] | join(", ")' "$R") +[ -n "$unfetched" ] && incomplete "could not fetch the upstream action.yml to compare inputs and outputs for: $unfetched" + +questions=() +evidence=() +perm=$(jq -r '[.permissions_changes[] | "\(.where): \(.from) → \(.to)"] | join("; ")' "$R") +[ -n "$perm" ] && questions+=("accept the workflow permission changes ($perm)") +# Fields are joined with the unit separator, not a tab. Tab is IFS whitespace, +# so read would merge empty fields and shift the later fields left. +while IFS=$'\x1f' read -r name from to untested runtime defaults selfhosted used; do + [ -n "$name" ] || continue + ev="$name $from → $to: inputs and outputs that the repo uses are unchanged; runtime $runtime${defaults:+; changed defaults: $defaults}${selfhosted:+; non-standard runner labels: $selfhosted}" + evidence+=("$ev") + if [ -n "$untested" ]; then + questions+=("accept $name $to in workflows that PR CI does not run ($untested)") + elif [ -n "$defaults" ]; then + questions+=("accept the changed input defaults of $name in $used ($defaults)") + fi +done < <(jq -r '.actions[] | select(.upstream.status == "ok") | select(.breaking or (.upstream.defaults_changed | length) > 0) | + [.name, (if (.replaced | length) > 1 then (.replaced | join(", ")) else .from end), .to, + ([.usages[] | select(.runs_on_pr | not) | .workflows[]] | unique | map(sub("^\\.github/workflows/"; "")) | join(", ")), + (.upstream.runs_using | join(" → ")), + ([.upstream.defaults_changed[] | "\(.input): \(.from // "none") → \(.to // "none")"] | join(", ")), + (.self_hosted_runners | join(", ")), + ([.usages[].workflows[]] | unique | map(sub("^\\.github/workflows/"; "")) | join(", "))] | map(. // "" | tostring) | join("\u001f")' "$R") + +if [ ${#questions[@]} -gt 0 ]; then + fingerprint "${questions[*]}" + q="$(join_by '; and ' "${questions[@]}")" + decide "${q^}?" "$(join_by ' | ' "${evidence[@]}")" +fi + +untested_minor=$(jq -r '[.actions[] | select(.breaking | not) | .usages[] | select(.runs_on_pr | not) | .workflows[]] | unique | join(", ")' "$R") +if [ -n "$untested_minor" ]; then + info "Interfaces match. Non-breaking update also used in workflows that PR CI does not run: $untested_minor" +fi +pass "Every usage runs on PR CI, and the inputs, outputs, and runtime match" diff --git a/scripts/dependency-pr/checks/actions-pinning.sh b/scripts/dependency-pr/checks/actions-pinning.sh new file mode 100755 index 000000000..b900bbedd --- /dev/null +++ b/scripts/dependency-pr/checks/actions-pinning.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# SEC-7924 (#668): third-party actions must be pinned to a full commit SHA with +# a version comment. GitHub-owned and LaunchDarkly-owned actions are exempt. +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +exempt="${PIN_EXEMPT_OWNERS:-actions github launchdarkly}" +: >"$ARTIFACTS/violations" +while IFS=$'\t' read -r file ref comment; do + case "$ref" in ./* | docker://*) continue ;; esac + owner="${ref%%/*}" + [[ " $exempt " == *" $owner "* ]] && continue + sha="${ref##*@}" + if ! [[ "$sha" =~ ^[0-9a-f]{40}$ ]]; then + printf '%s: %s (not pinned to a commit SHA)\n' "$file" "$ref" >>"$ARTIFACTS/violations" + elif [ -z "$comment" ]; then + printf '%s: %s (missing "# vX" version comment)\n' "$file" "$ref" >>"$ARTIFACTS/violations" + fi +done < <(rg --no-line-number --with-filename -o -g '*.yml' -g '*.yaml' \ + '^\s*-?\s*uses:\s*["'\'']?([^\s"'\''#]+)["'\'']?(?:\s*#\s*(\S+))?' -r '$1 $2' .github/workflows .github/actions 2>/dev/null | + sed -E 's/:/\t/' | sort -u) + +if [ -s "$ARTIFACTS/violations" ]; then + sort -u -o "$ARTIFACTS/violations" "$ARTIFACTS/violations" + findings_file "$ARTIFACTS/violations" + detail_block "$ARTIFACTS/violations" 20 + recommend "Pin third-party actions to a full commit SHA with a \`# vX.Y.Z\` comment." + fail "$(wc -l <"$ARTIFACTS/violations") unpinned third-party action reference(s)" +fi +pass "All third-party actions pinned to commit SHAs" diff --git a/scripts/dependency-pr/checks/binary-smoke.sh b/scripts/dependency-pr/checks/binary-smoke.sh new file mode 100755 index 000000000..b0bb5c254 --- /dev/null +++ b/scripts/dependency-pr/checks/binary-smoke.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Builds the CLI and exercises it offline: version, help for every command, +# and a dev-server start that serves the embedded UI and the /dev API. +source "$VERIFY_ROOT/lib/check.sh" + +bin="$ARTIFACTS/ldcli" +if ! build_ldcli "$bin" >"$ARTIFACTS/build.out" 2>&1; then + cat "$ARTIFACTS/build.out" + fingerprint "$(sed -E 's/:[0-9]+:[0-9]+:/:/' "$ARTIFACTS/build.out")" + fail "go build fails" +fi + +problems=() +version=$("$bin" --version 2>&1) || problems+=("--version exits non-zero") +detail "- \`ldcli --version\`: $version" + +help_dump "$bin" "$ARTIFACTS/help" +n_cmds=$(wc -l <"$ARTIFACTS/help/.commands") +if [ -s "$ARTIFACTS/help/.failures" ]; then + sort -o "$ARTIFACTS/help/.failures" "$ARTIFACTS/help/.failures" + problems+=("--help fails for: $(head -n5 "$ARTIFACTS/help/.failures" | paste -sd, -)") +fi +[ "$n_cmds" -lt 20 ] && problems+=("only $n_cmds commands discovered") + +port=$(free_port) || fail "no free port for dev-server" +mkdir -p "$ARTIFACTS/xdg-state" +# The flag is required but unused when no project is configured, so the +# server starts without reaching LaunchDarkly. +XDG_STATE_HOME="$ARTIFACTS/xdg-state" "$bin" dev-server start --port "$port" --analytics-opt-out \ + --access-token verify-smoke-placeholder >"$ARTIFACTS/dev-server.log" 2>&1 & +pid=$! +trap 'kill $pid 2>/dev/null; wait $pid 2>/dev/null' EXIT + +up=false +for _ in $(seq 1 60); do + if curl -fsS -o "$ARTIFACTS/ui.html" "http://127.0.0.1:$port/ui/" 2>/dev/null; then + up=true + break + fi + kill -0 $pid 2>/dev/null || break + sleep 0.5 +done +if [ "$up" = true ]; then + grep -q '
' "$ARTIFACTS/ui.html" || problems+=("/ui/ does not serve the app shell") + ui_kb=$(($(wc -c <"$ARTIFACTS/ui.html") / 1024)) + projects=$(curl -fsS "http://127.0.0.1:$port/dev/projects" 2>/dev/null) + jq -e 'type == "array"' <<<"$projects" >/dev/null 2>&1 || problems+=("/dev/projects did not return a JSON array: ${projects:0:80}") + detail "- dev-server served /ui/ (${ui_kb} KB) and /dev/projects (\`${projects:0:40}\`)" +else + tail -n 20 "$ARTIFACTS/dev-server.log" + detail_block "$ARTIFACTS/dev-server.log" 15 + problems+=("dev-server did not serve /ui/ within 30s") +fi + +if [ ${#problems[@]} -gt 0 ]; then + fingerprint "$(printf '%s\n' "${problems[@]}" | sed -E 's/\([0-9]+ KB\)//')" + fail "$(join_by '; ' "${problems[@]}")" +fi +pass "--version, --help for $n_cmds commands, dev-server UI and API all OK" diff --git a/scripts/dependency-pr/checks/ci-status.sh b/scripts/dependency-pr/checks/ci-status.sh new file mode 100755 index 000000000..4b8ae83c1 --- /dev/null +++ b/scripts/dependency-pr/checks/ci-status.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" + +if [ "$(jq -r '.ci == null' "$PR_META")" = "true" ]; then + skip "No PR metadata (branch mode without an open PR)" +fi + +summary=$(jq -r ' + [.ci[] | {name, state: ((.conclusion // .state // .status // "") | ascii_upcase)}] as $c + | { + failing: [$c[] | select(.state | IN("FAILURE", "TIMED_OUT", "CANCELLED", "ACTION_REQUIRED", "STARTUP_FAILURE", "ERROR")) | .name] | unique, + pending: [$c[] | select(.state | IN("QUEUED", "IN_PROGRESS", "PENDING", "WAITING", "REQUESTED", "EXPECTED", "")) | .name] | unique, + total: ($c | length) + }' "$PR_META") + +failing=$(jq -r '.failing | join(", ")' <<<"$summary") +pending=$(jq -r '.pending | join(", ")' <<<"$summary") +total=$(jq -r '.total' <<<"$summary") +head=$(jq -r '.head_sha[0:7]' "$PR_META") + +detail "- CI results are for the PR head \`$head\` as pushed, which may be behind the base branch." +if [ -n "$failing" ]; then + fail "Failing on PR head $head: $failing" +fi +if [ -n "$pending" ]; then + incomplete "CI is still running on PR head $head: $pending" +fi +if [ "$total" -eq 0 ]; then + incomplete "No CI results on PR head $head" +fi +pass "All $total CI checks green on PR head $head" diff --git a/scripts/dependency-pr/checks/cli-help-diff.sh b/scripts/dependency-pr/checks/cli-help-diff.sh new file mode 100755 index 000000000..d7f7b3b9e --- /dev/null +++ b/scripts/dependency-pr/checks/cli-help-diff.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# CLI-library bumps (cobra, pflag, viper, glamour) can change flags, defaults, +# or help rendering without failing any test. Diff the full help tree. +source "$VERIFY_ROOT/lib/check.sh" + +for side in base pr; do + wt="$BASE_WT" + [ "$side" = pr ] && wt="$PR_WT" + if ! (cd "$wt" && run go build -o "$ARTIFACTS/ldcli-$side" .); then + incomplete "could not build the $side binary" + fi + help_dump "$ARTIFACTS/ldcli-$side" "$ARTIFACTS/help-$side" +done + +if diff -u "$ARTIFACTS/help-base/all.txt" "$ARTIFACTS/help-pr/all.txt" >"$ARTIFACTS/help.diff"; then + pass "Help output identical for $(wc -l <"$ARTIFACTS/help-pr/.commands") commands" +fi +added=$(comm -13 "$ARTIFACTS/help-base/.commands" "$ARTIFACTS/help-pr/.commands" | paste -sd, -) +removed=$(comm -23 "$ARTIFACTS/help-base/.commands" "$ARTIFACTS/help-pr/.commands" | paste -sd, -) +sections=$(awk '/^[ +-]?### /{s=substr($0, 2)} /^[+-][^+-]/{print s}' "$ARTIFACTS/help.diff" | sed -E 's/^#* ?//' | sort -u) +n_sections=$(printf '%s\n' "$sections" | grep -c .) +[ -n "$added" ] && detail "- Commands added: $added" +[ -n "$removed" ] && detail "- Commands removed: $removed" +detail "- Help text changed for: $(printf '%s\n' "$sections" | head -n 15 | paste -sd, -)" +detail_block "$ARTIFACTS/help.diff" 60 +fingerprint_file "$ARTIFACTS/help.diff" +decide "Accept these changes to the CLI help and flags (see the diff)?" "Help output differs for $n_sections command(s)${removed:+; removed: $removed}${added:+; added: $added}" diff --git a/scripts/dependency-pr/checks/docker-image.sh b/scripts/dependency-pr/checks/docker-image.sh new file mode 100755 index 000000000..9a11ca6ba --- /dev/null +++ b/scripts/dependency-pr/checks/docker-image.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# The published image (Dockerfile.goreleaser) is only built at release time. +# This builds it from the PR with a binary made the way the release linux +# target makes it (CGO for SQLite, musl, static), then runs it. +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +dockerfile="Dockerfile.goreleaser" +while IFS= read -r u; do + [ -n "$u" ] || continue + name=$(jq -r '.name' <<<"$u") + tag=$(jq -r '.to' <<<"$u") + case "$name" in + */*.*/* | *.*/*) detail "- \`$name:$tag\`: not on Docker Hub, so the tag was not looked up" ;; + *) + repo="$name" + [[ "$repo" == */* ]] || repo="library/$repo" + code=$(curl -sS -o "$ARTIFACTS/tag.json" -w '%{http_code}' "https://hub.docker.com/v2/repositories/$repo/tags/$tag") || code=000 + case "$code" in + 200) detail "- \`$name:$tag\` exists (digest \`$(jq -r '.digest // "?"' "$ARTIFACTS/tag.json" | cut -c1-19)\`, pushed $(jq -r '.tag_last_pushed // .last_updated // "?"' "$ARTIFACTS/tag.json"))" ;; + 404) fingerprint "missing-tag:$name:$tag"; fail "Base image tag $name:$tag does not exist on Docker Hub" ;; + *) incomplete "Docker Hub lookup for $name:$tag failed (HTTP $code)" ;; + esac + ;; + esac +done < <(updates_for docker) + +if ! command -v docker >/dev/null 2>&1 || ! docker info >/dev/null 2>&1; then + incomplete "Docker is not available. The base tag exists, but the release image was not built or run" +fi +command -v musl-gcc >/dev/null 2>&1 || incomplete "musl-gcc is not installed (package musl-tools), so the static release binary was not built" + +ctx="$ARTIFACTS/ctx" +mkdir -p "$ctx" +if ! run env CGO_ENABLED=1 CC=musl-gcc GOOS=linux go build -ldflags '-s -w -extldflags "-static"' -o "$ctx/ldcli" . >"$ARTIFACTS/go-build.out" 2>&1; then + cat "$ARTIFACTS/go-build.out" + fingerprint "$(sed -E 's/:[0-9]+:[0-9]+:/:/' "$ARTIFACTS/go-build.out")" + detail_block "$ARTIFACTS/go-build.out" 20 + fail "The static musl build of ldcli fails" +fi +cp "$dockerfile" "$ctx/Dockerfile" +base_image=$(sed -n 's/^FROM[[:space:]]\{1,\}\([^[:space:]]*\).*/\1/p' "$dockerfile" | head -n1) +if ! run docker pull -q "$base_image" >"$ARTIFACTS/pull.out" 2>&1; then + cat "$ARTIFACTS/pull.out" + incomplete "could not pull $base_image ($(tail -n1 "$ARTIFACTS/pull.out" | cut -c1-120))" +fi +img="ldcli-verify:$SIDE-$$" +if ! run docker build -q -t "$img" "$ctx" >"$ARTIFACTS/build.out" 2>&1; then + cat "$ARTIFACTS/build.out" + fingerprint_file "$ARTIFACTS/build.out" + detail_block "$ARTIFACTS/build.out" 20 + fail "docker build of $dockerfile fails" +fi +cleanup() { + [ -n "${cid:-}" ] && docker rm -f "$cid" >/dev/null 2>&1 + docker rmi -f "$img" >/dev/null 2>&1 +} +trap cleanup EXIT + +problems=() +version=$(docker run --rm "$img" --version 2>&1) || problems+=("ldcli --version fails in the image: $version") +detail "- \`ldcli --version\` in the image: $version" +docker run --rm --entrypoint cat "$img" /etc/os-release 2>/dev/null | sed -n 's/^PRETTY_NAME=/- image OS: /p' | tr -d '"' >>"$ARTIFACTS/details.md" + +# HTTPS to LaunchDarkly must work with the CA bundle of the image. +if docker run --rm --entrypoint wget "$img" -q -O /dev/null https://app.launchdarkly.com >"$ARTIFACTS/tls.out" 2>&1; then + detail "- HTTPS to app.launchdarkly.com works with the CA bundle of the image" +elif grep -qiE 'certificate|ssl|tls' "$ARTIFACTS/tls.out"; then + problems+=("HTTPS fails in the image: $(head -n1 "$ARTIFACTS/tls.out")") +else + detail "- HTTPS from the image was not tested (no network from the container: $(head -n1 "$ARTIFACTS/tls.out"))" +fi + +# The dev server uses SQLite through CGO; start it inside the image. +port=$(free_port) || incomplete "no free port for the dev server" +cid=$(docker run -d -p "127.0.0.1:$port:8765" "$img" dev-server start --port 8765 --analytics-opt-out --access-token verify-smoke-placeholder 2>"$ARTIFACTS/run.err") +up=false +for _ in $(seq 1 60); do + if out=$(curl -fsS "http://127.0.0.1:$port/dev/projects" 2>/dev/null); then up=true; break; fi + sleep 0.5 +done +if [ "$up" = true ] && jq -e 'type == "array"' <<<"$out" >/dev/null 2>&1; then + detail "- dev-server in the image served /dev/projects (\`${out:0:40}\`)" +else + docker logs "$cid" >"$ARTIFACTS/dev-server.log" 2>&1 + detail_block "$ARTIFACTS/dev-server.log" 15 + problems+=("dev-server does not start in the image") +fi + +if [ ${#problems[@]} -gt 0 ]; then + fingerprint "$(printf '%s\n' "${problems[@]}")" + fail "$(join_by '; ' "${problems[@]}")" +fi +pass "The release image builds; ldcli, HTTPS, and the dev server (SQLite) work in it ($version)" diff --git a/scripts/dependency-pr/checks/downgrades.sh b/scripts/dependency-pr/checks/downgrades.sh new file mode 100755 index 000000000..6a80331c1 --- /dev/null +++ b/scripts/dependency-pr/checks/downgrades.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" + +downs=$(jq -r '[.updates[] | select(.semver == "downgrade") | "\(.name) \(.from) → \(.to)"] | join(", ")' "$CLASSIFICATION") +if [ -n "$downs" ]; then + detail "- Downgrades usually mean the branch is stale and the base branch already moved past these versions." + FIX_KIND=comment fix_recipe dependabot-rebase "@dependabot rebase" + recommend "Rebase the PR (comment \`@dependabot rebase\`) or drop the stale pins that would now be downgrades." + fail "Downgrades: $downs" +fi +pass "No downgrades" diff --git a/scripts/dependency-pr/checks/go-build-vet.sh b/scripts/dependency-pr/checks/go-build-vet.sh new file mode 100755 index 000000000..175a340ac --- /dev/null +++ b/scripts/dependency-pr/checks/go-build-vet.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +run go build ./... 2>&1 | tee "$ARTIFACTS/build.out" +if [ "${PIPESTATUS[0]}" -ne 0 ]; then + grep -E '\.go:[0-9]+' "$ARTIFACTS/build.out" | sed -E 's/:[0-9]+:[0-9]+:/:/' | sort -u >"$ARTIFACTS/build.errs" + fingerprint_file "$ARTIFACTS/build.errs" + detail_block "$ARTIFACTS/build.out" 30 + fail "go build fails: $(head -n1 "$ARTIFACTS/build.errs")" +fi + +run go vet ./... 2>&1 | tee "$ARTIFACTS/vet.out" +if [ "${PIPESTATUS[0]}" -ne 0 ]; then + grep -E '\.go:[0-9]+' "$ARTIFACTS/vet.out" | sed -E 's/:[0-9]+:[0-9]+:/:/' | sort -u >"$ARTIFACTS/vet.errs" + fingerprint_file "$ARTIFACTS/vet.errs" + detail_block "$ARTIFACTS/vet.out" 30 + fail "go vet reports $(wc -l <"$ARTIFACTS/vet.errs") finding(s)" +fi +pass "go build ./... and go vet ./... succeed" diff --git a/scripts/dependency-pr/checks/go-directive.sh b/scripts/dependency-pr/checks/go-directive.sh new file mode 100755 index 000000000..75e40f07a --- /dev/null +++ b/scripts/dependency-pr/checks/go-directive.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# A change to the go or toolchain directive changes the Go that CI uses +# (go.yml reads go-version-file: go.mod). The "go-directive" risk tag makes +# golangci-lint and the goreleaser-cross snapshot required gates. This check +# covers the rest: the new Go is available, and no workflow pins an older Go. +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +read -r gofrom goto < <(jq -r '"\(.go_directive.from // "none") \(.go_directive.to // "none")"' "$CLASSIFICATION") +read -r tfrom tto < <(jq -r '"\(.toolchain.from // "none") \(.toolchain.to // "none")"' "$CLASSIFICATION") + +if [ "$gofrom" = "$goto" ] && [ "$tfrom" = "$tto" ]; then + pass "go $gofrom (unchanged)" +fi + +detail "- go directive: \`$gofrom\` → \`$goto\`; toolchain: \`$tfrom\` → \`$tto\`" +# `go version` in the worktree selects (and if necessary downloads) the Go that go.mod asks for. +if ! used=$(go version 2>"$ARTIFACTS/go-version.err"); then + incomplete "could not get the Go version that go.mod requires: $(head -n1 "$ARTIFACTS/go-version.err")" +fi +detail "- Go used for this PR: \`$(awk '{print $3}' <<<"$used")\`" + +# Workflow steps that pin a literal Go version older than the new directive. +older() { [ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | head -n1)" = "$1" ] && [ "$1" != "$2" ]; } +stale=() +while IFS=: read -r file line value; do + v=$(sed -E "s/[\"' ]//g" <<<"$value") + [[ "$v" =~ ^[0-9]+\.[0-9]+ ]] || continue + older "$v" "$goto" && stale+=("$file:$line pins Go $v") +done < <(rg -n --no-heading -o 'go-version:\s*\S+' .github 2>/dev/null | sed -E 's/go-version:\s*//') + +if [ ${#stale[@]} -gt 0 ]; then + fingerprint "$(printf '%s\n' "${stale[@]}" | sed -E 's/:[0-9]+ / /')" + recommend "Raise the pinned Go version in these workflow steps to at least $goto: ${stale[*]}" + fail "Workflows pin a Go version older than the new go directive $goto: $(join_by '; ' "${stale[@]}")" +fi +info "go directive $gofrom → $goto; the new Go is available and no workflow pins an older Go. golangci-lint and the release snapshot are required gates for this change" diff --git a/scripts/dependency-pr/checks/go-generate-drift.sh b/scripts/dependency-pr/checks/go-generate-drift.sh new file mode 100755 index 000000000..a6fedf7eb --- /dev/null +++ b/scripts/dependency-pr/checks/go-generate-drift.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Catches the #720 failure mode: a generator bump merges green, but the +# committed output was never regenerated and regenerating breaks the build. +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +restore_tree +n_gen=$(rg -l '^//go:generate' --glob '*.go' . | wc -l) + +run go generate ./... 2>&1 | tee "$ARTIFACTS/generate.out" +if [ "${PIPESTATUS[0]}" -ne 0 ]; then + grep -vE '^[0-9]{4}/[0-9]{2}/[0-9]{2} ' "$ARTIFACTS/generate.out" | sort -u >"$ARTIFACTS/generate.errs" + fingerprint_file "$ARTIFACTS/generate.errs" + detail_block "$ARTIFACTS/generate.out" 30 + restore_tree + fail "go generate fails" +fi + +if [ -z "$(git status --porcelain)" ]; then + pass "No drift across $n_gen go:generate directive(s)" +fi + +git diff >"$ARTIFACTS/drift.patch" +git status --porcelain >"$ARTIFACTS/drift.files" +git diff --stat=100 --stat-graph-width=20 >"$ARTIFACTS/drift.stat" +n_files=$(wc -l <"$ARTIFACTS/drift.files") +detail "Files rewritten by \`go generate ./...\` (patch saved as drift.patch next to this check's status):" +detail_block "$ARTIFACTS/drift.stat" 20 + +build_ok=true +if ! go build ./... >"$ARTIFACTS/rebuild.out" 2>&1; then + build_ok=false + sed -E 's/:[0-9]+:[0-9]+:/:/' "$ARTIFACTS/rebuild.out" | sort -u >"$ARTIFACTS/rebuild.errs" + detail "The regenerated code does not compile:" + detail_block "$ARTIFACTS/rebuild.out" 15 +fi +fingerprint "$(cat "$ARTIFACTS/drift.stat" "$ARTIFACTS/rebuild.errs" 2>/dev/null)" +restore_tree + +if [ "$build_ok" = false ]; then + recommend "Regenerate (\`make generate\`) in this PR and bump the generator's runtime library alongside it (e.g. oapi-codegen with oapi-codegen/runtime) so the regenerated code compiles." + fail "go generate rewrites $n_files file(s) and the regenerated code does not compile" +fi +recommend "Run \`make generate\` and commit the regenerated files." +# shellcheck disable=SC2046 +fix_recipe go-generate "go generate ./..." $(awk '{print $2}' "$ARTIFACTS/drift.files") +fail "go generate rewrites $n_files file(s); regenerated code compiles" diff --git a/scripts/dependency-pr/checks/go-mod-tidy.sh b/scripts/dependency-pr/checks/go-mod-tidy.sh new file mode 100755 index 000000000..c63c54e33 --- /dev/null +++ b/scripts/dependency-pr/checks/go-mod-tidy.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +if ! run go mod tidy; then + restore_tree + fail "go mod tidy failed" +fi +if ! git diff --quiet -- go.mod go.sum; then + git diff -- go.mod go.sum >"$ARTIFACTS/tidy.patch" + git diff --stat -- go.mod go.sum >"$ARTIFACTS/tidy.stat" + detail "go mod tidy would change:" + detail_block "$ARTIFACTS/tidy.patch" 30 + fingerprint_file "$ARTIFACTS/tidy.patch" + n=$(grep -c '^[+-][^+-]' "$ARTIFACTS/tidy.patch") + restore_tree + recommend "Run \`go mod tidy\` and commit go.mod/go.sum." + fix_recipe go-mod-tidy "go mod tidy" go.mod go.sum + fail "go mod tidy changes go.mod/go.sum ($n lines)" +fi +restore_tree +pass "go.mod/go.sum are tidy" diff --git a/scripts/dependency-pr/checks/go-test.sh b/scripts/dependency-pr/checks/go-test.sh new file mode 100755 index 000000000..a8945bfe2 --- /dev/null +++ b/scripts/dependency-pr/checks/go-test.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +run go test ./... 2>&1 | tee "$ARTIFACTS/test.out" +rc=${PIPESTATUS[0]} +pkgs=$(grep -cE '^(ok|FAIL)\s' "$ARTIFACTS/test.out") + +if [ "$rc" -ne 0 ]; then + grep -E '^(--- FAIL|FAIL\s|panic:)' "$ARTIFACTS/test.out" | + sed -E 's/ \([0-9.]+s\)//; s/\s+[0-9.]+s$//; s/\s+\[[^]]*\]$//' | sort -u >"$ARTIFACTS/failures" + fingerprint_file "$ARTIFACTS/failures" + detail_block "$ARTIFACTS/failures" 30 + fail "$(grep -c '^FAIL\s' "$ARTIFACTS/failures") package(s) failing: $(grep '^FAIL\s' "$ARTIFACTS/failures" | awk '{print $2}' | sed 's#github.com/launchdarkly/ldcli/##' | paste -sd, -)" +fi +cached=$(grep -cE '^ok\s.*\(cached\)' "$ARTIFACTS/test.out") +[ "$cached" -gt 0 ] || cached="" +pass "$pkgs packages pass${cached:+ ($cached served from the go test cache)}" diff --git a/scripts/dependency-pr/checks/golangci-lint.sh b/scripts/dependency-pr/checks/golangci-lint.sh new file mode 100755 index 000000000..5dae417fa --- /dev/null +++ b/scripts/dependency-pr/checks/golangci-lint.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +if ! command -v golangci-lint >/dev/null 2>&1; then + incomplete "golangci-lint is not installed (CI runs v1.63.4 through pre-commit)" +fi +# Verifier runs can overlap. Without --allow-parallel-runners, the second +# golangci-lint stops with "parallel golangci-lint is running". +run golangci-lint run --allow-parallel-runners ./... 2>&1 | tee "$ARTIFACTS/lint.out" +rc=${PIPESTATUS[0]} +if [ "$rc" -ne 0 ]; then + grep -E '\.go:[0-9]+' "$ARTIFACTS/lint.out" | sed -E 's/:[0-9]+:[0-9]+:/:/' | sort -u >"$ARTIFACTS/lint.errs" + # A non-zero exit with no finding is a tool problem (a lock, a config or + # load error), not a property of the PR. + if [ ! -s "$ARTIFACTS/lint.errs" ]; then + incomplete "golangci-lint exited $rc without a finding: $(grep -m1 -iE 'error|level=' "$ARTIFACTS/lint.out" | cut -c1-160)" + fi + findings_file "$ARTIFACTS/lint.errs" + detail_block "$ARTIFACTS/lint.out" 30 + fail "golangci-lint reports $(wc -l <"$ARTIFACTS/lint.errs") finding(s)" +fi +pass "golangci-lint clean ($(golangci-lint --version | awk '{print $4}'))" diff --git a/scripts/dependency-pr/checks/govulncheck.sh b/scripts/dependency-pr/checks/govulncheck.sh new file mode 100755 index 000000000..569501ecb --- /dev/null +++ b/scripts/dependency-pr/checks/govulncheck.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Compares vulnerabilities between base and PR. Needs network for the vuln DB. +# +# govulncheck reports each advisory at its most precise level: a function that +# ldcli calls ("reachable"), a package that ldcli imports, or a module in the +# build. Only a new reachable advisory fails. The details list the advisories +# that the PR fixes at each level, and the reachable ones that stay. +source "$VERIFY_ROOT/lib/check.sh" + +bin=$(go_tool golang.org/x/vuln/cmd/govulncheck v1.1.4) || incomplete "could not install govulncheck" +scan() { + (cd "$1" && "$bin" -format json ./...) >"$ARTIFACTS/$2.raw.json" 2>"$ARTIFACTS/$2.err" || return 1 + jq -s '[.[] | select(.finding) | .finding + | {id: .osv, module: .trace[0].module, fixed: .fixed_version, + level: (if .trace[0].function then 3 elif .trace[0].package then 2 else 1 end)}] + | group_by(.id) | map(max_by(.level))' "$ARTIFACTS/$2.raw.json" >"$ARTIFACTS/$2.json" +} +scan "$BASE_WT" base || { cat "$ARTIFACTS/base.err"; incomplete "govulncheck could not run on base; see log"; } +scan "$PR_WT" pr || { cat "$ARTIFACTS/pr.err"; incomplete "govulncheck could not run on the PR; see log"; } +cat "$ARTIFACTS/pr.json" + +jq -n --slurpfile b "$ARTIFACTS/base.json" --slurpfile p "$ARTIFACTS/pr.json" ' + def lvl: {"3": "reachable", "2": "imported package", "1": "required module"}[tostring]; + def ids: map(.id); + ($b[0] | map({(.id): .}) | add // {}) as $bm | ($p[0] | map({(.id): .}) | add // {}) as $pm + | { + new_reachable: [$p[0][] | select(.level == 3 and (($bm[.id].level // 0) < 3)) | "\(.id) (\(.module))"], + new_other: [$p[0][] | select(.level < 3 and ($bm[.id] == null)) | "\(.id) (\(.module), \(.level | lvl))"], + fixed: [$b[0][] | select($pm[.id] == null) | {id, module, level}], + still_reachable: [$p[0][] | select(.level == 3) | "\(.id) (\(.module)\(if .fixed then ", fixed in \(.fixed)" else "" end))"] + } + | . + {fixed_by_level: (.fixed | group_by(.level) | map({level: (.[0].level | lvl), ids: map(.id)}) | reverse)}' >"$ARTIFACTS/delta.json" + +jq -r '.fixed_by_level[] | "- Fixed by this PR (\(.level)): \(.ids | join(", "))"' "$ARTIFACTS/delta.json" >>"$ARTIFACTS/details.md" +jq -r 'if (.still_reachable | length) > 0 then "- Still reachable on the PR: \(.still_reachable | join(", "))" else empty end' "$ARTIFACTS/delta.json" >>"$ARTIFACTS/details.md" +jq -r 'if (.new_other | length) > 0 then "- New, but not reachable: \(.new_other | join(", "))" else empty end' "$ARTIFACTS/delta.json" >>"$ARTIFACTS/details.md" + +new=$(jq -r '.new_reachable | join(", ")' "$ARTIFACTS/delta.json") +[ -n "$new" ] && fail "New reachable vulnerabilities: $new" +summary=$(jq -r ' + (.fixed | length) as $n + | [ "No new reachable vulnerabilities", + (if $n > 0 then "fixes \($n) advisor\(if $n == 1 then "y" else "ies" end) (" + (.fixed_by_level | map("\(.ids | length) \(.level)") | join(", ")) + ")" else empty end), + (if (.still_reachable | length) > 0 then "still reachable: " + (.still_reachable | map(split(" ")[0]) | join(", ")) else empty end) ] + | join("; ")' "$ARTIFACTS/delta.json") +pass "$summary" diff --git a/scripts/dependency-pr/checks/license-changes.sh b/scripts/dependency-pr/checks/license-changes.sh new file mode 100755 index 000000000..07997056c --- /dev/null +++ b/scripts/dependency-pr/checks/license-changes.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Licenses of the dependencies that this PR adds or changes. A license change, +# or a license outside license-policy.json, needs a decision from a person. +source "$VERIFY_ROOT/lib/check.sh" + +checked=0 +: >"$ARTIFACTS/problems.jsonl" +for eco in gomod npm-ui npm-wrapper; do + has_ecosystem "$eco" || continue + if ! python3 "$VERIFY_ROOT/lib/deps.py" licenses "$BASE_WT" "$PR_WT" "$eco" "$VERIFY_ROOT/license-policy.json" >"$ARTIFACTS/$eco.json" 2>"$ARTIFACTS/$eco.err"; then + cat "$ARTIFACTS/$eco.err" + incomplete "could not read the $eco licenses: $(tail -n1 "$ARTIFACTS/$eco.err" | cut -c1-160)" + fi + cat "$ARTIFACTS/$eco.json" + checked=$((checked + $(jq -r .checked "$ARTIFACTS/$eco.json"))) + jq -r '.all[] | "- `\(.package)`: \(if .changed then "\(.from) → " else "" end)\(.to)\(if .allowed then "" else " (not in the license policy)" end)\(if .dev then " (dev only)" else "" end)"' "$ARTIFACTS/$eco.json" | head -n 40 >>"$ARTIFACTS/details.md" + jq -c '.problems[]' "$ARTIFACTS/$eco.json" >>"$ARTIFACTS/problems.jsonl" +done + +if [ -s "$ARTIFACTS/problems.jsonl" ]; then + # One item per package family: lightningcss and its 11 lightningcss- + # binaries in #779 are one decision, not 12. + jq -s -r ' + def lic_label: if .changed then "the license change from \(.from) to \(.to)" else .to end; + sort_by(.name | length) + | reduce .[] as $p ([]; + ([to_entries[] | .value.root.name as $rn + | select(.value.root.version == $p.version and (.value.root | lic_label) == ($p | lic_label) + and ($p.name | startswith($rn + "-"))) | .key] | first) as $i + | if $i == null then . + [{root: $p, members: [$p]}] else .[$i].members += [$p] end) + | map((.members | length) as $n | (.members | all(.dev)) as $dev + | "\(.root | lic_label) for \(.root.name) \(.root.version)" + + (if $n > 1 then " and \($n - 1) \(.root.name)-* package(s)" else "" end) + + (if $dev then " (dev-only build tools)" elif $n == 1 and (.root.file // null) then " (\(.root.file))" else "" end)) + | join("; ")' "$ARTIFACTS/problems.jsonl" >"$ARTIFACTS/question.txt" + jq -s -r '.[] | .package' "$ARTIFACTS/problems.jsonl" | sort -u >"$ARTIFACTS/problem-packages" + findings_file "$ARTIFACTS/problem-packages" + dev_note="" + if jq -s -e 'all(.[]; .dev)' "$ARTIFACTS/problems.jsonl" >/dev/null; then + dev_note=" Only devDependencies use these packages, so they do not ship in the UI bundle." + fi + decide "Accept $(cat "$ARTIFACTS/question.txt")?" \ + "Packages: $(paste -sd' ' "$ARTIFACTS/problem-packages").$dev_note The license policy (scripts/dependency-pr/license-policy.json) allows $(jq -r '.allowed | join(", ")' "$VERIFY_ROOT/license-policy.json")." +fi +[ "$checked" -eq 0 ] && pass "No added or changed dependency to check" +pass "All $checked added or changed dependencies use a license in the policy, with no license change" diff --git a/scripts/dependency-pr/checks/npm-wrapper-install.sh b/scripts/dependency-pr/checks/npm-wrapper-install.sh new file mode 100755 index 000000000..30c312a84 --- /dev/null +++ b/scripts/dependency-pr/checks/npm-wrapper-install.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Exercises the root npm package's install path: go-npm's postinstall downloads +# the GitHub release binary for the version in package.json and copies it into +# $npm_config_prefix/bin (newer npm has no `npm bin`, so go-npm falls back to +# the prefix). Uses the PR's locked go-npm version and a scratch prefix. +source "$VERIFY_ROOT/lib/check.sh" + +version=$(jq -r '.version' "$WT/package.json") +range=$(jq -r '.dependencies["@go-task/go-npm"] // "none"' "$WT/package.json") +tmp="$ARTIFACTS/install" +rm -rf "$tmp" && mkdir -p "$tmp/prefix/bin" +cp "$WT/package.json" "$WT/package-lock.json" "$tmp/" + +errs() { + grep -E 'npm error|Error' "$1" | grep -v 'A complete log' | sed -E "s#$tmp/?##g" | sort -u >"$ARTIFACTS/errs" + fingerprint_file "$ARTIFACTS/errs" + detail_block "$ARTIFACTS/errs" 20 +} + +if ! (cd "$tmp" && run npm ci --ignore-scripts --no-audit --no-fund) >"$ARTIFACTS/npm-ci.out" 2>&1; then + cat "$ARTIFACTS/npm-ci.out" + errs "$ARTIFACTS/npm-ci.out" + fail "npm ci of the wrapper package fails" +fi +locked=$(jq -r '.version' "$tmp/node_modules/@go-task/go-npm/package.json") +detail "- go-npm: locked \`$locked\`; users installing from npm resolve \`$range\` (the lockfile is not published)" + +if ! (cd "$tmp" && npm_config_prefix="$tmp/prefix" run npm run postinstall) >"$ARTIFACTS/postinstall.out" 2>&1; then + cat "$ARTIFACTS/postinstall.out" + errs "$ARTIFACTS/postinstall.out" + fail "go-npm postinstall fails (downloads v$version)" +fi +cat "$ARTIFACTS/postinstall.out" + +bin="$tmp/prefix/bin/ldcli" +[ -x "$bin" ] || { fingerprint "no-binary"; fail "postinstall finished but $bin is missing"; } +out=$("$bin" --version 2>&1) +detail "- installed \`ldcli --version\`: $out" +if ! grep -q "$version" <<<"$out"; then + fingerprint "version-mismatch" + fail "Installed binary reports '$out', expected $version" +fi + +pack=$(cd "$WT" && npm pack --dry-run --json 2>/dev/null | + jq -r '.[0] | "\(.entryCount) files, \(.unpackedSize / 1024 | floor) KB unpacked; top level: \([.files[].path | split("/")[0]] | unique | join(", "))"') +detail "- \`npm pack\`: ${pack:-unavailable}" +pass "go-npm $locked installs a working ldcli $version" diff --git a/scripts/dependency-pr/checks/pr-disclosure.sh b/scripts/dependency-pr/checks/pr-disclosure.sh new file mode 100755 index 000000000..45378f811 --- /dev/null +++ b/scripts/dependency-pr/checks/pr-disclosure.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# A grouped Dependabot update lists its updates in the PR description. The +# commit can change more direct dependencies than the list names: #779 named +# dompurify and uuid, but also moved react-router, vite, and +# @launchpad-ui/components to new major versions. A reviewer who reads only the +# description does not see those updates. +source "$VERIFY_ROOT/lib/check.sh" + +if [ "$(jq -r '.title == null and .body == null' "$PR_META")" = true ]; then + skip "No PR description to compare (no PR metadata)" +fi +read -r security ndirect < <(jq -r '"\(.security) \(.direct_update_count)"' "$CLASSIFICATION") +grouped=false +jq -r '.title // ""' "$PR_META" | grep -qiE '\bgroup\b' && grouped=true +if [ "$grouped" != true ] && [ "$ndirect" -le 1 ]; then + skip "Not a grouped update, and only one direct update" +fi +kind="update" +[ "$grouped" = true ] && kind="grouped update" +[ "$security" = true ] && kind="security $kind" + +# Only Dependabot's own summary counts as disclosure. Bot blocks such as the +# Cursor Bugbot summary ( … ) +# describe the whole diff, and the
sections quote upstream notes. +jq -r '(.title // "") + "\n" + (.body // "")' "$PR_META" | + perl -0pe 's/.*?//gs; s///gis' \ + >"$ARTIFACTS/description.txt" + +# A name counts as disclosed if the description has it as a whole token. For +# an action in a subdirectory (owner/repo/path), owner/repo also counts. +jq --rawfile text "$ARTIFACTS/description.txt" ' + def esc: gsub("(?[.*+?^${}()|\\[\\]\\\\])"; "\\\(.c)"); + def named: ([.name] + (if .ecosystem == "github-actions" then [.name | split("/")[0:2] | join("/")] else [] end)) + | any(.[]; . as $n | $text | test("(^|[^A-Za-z0-9@/_.-])" + ($n | esc) + "($|[^A-Za-z0-9/_-])")); + {named: [.updates[] | select(named) | {name, from, to, ecosystem}], + undisclosed: [.updates[] | select(.direct and (named | not)) | {name, from, to, semver, ecosystem}], + total: ([.updates[] | select(.direct)] | length)}' "$CLASSIFICATION" >"$ARTIFACTS/disclosure.json" +total=$(jq '.total' "$ARTIFACTS/disclosure.json") +n=$(jq '.undisclosed | length' "$ARTIFACTS/disclosure.json") +[ "$n" -eq 0 ] && pass "The PR description names all $total direct updates of this $kind" + +# A Go module that a named update requires at the new version or higher is +# forced by that update (go.uber.org/mock v0.6.0 raises golang.org/x/term in +# #621). A focused PR cannot leave it out, so it needs no decision. +echo '{}' >"$ARTIFACTS/forced.json" +jq '[.named[] | select(.ecosystem == "gomod" and .to != null)]' "$ARTIFACTS/disclosure.json" >"$ARTIFACTS/roots.json" +jq '[.undisclosed[] | select(.ecosystem == "gomod" and .to != null)]' "$ARTIFACTS/disclosure.json" >"$ARTIFACTS/candidates.json" +if [ "$(jq 'length' "$ARTIFACTS/roots.json")" -gt 0 ] && [ "$(jq 'length' "$ARTIFACTS/candidates.json")" -gt 0 ]; then + python3 "$VERIFY_ROOT/lib/deps.py" forced "$PR_WT" "$ARTIFACTS/roots.json" "$ARTIFACTS/candidates.json" \ + >"$ARTIFACTS/forced.json" 2>"$ARTIFACTS/forced.err" || + { cat "$ARTIFACTS/forced.err"; echo '{}' >"$ARTIFACTS/forced.json"; } +fi +jq -r --slurpfile f "$ARTIFACTS/forced.json" '.undisclosed[] | select($f[0][.name]) + | ($f[0][.name]) as $x + | "\(.name) \(.from // "∅") → \(.to // "∅"), required by \($x.by)\(if ($x.via | length) > 0 then " through " + ($x.via | join(", ")) else "" end)"' \ + "$ARTIFACTS/disclosure.json" >"$ARTIFACTS/forced.txt" +jq -r --slurpfile f "$ARTIFACTS/forced.json" '.undisclosed[] | select($f[0][.name] | not) + | "\(.name) \(.from // "∅") → \(.to // "∅") (\(.semver))"' "$ARTIFACTS/disclosure.json" >"$ARTIFACTS/undisclosed" + +[ -s "$ARTIFACTS/forced.txt" ] && sed 's/^/- Not named, but forced: /' "$ARTIFACTS/forced.txt" >>"$ARTIFACTS/details.md" +if [ -s "$ARTIFACTS/undisclosed" ]; then + sed 's/^/- Not named: /' "$ARTIFACTS/undisclosed" >>"$ARTIFACTS/details.md" + m=$(wc -l <"$ARTIFACTS/undisclosed") + list=$(paste -sd';' "$ARTIFACTS/undisclosed" | sed 's/;/; /g') + named="$((total - n)) of them" + [ "$n" -eq "$total" ] && named="none of them" + findings_file "$ARTIFACTS/undisclosed" + recommend "If these updates are not wanted, close the PR and update the named packages in a focused PR." + decide "Accept the $m direct update(s) that the PR description does not name: $list?" \ + "This $kind changes $total direct dependencies. Dependabot's description names $named." +fi +info "The description does not name $n of $total direct update(s), but the updates that it names require them: $(paste -sd';' "$ARTIFACTS/forced.txt" | sed 's/;/; /g')" diff --git a/scripts/dependency-pr/checks/pr-state.sh b/scripts/dependency-pr/checks/pr-state.sh new file mode 100755 index 000000000..698249be1 --- /dev/null +++ b/scripts/dependency-pr/checks/pr-state.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" + +merge_status=$(jq -r '.merge.status' "$PR_META") +behind=$(jq -r '.behind_by // 0' "$PR_META") +author=$(jq -r '.author // ""' "$PR_META") +base_ref=$(jq -r '.base_ref' "$PR_META") + +if [ "$merge_status" = "conflict" ]; then + files=$(jq -r '.merge.conflicts | join(", ")' "$PR_META") + [ "$behind" -gt 0 ] && detail "- Branch is $behind commit(s) behind \`$base_ref\`." + detail "- Conflicting files: $files. The PR does not merge into \`$base_ref\`, so the checks ran on the PR head as it is, compared with its merge base." + recommend "Rebase the PR (comment \`@dependabot rebase\`) to resolve conflicts with \`$base_ref\`." + FIX_KIND=comment fix_recipe dependabot-rebase "@dependabot rebase" + fail "Conflicts with $base_ref ($files)" +fi + +if [ "$behind" -gt 0 ]; then + detail "- Branch is $behind commit(s) behind \`$base_ref\`. The checks ran on the PR merged into the current \`$base_ref\`." +fi + +# Generated outputs are compared with a fresh build by the drift checks, so +# they need no human review here. +generated_re='^(internal/dev_server/ui/dist/|internal/dev_server/api/server\.gen\.go$|cmd/resources/resource_cmds\.go$|.*/mocks?/|.*mocks?\.go$)' +other=$(jq -r '.other_files[]' "$CLASSIFICATION" | grep -vE "$generated_re" | paste -sd, - | sed 's/,/, /g') +generated=$(jq -r '.other_files[]' "$CLASSIFICATION" | grep -E "$generated_re" | paste -sd, - | sed 's/,/, /g') +[ -n "$generated" ] && detail "- Generated files in the PR (compared with a fresh build by the drift checks): $generated" +case "$author" in + app/dependabot | dependabot\[bot\] | dependabot | "") ;; + *) detail "- The author is $author, not Dependabot." ;; +esac + +if [ -n "$other" ]; then + detail "- Files outside dependency manifests: $other" + decide "Accept the changes to files outside the dependency manifests ($other)?" "The PR changes $other in addition to the dependency update." +fi +if [ -n "$generated" ]; then + info "Merges cleanly into $base_ref; changes manifests and generated files only" +fi +pass "Merges cleanly into $base_ref; only dependency manifests changed" diff --git a/scripts/dependency-pr/checks/registry.json b/scripts/dependency-pr/checks/registry.json new file mode 100644 index 000000000..4385d74f9 --- /dev/null +++ b/scripts/dependency-pr/checks/registry.json @@ -0,0 +1,42 @@ +{ + "schema": 2, + "_doc": "Baseline checks in execution order. when: ecosystems that trigger the check ('any' = always). packages: optional regex on updated package names. compare_base: when the PR result is fail or decide, run the check on base too, to find pre-existing problems. required: if the check does not run, the verdict is 'incomplete'. gate: the check must pass; a pre-existing failure does not satisfy it. required_for_tags: required and a gate, but only when an update has one of these risk tags. profile: fast (default) or full.", + "checks": [ + { "id": "pr-state", "title": "PR merges cleanly and changes only dependency files", "script": "pr-state.sh", "when": ["any"], "compare_base": false, "required": false, "timeout": 60 }, + { "id": "pr-disclosure", "title": "The PR description names every direct update", "script": "pr-disclosure.sh", "when": ["any"], "compare_base": false, "required": false, "timeout": 60 }, + { "id": "ci-status", "title": "CI results on the PR head", "script": "ci-status.sh", "when": ["any"], "compare_base": false, "required": false, "timeout": 60 }, + { "id": "downgrades", "title": "No dependency goes to a lower version than base", "script": "downgrades.sh", "when": ["any"], "compare_base": false, "required": false, "timeout": 60 }, + { "id": "upstream-changes", "title": "Upstream release notes and diff of each direct update", "script": "upstream-changes.sh", "when": ["gomod", "npm-ui", "npm-wrapper", "github-actions"], "compare_base": false, "required": false, "timeout": 300 }, + { "id": "transitive-changes", "title": "Transitive dependency changes (new, removed, major, install scripts)", "script": "transitive-changes.sh", "when": ["gomod", "npm-ui", "npm-wrapper"], "compare_base": false, "required": true, "timeout": 600 }, + { "id": "license-changes", "title": "Licenses of new and changed dependencies", "script": "license-changes.sh", "when": ["gomod", "npm-ui", "npm-wrapper"], "compare_base": false, "required": true, "timeout": 600 }, + + { "id": "go-mod-tidy", "title": "go mod tidy makes no change to go.mod or go.sum", "script": "go-mod-tidy.sh", "when": ["gomod"], "compare_base": true, "required": true, "timeout": 600 }, + { "id": "go-directive", "title": "go and toolchain directives (unchanged, or the new version is supported)", "script": "go-directive.sh", "when": ["gomod"], "compare_base": false, "required": true, "timeout": 600 }, + { "id": "go-build-vet", "title": "go build and go vet", "script": "go-build-vet.sh", "when": ["gomod"], "compare_base": true, "required": true, "gate": true, "timeout": 900 }, + { "id": "go-test", "title": "go test ./... (isolated environment)", "script": "go-test.sh", "when": ["gomod"], "compare_base": true, "required": true, "gate": true, "timeout": 1200 }, + { "id": "go-generate-drift", "title": "go generate makes no change, and the generated code builds", "script": "go-generate-drift.sh", "when": ["gomod"], "compare_base": true, "required": true, "timeout": 900 }, + { "id": "govulncheck", "title": "govulncheck: no new reachable vulnerabilities", "script": "govulncheck.sh", "when": ["gomod"], "compare_base": false, "required": false, "timeout": 900 }, + { "id": "release-snapshot", "title": "All release targets cross-compile in goreleaser-cross (CGO)", "script": "release-snapshot.sh", "when": ["gomod"], "compare_base": true, "required": false, "required_for_tags": ["cgo", "go-directive"], "profile": "full", "timeout": 2400 }, + { "id": "golangci-lint", "title": "golangci-lint (pre-commit configuration)", "script": "golangci-lint.sh", "when": ["gomod"], "compare_base": true, "required": false, "required_for_tags": ["go-directive"], "profile": "full", "timeout": 900 }, + + { "id": "ui-npm-ci", "title": "npm ci resolves (no ERESOLVE or peer conflicts)", "script": "ui-npm-ci.sh", "when": ["npm-ui"], "compare_base": true, "required": true, "gate": true, "timeout": 900 }, + { "id": "ui-lint", "title": "UI lint (eslint)", "script": "ui-lint.sh", "when": ["npm-ui"], "compare_base": true, "required": true, "timeout": 600 }, + { "id": "ui-prettier", "title": "UI format (prettier --check)", "script": "ui-prettier.sh", "when": ["npm-ui"], "compare_base": true, "required": true, "timeout": 600 }, + { "id": "ui-test", "title": "UI tests (vitest)", "script": "ui-test.sh", "when": ["npm-ui"], "compare_base": true, "required": true, "gate": true, "timeout": 900 }, + { "id": "ui-build-drift", "title": "UI builds, and the committed dist/ is current", "script": "ui-build-drift.sh", "when": ["npm-ui"], "compare_base": true, "required": true, "timeout": 900 }, + { "id": "ui-npm-ls", "title": "npm ls: the dependency tree is valid (peers satisfied)", "script": "ui-npm-ls.sh", "when": ["npm-ui"], "compare_base": true, "required": false, "timeout": 300 }, + { "id": "ui-npm-audit", "title": "npm audit (runtime dependencies): no new advisories", "script": "ui-npm-audit.sh", "when": ["npm-ui"], "compare_base": false, "required": false, "timeout": 300 }, + { "id": "ui-dep-usage", "title": "The updated UI dependencies are used", "script": "ui-dep-usage.sh", "when": ["npm-ui"], "compare_base": false, "required": false, "timeout": 120 }, + + { "id": "npm-wrapper-install", "title": "The npm package installs, and its postinstall gets a working binary", "script": "npm-wrapper-install.sh", "when": ["npm-wrapper"], "compare_base": true, "required": true, "gate": true, "timeout": 600 }, + + { "id": "actions-pinning", "title": "Third-party actions use a commit SHA (SEC-7924)", "script": "actions-pinning.sh", "when": ["github-actions"], "compare_base": true, "required": true, "timeout": 120 }, + { "id": "actionlint", "title": "actionlint", "script": "actionlint.sh", "when": ["github-actions"], "compare_base": true, "required": true, "timeout": 600 }, + { "id": "actions-coverage", "title": "Updated actions: CI coverage, inputs, outputs, defaults, runtime, permissions", "script": "actions-coverage.sh", "when": ["github-actions"], "compare_base": false, "required": true, "timeout": 300 }, + + { "id": "docker-image", "title": "The base image exists, and the release image builds and runs", "script": "docker-image.sh", "when": ["docker"], "compare_base": true, "required": true, "gate": true, "timeout": 900 }, + + { "id": "binary-smoke", "title": "Binary smoke test: --version, help for all commands, dev-server serves the UI and API", "script": "binary-smoke.sh", "when": ["gomod", "npm-ui", "docker"], "compare_base": true, "required": true, "gate": true, "timeout": 600 }, + { "id": "cli-help-diff", "title": "CLI help output is the same as on base", "script": "cli-help-diff.sh", "when": ["gomod"], "compare_base": false, "required": false, "timeout": 600 } + ] +} diff --git a/scripts/dependency-pr/checks/release-snapshot.sh b/scripts/dependency-pr/checks/release-snapshot.sh new file mode 100755 index 000000000..f0f4e5a5f --- /dev/null +++ b/scripts/dependency-pr/checks/release-snapshot.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Cross-compiles every release target the way releases do: goreleaser inside +# goreleaser-cross (CGO for SQLite with musl static, mingw, osxcross). PR CI +# only builds linux/amd64 with the host gcc. A CGO-off cross build is not a +# substitute: internal/dev_server/db/backup needs CGO to compile at all. +# +# The release image (ghcr.io/launchdarkly/goreleaser-cross) is a private org +# package. If it cannot be pulled (no ghcr.io credentials), the check uses the +# public upstream image that it is based on, plus the musl.cc toolchains that it +# adds under /musl. The publish action downloaded the same tarballs before #543. +# Both are pinned by digest. Fidelity gaps of the fallback: the public image and +# musl.cc replace the LaunchDarkly image, which can have extras. With either +# image, the binaries are not executed, and goreleaser build skips the dockers +# and brews steps. +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT" || exit 1 + +PUBLIC_IMAGE="${VERIFY_GORELEASER_PUBLIC_IMAGE:-goreleaser/goreleaser-cross@sha256:c4bfde12925cd9e23faae3e355ec7656119348ace0b9075ba7ba94e51229ab8f}" # v1.24.2 +MUSL_TOOLCHAINS=( + "x86_64 c5d410d9f82a4f24c549fe5d24f988f85b2679b452413a9f7e5f7b956f2fe7ea" + "aarch64 c909817856d6ceda86aa510894fa3527eac7989f0ef6e87b5721c58737a06c38" + "i686 93bd5504d5d0349258c43d7a668b506acbd627d31c0843a96e4b4c47b27a0180" +) + +if ! command -v docker >/dev/null 2>&1 || ! docker info >/dev/null 2>&1; then + incomplete "Docker is not available, so the goreleaser-cross build did not run" +fi +# Trees from before #629 (a conflicting PR is tested at its old head) use a tag +# such as :v1.24.2 instead of a digest. +release_image="${VERIFY_GORELEASER_IMAGE:-$(rg -o --no-filename 'ghcr\.io/launchdarkly/goreleaser-cross(@sha256:[0-9a-f]{64}|:[A-Za-z0-9._-]+)' .github/actions/publish/action.yml | head -n1)}" +[ -n "$release_image" ] || incomplete "could not find the goreleaser-cross image in .github/actions/publish/action.yml" + +# musl_root: prints a directory that holds the merged musl.cc cross toolchains. +musl_root() { + local cache="${XDG_CACHE_HOME:-$HOME/.cache}/ldcli-verify/musl-cross" entry arch sum tgz + if [ ! -f "$cache/root/.complete" ]; then + rm -rf "$cache/root" && mkdir -p "$cache/dl" "$cache/root" || return 1 + for entry in "${MUSL_TOOLCHAINS[@]}"; do + read -r arch sum <<<"$entry" + tgz="$cache/dl/$arch-linux-musl-cross.tgz" + if ! echo "$sum $tgz" | sha256sum -c --status 2>/dev/null; then + curl -fsSL --retry 5 --retry-all-errors --retry-delay 5 "https://musl.cc/$arch-linux-musl-cross.tgz" -o "$tgz" >&2 || return 1 + echo "$sum $tgz" | sha256sum -c --status || { echo "checksum mismatch for $tgz" >&2; return 1; } + fi + tar -xzf "$tgz" -C "$cache/root" --strip-components=1 || return 1 + done + touch "$cache/root/.complete" + fi + printf '%s\n' "$cache/root" +} + +# A failed pull is a gap in this environment, not a property of the PR. It must +# never count as a failure, because base repeats it and the verdict then files +# it as pre-existing. +extra=() +if docker image inspect "$release_image" >/dev/null 2>&1 || run docker pull -q "$release_image" >"$ARTIFACTS/pull.out" 2>&1; then + image="$release_image" + image_label="the release image" + detail "- Image: release image \`${image:0:60}…\`" +else + cat "$ARTIFACTS/pull.out" + image="$PUBLIC_IMAGE" + image_label="the public image with musl.cc toolchains, not the release image" + detail "- Release image not pullable ($(tail -n1 "$ARTIFACTS/pull.out" | cut -c1-120)). Fallback: public \`goreleaser/goreleaser-cross:v1.24.2\` (\`${image:0:60}…\`) with sha256-pinned musl.cc toolchains at \`/musl\`." + if ! docker image inspect "$image" >/dev/null 2>&1 && ! run docker pull -q "$image" >"$ARTIFACTS/pull-public.out" 2>&1; then + cat "$ARTIFACTS/pull-public.out" + incomplete "could not pull the goreleaser-cross release image or the public fallback image ($(tail -n1 "$ARTIFACTS/pull-public.out" | cut -c1-120))" + fi + musl=$(musl_root) || incomplete "could not fetch the musl.cc cross toolchains for the public goreleaser-cross image" + extra+=(-v "$musl:/musl:ro") +fi + +# A worktree's .git file points into the main repository's git dir, so both are +# mounted at their real paths (the publish action mounts "$PWD:$PWD" likewise). +common=$(cd "$WT" && cd "$(git rev-parse --git-common-dir)" && pwd) +run docker run --rm -v "$WT:$WT" -v "$common:$common" "${extra[@]}" \ + -v ldcli-verify-gomod:/root/go/pkg/mod -v ldcli-verify-gocache:/root/.cache/go-build \ + -w "$WT" --entrypoint bash "$image" -c \ + "git config --global --add safe.directory '*' && goreleaser build --snapshot --clean --config .goreleaser.yaml && for f in dist/*/ldcli*; do printf '%s: ' \"\$f\"; file -b \"\$f\" | cut -d, -f1-2; done" \ + 2>&1 | tee "$ARTIFACTS/goreleaser.out" +rc=${PIPESTATUS[0]} +# The container runs as root; remove its dist/ before restoring the tree. +docker run --rm -v "$WT:$WT" --entrypoint rm "$image" -rf "$WT/dist" >/dev/null 2>&1 +restore_tree + +if [ "$rc" -ne 0 ]; then + grep -E 'error|failed|\.go:[0-9]+' "$ARTIFACTS/goreleaser.out" | sed -E 's/:[0-9]+:[0-9]+:/:/; s/[0-9.]+m?s\b//g' | sort -u >"$ARTIFACTS/errs" + fingerprint_file "$ARTIFACTS/errs" + detail_block "$ARTIFACTS/errs" 30 + fail "goreleaser snapshot build fails" +fi +targets=$(grep -cE '^dist/' "$ARTIFACTS/goreleaser.out") +[ "$targets" -gt 0 ] || incomplete "goreleaser exited 0, but no binary was found under dist/" +grep -E '^dist/' "$ARTIFACTS/goreleaser.out" | sed 's/^/- /' >>"$ARTIFACTS/details.md" +if [ "$image" != "$release_image" ]; then + detail "- Fidelity gap: the public image plus musl.cc replace the LaunchDarkly image, which can have extras that this build does not have." +fi +detail "- Fidelity gap: the binaries are built, but not executed." +detail "- Fidelity gap: \`goreleaser build\` does not run the \`dockers\`, \`docker_manifests\`, or \`brews\` steps." +pass "All $targets release targets build in goreleaser-cross ($image_label)" diff --git a/scripts/dependency-pr/checks/transitive-changes.sh b/scripts/dependency-pr/checks/transitive-changes.sh new file mode 100755 index 000000000..8251396e4 --- /dev/null +++ b/scripts/dependency-pr/checks/transitive-changes.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Shows what changes beyond the direct update: modules compiled into ldcli +# (Go) or lockfile entries (npm). New npm install scripts need a decision, +# because npm runs them on every install. +source "$VERIFY_ROOT/lib/check.sh" + +summaries=() +scripts=() +for eco in gomod npm-ui npm-wrapper; do + has_ecosystem "$eco" || continue + if ! python3 "$VERIFY_ROOT/lib/deps.py" transitive "$BASE_WT" "$PR_WT" "$eco" >"$ARTIFACTS/$eco.json" 2>"$ARTIFACTS/$eco.err"; then + cat "$ARTIFACTS/$eco.err" + incomplete "could not compare the $eco dependency graph: $(tail -n1 "$ARTIFACTS/$eco.err" | cut -c1-160)" + fi + cat "$ARTIFACTS/$eco.json" + read -r n_add n_rm n_chg n_major < <(jq -r '"\(.added | length) \(.removed | length) \(.changed | length) \([.changed[] | select(.semver == "major")] | length)"' "$ARTIFACTS/$eco.json") + summaries+=("$eco: $n_chg changed ($n_major major), $n_add added, $n_rm removed") + { + printf -- '- %s (%s): %s changed, %s added, %s removed\n' "$eco" "$(jq -r .scope "$ARTIFACTS/$eco.json")" "$n_chg" "$n_add" "$n_rm" + jq -r '.changed[] | " - `\(.name)` \(.from) → \(.to) (\(.semver))"' "$ARTIFACTS/$eco.json" | head -n 30 + jq -r '.added[] | " - added `\(.)`"' "$ARTIFACTS/$eco.json" | head -n 20 + jq -r '.removed[] | " - removed `\(.)`"' "$ARTIFACTS/$eco.json" | head -n 20 + } >>"$ARTIFACTS/details.md" + while IFS= read -r s; do [ -n "$s" ] && scripts+=("$s"); done < <(jq -r '.install_scripts[]' "$ARTIFACTS/$eco.json") +done + +if [ ${#scripts[@]} -gt 0 ]; then + fingerprint "${scripts[*]}" + decide "Allow the new npm install scripts in ${scripts[*]}?" "npm runs these scripts on every install. They are new in this PR: ${scripts[*]}." +fi +if [ ${#summaries[@]} -eq 0 ]; then + skip "no Go or npm ecosystem in this PR" +fi +info "$(join_by '; ' "${summaries[@]}")" diff --git a/scripts/dependency-pr/checks/ui-build-drift.sh b/scripts/dependency-pr/checks/ui-build-drift.sh new file mode 100755 index 000000000..c63f908a0 --- /dev/null +++ b/scripts/dependency-pr/checks/ui-build-drift.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# The rebuilt dist/ is left in place on purpose: binary-smoke runs later and +# embeds it, which tests what main would serve once dist is rebuilt. +source "$VERIFY_ROOT/lib/check.sh" +ensure_ui_deps || incomplete "npm ci failed, so the build did not run" +cd "$WT/$UI_DIR_REL" || exit 1 + +run npm run build 2>&1 | tee "$ARTIFACTS/build.out" +if [ "${PIPESTATUS[0]}" -ne 0 ]; then + grep -E 'error|Error' "$ARTIFACTS/build.out" | sed -E "s#$WT/##g; s/\([0-9]+,[0-9]+\)//" | sort -u >"$ARTIFACTS/build.errs" + fingerprint_file "$ARTIFACTS/build.errs" + detail_block "$ARTIFACTS/build.out" 30 + fail "npm run build fails" +fi + +changed=$(git -C "$WT" status --porcelain -- "$UI_DIR_REL") +if [ -n "$changed" ]; then + git -C "$WT" diff --stat -- "$UI_DIR_REL" >"$ARTIFACTS/drift.stat" + printf '%s\n' "$changed" >>"$ARTIFACTS/drift.stat" + fingerprint "$(git -C "$WT" diff -- "$UI_DIR_REL")" + detail "Build output differs from the committed files:" + detail_block "$ARTIFACTS/drift.stat" 15 + fix_recipe ui-dist-rebuild "cd internal/dev_server/ui && npm ci && npm run build" internal/dev_server/ui/dist/ + recommend "Rebuild the UI and commit dist: \`cd internal/dev_server/ui && npm ci && npm run build\` (the dev-server UI CI job fails until then)." + fail "Committed dist/ is stale: the build rewrites $(printf '%s\n' "$changed" | wc -l) file(s)" +fi +pass "Build succeeds; committed dist/ matches" diff --git a/scripts/dependency-pr/checks/ui-dep-usage.sh b/scripts/dependency-pr/checks/ui-dep-usage.sh new file mode 100755 index 000000000..1cbca810a --- /dev/null +++ b/scripts/dependency-pr/checks/ui-dep-usage.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# A dependency that is never imported (react-window in #831) passes CI no matter +# how breaking the bump is. Flag it so the reviewer can remove it instead. +source "$VERIFY_ROOT/lib/check.sh" +ui="$WT/$UI_DIR_REL" +ensure_ui_deps >/dev/null 2>&1 || true + +re_escape() { printf '%s' "$1" | sed -E 's/[][\.*^$+?(){}|/]/\\&/g'; } + +imported() { + local pat + pat=$(re_escape "$1") + rg -q -e "(from|import|require\()\s*['\"]${pat}(/[^'\"]*)?['\"]" \ + "$ui/src" "$ui"/*.config.* "$ui"/eslint.config.js "$ui/index.html" 2>/dev/null +} + +mentioned_in_tooling() { + local pat + pat=$(re_escape "$1") + rg -q -e "$pat" "$ui"/*.config.* "$ui"/tsconfig*.json "$ui"/eslint.config.js 2>/dev/null && return 0 + jq -e --arg n "$1" '.scripts // {} | to_entries | any(.value | contains($n))' "$ui/package.json" >/dev/null && return 0 + # Tools invoked through package.json scripts by their bin name. + local bins + bins=$(jq -r '.bin // {} | if type == "string" then empty else keys[] end' "$ui/node_modules/$1/package.json" 2>/dev/null) + for b in $bins; do + jq -e --arg b "$b" '.scripts // {} | to_entries | any(.value | test("(^|[ &|;])" + $b + "( |$)"))' "$ui/package.json" >/dev/null && return 0 + done + return 1 +} + +unused=() +while IFS= read -r u; do + [ -n "$u" ] || continue + name=$(jq -r '.name' <<<"$u") + dev=$(jq -r '.dev' <<<"$u") + target="$name" + if [[ "$name" == @types/* ]]; then + target="${name#@types/}" + [[ "$target" == *__* ]] && target="@${target/__//}" + fi + meta="$ui/node_modules/$name/package.json" + if [ -f "$meta" ]; then + engines=$(jq -r '.engines.node // empty' "$meta") + peers=$(jq -r '.peerDependencies // {} | to_entries | map("\(.key)@\(.value)") | join(", ")' "$meta") + deprecated=$(jq -r '.deprecated // empty' "$meta") + [ -n "$engines" ] && detail "- \`$name\` requires node \`$engines\`" + [ -n "$peers" ] && detail "- \`$name\` peers: $peers" + [ -n "$deprecated" ] && detail "- \`$name\` is deprecated: $deprecated" + fi + if imported "$target"; then + detail "- \`$name\` is imported by the UI" + elif [ "$dev" = "true" ] && mentioned_in_tooling "$target"; then + detail "- \`$name\` is used by tooling/config" + else + unused+=("$name") + detail "- \`$name\` is not imported in src/ or referenced by tooling" + fi +done < <(updates_for npm-ui | jq -c 'select(.direct)') + +if [ ${#unused[@]} -gt 0 ]; then + # "a", "a and b", "a, b, and c" + names="${unused[0]}" it="it" + if [ ${#unused[@]} -eq 2 ]; then + names="${unused[0]} and ${unused[1]}" it="them" + elif [ ${#unused[@]} -gt 2 ]; then + names="$(join_by ', ' "${unused[@]:0:${#unused[@]}-1}"), and ${unused[-1]}" it="them" + fi + recommend "Remove unused dependencies instead of bumping them: $names" + FIX_NEEDS_DECISION=1 fix_recipe ui-remove-unused "cd internal/dev_server/ui && npm uninstall ${unused[*]} && npm run build" internal/dev_server/ui/package.json internal/dev_server/ui/package-lock.json internal/dev_server/ui/dist/ + decide "Remove $names from package.json instead of updating $it?" "Nothing in src/, the build configuration, or the package scripts uses $names. The update has no effect at run time." +fi +pass "All updated direct dependencies are used" diff --git a/scripts/dependency-pr/checks/ui-lint.sh b/scripts/dependency-pr/checks/ui-lint.sh new file mode 100755 index 000000000..6cc543e60 --- /dev/null +++ b/scripts/dependency-pr/checks/ui-lint.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +ensure_ui_deps || incomplete "npm ci failed, so eslint did not run" +cd "$WT/$UI_DIR_REL" || exit 1 + +run npm run lint 2>&1 | tee "$ARTIFACTS/lint.out" +if [ "${PIPESTATUS[0]}" -ne 0 ]; then + sed -E "s#$WT/##g" "$ARTIFACTS/lint.out" | grep -E 'error|warning' | sed -E 's/^\s*[0-9]+:[0-9]+\s+//' | sort -u >"$ARTIFACTS/lint.errs" + fingerprint_file "$ARTIFACTS/lint.errs" + detail_block "$ARTIFACTS/lint.out" 30 + fail "eslint reports problems ($(grep -oE '[0-9]+ problems?' "$ARTIFACTS/lint.out" | tail -n1))" +fi +pass "eslint clean" diff --git a/scripts/dependency-pr/checks/ui-npm-audit.sh b/scripts/dependency-pr/checks/ui-npm-audit.sh new file mode 100755 index 000000000..ef707938c --- /dev/null +++ b/scripts/dependency-pr/checks/ui-npm-audit.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Compares npm audit for runtime dependencies between base and PR. Needs the registry. +# +# The comparison uses advisory IDs. npm audit also flags each package that +# depends on a vulnerable package (@launchpad-ui/core "via" +# @launchpad-ui/navigation "via" react-router in #723). Such a package has no +# advisory of its own, and a new path to an old advisory is not a new advisory. +source "$VERIFY_ROOT/lib/check.sh" + +audit() { + (cd "$1/$UI_DIR_REL" && npm audit --omit=dev --json 2>/dev/null) | + jq '{error: (.error.summary // null), + advisories: ([(.vulnerabilities // {})[] | .via[] | objects + | {id: ((.url // "" | capture("(?GHSA-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4})").g) // "npm-\(.source)"), + package: .name, severity, title}] + | group_by(.id) | map(.[0] + {packages: (map(.package) | unique)}))}' +} +audit "$BASE_WT" >"$ARTIFACTS/base.json" || incomplete "npm audit did not run on base" +audit "$PR_WT" >"$ARTIFACTS/pr.json" || incomplete "npm audit did not run on the PR" +cat "$ARTIFACTS/pr.json" +for s in base pr; do + err=$(jq -r '.error // empty' "$ARTIFACTS/$s.json") + [ -n "$err" ] && incomplete "npm audit failed on $s: $err" +done + +jq -n --slurpfile b "$ARTIFACTS/base.json" --slurpfile p "$ARTIFACTS/pr.json" ' + def rank: {"info":0,"low":1,"moderate":2,"high":3,"critical":4}[.] // 0; + def show: "\(.id) (\(.packages | join(", ")), \(.severity))"; + ($b[0].advisories | map(.id)) as $bids | ($p[0].advisories | map(.id)) as $pids + | ($p[0].advisories | map(select(.id as $i | $bids | index($i) | not))) as $new + | { + new: [$new[] | show], + new_serious: [$new[] | select((.severity | rank) >= 3) | show], + fixed: [$b[0].advisories[] | select(.id as $i | $pids | index($i) | not) | show], + total_pr: ($pids | length), total_base: ($bids | length) + }' >"$ARTIFACTS/delta.json" + +fixed=$(jq -r '.fixed | join(", ")' "$ARTIFACTS/delta.json") +[ -n "$fixed" ] && detail "- Advisories resolved by this PR: $fixed" +serious=$(jq -r '.new_serious | join(", ")' "$ARTIFACTS/delta.json") +new=$(jq -r '.new | join(", ")' "$ARTIFACTS/delta.json") +read -r tb tp < <(jq -r '"\(.total_base) \(.total_pr)"' "$ARTIFACTS/delta.json") +[ -n "$new" ] && detail "- New advisories on the PR: $new" +[ -n "$serious" ] && fail "New high/critical advisories: $serious" +[ -n "$new" ] && decide "Accept these new advisories in runtime UI dependencies: $new?" "npm audit (runtime dependencies only) reports advisories on the PR that base does not have: $new" +pass "No new advisories (base $tb, PR $tp advisories)" diff --git a/scripts/dependency-pr/checks/ui-npm-ci.sh b/scripts/dependency-pr/checks/ui-npm-ci.sh new file mode 100755 index 000000000..3bc99c344 --- /dev/null +++ b/scripts/dependency-pr/checks/ui-npm-ci.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +cd "$WT/$UI_DIR_REL" || exit 1 + +detail "- node $(node --version), npm $(npm --version). CI uses \`node-version: lts/*\`." +run npm ci --no-audit --no-fund 2>&1 | tee "$ARTIFACTS/npm-ci.out" +if [ "${PIPESTATUS[0]}" -ne 0 ]; then + # Drop log-file paths and timestamps so base and PR fingerprints compare. + grep -E 'ERESOLVE|While resolving|Found:|Could not resolve|peer |Conflicting peer|code E' "$ARTIFACTS/npm-ci.out" | + grep -v 'A complete log' | sed -E "s#$WT/##g" | sort -u >"$ARTIFACTS/npm-ci.errs" + fingerprint_file "$ARTIFACTS/npm-ci.errs" + detail_block "$ARTIFACTS/npm-ci.errs" 25 + if grep -q ERESOLVE "$ARTIFACTS/npm-ci.out"; then + # Only "npm error" lines describe the conflict that stopped the install. + # npm also prints "npm warn" lines with the same wording for conflicts it + # could work around (#729 named the wrong peer). + conflict=$(grep -E '^npm error' "$ARTIFACTS/npm-ci.out" | grep -m1 -E 'Could not resolve dependency|Conflicting peer dependency' -A2 | grep -oE '(peer )?[@a-z0-9/._-]+@"?[^ "]+"?( from [@a-z0-9/._-]+@[^ ]+)?' | grep -v '^node_modules/' | head -n2 | paste -sd' ' -) + recommend "Resolve the peer-dependency conflict with a coordinated upgrade or a scoped \`overrides\` entry (see #777), or close in favor of a focused PR." + fail "npm ci fails with ERESOLVE peer conflict${conflict:+: $conflict}" + fi + fail "npm ci fails: $(grep -m1 'npm error' "$ARTIFACTS/npm-ci.out" | sed 's/^npm error //')" +fi +sha256sum node_modules/.verify-lock-hash +pass "npm ci succeeds" diff --git a/scripts/dependency-pr/checks/ui-npm-ls.sh b/scripts/dependency-pr/checks/ui-npm-ls.sh new file mode 100755 index 000000000..fcd3d37e9 --- /dev/null +++ b/scripts/dependency-pr/checks/ui-npm-ls.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +ensure_ui_deps || incomplete "npm ci failed, so npm ls did not run" +cd "$WT/$UI_DIR_REL" || exit 1 + +run npm ls --all >/dev/null 2>"$ARTIFACTS/ls.err" +if [ $? -ne 0 ]; then + cat "$ARTIFACTS/ls.err" + grep -E 'npm error (invalid|missing|extraneous|peer)' "$ARTIFACTS/ls.err" | sed -E "s#$WT/##g" | sort -u >"$ARTIFACTS/problems" + findings_file "$ARTIFACTS/problems" + detail_block "$ARTIFACTS/problems" 20 + recommend "Fix the dependency tree (coordinated upgrade or a scoped \`overrides\` entry) so that \`npm ls --all\` reports no new problems." + fail "npm ls reports $(wc -l <"$ARTIFACTS/problems") problem(s): $(head -n2 "$ARTIFACTS/problems" | sed -E 's/^npm error //; s# /?internal/dev_server/ui/node_modules/[^ ]*##' | paste -sd';' -)" +fi +pass "npm ls --all reports a valid tree" diff --git a/scripts/dependency-pr/checks/ui-prettier.sh b/scripts/dependency-pr/checks/ui-prettier.sh new file mode 100755 index 000000000..90449976a --- /dev/null +++ b/scripts/dependency-pr/checks/ui-prettier.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +ensure_ui_deps || incomplete "npm ci failed, so prettier did not run" +cd "$WT/$UI_DIR_REL" || exit 1 + +run npx --no-install prettier . --check 2>&1 | tee "$ARTIFACTS/prettier.out" +if [ "${PIPESTATUS[0]}" -ne 0 ]; then + grep -E '^\[warn\] ' "$ARTIFACTS/prettier.out" | grep -v 'Code style issues' | sort -u >"$ARTIFACTS/prettier.files" + findings_file "$ARTIFACTS/prettier.files" + detail_block "$ARTIFACTS/prettier.files" 20 + fix_recipe ui-prettier "cd internal/dev_server/ui && npm ci && npm run prettier:write" internal/dev_server/ui/ + recommend "Run \`npm run prettier:write\` in internal/dev_server/ui and commit (a prettier bump can reformat files)." + fail "prettier would reformat $(wc -l <"$ARTIFACTS/prettier.files") file(s)" +fi +pass "prettier --check clean ($(npx --no-install prettier --version))" diff --git a/scripts/dependency-pr/checks/ui-test.sh b/scripts/dependency-pr/checks/ui-test.sh new file mode 100755 index 000000000..4af6608f4 --- /dev/null +++ b/scripts/dependency-pr/checks/ui-test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +ensure_ui_deps || incomplete "npm ci failed, so the tests did not run" +cd "$WT/$UI_DIR_REL" || exit 1 + +run npm test 2>&1 | tee "$ARTIFACTS/test.out" +rc=${PIPESTATUS[0]} +counts=$(grep -E '^\s*Tests\s' "$ARTIFACTS/test.out" | tail -n1 | sed -E 's/\s+\(.*//; s/^\s*Tests\s+//') +if [ "$rc" -ne 0 ]; then + grep -E '(FAIL|×|✗)\s' "$ARTIFACTS/test.out" | sed -E "s#$WT/##g; s/ [0-9]+ms$//" | sort -u >"$ARTIFACTS/failures" + fingerprint_file "$ARTIFACTS/failures" + detail_block "$ARTIFACTS/failures" 30 + fail "vitest fails${counts:+ ($counts)}" +fi +pass "vitest passes${counts:+ ($counts)}" diff --git a/scripts/dependency-pr/checks/upstream-changes.sh b/scripts/dependency-pr/checks/upstream-changes.sh new file mode 100755 index 000000000..4e931b1fe --- /dev/null +++ b/scripts/dependency-pr/checks/upstream-changes.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Evidence for the impact review: for each direct update, the upstream +# repository, the compare link, and the release notes in the version range. +# The agent reads the notes (state/checks/upstream-changes/pr/notes/) and +# records its conclusions in agent/impact.json. This check never decides. +# +# The summary gives the real note coverage. Notes that stop before the old +# version are "partial", so the reviewer reads the compare diff for the rest. +source "$VERIFY_ROOT/lib/check.sh" + +command -v gh >/dev/null 2>&1 || incomplete "gh is not installed, so upstream notes were not collected" +if ! python3 "$VERIFY_ROOT/lib/upstream.py" "$CLASSIFICATION" "$WT" "$ARTIFACTS/notes" >"$ARTIFACTS/report.json" 2>"$ARTIFACTS/err"; then + cat "$ARTIFACTS/err" + incomplete "could not collect upstream notes: $(tail -n1 "$ARTIFACTS/err" | cut -c1-160)" +fi +cat "$ARTIFACTS/report.json" + +n=$(jq length "$ARTIFACTS/report.json") +[ "$n" -eq 0 ] && skip "no direct update with an upstream source" + +jq -r '.[] | "- `\(.name)` \(.from) → \(.to): \(if .repo then "[\(.repo)](https://github.com/\(.repo))" else "source not found" end)\(if .compare_url then ", [\(.commits) commits, \(.files_changed) files](\(.compare_url))" else "" end)\(if .tags_missing then ", no upstream tag for " + (.tags_missing | join(" and ")) + ", so no compare link" else "" end), notes: \( + if .notes_coverage == "full" then "\(.notes_source), full range" + elif .notes_coverage == "partial" then "\(.notes_source), **partial** (back to \(.notes_lowest // "?") only, not \(.from))" + else "**none found**" end)\(if (.keywords // {}) != {} then ", mentions: " + ([.keywords | to_entries[] | "\(.key) (\(.value))"] | join(", ")) else "" end)"' \ + "$ARTIFACTS/report.json" >>"$ARTIFACTS/details.md" + +coverage=$(jq -r ' + (map(select(.notes_coverage == "full")) | length) as $full + | [ "Upstream notes cover the full range for \($full) of \(length) direct update(s)", + (map(select(.notes_coverage == "partial")) | if length > 0 then "partial for " + (map("\(.name) (back to \(.notes_lowest // "?"), not \(.from))") | join(", ")) else empty end), + (map(select(.notes_coverage == "none")) | if length > 0 then "none for " + (map(.name) | join(", ")) else empty end) ] + | join("; ")' "$ARTIFACTS/report.json") +flags=$(jq -r '[.[] | (.keywords // {}) | keys[]] | group_by(.) | map("\(.[0]) (\(length))") | join(", ")' "$ARTIFACTS/report.json") +gaps=$(jq '[.[] | select(.notes_coverage != "full")] | length' "$ARTIFACTS/report.json") +tail="" +if [ "$(jq -r '.tier' "$CLASSIFICATION")" != low ]; then + tail=". The impact review must cover them" + [ "$gaps" -gt 0 ] && tail="$tail, and read the compare diff where the notes are partial or missing" +fi +info "$coverage${flags:+; notes mention: $flags}$tail" diff --git a/scripts/dependency-pr/lib/actions_analysis.py b/scripts/dependency-pr/lib/actions_analysis.py new file mode 100755 index 000000000..8e852f49c --- /dev/null +++ b/scripts/dependency-pr/lib/actions_analysis.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""For each updated action: where the repo uses it, whether those workflows run +on pull_request, and (with gh) how the upstream action.yml changed between the +two refs: inputs, input defaults, outputs that the repo reads, and runtime. +It also compares the `permissions` blocks of all workflows between base and PR. + +Usage: actions_analysis.py (prints JSON) +""" +import base64 +import glob +import json +import os +import re +import subprocess +import sys + +import yaml + +GITHUB_HOSTED = re.compile(r"^(ubuntu|windows|macos)-[\w.-]+$") + + +def load(path): + with open(path) as f: + return yaml.safe_load(f) or {} + + +def triggers(doc): + # PyYAML (YAML 1.1) parses the bare key `on` as boolean True. + on = doc.get("on", doc.get(True)) + if isinstance(on, str): + return [on] + if isinstance(on, list): + return on + if isinstance(on, dict): + return list(on.keys()) + return [] + + +def jobs_of(doc): + for job_id, job in (doc.get("jobs") or {}).items(): + if isinstance(job, dict): + yield job_id, job + runs = doc.get("runs") or {} + if runs.get("steps"): + yield "(composite)", {"steps": runs["steps"]} + + +def steps_of(job): + if isinstance(job.get("uses"), str): + yield {"uses": job["uses"], "with": job.get("with") or {}} + for step in job.get("steps") or []: + if isinstance(step, dict): + yield step + + +def workflows(wt): + return {os.path.relpath(p, wt): load(p) for p in sorted(glob.glob(os.path.join(wt, ".github/workflows/*.y*ml")))} + + +def permissions(wt): + out = {} + for f, d in workflows(wt).items(): + out[f"{f} (workflow)"] = d.get("permissions") + for job_id, job in jobs_of(d): + out[f"{f} job {job_id}"] = job.get("permissions") + return out + + +def on_github_com(value): + """A default as github.com evaluates it. Actions write GHES fallbacks as + `github.server_url == 'https://github.com' && X || Y`, which is X there.""" + if not isinstance(value, str): + return value + m = re.fullmatch(r"\$\{\{\s*(.*?)\s*\}\}", value.strip(), re.S) + if not m: + return value.strip() + expr = m.group(1) + ghes = re.fullmatch(r"github\.server_url\s*==\s*'https://github\.com'\s*&&\s*(.+?)\s*\|\|\s*.+", expr, re.S) + return "${{ " + (ghes.group(1) if ghes else expr).strip() + " }}" + + +def gh_action_yml(name, ref): + parts = name.split("/") + repo, sub = "/".join(parts[:2]), "/".join(parts[2:]) + for fname in ("action.yml", "action.yaml"): + path = f"{sub}/{fname}" if sub else fname + try: + out = subprocess.run(["gh", "api", f"repos/{repo}/contents/{path}?ref={ref}", "--jq", ".content"], + capture_output=True, text=True, timeout=60) + except (OSError, subprocess.TimeoutExpired): + return None + if out.returncode == 0 and out.stdout.strip(): + return yaml.safe_load(base64.b64decode(out.stdout.strip())) + return None + + +def main(): + wt, updates_path, base_wt = sys.argv[1], sys.argv[2], sys.argv[3] + with open(updates_path) as f: + updates = json.load(f) + + wfs = workflows(wt) + composites = {} + for p in sorted(glob.glob(os.path.join(wt, ".github/actions/**/action.y*ml"), recursive=True)): + composites[os.path.relpath(os.path.dirname(p), wt)] = (os.path.relpath(p, wt), load(p)) + on_pr = {f: bool({"pull_request", "pull_request_target", "merge_group"} & set(map(str, triggers(d)))) for f, d in wfs.items()} + callers, runners = {}, {} + for f, d in wfs.items(): + for _, job in jobs_of(d): + ro = job.get("runs-on") + runners.setdefault(f, set()).update(ro if isinstance(ro, list) else [ro] if ro else []) + for step in steps_of(job): + uses = step.get("uses", "") + if uses.startswith("./"): + callers.setdefault(uses[2:].rstrip("/"), []).append(f) + + report = [] + for u in updates: + name = u["name"] + usages = [] + sources = [(f, d, None) for f, d in wfs.items()] + [(path, d, cdir) for cdir, (path, d) in composites.items()] + for f, d, cdir in sources: + for job_id, job in jobs_of(d): + job_text = json.dumps(job) + for step in steps_of(job): + if step.get("uses", "").split("@")[0] != name: + continue + wf_list = [f] if cdir is None else callers.get(cdir, []) + outputs = sorted(set(re.findall(r"steps\.%s\.outputs\.([\w-]+)" % re.escape(step["id"]), job_text))) if step.get("id") else [] + usages.append({ + "file": f, "job": job_id, "with": sorted((step.get("with") or {}).keys()), + "outputs_read": outputs, "workflows": wf_list, + "runs_on_pr": any(on_pr.get(w, False) for w in wf_list), + "runners": sorted({str(r) for w in wf_list for r in runners.get(w, set())}), + }) + entry = {"name": name, "from": u.get("from"), "to": u.get("to"), "breaking": bool(u.get("breaking")), "usages": usages, + "replaced": [r["version"] for r in u.get("replaced") or []]} + + # Every old ref that the PR replaces is compared with the new ref (a repo + # can pin v4 in one workflow and v5 in another). + old_refs = [r["ref"] for r in u.get("replaced") or []] or ([u["from_ref"]] if u.get("from_ref") else []) + new_ref = u.get("to_ref") + if old_refs and new_ref: + olds = [gh_action_yml(name, r) for r in old_refs] + new = gh_action_yml(name, new_ref) + if new is None or any(o is None for o in olds): + entry["upstream"] = {"status": "unavailable"} + else: + new_in, new_out = new.get("inputs") or {}, new.get("outputs") or {} + provided = {k for us in usages for k in us["with"]} + read = {o for us in usages for o in us["outputs_read"]} + default = lambda spec: (spec or {}).get("default") if isinstance(spec, dict) else None + defaults_changed, defaults_same, new_required, outputs_missing = [], [], set(), set() + for old in olds: + old_in, old_out = old.get("inputs") or {}, old.get("outputs") or {} + new_required |= {k for k, v in new_in.items() + if k not in old_in and isinstance(v, dict) and v.get("required") + and "default" not in v and k not in provided} + for k, v in sorted(new_in.items()): + change = {"input": k, "from": default(old_in.get(k)), "to": default(v)} + if k not in old_in or k in provided or str(change["from"]) == str(change["to"]): + continue + if str(on_github_com(change["from"])) == str(on_github_com(change["to"])): + if change not in defaults_same: + defaults_same.append(change) + elif change not in defaults_changed: + defaults_changed.append(change) + # An output that neither action.yml declares is set at run time + # (release-please-action), so only a declared output can go missing. + outputs_missing |= {o for o in read if o in old_out and o not in new_out} + old_runtimes = sorted({str((o.get("runs") or {}).get("using")) for o in olds}) + entry["upstream"] = { + "status": "ok", + "compared_refs": old_refs, + "inputs_used": sorted(provided), + "removed_but_used": sorted(k for k in provided + if k not in new_in and any(k in (o.get("inputs") or {}) for o in olds)), + "new_required": sorted(new_required), + "defaults_changed": defaults_changed, + "defaults_same_on_github_com": defaults_same, + "outputs_read": sorted(read), + "outputs_undeclared": sorted(o for o in read if o not in new_out and all(o not in (x.get("outputs") or {}) for x in olds)), + "outputs_missing": sorted(outputs_missing), + "runs_using": [", ".join(old_runtimes), str((new.get("runs") or {}).get("using"))], + } + entry["self_hosted_runners"] = sorted({r for us in usages for r in us["runners"] if not GITHUB_HOSTED.match(r)}) + report.append(entry) + + pb, pp = permissions(base_wt), permissions(wt) + perm_changes = [{"where": k, "from": pb.get(k), "to": pp.get(k)} for k in sorted(set(pb) | set(pp)) if pb.get(k) != pp.get(k)] + json.dump({"workflows_on_pr": on_pr, "actions": report, "permissions_changes": perm_changes}, sys.stdout, indent=2, default=str) + + +if __name__ == "__main__": + main() diff --git a/scripts/dependency-pr/lib/check.sh b/scripts/dependency-pr/lib/check.sh new file mode 100644 index 000000000..6bba601df --- /dev/null +++ b/scripts/dependency-pr/lib/check.sh @@ -0,0 +1,200 @@ +# Helpers sourced by every check script (baseline and generated). +# +# Environment provided by the runner: +# WT worktree under test (the check's working directory) +# SIDE "pr" or "base" +# BASE_WT PR_WT both worktrees, for checks that compare sides +# ARTIFACTS per-check, per-side directory for status and evidence +# CLASSIFICATION path to classification.json +# PR_META path to pr.json (PR metadata, CI rollup, merge state) +# PROFILE "fast" or "full" +# VERIFY_ROOT scripts/dependency-pr +# +# Protocol: finish with exactly one of these. A script that exits without one +# (crash, timeout, `set -e` abort) is recorded as "error". +# pass "summary" the check found no problem +# info "summary" no problem, but the summary is worth showing +# fail "summary" the PR must not merge as it is (add `recommend` for the fix) +# decide "question" "evidence" only a person can make this choice; ask one exact question +# incomplete "reason" the check could not run here (missing tool, no network) +# skip "reason" the check does not apply to this PR +# Everything printed to stdout/stderr goes to the check's log. + +: "${ARTIFACTS:?ARTIFACTS must be set by the runner}" +: "${WT:?WT must be set by the runner}" +mkdir -p "$ARTIFACTS" + +_finish() { + printf '%s\n' "$1" >"$ARTIFACTS/status" + shift + printf '%s\n' "$*" >"$ARTIFACTS/summary" + exit 0 +} +pass() { _finish pass "$@"; } +info() { _finish info "$@"; } +fail() { _finish fail "$@"; } +skip() { _finish skip "$@"; } +incomplete() { _finish incomplete "$@"; } +decide() { + printf '%s\n' "$1" >"$ARTIFACTS/question" + shift + _finish decide "$@" +} + +# The fix for a failure, shown in the comment. +recommend() { printf '%s\n' "$*" >>"$ARTIFACTS/recommendations"; } + +# A machine-applicable fix for a failure, copied into result.json so that a +# later step (apply-fixes.sh) can run it, commit the expected paths, and +# re-run verify.sh. Only record deterministic, mechanical commands. +# fix_recipe ... +# Set FIX_NEEDS_DECISION=1 when a person must approve the fix first. +# Set FIX_KIND=comment when the fix is a PR comment (e.g. "@dependabot rebase"). +fix_recipe() { + local id="$1" cmd="$2" + shift 2 + jq -n --arg id "$id" --arg cmd "$cmd" --arg decision "${FIX_NEEDS_DECISION:-0}" --arg kind "${FIX_KIND:-commit}" \ + '{id: $id, kind: $kind, command: $cmd, paths: $ARGS.positional, needs_decision: ($decision == "1")}' \ + --args "$@" >"$ARTIFACTS/fix.json" +} + +# Markdown lines shown under the check in the comment's details section. +detail() { printf '%s\n' "$*" >>"$ARTIFACTS/details.md"; } +detail_block() { + # detail_block [max_lines] + local f="$1" max="${2:-40}" + { + printf '```\n' + head -n "$max" "$f" + local n + n=$(wc -l <"$f") + [ "$n" -gt "$max" ] && printf '… (%s more lines, see log)\n' "$((n - max))" + printf '```\n' + } >>"$ARTIFACTS/details.md" +} + +# Fingerprint of a failure. When a check fails on the PR it is re-run on base; +# an identical fingerprint there means the failure is pre-existing on main. +# Keep it free of absolute paths, timings, and line numbers that can shift. +fingerprint() { printf '%s' "$*" | sha256sum | cut -c1-16 >"$ARTIFACTS/fingerprint"; } +fingerprint_file() { sha256sum <"$1" | cut -c1-16 >"$ARTIFACTS/fingerprint"; } +# For a failure that is a set of independent findings (one per line, without +# line numbers): if every PR finding also fails on base, the verdict counts the +# failure as pre-existing, even when the PR removes some findings. +findings_file() { + sort -u "$1" >"$ARTIFACTS/findings.list" + fingerprint_file "$ARTIFACTS/findings.list" +} + +join_by() { + local sep="$1" out="" item + shift + for item in "$@"; do out="${out:+$out$sep}$item"; done + printf '%s' "$out" +} + +run() { + printf '+ %s\n' "$*" >&2 + "$@" +} + +# go_tool : prints the path of a pinned Go tool, +# installing it once into a shared cache so its download output stays out of +# check results. +go_tool() { + local pkg="$1" version="$2" name dir + name="$(basename "$pkg")" + command -v "$name" >/dev/null 2>&1 && { command -v "$name"; return 0; } + dir="${XDG_CACHE_HOME:-$HOME/.cache}/ldcli-verify/bin/$name-$version" + if [ ! -x "$dir/$name" ]; then + mkdir -p "$dir" + GOBIN="$dir" go install "$pkg@$version" >&2 || return 1 + fi + printf '%s\n' "$dir/$name" +} + +# Updates of one ecosystem from classification.json as compact JSON lines. +updates_for() { + jq -c --arg e "$1" '.updates[] | select(.ecosystem == $e)' "$CLASSIFICATION" +} + +has_ecosystem() { + jq -e --arg e "$1" '.ecosystems | index($e) != null' "$CLASSIFICATION" >/dev/null +} + +UI_DIR_REL="internal/dev_server/ui" + +# Installs UI dependencies once per lockfile content. +ensure_ui_deps() { + local dir="$WT/$UI_DIR_REL" stamp hash + stamp="$dir/node_modules/.verify-lock-hash" + hash=$(sha256sum <"$dir/package-lock.json" | cut -c1-16) + if [ -f "$stamp" ] && [ "$(cat "$stamp")" = "$hash" ]; then + return 0 + fi + (cd "$dir" && run npm ci --no-audit --no-fund) || return 1 + printf '%s\n' "$hash" >"$stamp" +} + +build_ldcli() { + # build_ldcli + (cd "$WT" && run go build -o "$1" .) +} + +free_port() { + local p + for _ in $(seq 1 50); do + p=$((20000 + RANDOM % 20000)) + if ! (exec 3<>"/dev/tcp/127.0.0.1/$p") 2>/dev/null; then + printf '%s\n' "$p" + return 0 + fi + done + return 1 +} + +JOBS="$(nproc 2>/dev/null || echo 4)" + +_subcommands_of() { + "$1" __complete "$2" "" 2>/dev/null | + awk -F'\t' -v p="$2" '!/^:/ && $1 != "" && $1 !~ /^-/ && $1 != "help" {print p " " $1}' +} +_help_one() { + # _help_one ; unquoted $3 splits "flags list" into args. + local f + f="$2/$(printf '%s' "$3" | tr ' ' '_').txt" + # shellcheck disable=SC2086 + { printf '### ldcli %s\n' "$3"; "$1" $3 --help 2>&1; } >"$f" || printf '%s\n' "$3" >>"$2/.failures" +} +export -f _subcommands_of _help_one + +# Top-level commands and their direct subcommands, via cobra's hidden +# completion command, one per line ("flags", "flags list", ...). +list_commands() { + local bin="$1" top + top=$("$bin" __complete "" 2>/dev/null | awk -F'\t' '!/^:/ && $1 != "" && $1 != "help" {print $1}') + { + printf '%s\n' "$top" + printf '%s\n' "$top" | xargs -P "$JOBS" -I{} bash -c '_subcommands_of "$0" "$1"' "$bin" {} + } | sort -u +} + +# help_dump : writes /all.txt (help for every command, sorted), +# /.commands and /.failures (commands whose --help exits non-zero). +help_dump() { + local bin="$1" dir="$2" c + mkdir -p "$dir/cmd" + : >"$dir/.failures" + list_commands "$bin" >"$dir/.commands" + xargs -P "$JOBS" -I{} bash -c '_help_one "$0" "$1" "$2"' "$bin" "$dir/cmd" {} <"$dir/.commands" + mv "$dir/cmd/.failures" "$dir/.failures" 2>/dev/null || true + { + printf '### ldcli\n' + "$bin" --help 2>&1 + while IFS= read -r c; do cat "$dir/cmd/$(printf '%s' "$c" | tr ' ' '_').txt"; done <"$dir/.commands" + } >"$dir/all.txt" +} + +restore_tree() { + git -C "$WT" checkout -q -- . && git -C "$WT" clean -fdq +} diff --git a/scripts/dependency-pr/lib/classify.sh b/scripts/dependency-pr/lib/classify.sh new file mode 100755 index 000000000..bfd654b83 --- /dev/null +++ b/scripts/dependency-pr/lib/classify.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# Usage: classify.sh +# +# Compares the base and PR worktrees (not the PR diff) so updates reflect what +# would actually change on the base branch after merging. +set -euo pipefail +source "$(dirname "$0")/common.sh" + +BASE_WT="$1" PR_WT="$2" CHANGED="$3" PR_JSON="$4" OUT="$5" +UI_DIR="internal/dev_server/ui" +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +go_mod_json() { + if [ -f "$1/go.mod" ]; then (cd "$1" && go mod edit -json); else echo '{}'; fi +} + +npm_json() { + # npm_json : {deps, dev, lock} for one package directory + local d="$1/$2" pkg lock + pkg="$d/package.json" + lock="$d/package-lock.json" + [ -f "$pkg" ] || pkg=/dev/null + [ -f "$lock" ] || lock=/dev/null + jq -n --slurpfile p <(cat "$pkg" 2>/dev/null || echo '{}') --slurpfile l <(cat "$lock" 2>/dev/null || echo '{}') ' + ($p[0] // {}) as $p | ($l[0] // {}) as $l + | { + deps: ($p.dependencies // {}), + dev: ($p.devDependencies // {}), + lock: (($l.packages // {}) | to_entries + | map(select(.key | test("^node_modules/(@[^/]+/)?[^/]+$"))) + | map({key: (.key | sub("^node_modules/"; "")), value: .value.version}) + | from_entries) + }' +} + +actions_uses() { + # Prints "namerefcomment-version" for each non-local `uses:`. + local dirs=() + [ -d "$1/.github/workflows" ] && dirs+=("$1/.github/workflows") + [ -d "$1/.github/actions" ] && dirs+=("$1/.github/actions") + [ ${#dirs[@]} -gt 0 ] || return 0 + rg --no-filename --no-line-number -o -g '*.yml' -g '*.yaml' \ + '^\s*-?\s*uses:\s*["'\'']?([^\s"'\''#]+)["'\'']?(?:\s*#\s*(\S+))?' -r '$1 $2' "${dirs[@]}" | + awk -F'\t' '$1 !~ /^\.\// && $1 !~ /^docker:\/\// && index($1, "@") > 0 { + at = index($1, "@"); printf "%s\t%s\t%s\n", substr($1, 1, at - 1), substr($1, at + 1), $2 }' | + sort -u +} + +actions_json() { + actions_uses "$1" | jq -R -s ' + split("\n") | map(select(length > 0) | split("\t")) + | map({name: .[0], ref: .[1], + version: (if (.[1] | test("^[0-9a-f]{40}$")) and ((.[2] // "") != "") then .[2] else .[1] end)}) + | group_by(.name) | map({key: .[0].name, value: {refs: (map(.ref) | unique), versions: (map(.version) | unique), + pairs: (map({ref, version}) | unique)}}) + | from_entries' +} + +docker_json() { + local f out='{}' + for f in "$1"/Dockerfile*; do + [ -f "$f" ] || continue + out=$(rg --no-line-number -o '^FROM\s+(\S+)' -r '$1' "$f" | jq -R -s --argjson acc "$out" ' + split("\n") | map(select(length > 0)) + | map(sub("@sha256:.*$"; "") | capture("^(?[^:]+)(:(?.+))?$") | {key: .name, value: (.tag // "latest")}) + | from_entries | $acc + .') + done + printf '%s\n' "$out" +} + +go_mod_json "$BASE_WT" >"$TMP/go.base.json" +go_mod_json "$PR_WT" >"$TMP/go.pr.json" +npm_json "$BASE_WT" "$UI_DIR" >"$TMP/ui.base.json" +npm_json "$PR_WT" "$UI_DIR" >"$TMP/ui.pr.json" +npm_json "$BASE_WT" "." >"$TMP/root.base.json" +npm_json "$PR_WT" "." >"$TMP/root.pr.json" +actions_json "$BASE_WT" >"$TMP/actions.base.json" +actions_json "$PR_WT" >"$TMP/actions.pr.json" +docker_json "$BASE_WT" >"$TMP/docker.base.json" +docker_json "$PR_WT" >"$TMP/docker.pr.json" + +jq -n -L "$VERIFY_ROOT/lib" \ + --rawfile changed "$CHANGED" \ + --slurpfile pr "$PR_JSON" \ + --slurpfile risk "$VERIFY_ROOT/risk-map.json" \ + --slurpfile gob "$TMP/go.base.json" --slurpfile gop "$TMP/go.pr.json" \ + --slurpfile uib "$TMP/ui.base.json" --slurpfile uip "$TMP/ui.pr.json" \ + --slurpfile rootb "$TMP/root.base.json" --slurpfile rootp "$TMP/root.pr.json" \ + --slurpfile actb "$TMP/actions.base.json" --slurpfile actp "$TMP/actions.pr.json" \ + --slurpfile dockb "$TMP/docker.base.json" --slurpfile dockp "$TMP/docker.pr.json" ' +include "semver"; + +def vsort: sort_by(vparse | if . == null then [-1] else .nums end); +def maxv: vsort | last; +def vsort_pairs: sort_by(.version | vparse | if . == null then [-1] else .nums end); + +def go_updates: + def reqmap: (.Require // []) | map({key: .Path, value: {v: .Version, indirect: (.Indirect // false)}}) | from_entries; + ($gob[0] | reqmap) as $b | ($gop[0] | reqmap) as $p + | [ (($b | keys) + ($p | keys) | unique)[] as $k + | select($b[$k].v != $p[$k].v) + | {ecosystem: "gomod", name: $k, from: $b[$k].v, to: $p[$k].v, + direct: ((($p[$k] // $b[$k]).indirect) | not), dev: false} ]; + +def npm_updates($eco; $b; $p): + [ (($b.lock | keys) + ($p.lock | keys) | unique)[] as $k + | select($b.lock[$k] != $p.lock[$k]) + | ([$b.deps, $p.deps] | any(has($k))) as $rt + | ([$b.dev, $p.dev] | any(has($k))) as $dv + | {ecosystem: $eco, name: $k, from: $b.lock[$k], to: $p.lock[$k], + direct: ($rt or $dv), dev: ($dv and ($rt | not))} ]; + +# A workflow can pin one action at several refs (v4 in one file, v5 in +# another). "replaced" holds every old ref that the PR removes, lowest version +# first, and "from" is the lowest of them, so the compared range covers all. +def action_updates: + $actb[0] as $b | $actp[0] as $p + | [ (($b | keys) + ($p | keys) | unique)[] as $k + | select($b[$k].refs != $p[$k].refs) + | ($b[$k].pairs // []) as $bp | ($p[$k].pairs // []) as $pp + | ($pp | map(.ref)) as $prefs + | ($bp | map(select(.ref as $r | $prefs | index($r) | not)) | vsort_pairs) as $replaced + | (if ($replaced | length) > 0 then $replaced[0] else ($bp | vsort_pairs | last) end) as $old + | ($pp | vsort_pairs | last) as $new + | {ecosystem: "github-actions", name: $k, + from: ($old.version // null), to: ($new.version // null), + from_ref: ($old.ref // null), to_ref: ($new.ref // null), + replaced: $replaced, + from_all: ($b[$k].versions // []), to_all: ($p[$k].versions // []), + from_refs: ($b[$k].refs // []), to_refs: ($p[$k].refs // []), + direct: true, dev: false} ]; + +def docker_updates: + $dockb[0] as $b | $dockp[0] as $p + | [ (($b | keys) + ($p | keys) | unique)[] as $k + | select($b[$k] != $p[$k]) + | {ecosystem: "docker", name: $k, from: $b[$k], to: $p[$k], direct: true, dev: false} ]; + +def ecosystems_from($files): + [ $files[] + | if test("^go\\.(mod|sum)$") then "gomod" + elif test("^internal/dev_server/ui/package(-lock)?\\.json$") then "npm-ui" + elif test("^package(-lock)?\\.json$") then "npm-wrapper" + elif test("^\\.github/(workflows|actions)/") then "github-actions" + elif test("(^|/)Dockerfile[^/]*$") then "docker" + else empty end ] | unique; + +def manifest_file: test("^go\\.(mod|sum)$|(^|/)package(-lock)?\\.json$|^\\.github/(workflows|actions)/|(^|/)Dockerfile[^/]*$"); + +($changed | split("\n") | map(select(length > 0))) as $files +| ($pr[0] // {}) as $pr +| (go_updates + npm_updates("npm-ui"; $uib[0]; $uip[0]) + npm_updates("npm-wrapper"; $rootb[0]; $rootp[0]) + + action_updates + docker_updates) as $raw +| $risk[0].rules as $rules +| [ $raw[] + | semver_class(.from; .to) as $c + | (if $c == "none" and .ecosystem == "github-actions" then "digest" else $c end) as $c + | . + {semver: $c, breaking: is_breaking(.from; .to)} + | . as $u + | [ $rules[] | . as $rule | select($u.name | test($rule.match)) ] as $hits + | ( if $c == "downgrade" then "medium" + elif .breaking and .direct then "high" + elif $c == "minor" and .direct and (.dev | not) then "medium" + else "low" end ) as $base + | ($hits | map(.tier) | reduce .[] as $t ("low"; max_tier(.; $t))) as $rt + | (if .direct then $rt else max_tier("low"; (if $rt == "high" then "medium" else $rt end)) end) as $rt + | . + {tier: max_tier($base; $rt), + tags: ($hits | map(.tags[]) | unique), + risk_notes: ($hits | map(.why) | unique)} ] as $updates +| ($updates | map(select(.direct)) | length) as $ndirect +| {from: ($gob[0].Go // null), to: ($gop[0].Go // null)} as $godir +| {from: ($gob[0].Toolchain.Name // null), to: ($gop[0].Toolchain.Name // null)} as $tool +| (ecosystems_from($files) + ($updates | map(.ecosystem)) | unique) as $ecos +| ( [ ($updates[] | select(.tier != "low") | "\(.name) \(.from // "∅") → \(.to // "∅"): \(.tier) (\(.semver)\(if .breaking then ", breaking range" else "" end)\(if (.tags | length) > 0 then "; " + (.tags | join(", ")) else "" end))"), + (if $godir.from != $godir.to then "go directive \($godir.from) → \($godir.to): high" else empty end), + (if $tool.from != $tool.to then "toolchain \($tool.from) → \($tool.to): high" else empty end), + (if ($ecos | index("docker")) then "docker base image: at least medium" else empty end), + (if $ndirect > 3 then "\($ndirect) direct updates in one PR: high" else empty end), + (if ($updates | length) == 0 then "no dependency change detected between base and PR: medium" else empty end) + ] ) as $reasons +| ( ($updates | map(.tier)) + [ + (if $godir.from != $godir.to or $tool.from != $tool.to then "high" else "low" end), + (if ($ecos | index("docker")) then "medium" else "low" end), + (if $ndirect > 3 then "high" else "low" end), + (if ($updates | length) == 0 then "medium" else "low" end) + ] | reduce .[] as $t ("low"; max_tier(.; $t)) ) as $tier +| { + schema: 1, + ecosystems: $ecos, + changed_files: $files, + other_files: ($files | map(select(manifest_file | not))), + updates: $updates, + direct_update_count: $ndirect, + go_directive: $godir, + toolchain: $tool, + group: ($ndirect > 1 or (($pr.title // "") | test("group"; "i"))), + security: ((($pr.title // "") + " " + ($pr.body // "")) | test("GHSA-|CVE-[0-9]{4}-|\\[security\\]"; "i")), + tier: $tier, + tags: (($updates | map(.tags[])) + (if $godir.from != $godir.to or $tool.from != $tool.to then ["go-directive"] else [] end) | unique), + tier_reasons: $reasons + }' >"$OUT" diff --git a/scripts/dependency-pr/lib/common.sh b/scripts/dependency-pr/lib/common.sh new file mode 100644 index 000000000..76b28ae39 --- /dev/null +++ b/scripts/dependency-pr/lib/common.sh @@ -0,0 +1,62 @@ +# Shared helpers for scripts/dependency-pr. Source, do not execute. + +VERIFY_ROOT="${VERIFY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +export VERIFY_ROOT + +log() { printf '[verify] %s\n' "$*" >&2; } + +die_infra() { + printf '[verify] ERROR: %s\n' "$*" >&2 + exit 2 +} + +require_tools() { + local missing=() + for t in "$@"; do + command -v "$t" >/dev/null 2>&1 || missing+=("$t") + done + [ ${#missing[@]} -eq 0 ] || die_infra "missing required tools: ${missing[*]}" +} + +# Local credentials and CLI config leak into `go test` and the binary smoke +# test (several cmd/ tests fail when LD_ACCESS_TOKEN or ~/.config/ldcli exist), +# so every check runs with LD_* unset and private XDG config/state/data dirs. +# Build and module caches are kept so runs stay fast. +hermetic_env_args() { + local sandbox="$1" + mkdir -p "$sandbox/config" "$sandbox/state" "$sandbox/data" + local args=() + local v + while IFS= read -r v; do + args+=("-u" "$v") + done < <(env | sed -n 's/^\(LD_[A-Za-z0-9_]*\)=.*/\1/p') + args+=( + "GH_CONFIG_DIR=${GH_CONFIG_DIR:-${XDG_CONFIG_HOME:-$HOME/.config}/gh}" + "XDG_CONFIG_HOME=$sandbox/config" + "XDG_STATE_HOME=$sandbox/state" + "XDG_DATA_HOME=$sandbox/data" + "GOCACHE=${GOCACHE_REAL}" + "GOMODCACHE=${GOMODCACHE_REAL}" + "LD_ANALYTICS_OPT_OUT=true" + "CI=true" + "NO_COLOR=1" + "npm_config_fund=false" + "npm_config_update_notifier=false" + ) + printf '%s\n' "${args[@]}" +} + +init_go_cache_env() { + if command -v go >/dev/null 2>&1; then + GOCACHE_REAL="$(go env GOCACHE)" + GOMODCACHE_REAL="$(go env GOMODCACHE)" + else + GOCACHE_REAL="${HOME}/.cache/go-build" + GOMODCACHE_REAL="${HOME}/go/pkg/mod" + fi + export GOCACHE_REAL GOMODCACHE_REAL +} + +now_s() { date +%s; } + +sanitize() { printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '-'; } diff --git a/scripts/dependency-pr/lib/deps.py b/scripts/dependency-pr/lib/deps.py new file mode 100755 index 000000000..e9341123f --- /dev/null +++ b/scripts/dependency-pr/lib/deps.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Compares the dependency graph and licenses of base and PR. + +Usage: + deps.py transitive + deps.py licenses + deps.py forced (Go modules only) +ecosystem: gomod | npm-ui | npm-wrapper. Prints JSON. + +For Go, the graph is the set of modules compiled into ldcli (go list -deps), +not every module in go.sum. For npm, it is every entry in package-lock.json. +""" +import json +import os +import re +import subprocess +import sys + +NPM_DIRS = {"npm-ui": "internal/dev_server/ui", "npm-wrapper": "."} + + +def run(cmd, cwd): + out = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if out.returncode != 0: + raise RuntimeError(f"{' '.join(cmd)} failed: {out.stderr.strip()[:300]}") + return out.stdout + + +def vkey(v): + nums = re.findall(r"\d+", v or "")[:3] + return tuple(int(n) for n in nums) + (0,) * (3 - len(nums)) + + +def semver(a, b): + if not a: + return "added" + if not b: + return "removed" + x, y = vkey(a), vkey(b) + if x == y: + return "none" if a == b else "prerelease" + if x > y: + return "downgrade" + if x[0] != y[0] or (x[0] == 0 and x[1] != y[1]): + return "major" + return "minor" if x[1] != y[1] else "patch" + + +# ---------------------------------------------------------------- Go + +def go_built(wt): + out = run(["go", "list", "-deps", "-f", "{{with .Module}}{{if not .Main}}{{.Path}} {{.Version}}{{end}}{{end}}", "./..."], wt) + mods = {} + for line in out.splitlines(): + if line.strip(): + path, version = line.split() + mods[path] = version + return mods + + +def go_dir(wt, path, version): + out = run(["go", "mod", "download", "-json", f"{path}@{version}"], wt) + return json.loads(out).get("Dir") + + +# ---------------------------------------------------------------- npm + +def npm_entries(wt, eco): + lock = os.path.join(wt, NPM_DIRS[eco], "package-lock.json") + if not os.path.exists(lock): + return {} + with open(lock) as f: + packages = json.load(f).get("packages", {}) + entries = {} + for key, meta in packages.items(): + if not key: + continue + name = key.rsplit("node_modules/", 1)[-1] + version = meta.get("version") + if not version or meta.get("link"): + continue + lic = meta.get("license") + if isinstance(lic, dict): + lic = lic.get("type") + entries.setdefault(f"{name}@{version}", { + "name": name, "version": version, "license": lic, + "install_script": bool(meta.get("hasInstallScript")), "dev": bool(meta.get("dev")), + }) + return entries + + +# ---------------------------------------------------------------- licenses + +def identify(text): + t = re.sub(r"\s+", " ", text.lower()) + if "apache license" in t and "version 2.0" in t: + return "Apache-2.0" + if "mozilla public license" in t: + return "MPL-2.0" + if "gnu affero general public license" in t: + return "AGPL-3.0" + if "gnu lesser general public license" in t: + return "LGPL" + if "gnu general public license" in t: + return "GPL" + if "permission is hereby granted, free of charge" in t: + return "MIT" + if "permission to use, copy, modify, and/or distribute this software for any purpose" in t: + return "ISC" if "isc" in t or "with or without fee" in t else "0BSD" + if "redistribution and use in source and binary forms" in t: + return "BSD-3-Clause" if ("neither the name" in t or "names of its contributors" in t) else "BSD-2-Clause" + if "this is free and unencumbered software released into the public domain" in t: + return "Unlicense" + if "creative commons" in t and "cc0" in t: + return "CC0-1.0" + return "unknown" + + +def go_license(wt, path, version): + d = go_dir(wt, path, version) + if not d or not os.path.isdir(d): + return "unknown", None + for name in sorted(os.listdir(d)): + if re.match(r"^(licen[cs]e|copying)(\.(md|txt))?$", name, re.I): + with open(os.path.join(d, name), errors="replace") as f: + return identify(f.read()), name + return "unknown", None + + +def allowed(expr, policy): + if not expr or expr == "unknown": + return False + expr = expr.strip("() ") + if " OR " in expr: + return any(allowed(p, policy) for p in expr.split(" OR ")) + if " AND " in expr: + return all(allowed(p, policy) for p in expr.split(" AND ")) + return expr in policy["allowed"] + + +# ---------------------------------------------------------------- commands + +def transitive(base, pr, eco): + if eco == "gomod": + b, p = go_built(base), go_built(pr) + changed = [{"name": n, "from": b[n], "to": p[n], "semver": semver(b[n], p[n])} + for n in sorted(set(b) & set(p)) if b[n] != p[n]] + return { + "ecosystem": eco, "scope": "modules compiled into ldcli", + "added": [f"{n}@{p[n]}" for n in sorted(set(p) - set(b))], + "removed": [f"{n}@{b[n]}" for n in sorted(set(b) - set(p))], + "changed": changed, "install_scripts": [], + } + b, p = npm_entries(base, eco), npm_entries(pr, eco) + names_b = {e["name"] for e in b.values()} + added = sorted(set(p) - set(b)) + removed = sorted(set(b) - set(p)) + changed = [] + for name in sorted({p[k]["name"] for k in added} & {b[k]["name"] for k in removed}): + old = sorted(b[k]["version"] for k in removed if b[k]["name"] == name) + new = sorted(p[k]["version"] for k in added if p[k]["name"] == name) + changed.append({"name": name, "from": ", ".join(old), "to": ", ".join(new), "semver": semver(old[-1], new[-1])}) + changed_names = {c["name"] for c in changed} + # A version change of a package that already ran an install script on base + # (esbuild in #779) adds no new script to approve. + scripted_b = {e["name"] for e in b.values() if e["install_script"]} + return { + "ecosystem": eco, "scope": "package-lock.json", + "added": [k for k in added if p[k]["name"] not in changed_names], + "new_packages": sorted({p[k]["name"] for k in added if p[k]["name"] not in names_b}), + "removed": [k for k in removed if b[k]["name"] not in changed_names], + "changed": changed, + "install_scripts": [k for k in added if p[k]["install_script"] and p[k]["name"] not in scripted_b], + } + + +def licenses(base, pr, eco, policy): + findings = [] + if eco == "gomod": + b, p = go_built(base), go_built(pr) + for name in sorted(p): + if b.get(name) == p[name]: + continue + new, new_file = go_license(pr, name, p[name]) + old = go_license(base, name, b[name])[0] if name in b else None + findings.append({"package": f"{name}@{p[name]}", "name": name, "version": p[name], + "from": old, "to": new, "file": new_file, "dev": False}) + else: + b, p = npm_entries(base, eco), npm_entries(pr, eco) + old_by_name = {} + for e in b.values(): + old_by_name.setdefault(e["name"], e["license"]) + for key in sorted(set(p) - set(b)): + e = p[key] + findings.append({"package": key, "name": e["name"], "version": e["version"], + "from": old_by_name.get(e["name"]), "to": e["license"] or "unknown", "file": None, + "dev": e["dev"]}) + for f in findings: + f["changed"] = f["from"] is not None and f["from"] != f["to"] + f["allowed"] = allowed(f["to"], policy) + return {"ecosystem": eco, "checked": len(findings), + "problems": [f for f in findings if f["changed"] or not f["allowed"]], + "all": findings} + + +def forced(pr, disclosed, candidates): + """For each candidate Go module update, finds a disclosed update that needs + at least the candidate's new version through the PR's module graph (e.g. + go.uber.org/mock v0.6.0 → x/tools → x/net → golang.org/x/term v0.34.0).""" + edges = {} + for line in run(["go", "mod", "graph"], pr).splitlines(): + parts = line.split() + if len(parts) == 2: + edges.setdefault(parts[0], []).append(parts[1]) + out = {} + for root in disclosed: + start = f"{root['name']}@{root['to']}" + seen, queue, parent = {start}, [start], {} + while queue: + node = queue.pop(0) + for dep in edges.get(node, []): + if dep in seen: + continue + seen.add(dep) + parent[dep] = node + queue.append(dep) + for c in candidates: + if c["name"] in out: + continue + for node in seen: + name, _, version = node.rpartition("@") + if name == c["name"] and vkey(version) >= vkey(c["to"]): + path, n = [], node + while n != start: + n = parent[n] + path.append(n) + out[c["name"]] = {"by": f"{root['name']} {root['to']}", "requires": version, + "via": [p.replace("@", " ") for p in reversed(path[:-1])]} + break + return out + + +def main(): + cmd = sys.argv[1] + if cmd == "forced": + with open(sys.argv[3]) as f: + disclosed = json.load(f) + with open(sys.argv[4]) as f: + candidates = json.load(f) + json.dump(forced(sys.argv[2], disclosed, candidates), sys.stdout, indent=2) + return + base, pr, eco = sys.argv[2:5] + if cmd == "transitive": + result = transitive(base, pr, eco) + elif cmd == "licenses": + with open(sys.argv[5]) as f: + result = licenses(base, pr, eco, json.load(f)) + else: + raise SystemExit(f"unknown command {cmd}") + json.dump(result, sys.stdout, indent=2) + + +if __name__ == "__main__": + main() diff --git a/scripts/dependency-pr/lib/semver.jq b/scripts/dependency-pr/lib/semver.jq new file mode 100644 index 000000000..0ef4aa338 --- /dev/null +++ b/scripts/dependency-pr/lib/semver.jq @@ -0,0 +1,48 @@ +# jq module: version parsing and update classification. +# Use with: jq -L "$VERIFY_ROOT/lib" 'include "semver"; ...' + +# Accepts "v1.2.3", "1.2", "release-secrets-v1.2.0", Go pseudo-versions, +# and npm/semver prerelease suffixes. Returns null when no number is found. +def vparse: + tostring as $raw + | ($raw | sub("^[^0-9]*"; "")) as $s + | ($s | capture("^(?[0-9]+(\\.[0-9]+)*)(?.*)$")) // null + | if . == null then null + else { + raw: $raw, + nums: ((.core | split(".") | map(tonumber)) + [0, 0, 0])[0:3], + pre: .rest, + pseudo: (.rest | test("[0-9]{14}-[0-9a-f]{12}$")) + } + end; + +# One of: added, removed, none, major, minor, patch, prerelease, pseudo, +# downgrade, unknown. +def semver_class($from; $to): + if $from == null then "added" + elif $to == null then "removed" + else + ($from | vparse) as $f | ($to | vparse) as $t + | if $f == null or $t == null then (if $from == $to then "none" else "unknown" end) + elif $f.nums == $t.nums then + (if $f.pre == $t.pre then "none" + elif $f.pseudo or $t.pseudo then "pseudo" + else "prerelease" end) + elif $f.nums > $t.nums then "downgrade" + elif $f.nums[0] != $t.nums[0] then "major" + elif $f.nums[1] != $t.nums[1] then "minor" + else "patch" + end + end; + +# Semver allows breaking changes on a major bump, and on a minor bump while +# the major version is 0. +def is_breaking($from; $to): + semver_class($from; $to) as $c + | if $c == "major" then true + elif $c == "minor" then (($from | vparse).nums[0] == 0) + else false + end; + +def tier_rank: {"low": 0, "medium": 1, "high": 2}[.] // 0; +def max_tier($a; $b): if ($a | tier_rank) >= ($b | tier_rank) then $a else $b end; diff --git a/scripts/dependency-pr/lib/upstream.py b/scripts/dependency-pr/lib/upstream.py new file mode 100644 index 000000000..a011149a7 --- /dev/null +++ b/scripts/dependency-pr/lib/upstream.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Collects upstream evidence for each direct update: the source repository, +the compare link and size between the two versions, and the release notes +(or CHANGELOG section) in that range. It flags notes that mention breaking +changes, security fixes, deprecations, new requirements, or licenses. + +Usage: upstream.py (prints JSON) +Needs: gh (authenticated), go and npm for resolving sources. +""" +import base64 +import json +import os +import re +import subprocess +import sys + +KEYWORDS = { + "breaking": r"breaking|backwards?[- ]incompatible|\bmajor change|\bremoved?\b.*\b(api|support|options?|methods?|functions?|packages?|components?|exports?|props?|inputs?|outputs?)\b|\brewrite\b", + "security": r"security|vulnerab|\bcve-\d|\bghsa-", + "deprecation": r"deprecat", + "requirements": r"minimum (go|node|version)|requires? (go|node)|engines|drop(ped|s)? support|node ?\d\d", + "license": r"\blicen[cs]e", +} +VERSION_RE = re.compile(r"(\d+(?:\.\d+)+(?:-[0-9A-Za-z.]+)?)(?!.*\d+\.\d+)") + + +def sh(cmd, cwd=None): + out = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=120) + return out.stdout if out.returncode == 0 else None + + +def gh_json(path): + out = sh(["gh", "api", path]) + return json.loads(out) if out else None + + +def vtuple(v): + core = re.search(r"\d+(?:\.\d+)*", v or "") + if not core: + return None + nums = [int(n) for n in core.group(0).split(".")][:3] + return tuple(nums + [0] * (3 - len(nums))) + + +def split_tag(tag): + m = VERSION_RE.search(tag or "") + if not m: + return None, None + return tag[: m.start()].rstrip("v"), m.group(1) + + +def github_repo(url): + m = re.search(r"github\.com[/:]([^/]+)/([^/#?]+?)(?:\.git)?(?:[/#?]|$)", url or "") + return f"{m.group(1)}/{m.group(2)}" if m else None + + +def go_repo(name, origin_url): + # golang.org/x/ is served from go.googlesource.com and mirrored to github.com/golang/. + m = re.match(r"^golang\.org/x/([^/]+)", name) + if m: + return f"golang/{m.group(1)}" + return github_repo(origin_url) or github_repo("https://" + name) + + +def resolve(update, wt): + """Returns (repo, from_ref, to_ref, subdir). subdir is the package directory in a monorepo.""" + eco, name = update["ecosystem"], update["name"] + frm, to = update.get("from"), update.get("to") + if eco == "gomod": + info = {} + for side, v in (("from", frm), ("to", to)): + out = sh(["go", "mod", "download", "-json", f"{name}@{v}"], cwd=wt) if v else None + origin = (json.loads(out).get("Origin") or {}) if out else {} + info[side] = origin + repo = go_repo(name, info["to"].get("URL")) + tag = lambda o, v: (o.get("Ref") or "").replace("refs/tags/", "") or v + return repo, tag(info["from"], frm), tag(info["to"], to), None + if eco in ("npm-ui", "npm-wrapper"): + meta = {} + for side, v in (("from", frm), ("to", to)): + out = sh(["npm", "view", f"{name}@{v}", "repository.url", "repository.directory", "gitHead", "--json"]) if v else None + try: + meta[side] = json.loads(out) if out else {} + except json.JSONDecodeError: + meta[side] = {} + if isinstance(meta[side], str): + meta[side] = {"repository.url": meta[side]} + repo = github_repo(meta["to"].get("repository.url")) + subdir = (meta["to"].get("repository.directory") or "").strip("/") or None + + def tag(v): + # Without gitHead, use the first tag form that exists: "@1.2.3" + # (react-router, @launchpad-ui/*), "v1.2.3" (vite), or "1.2.3". + forms = ([f"{name}@{v}"] if subdir else []) + [f"v{v}", v] + if repo: + for t in forms: + if sh(["gh", "api", f"repos/{repo}/git/ref/tags/{t}", "--jq", ".ref"]): + return t + return None + return forms[0] + return repo, meta["from"].get("gitHead") or tag(frm), meta["to"].get("gitHead") or tag(to), subdir + if eco == "github-actions": + repo = "/".join(name.split("/")[:2]) + sub = "/".join(name.split("/")[2:]) or None + return repo, update.get("from_ref") or frm, update.get("to_ref") or to, sub + return None, None, None, None + + +HEAD_VERSION = re.compile(r"^#+\s.*?\bv?(\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?)") +MAX_CHANGELOG_LINES = 6000 +MAX_RELEASE_PAGES = 10 + + +def changelog_section(repo, path, frm, to, to_tag): + """Returns (text, coverage). The section starts at the heading of `to` and + ends at the first release heading at or below `frm`. coverage is + {"status": "full" | "partial", "lowest": }.""" + data = None + # A monorepo can publish a version without a tag (@launchpad-ui/core + # 0.59.17). The default branch has the same CHANGELOG with newer entries. + for ref in ([to_tag] if to_tag else []) + [None]: + data = gh_json(f"repos/{repo}/contents/{path}" + (f"?ref={ref}" if ref else "")) + if data and "content" in data: + break + if not data or "content" not in data: + return "", None + text = base64.b64decode(data["content"]).decode("utf-8", "replace").splitlines() + want = to.lstrip("v") + start = next((i for i, l in enumerate(text) if HEAD_VERSION.match(l) and HEAD_VERSION.match(l).group(1) == want), None) + if start is None: + start = next((i for i, l in enumerate(text) if l.startswith("#") and want in l), None) + if start is None: + return "", None + lo, lowest = vtuple(frm), want + for i in range(start + 1, min(len(text), start + MAX_CHANGELOG_LINES)): + m = HEAD_VERSION.match(text[i]) + if not m: + continue + if lo and vtuple(m.group(1)) <= lo: + return "\n".join(text[start:i]), {"status": "full", "lowest": lowest} + lowest = m.group(1) + end = min(len(text), start + MAX_CHANGELOG_LINES) + return "\n".join(text[start:end]), {"status": "partial", "lowest": lowest} + + +def release_notes(repo, frm, to, to_tag, name=None, subdir=None): + """Returns (source, text, coverage).""" + # In a monorepo, the package CHANGELOG has the per-version notes. The GitHub + # releases can only link to it (react-router) or tag other packages. + if subdir: + text, cov = changelog_section(repo, f"{subdir}/CHANGELOG.md", frm, to, to_tag) + if text: + return f"{subdir}/CHANGELOG.md", text, cov + lo, hi = vtuple(frm), vtuple(to) + want_prefix = split_tag(to_tag)[0] if to_tag and not re.fullmatch(r"[0-9a-f]{40}", to_tag) else None + pkg_prefix = f"{name}@" if subdir and name else None + notes, reached, exhausted = [], False, False + for page in range(1, MAX_RELEASE_PAGES + 1): + releases = gh_json(f"repos/{repo}/releases?per_page=100&page={page}") or [] + for rel in releases: + tag = rel.get("tag_name") or "" + if pkg_prefix and not tag.startswith(pkg_prefix): + continue + prefix, version = split_tag(tag) + if want_prefix is not None and prefix != want_prefix: + continue + v = vtuple(version) + if not v or not lo or not hi: + continue + if v <= lo: + reached = True + elif v <= hi: + notes.append((v, version, f"## {tag}\n\n{rel.get('body') or ''}\n")) + if len(releases) < 100: + exhausted = True + break + if reached: + break + if notes: + notes.sort(reverse=True) + cov = {"status": "full" if reached or exhausted else "partial", "lowest": notes[-1][1]} + return "release notes", "\n".join(n for _, _, n in notes), cov + for path in ("CHANGELOG.md", "CHANGES.md", "HISTORY.md"): + text, cov = changelog_section(repo, path, frm, to, to_tag) + if text: + return path, text, cov + return None, "", None + + +def commit_log(cmp): + """Commit subjects from a compare result, for repositories without notes (golang.org/x/*).""" + commits = (cmp or {}).get("commits") or [] + if not commits: + return "", None + lines = [f"- {c['sha'][:7]} {(c.get('commit') or {}).get('message', '').splitlines()[0]}" for c in commits] + head = f"## Commits ({len(commits)} of {cmp.get('total_commits')})" + status = "full" if len(commits) >= (cmp.get("total_commits") or 0) else "partial" + return head + "\n\n" + "\n".join(reversed(lines)) + "\n", {"status": status, "lowest": None} + + +def main(): + cls_path, wt, notes_dir = sys.argv[1:4] + os.makedirs(notes_dir, exist_ok=True) + with open(cls_path) as f: + cls = json.load(f) + report = [] + for u in cls["updates"]: + if not u.get("direct") or u["ecosystem"] == "docker" or not u.get("from") or not u.get("to"): + continue + entry = {"name": u["name"], "ecosystem": u["ecosystem"], "from": u["from"], "to": u["to"]} + repo, from_ref, to_ref, subdir = resolve(u, wt) + entry["repo"] = repo + entry["notes_coverage"] = "none" + if repo: + missing = [v for v, r in ((u["from"], from_ref), (u["to"], to_ref)) if not r] + if missing: + entry["tags_missing"] = missing + cmp = gh_json(f"repos/{repo}/compare/{from_ref}...{to_ref}") if from_ref and to_ref else None + if cmp: + entry["compare_url"] = cmp.get("html_url") + entry["commits"] = cmp.get("total_commits") + entry["files_changed"] = len(cmp.get("files") or []) + npm_subdir = subdir if u["ecosystem"] in ("npm-ui", "npm-wrapper") else None + source, text, cov = release_notes(repo, u["from"], u["to"], to_ref, u["name"], npm_subdir) + if not text and u["ecosystem"] == "gomod": + text, cov = commit_log(cmp) + source = "commit messages" if text else None + entry["notes_source"] = source + entry["notes_coverage"] = (cov or {}).get("status", "none") + entry["notes_lowest"] = (cov or {}).get("lowest") + if text: + fname = re.sub(r"[^A-Za-z0-9._-]", "_", u["name"]) + ".md" + with open(os.path.join(notes_dir, fname), "w") as f: + f.write(text) + entry["notes_file"] = fname + low = text.lower() + entry["keywords"] = {k: len(re.findall(p, low)) for k, p in KEYWORDS.items() if re.search(p, low)} + report.append(entry) + json.dump(report, sys.stdout, indent=2) + + +if __name__ == "__main__": + main() diff --git a/scripts/dependency-pr/lib/verdict.jq b/scripts/dependency-pr/lib/verdict.jq new file mode 100644 index 000000000..7ce851ff8 --- /dev/null +++ b/scripts/dependency-pr/lib/verdict.jq @@ -0,0 +1,164 @@ +# jq module: turns check records, classification, and agent notes into a verdict. +# Use with: jq -L "$VERIFY_ROOT/lib" 'include "verdict"; ...' +# +# Verdicts, strongest first: +# block the PR must not merge as it is (breakage, or a fix is required) +# needs-human verification is complete, but a person must answer a specific question +# incomplete a required check, gate, or review did not run or proved nothing; a rerun or the agent closes it, not a person +# +# A gate (registry "gate": true, or a matched "required_for_tags") must pass. +# "pre-existing" or "incomplete" never satisfies a gate. +# safe-to-merge every required check ran and passed +include "semver"; + +def nonpass: . == "fail" or . == "decide"; + +# A failure that also happens on base with the same fingerprint is +# pre-existing and does not count against the PR. For a check that lists +# independent findings, a PR whose findings all fail on base too is also +# pre-existing (the PR can remove findings, but adds none). +def findings_subset: + ((.pr.findings // []) | length) > 0 and (.base.findings // null) != null + and (((.pr.findings // []) - .base.findings) | length) == 0; + +def baseline_outcome: + .pr.status as $p | (.base.status // null) as $b + | if ($p | IN("pass", "info", "skip", "incomplete", "error")) then $p + elif $b == null then $p + elif ($b | IN("pass", "info")) then "regression" + elif ($b | nonpass) then + (if (.pr.fingerprint // .pr.summary) == (.base.fingerprint // .base.summary) or findings_subset + then "pre-existing" else "changed" end) + else "base-inconclusive" + end; + +# A discriminating check counts ("proven") only when it fails on the old +# version and passes on the new one. +def generated_outcome: + .pr.status as $p | .base.status as $b + | def passed: IN("pass", "info"); + if $p == "error" or $b == "error" then "error" + elif ($p | IN("skip", "incomplete")) or ($b | IN("skip", "incomplete")) then "incomplete" + elif .kind == "discriminating" then + (if ($b | passed | not) and ($p | passed) then "proven" + elif ($b | passed) and ($p | passed) then "not-discriminating" + elif ($b | passed) then "regression" + else "fails-both" end) + else + (if ($b | passed) and ($p | passed) then "holds" + elif ($b | passed) then "regression" + else "invalid" end) + end; + +def tag_required($cls): [ (.required_for_tags // [])[] | select(. as $t | $cls.tags | index($t) != null) ]; + +def build_result($meta; $cls; $checks; $gen; $impact; $profile; $generated_at): + ($checks | map(. + {outcome: baseline_outcome})) as $checks + | ($gen | map(. + {outcome: generated_outcome} | . + {counted: (.outcome == "proven")})) as $gen + | ($gen | map(select(.counted)) | length) as $proven + | (if $impact != null and ($impact.tier // null) != null + then max_tier($cls.tier; $impact.tier) else $cls.tier end) as $tier + | ($cls.updates | map(select(.direct)) | map(.name)) as $direct + + | [ ( $checks[] + | select(.pr.status == "fail" and (.outcome | IN("pre-existing") | not)) + | {source: .id, title, + problem: (.pr.summary + + (if .outcome == "changed" then " (base fails too, but this PR changes the result)" + elif .outcome == "base-inconclusive" then " (could not compare with base)" + else "" end)), + fix: (.pr.recommendations // []), recipe: .pr.fix} ), + ( $gen[] | select(.outcome == "regression") + | {source: "generated:\(.id)", title, problem: .pr.summary, + fix: (.pr.recommendations // []), recipe: null} ), + ( ($impact.findings // [])[] | select(.severity == "block") + | {source: "impact", title: "Impact review", problem: .text, fix: [], recipe: null} ) + ] as $blocks + + | [ ( $checks[] + | select(.pr.status == "decide" and .outcome != "pre-existing") + | {source: .id, question: (.pr.question // .title), evidence: ([.pr.summary] + (.pr.recommendations // [])), + recipe: .pr.fix} ), + ( if $tier != "low" and ($impact.behavior_changes_reachable // false) and $proven == 0 and (($impact.no_local_proof // "") != "") then + {source: "impact", question: "Accept the upstream behavior changes that reach ldcli, which no local check can prove?", + evidence: ([$impact.no_local_proof] + [($impact.changelog // [])[] | "\(.package) \(.range // ""): \(.notes)"]), recipe: null} + else empty end ), + ( ($impact.findings // [])[] | select(.severity == "decide" or .severity == "warn") + | {source: "impact", question: (.question // .text), evidence: ([.text] + (.evidence // []) | unique), + recipe: null} ) + ] as $decisions + + # A reachable change that no local check can prove is a decision. If a check + # already asks a question, add the reason to its evidence instead of asking twice. + | ($decisions | map(select(.source == "impact" and (.question | startswith("Accept the upstream behavior changes"))))) as $nlp + | ($decisions | map(select((.source == "impact" and (.question | startswith("Accept the upstream behavior changes"))) | not))) as $others + | (if ($nlp | length) > 0 and ($others | length) > 0 + then ($others | .[0].evidence += $nlp[0].evidence) + else $decisions end) as $decisions + + | [ ( $checks[] | select(.outcome == "error") + | {source: .id, reason: "\(.title): the check crashed (\(.pr.summary))"} ), + ( $checks[] | select(.outcome == "incomplete" and .required) + | {source: .id, reason: "\(.title): \(.pr.summary)"} ), + ( $checks[] | select((.outcome == "incomplete" or (.pr.profile_skipped // false)) and (.required | not)) + | tag_required($cls) as $hit | select(($hit | length) > 0) + | {source: .id, + reason: "\(.title): \(.pr.summary). This check is required for \($hit | join(", ")) updates."} ), + ( $checks[] | select(.outcome == "pre-existing" and ((.gate // false) or ((tag_required($cls) | length) > 0))) + | {source: .id, + reason: "\(.title) failed the same way on base and on the PR, so this gate does not show that the update works (\(.pr.summary))"} ), + ( $gen[] | select(.outcome | IN("error", "incomplete", "fails-both", "invalid")) + | {source: "generated:\(.id)", + reason: (({"error": "the generated check crashed", + "incomplete": "the generated check could not run", + "fails-both": "the discriminating check fails on both versions; fix the check or the analysis", + "invalid": "the guard fails on base; fix the check"}[.outcome]) + " (\(.title))")} ), + ( if $tier != "low" then + ( if $impact == null then + {source: "impact", reason: "The \($tier)-risk update needs an impact review (agent/impact.json): upstream notes, reach into ldcli, and breaking changes"} + else + ( [ $direct[] | select(. as $n | ($impact.changelog // []) | map(.package) | index($n) | not) ] as $missing + | if ($missing | length) > 0 then + {source: "impact", reason: "The impact review does not cover these direct updates: \($missing | join(", "))"} + else empty end ), + ( if ($impact | has("behavior_changes_reachable") | not) then + {source: "impact", reason: "The impact review must state behavior_changes_reachable (true or false)"} + # A guard regression or a failing gate (npm ci, build) already shows that the + # change reaches ldcli, and no discriminating check can pass on a PR that does not install. + elif $impact.behavior_changes_reachable == true and $proven == 0 and (($impact.no_local_proof // "") == "") + and ([$gen[] | select(.outcome == "regression")] | length) == 0 + and ([$checks[] | select((.outcome | IN("regression", "changed")) + and ((.gate // false) or ((tag_required($cls) | length) > 0)))] | length) == 0 then + {source: "impact", reason: "The impact review found upstream behavior changes that reach ldcli, but no generated check proved one (it must fail on the old version and pass on the new one)"} + else empty end ) + end ) + else empty end ) + ] as $incomplete + + | [ $checks[] | select((.outcome == "incomplete" or (.pr.profile_skipped // false)) and (.required | not) and ((tag_required($cls) | length) == 0)) + | {source: .id, reason: "\(.title): \(.pr.summary)"} ] as $not_run + + | (if ($blocks | length) > 0 then "block" + elif ($decisions | length) > 0 then "needs-human" + elif ($incomplete | length) > 0 then "incomplete" + else "safe-to-merge" end) as $verdict + | { + schema: 2, + generated_at: $generated_at, + profile: $profile, + pr: $meta, + classification: $cls, + tier: $tier, + verdict: $verdict, + exit_code: ({"safe-to-merge": 0, "needs-human": 1, "block": 1, "incomplete": 2}[$verdict]), + blocks: $blocks, + decisions: $decisions, + incomplete: $incomplete, + not_run: $not_run, + fixes: [ $blocks[], $decisions[] | .recipe | select(. != null) ] | unique_by(.id), + pre_existing: [ $checks[] | select(.outcome == "pre-existing") | {id, title, summary: .pr.summary} ], + checks: $checks, + generated_checks: $gen, + generated_proven: $proven, + impact: $impact + }; diff --git a/scripts/dependency-pr/license-policy.json b/scripts/dependency-pr/license-policy.json new file mode 100644 index 000000000..a9ac16638 --- /dev/null +++ b/scripts/dependency-pr/license-policy.json @@ -0,0 +1,4 @@ +{ + "_doc": "Licenses that need no review when a dependency is added or changed. A new or changed dependency with any other license, or with a license change, becomes a decision for a person.", + "allowed": ["MIT", "ISC", "0BSD", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "Unlicense", "CC0-1.0", "BlueOak-1.0.0", "Python-2.0", "CC-BY-4.0"] +} diff --git a/scripts/dependency-pr/post-comment.sh b/scripts/dependency-pr/post-comment.sh new file mode 100755 index 000000000..3b38fbab4 --- /dev/null +++ b/scripts/dependency-pr/post-comment.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Posts (or updates) the single verifier comment on a PR. This is the only +# script in scripts/dependency-pr that writes to GitHub, and it only comments: +# it never approves, requests changes, or merges. verify.sh never calls it. +# +# Usage: post-comment.sh --out-dir DIR [--repo OWNER/NAME] [--dry-run] [--force] +# --out-dir DIR a verify.sh output directory (needs result.json and comment.md) +# --dry-run print what would be posted and where, without writing +# --force post even if the PR head moved since verification +# +# Exit: 0 posted (or dry run), 1 refused (stale result, not a PR run), 2 error. +set -euo pipefail + +OUT="" REPO="" DRY_RUN=false FORCE=false +MARKER="" + +while [ $# -gt 0 ]; do + case "$1" in + --out-dir) OUT="${2:?}"; shift 2 ;; + --repo) REPO="${2:?}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + --force) FORCE=true; shift ;; + -h | --help) sed -n '2,/^set -euo/p' "$0" | sed '$d; s/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +[ -n "$OUT" ] || { echo "--out-dir is required" >&2; exit 2; } +result="$OUT/result.json" comment="$OUT/comment.md" +[ -f "$result" ] && [ -f "$comment" ] || { echo "missing $result or $comment" >&2; exit 2; } +grep -qF "$MARKER" "$comment" || { echo "$comment lacks the verifier marker" >&2; exit 2; } + +pr=$(jq -r '.pr.number // empty' "$result") +[ -n "$pr" ] || { echo "result.json has no PR number (branch run without a PR); nothing to post" >&2; exit 1; } +REPO="${REPO:-$(jq -r '.pr.repo' "$result")}" +verified=$(jq -r '.pr.head_sha' "$result") + +current=$(gh pr view "$pr" --repo "$REPO" --json headRefOid --jq .headRefOid) || { echo "cannot read PR #$pr" >&2; exit 2; } +if [ "$current" != "$verified" ] && [ "$FORCE" != true ]; then + echo "refusing to post: PR #$pr head is now ${current:0:7}, but the result is for ${verified:0:7}. Re-run verify.sh." >&2 + exit 1 +fi + +existing=$(gh api "repos/$REPO/issues/$pr/comments" --paginate \ + --jq ".[] | select(.body | contains(\"$MARKER\")) | .id" | tail -n1) + +if [ "$DRY_RUN" = true ]; then + if [ -n "$existing" ]; then + echo "[dry-run] would update comment $existing on $REPO#$pr ($(wc -c <"$comment") bytes)" + else + echo "[dry-run] would create a comment on $REPO#$pr ($(wc -c <"$comment") bytes)" + fi + exit 0 +fi + +body=$(jq -Rs '{body: .}' "$comment") +if [ -n "$existing" ]; then + gh api -X PATCH "repos/$REPO/issues/comments/$existing" --input - <<<"$body" --jq .html_url +else + gh api -X POST "repos/$REPO/issues/$pr/comments" --input - <<<"$body" --jq .html_url +fi diff --git a/scripts/dependency-pr/render-comment.sh b/scripts/dependency-pr/render-comment.sh new file mode 100755 index 000000000..42fdd56af --- /dev/null +++ b/scripts/dependency-pr/render-comment.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Usage: render-comment.sh — prints the PR comment as Markdown. +set -euo pipefail + +RESULT="${1:?usage: render-comment.sh }" +MAX_CHARS=60000 # GitHub caps comments at 65536 characters. + +render() { + local with_details="$1" + jq -r --argjson with_details "$with_details" ' + def esc: tostring | gsub("\\|"; "\\|") | gsub("\n"; " "); + def rlabel: + {"pass": "pass", "info": "pass (note)", "fail": "**FAIL**", "decide": "decision", "skip": "does not apply", + "incomplete": "**not run**", "error": "**ERROR**", + "regression": "**FAIL** (passes on base)", "changed": "**FAIL** (differs from base)", + "pre-existing": "fails on base too", "base-inconclusive": "**FAIL** (base not compared)"}[.] // .; + def glabel: + {"proven": "proven (fails on base, passes on PR)", "not-discriminating": "passes on both: does not count", + "regression": "**REGRESSION** (passes on base, fails on PR)", "fails-both": "**fails on both**", + "holds": "holds on both", "invalid": "**invalid** (fails on base)", + "error": "**ERROR**", "incomplete": "**not run**"}[.] // .; + def sha7: if . == null then "?" else .[0:7] end; + + . as $r + | $r.pr as $pr + | $r.classification as $c + | ($c.updates | map(select(.direct))) as $direct + | ($c.updates | map(select(.direct | not))) as $transitive + | [ + "", + "## Dependency verification: \({"safe-to-merge": "SAFE TO MERGE", "needs-human": "NEEDS A HUMAN DECISION", "block": "BLOCK", "incomplete": "INCOMPLETE"}[$r.verdict])", + "", + "Risk tier **\($r.tier)**\(if ($c.tags | length) > 0 then " (" + ($c.tags | join(", ")) + ")" else "" end) · ecosystems: \($c.ecosystems | join(", ")) · head `\($pr.head_sha | sha7)` " + + (if $pr.merge.status == "merged" + then "tested as merged into `\($pr.base_ref)` @ `\($pr.base_sha | sha7)`" + else "conflicts with `\($pr.base_ref)`, so it was tested as it is against merge base `\($pr.merge_base | sha7)`" end), + "", + (if $r.verdict == "safe-to-merge" then + "Every required check ran and passed. No decision is necessary.", "" + else empty end), + + (if ($r.blocks | length) > 0 then + "### Must fix before merge", "", + ($r.blocks[] + | "- **\(.title)**: \(.problem)", + (.fix[] | " - Fix: \(.)"), + (if .recipe != null and .recipe.kind == "commit" then + " - Mechanical fix: `\(.recipe.command)` (changes \(.recipe.paths | map("`\(.)`") | join(", ")))" + else empty end)), + "" + else empty end), + + (if ($r.decisions | length) > 0 then + "### Decisions for a human", "", + "Verification is complete for these items. Each one needs a choice, not more checks.", "", + ($r.decisions | to_entries[] + | "\(.key + 1). **\(.value.question)**", + (.value.evidence[] | " - \(.)")), + "" + else empty end), + + (if ($r.incomplete | length) > 0 then + "### Incomplete verification", "", + "These items did not run. Run `verify.sh` again where the missing tool is available, or finish the agent review. A person does not have to do these checks.", "", + ($r.incomplete[] | "- \(.reason)"), + "" + else empty end), + + "### Updates", + "", + (if ($direct | length) == 0 and ($transitive | length) == 0 then "No dependency version change was found.", "" + else + "| Package | From | To | Change | Ecosystem | Tier |", + "|---|---|---|---|---|---|", + ($direct[] | "| `\(.name)` | \(.from // "∅") | \(.to // "∅") | \(.semver)\(if .breaking and .semver != "major" then " (0.x: breaking allowed)" else "" end) | \(.ecosystem)\(if .dev then ", dev" else "" end) | \(.tier) |"), + (if ($transitive | length) > 0 then + "", + "
\($transitive | length) transitive update(s)", + "", + ($transitive[0:60][] | "- `\(.name)` \(.from // "∅") → \(.to // "∅") (\(.semver))"), + (if ($transitive | length) > 60 then "- …" else empty end), + "
" + else empty end), + "" + end), + (if ($c.go_directive.from != $c.go_directive.to) then "Go directive: `\($c.go_directive.from)` → `\($c.go_directive.to)`", "" else empty end), + + "### What the verifier checked", + "", + "| Check | Result | Summary |", + "|---|---|---|", + ($r.checks[] | select((.pr.profile_skipped // false) | not) | select(.outcome != "skip") + | "| \(.title | esc) | \(.outcome | rlabel) | \(.pr.summary | esc) |"), + (if ($r.not_run | length) > 0 then + "", "Optional checks that did not run: \($r.not_run | map(.source) | join(", "))." + else empty end), + "", + + (if ($r.pre_existing | length) > 0 then + "### Already failing on `\($pr.base_ref)` (not caused by this PR)", + "", + ($r.pre_existing[] | "- **\(.title)**: \(.summary)"), + "" + else empty end), + + "### Generated checks", + "", + (if ($r.generated_checks | length) == 0 then + "None for this PR.", "" + else + "A generated check counts only when it fails on the old version and passes on the new one. \($r.generated_proven) of \($r.generated_checks | length) proven.", + "", + "| Check | Kind | Base | PR | Outcome |", + "|---|---|---|---|---|", + ($r.generated_checks[] | "| \(.title | esc) | \(.kind) | \(.base.status) | \(.pr.status) | \(.outcome | glabel) |"), + "" + end), + + (if $r.impact != null then + "### Impact review", + "", + ($r.impact.summary // empty), + "", + (($r.impact.changelog // [])[] | "- **\(.package)** \(.range // ""): \(.notes)\(if .breaking then " **(breaking)**" else "" end)\(if .url then " ([source](\(.url)))" else "" end)"), + (if (($r.impact.usage // []) | length) > 0 then "- Used in: " + ($r.impact.usage | map("`\(.)`") | join(", ")) else empty end), + (if ($r.impact | has("behavior_changes_reachable")) then + "- Upstream behavior changes that reach ldcli: \(if $r.impact.behavior_changes_reachable then "yes" else "no" end)" + else empty end), + (($r.impact.findings // [])[] | select(.severity == "info") | "- Note: \(.text)\(if ((.evidence // []) | length) > 0 then " (" + (.evidence | join(", ")) + ")" else "" end)"), + "" + else empty end), + + (if $with_details then + ([$r.checks[], ($r.generated_checks[] | . + {id: ("generated: " + .id)})] + | map(select(.pr.details != null or (.base.details // null) != null))) as $d + | if ($d | length) > 0 then + "
Check details", + "", + ($d[] | "#### \(.id)", "", (.pr.details // .base.details), ""), + "
", + "" + else empty end + else + "_The check details are not shown because of the GitHub comment size limit. See the run logs._", "" + end), + + "Generated by `scripts/dependency-pr/verify.sh` (profile \($r.profile)) at \($r.generated_at). The verifier does not approve or merge." + ] + | .[]' "$RESULT" +} + +out=$(render true) +if [ "${#out}" -gt "$MAX_CHARS" ]; then + out=$(render false) +fi +printf '%s\n' "$out" diff --git a/scripts/dependency-pr/risk-map.json b/scripts/dependency-pr/risk-map.json new file mode 100644 index 000000000..351a1a3ec --- /dev/null +++ b/scripts/dependency-pr/risk-map.json @@ -0,0 +1,25 @@ +{ + "rules": [ + { "match": "^github.com/oapi-codegen/", "tags": ["codegen"], "tier": "high", "why": "Generator output (server.gen.go) is committed; tool and runtime must move together (#720)." }, + { "match": "^github.com/getkin/kin-openapi$", "tags": ["codegen"], "tier": "high", "why": "Drives cmd/resources/gen_resources.go and oapi-codegen; can rewrite ~600 KB of generated commands." }, + { "match": "^github.com/iancoleman/strcase$", "tags": ["codegen"], "tier": "high", "why": "Used by the resource command generator; changes command and flag names." }, + { "match": "^go.uber.org/mock$", "tags": ["codegen", "mocks"], "tier": "medium", "why": "Six go:generate mockgen directives produce committed mocks." }, + { "match": "^github.com/mattn/go-sqlite3$", "tags": ["cgo", "dev-server"], "tier": "medium", "why": "CGO; PR CI builds linux/amd64 glibc only while releases cross-compile (musl static, mingw, osxcross)." }, + { "match": "^github.com/launchdarkly/(go-server-sdk|go-sdk-common|go-server-sdk-evaluation)", "tags": ["ld-sdk", "dev-server"], "tier": "medium", "why": "The dev server proxies SDK streaming and evaluation." }, + { "match": "^github.com/launchdarkly/api-client-go", "tags": ["ld-sdk", "codegen"], "tier": "medium", "why": "Generated resource commands call the API client." }, + { "match": "^github.com/launchdarkly/sdk-meta", "tags": ["ld-sdk", "setup"], "tier": "medium", "why": "Feeds the setup/quickstart SDK lists." }, + { "match": "^github.com/spf13/(cobra|pflag|viper)$", "tags": ["cli-surface"], "tier": "medium", "why": "Flag parsing, config precedence, usage templates." }, + { "match": "^github.com/go-viper/mapstructure", "tags": ["cli-surface"], "tier": "medium", "why": "Config decoding behind viper." }, + { "match": "^github.com/charmbracelet/", "tags": ["tui"], "tier": "medium", "why": "Interactive setup/quickstart flows with little test coverage." }, + { "match": "^@launchpad-ui/", "tags": ["ui-design-system", "peer-deps"], "tier": "medium", "why": "Peer-dependency tangles with React 18 / react-router-dom v6 overrides (#638, #642, #723)." }, + { "match": "^(react|react-dom|react-router|react-router-dom|@remix-run/router)$", "tags": ["ui-framework", "peer-deps"], "tier": "medium", "why": "Framework peers of @launchpad-ui; overrides pin react-router-dom v6." }, + { "match": "^launchdarkly-js-client-sdk$", "tags": ["ld-sdk", "dev-server-ui"], "tier": "medium", "why": "UI evaluates flags against the dev server." }, + { "match": "^(vite|vite-plugin-singlefile|@vitejs/plugin-react|typescript)$", "tags": ["ui-build"], "tier": "medium", "why": "Changes the embedded single-file dist/index.html." }, + { "match": "^googleapis/release-please-action$", "tags": ["release-tooling"], "tier": "high", "why": "Release workflow never runs on pull_request." }, + { "match": "^launchdarkly/gh-actions", "tags": ["release-tooling"], "tier": "medium", "why": "Release secrets / shared org workflows." }, + { "match": "^actions/(attest|upload-artifact|setup-node)$", "tags": ["release-tooling"], "tier": "medium", "why": "Used by publish/attestation steps that only run at release." }, + { "match": "^docker/", "tags": ["release-tooling"], "tier": "medium", "why": "Image build happens only at release." }, + { "match": "^alpine$", "tags": ["docker-base-image", "release-tooling"], "tier": "medium", "why": "Base of the published Docker image; never built on PRs." }, + { "match": "^@go-task/go-npm$", "tags": ["npm-wrapper", "release-tooling"], "tier": "high", "why": "postinstall downloads the release binary for every npm user." } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/agent/impact.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/agent/impact.json new file mode 100644 index 000000000..5ba960283 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/agent/impact.json @@ -0,0 +1,37 @@ +{ + "schema": 1, + "summary": "actions/setup-go moves to v6 in go.yml (from v4) and dependency-scan.yml (from v5). Both workflows run on pull_request, and PR CI passed on the PR head with v6. The breaking changes are the node24 runtime (runner v2.327.1 or later) and new toolchain selection that reads the go.mod toolchain directive. ldcli go.mod has no toolchain directive, so the selected Go version does not change. The v4 step in go.yml declares node16, which GitHub no longer supports. The PR removes that actionlint finding.", + "tier": "high", + "changelog": [ + { + "package": "actions/setup-go", + "range": "v4/v5..v6", + "notes": "v5.0.0: runtime node16 to node20. v5.1.0: architecture added to the cache key (one cold cache). v5.2 to v5.6: manifest from raw API, @actions/cache 4 (new cache service), downloads fall back to go.dev/dl. v6.0.0 breaking: node24 runtime (runner v2.327.1 or later), and toolchain handling (PR #460) that uses the go.mod 'toolchain' directive when GOTOOLCHAIN is not 'local'. Inputs that ldcli passes (go-version-file) and the outputs (none read) do not change.", + "breaking": true, + "url": "https://github.com/actions/setup-go/releases/tag/v6.0.0" + } + ], + "usage": [ + ".github/workflows/go.yml (build job: setup-go@v4 to v6, go-version-file: go.mod)", + ".github/workflows/dependency-scan.yml (setup-go@v5 to v6, go-version-file: go.mod)" + ], + "behavior_changes_reachable": true, + "findings": [ + { + "severity": "info", + "text": "Both workflows that use setup-go run on pull_request, and all 9 CI checks passed on the PR head c3c375a. No release workflow or composite action in .github/actions uses setup-go.", + "evidence": [ + "result.json ci-status", + "rg 'setup-go' .github" + ] + }, + { + "severity": "info", + "text": "Toolchain selection: setup-go v6 parseGoVersionFile reads '^toolchain go…' first. go.mod has 'go 1.25' and no toolchain line, so v6 installs the same Go as v4 and v5 (guard go-mod-no-toolchain-directive). If someone adds a toolchain line later, CI will install that toolchain.", + "evidence": [ + "https://github.com/actions/setup-go/blob/v6/src/installer.ts", + "go.mod" + ] + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/generated/checks.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/generated/checks.json new file mode 100644 index 000000000..e75cfd1e4 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/generated/checks.json @@ -0,0 +1,21 @@ +{ + "schema": 1, + "checks": [ + { + "id": "updated-action-runtime-supported", + "title": "Each step that uses actions/setup-go declares a runtime that GitHub Actions supports", + "kind": "discriminating", + "rationale": "go.yml (PR CI build job) runs actions/setup-go@v4, which declares the node16 runtime. GitHub removed node16 from runners and forces such actions onto a newer Node.js, and actionlint 1.7.7 reports the step as too old to run. setup-go v6 declares node24. The check runs actionlint on the worktree and keeps only the runtime findings for the updated action.", + "script": "updated-action-runtime-supported.sh", + "timeout": 300 + }, + { + "id": "go-mod-no-toolchain-directive", + "title": "setup-go v6 toolchain selection reads the same go directive as v4 and v5", + "kind": "guard", + "rationale": "setup-go v6 (PR #460) reads the go.mod 'toolchain' directive before the 'go' directive. go.yml and dependency-scan.yml pass go-version-file: go.mod. If go.mod has no toolchain line, v6 installs the same Go version as v4 and v5. go.yml relies on this: its comment says golangci-lint breaks with a Go newer than go.mod declares.", + "script": "go-mod-no-toolchain-directive.sh", + "timeout": 60 + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/generated/go-mod-no-toolchain-directive.sh b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/generated/go-mod-no-toolchain-directive.sh new file mode 100644 index 000000000..10c465f29 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/generated/go-mod-no-toolchain-directive.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" + +mapfile -t files < <(cd "$WT" && grep -rhoE 'go-version-file: *[^ #]+' .github | sed 's/go-version-file: *//' | tr -d "\"'" | sort -u) +[ ${#files[@]} -gt 0 ] || skip "no workflow passes go-version-file to actions/setup-go" +for f in "${files[@]}"; do + [ -f "$WT/$f" ] || fail "go-version-file $f does not exist" + godir=$(grep -m1 -E '^go [0-9]' "$WT/$f" | awk '{print $2}') + tc=$(grep -m1 -E '^toolchain ' "$WT/$f" | awk '{print $2}') + detail "- \`$f\`: go directive \`${godir:-none}\`, toolchain directive \`${tc:-none}\`" + [ -n "$godir" ] || fail "$f has no go directive, so setup-go cannot select a version" + [ -z "$tc" ] || fail "$f has 'toolchain $tc'; setup-go v6 installs that instead of go $godir" +done +pass "go-version-file (${files[*]}) has a go directive and no toolchain directive, so setup-go v4, v5, and v6 select the same Go version" diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/generated/updated-action-runtime-supported.sh b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/generated/updated-action-runtime-supported.sh new file mode 100644 index 000000000..d54ae4b08 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-717/generated/updated-action-runtime-supported.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" + +bin=$(go_tool github.com/rhysd/actionlint/cmd/actionlint v1.7.7) || incomplete "could not install actionlint" +mapfile -t names < <(updates_for github-actions | jq -r '.name' | sort -u) +[ ${#names[@]} -gt 0 ] || skip "no GitHub Actions updates" +(cd "$WT" && "$bin" -oneline -no-color) >"$ARTIFACTS/actionlint.out" 2>&1 +rc=$? +cat "$ARTIFACTS/actionlint.out" +[ "$rc" -le 1 ] || incomplete "actionlint could not run (exit $rc)" +: >"$ARTIFACTS/findings" +for n in "${names[@]}"; do + grep -F "the runner of \"$n@" "$ARTIFACTS/actionlint.out" | grep -F 'too old to run' >>"$ARTIFACTS/findings" || true +done +uses=$(cd "$WT" && grep -rhoE "uses: *($(IFS='|'; echo "${names[*]}" | sed 's/[.]/\\./g'))@[^ #]+" .github | sed 's/uses: *//' | sort | uniq -c | sed 's/^ *//' | paste -sd, -) +detail "- Steps that use the updated action: ${uses:-none}" +if [ -s "$ARTIFACTS/findings" ]; then + detail_block "$ARTIFACTS/findings" 10 + fail "actionlint: $(wc -l <"$ARTIFACTS/findings") step(s) use a version of ${names[*]} whose runtime GitHub Actions no longer supports" +fi +pass "every step that uses ${names[*]} declares a runtime that GitHub Actions supports (actionlint [action] rule)" diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-718/agent/impact.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-718/agent/impact.json new file mode 100644 index 000000000..996a6d927 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-718/agent/impact.json @@ -0,0 +1,38 @@ +{ + "schema": 1, + "summary": "manual-publish.yml moves launchdarkly/gh-actions/actions/release-secrets from release-secrets-v1.0.1 to release-secrets-v1.2.0. release-please.yml already uses v1.2.0. The verifier classified the update as v1.2.0 to v1.2.0 (a digest with 0 commits), but the real range is v1.0.1 to v1.2.0. Inside the composite, aws-actions/configure-aws-credentials moves from v1-node16 to v4 and the SSM getparameters action moves from the floating v1 tag (node16) to a pinned node20 commit. A new optional input s3_path_pairs is added. The inputs that ldcli passes (aws_assume_role, ssm_parameter_pairs) do not change. The same v1.2.0 step with the same inputs passed in the v3.12.0 release.", + "tier": "high", + "tier_reasons": [ + "Release tooling with a real version change (v1.0.1 to v1.2.0, which includes configure-aws-credentials v1 to v4). The classifier compared v1.2.0 with itself and reported a digest." + ], + "changelog": [ + { + "package": "launchdarkly/gh-actions/actions/release-secrets", + "range": "release-secrets-v1.0.1..release-secrets-v1.2.0", + "notes": "fix (#10): pin dkershner6/aws-ssm-getparameters-action to commit 4fcb4872 (node20) instead of v1 (node16). feat (#16): optional s3_path_pairs input and an S3 download step that runs only when the input is set. feat 1.2.0 (#35): aws-actions/configure-aws-credentials v1-node16 to v4 (node20), with the same audience, role-to-assume, and aws-region. No input was removed or made required.", + "breaking": false, + "url": "https://github.com/launchdarkly/gh-actions/compare/release-secrets-v1.0.1...release-secrets-v1.2.0" + } + ], + "usage": [ + ".github/workflows/manual-publish.yml (job release-ldcli, step 'Get Docker token': aws_assume_role, ssm_parameter_pairs; the job has id-token: write)", + ".github/workflows/release-please.yml (job release-ldcli, already on release-secrets-v1.2.0 with the same inputs)" + ], + "behavior_changes_reachable": true, + "findings": [ + { + "severity": "info", + "text": "The exact ref and inputs already run in production. In the v3.12.0 release (release-please.yml run 35993978650, 2026-09-24), the 'Get Docker token' step with release-secrets-v1.2.0 passed, and the job then published the images. manual-publish.yml uses the same AWS_ROLE_ARN variable and the same SSM parameters.", + "evidence": [ + "https://github.com/launchdarkly/ldcli/actions/runs/35993978650" + ] + }, + { + "severity": "info", + "text": "configure-aws-credentials v4 needs the id-token: write permission for OIDC. The release-ldcli job in manual-publish.yml has it, and the permissions block does not change.", + "evidence": [ + ".github/workflows/manual-publish.yml" + ] + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-718/generated/checks.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-718/generated/checks.json new file mode 100644 index 000000000..a69b6f299 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-718/generated/checks.json @@ -0,0 +1,13 @@ +{ + "schema": 1, + "checks": [ + { + "id": "release-secrets-nested-runtimes-supported", + "title": "release-secrets and the actions it runs declare supported runtimes, and the inputs that ldcli passes exist", + "kind": "discriminating", + "rationale": "manual-publish.yml (job release-ldcli, step 'Get Docker token') runs the release-secrets composite. At release-secrets-v1.0.1 the composite runs aws-actions/configure-aws-credentials@v1-node16 and dkershner6/aws-ssm-getparameters-action@v1, which both declare node16. GitHub removed node16 from runners. At v1.2.0 they are configure-aws-credentials@v4 and a pinned getparameters commit, both node20. The check reads action.yml at the refs that each side pins and also checks aws_assume_role and ssm_parameter_pairs. Uses the network (gh api).", + "script": "release-secrets-nested-runtimes-supported.sh", + "timeout": 180 + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-718/generated/release-secrets-nested-runtimes-supported.sh b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-718/generated/release-secrets-nested-runtimes-supported.sh new file mode 100644 index 000000000..c73c08829 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-718/generated/release-secrets-nested-runtimes-supported.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Network: reads action.yml files through gh. +source "$VERIFY_ROOT/lib/check.sh" +command -v gh >/dev/null 2>&1 || incomplete "gh is not installed" +python3 -c 'import yaml' 2>/dev/null || incomplete "python3 with PyYAML is required" +updates_for github-actions | jq -r '.name' | sort -u >"$ARTIFACTS/names" +[ -s "$ARTIFACTS/names" ] || skip "no GitHub Actions updates" + +python3 - "$WT" "$ARTIFACTS/names" >"$ARTIFACTS/report.json" <<'PY' +import base64, glob, json, subprocess, sys, yaml +wt, names = sys.argv[1], [l.strip() for l in open(sys.argv[2]) if l.strip()] +UNSUPPORTED = {"node12", "node16"} + +def action_yml(spec): + name, ref = spec.split("@", 1) + parts = name.split("/") + repo, sub = "/".join(parts[:2]), "/".join(parts[2:]) + for fn in ("action.yml", "action.yaml"): + path = f"{sub}/{fn}" if sub else fn + r = subprocess.run(["gh", "api", f"repos/{repo}/contents/{path}?ref={ref}", "--jq", ".content"], capture_output=True, text=True) + if r.returncode == 0 and r.stdout.strip(): + return yaml.safe_load(base64.b64decode(r.stdout.strip())) + return None + +usages = {} +for p in sorted(glob.glob(f"{wt}/.github/**/*.y*ml", recursive=True)): + d = yaml.safe_load(open(p)) or {} + jobs = list((d.get("jobs") or {}).values()) + [{"steps": (d.get("runs") or {}).get("steps")}] if isinstance(d, dict) else [] + for job in jobs: + for st in (job or {}).get("steps") or []: + u = st.get("uses", "") if isinstance(st, dict) else "" + if u.split("@")[0] in names and "@" in u: + usages.setdefault(u, []).append({"file": p[len(wt) + 1:], "with": sorted((st.get("with") or {}).keys())}) + +report = {"usages": [], "unavailable": []} +for spec, uses in sorted(usages.items()): + a = action_yml(spec) + if a is None: + report["unavailable"].append(spec) + continue + runs = a.get("runs") or {} + entry = {"spec": spec, "files": sorted({u["file"] for u in uses}), "using": runs.get("using"), "nested": [], "unknown_inputs": []} + declared = set((a.get("inputs") or {}).keys()) + entry["unknown_inputs"] = sorted({k for u in uses for k in u["with"] if k not in declared}) + for st in runs.get("steps") or []: + nu = st.get("uses", "") if isinstance(st, dict) else "" + if not nu or nu.startswith("./") or nu.startswith("docker://"): + continue + na = action_yml(nu) + if na is None: + report["unavailable"].append(nu) + continue + entry["nested"].append({"uses": nu, "using": (na.get("runs") or {}).get("using")}) + entry["unsupported"] = ([spec] if entry["using"] in UNSUPPORTED else []) + [n["uses"] for n in entry["nested"] if n["using"] in UNSUPPORTED] + report["usages"].append(entry) +print(json.dumps(report, indent=2)) +PY +[ $? -eq 0 ] || incomplete "could not analyze the action metadata" +cat "$ARTIFACTS/report.json" +[ "$(jq '.unavailable | length' "$ARTIFACTS/report.json")" -eq 0 ] || + incomplete "could not read action.yml for: $(jq -r '.unavailable | join(", ")' "$ARTIFACTS/report.json")" +jq -r '.usages[] | "- `\(.spec)` in \(.files | join(", ")): \(.using); nested: \([.nested[] | "`\(.uses)` \(.using)"] | join(", "))"' "$ARTIFACTS/report.json" >>"$ARTIFACTS/details.md" +bad_inputs=$(jq -r '[.usages[] | select(.unknown_inputs | length > 0) | "\(.spec): \(.unknown_inputs | join(", "))"] | join("; ")' "$ARTIFACTS/report.json") +[ -z "$bad_inputs" ] || fail "workflows pass inputs that the action does not declare: $bad_inputs" +bad=$(jq -r '[.usages[].unsupported[]] | unique | join(", ")' "$ARTIFACTS/report.json") +[ -z "$bad" ] || fail "the updated action runs nested actions on a Node.js runtime that GitHub no longer supports: $bad" +pass "every use of $(paste -sd, "$ARTIFACTS/names") and its nested actions declares a supported runtime, and the inputs passed exist" diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-721/agent/impact.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-721/agent/impact.json new file mode 100644 index 000000000..14b7229df --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-721/agent/impact.json @@ -0,0 +1,35 @@ +{ + "schema": 1, + "summary": "actions/setup-python moves from v3 to v6 in one step of go.yml, the PR CI build job, before pre-commit/action. ldcli passes no inputs and reads no outputs. Two changes reach the step. The runtime moves from node16 (no longer supported by GitHub) to node24. Since v4 there is no default python-version, so the step now uses the runner's Python on PATH instead of installing the latest 3.x. PR CI ran the step with v6 and passed.", + "tier": "high", + "changelog": [ + { + "package": "actions/setup-python", + "range": "v3..v6", + "notes": "v4.0.0: no default python-version. Without python-version or python-version-file the action reads .python-version, and if that file is missing it uses the Python on PATH. v5.0.0: node16 to node20. v5.x: cache and setuptools security fixes. v6.0.0 breaking: node24 runtime (runner v2.327.1 or later), and better .python-version parsing. The token default is now empty on GitHub Enterprise Server. On github.com it is still github.token.", + "breaking": true, + "url": "https://github.com/actions/setup-python/releases/tag/v6.0.0" + } + ], + "usage": [ + ".github/workflows/go.yml (build job: actions/setup-python@v3 with no inputs, then pre-commit/action v3.0.1)" + ], + "behavior_changes_reachable": true, + "findings": [ + { + "severity": "info", + "text": "The only step that uses setup-python is in go.yml, which runs on pull_request. CI run 28040393667 on the PR head 0676ba1 passed 'Run actions/setup-python@v6' and the pre-commit step. The run is from 2026-06-23, so it predates later changes on main, and GitHub has expired its logs.", + "evidence": [ + "https://github.com/launchdarkly/ldcli/actions/runs/28040393667" + ] + }, + { + "severity": "info", + "text": "With no python-version input and no .python-version file in the repository, v6 uses the Python that the ubuntu-latest image puts on PATH (Python 3.12 on ubuntu-24.04). pre-commit-hooks v4.6.0 supports Python 3.8 and later, and the golangci-lint hook uses Go, not Python. If a person wants a fixed Python, set python-version in go.yml.", + "evidence": [ + "https://github.com/actions/setup-python/releases/tag/v4.0.0", + ".pre-commit-config.yaml" + ] + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-721/generated/checks.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-721/generated/checks.json new file mode 100644 index 000000000..a66ff0b57 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-721/generated/checks.json @@ -0,0 +1,13 @@ +{ + "schema": 1, + "checks": [ + { + "id": "updated-action-runtime-supported", + "title": "Each step that uses actions/setup-python declares a runtime that GitHub Actions supports", + "kind": "discriminating", + "rationale": "go.yml (PR CI build job) runs actions/setup-python@v3 before pre-commit/action. v3 declares the node16 runtime. GitHub removed node16 from runners and forces such actions onto a newer Node.js, and actionlint 1.7.7 reports the step as too old to run. setup-python v6 declares node24. The check runs actionlint on the worktree and keeps only the runtime findings for the updated action.", + "script": "updated-action-runtime-supported.sh", + "timeout": 300 + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-721/generated/updated-action-runtime-supported.sh b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-721/generated/updated-action-runtime-supported.sh new file mode 100644 index 000000000..d54ae4b08 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-721/generated/updated-action-runtime-supported.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" + +bin=$(go_tool github.com/rhysd/actionlint/cmd/actionlint v1.7.7) || incomplete "could not install actionlint" +mapfile -t names < <(updates_for github-actions | jq -r '.name' | sort -u) +[ ${#names[@]} -gt 0 ] || skip "no GitHub Actions updates" +(cd "$WT" && "$bin" -oneline -no-color) >"$ARTIFACTS/actionlint.out" 2>&1 +rc=$? +cat "$ARTIFACTS/actionlint.out" +[ "$rc" -le 1 ] || incomplete "actionlint could not run (exit $rc)" +: >"$ARTIFACTS/findings" +for n in "${names[@]}"; do + grep -F "the runner of \"$n@" "$ARTIFACTS/actionlint.out" | grep -F 'too old to run' >>"$ARTIFACTS/findings" || true +done +uses=$(cd "$WT" && grep -rhoE "uses: *($(IFS='|'; echo "${names[*]}" | sed 's/[.]/\\./g'))@[^ #]+" .github | sed 's/uses: *//' | sort | uniq -c | sed 's/^ *//' | paste -sd, -) +detail "- Steps that use the updated action: ${uses:-none}" +if [ -s "$ARTIFACTS/findings" ]; then + detail_block "$ARTIFACTS/findings" 10 + fail "actionlint: $(wc -l <"$ARTIFACTS/findings") step(s) use a version of ${names[*]} whose runtime GitHub Actions no longer supports" +fi +pass "every step that uses ${names[*]} declares a runtime that GitHub Actions supports (actionlint [action] rule)" diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/agent/impact.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/agent/impact.json new file mode 100644 index 000000000..59d334bab --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/agent/impact.json @@ -0,0 +1,39 @@ +{ + "schema": 1, + "summary": "googleapis/release-please-action moves from v4.4.0 to v5.0.0 in the two steps of the release-please job in release-please.yml. That workflow runs only on push to main, so PR CI never runs it. The one breaking change is the node24 runtime. The bundled release-please library moves from 17.1.3 to 17.6.0, which changes how release PRs are created (file updates through the GitHub API, no empty PRs). The inputs that ldcli passes (token, skip-github-release, skip-github-pull-request) and the outputs that it reads (release_created, tag_name) do not change.", + "tier": "high", + "changelog": [ + { + "package": "googleapis/release-please-action", + "range": "v4.4.0..v5.0.0", + "notes": "v4.4.1: release-please 17.1.3 to 17.3.0. v5.0.0 breaking: runtime node20 to node24 (action.yml runs.using). Also release-please 17.3.0 to 17.6.0. action.yml inputs are the same, and neither version declares outputs (src/index.ts sets them). Library changes in 17.2 to 17.6: force-tag and include-v-in-release-name options (off by default), a fix for the prerelease versioning strategy, a commit search depth option, no PR when nothing changed (17.4.1), --no-verify in git operations (17.5.1), the git fetch depth limited to cloneDepth (17.5.2), and file updates through the GitHub API (17.6.0).", + "breaking": true, + "url": "https://github.com/googleapis/release-please-action/compare/16a9c90856f42705d54a6fda1823352bdc62cf38...45996ed1f6d02564a971a2fa1b5860e934307cf7" + } + ], + "usage": [ + ".github/workflows/release-please.yml (job release-please: step 'release' with skip-github-pull-request, step 'release-prs' with skip-github-release; job outputs release_created and tag_name gate release-ldcli, release-ldcli-npm, and publish-release)", + "release-please-config.json", + ".release-please-manifest.json" + ], + "behavior_changes_reachable": true, + "no_local_proof": "release-please.yml runs only on push to main, and the action needs a GitHub token with write access to create releases and PRs. No local check can run it. The guards confirm that the outputs are still set and that the configuration is valid for the bundled library.", + "findings": [ + { + "severity": "info", + "text": "Library 17.6.0 creates release PR file updates through the GitHub API instead of git. The release-prs step already uses GITHUB_TOKEN with contents: write and pull-requests: write, which is enough for the API. The permissions block does not change.", + "evidence": [ + "https://github.com/googleapis/release-please/releases/tag/v17.6.0", + ".github/workflows/release-please.yml" + ] + }, + { + "severity": "info", + "text": "Follow-up after merge: the next push to main runs the release-prs step with v5. Confirm that it updates the release PR. manual-publish.yml does not use release-please, so a manual-publish dry run does not test this change.", + "evidence": [ + ".github/workflows/release-please.yml", + ".github/workflows/manual-publish.yml" + ] + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/generated/checks.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/generated/checks.json new file mode 100644 index 000000000..142052f47 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/generated/checks.json @@ -0,0 +1,21 @@ +{ + "schema": 1, + "checks": [ + { + "id": "release-please-outputs-still-set", + "title": "The pinned release-please-action sets every output that release-please.yml reads", + "kind": "guard", + "rationale": "release-please.yml reads steps.release.outputs.release_created and tag_name to gate tag creation and every release job. release-please-action declares no outputs in action.yml and sets them in src/index.ts (setPathOutput, tagName to tag_name). The check reads the source at the commit that each side pins. Uses the network (gh api).", + "script": "release-please-outputs-still-set.sh", + "timeout": 120 + }, + { + "id": "release-please-config-schema", + "title": "release-please-config.json is valid for the release-please library that the pinned action bundles", + "kind": "guard", + "rationale": "release-please-action v5.0.0 bundles release-please 17.6.0 (v4.4.0 bundles 17.1.3). The library reads release-please-config.json and .release-please-manifest.json on every push to main. The check validates the configuration against schemas/config.json of the bundled library version. Uses the network (gh api, npx ajv-cli).", + "script": "release-please-config-schema.sh", + "timeout": 300 + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/generated/release-please-config-schema.sh b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/generated/release-please-config-schema.sh new file mode 100644 index 000000000..c4dbbc296 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/generated/release-please-config-schema.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Network: reads the action lockfile and the release-please schema through gh, and runs ajv-cli through npx. +source "$VERIFY_ROOT/lib/check.sh" +command -v gh >/dev/null 2>&1 || incomplete "gh is not installed" +command -v npx >/dev/null 2>&1 || incomplete "npx is not installed" + +ref=$(grep -rhoE 'googleapis/release-please-action@[0-9a-f]{40}' "$WT/.github/workflows" | head -n1 | cut -d@ -f2) +[ -n "$ref" ] || skip "no workflow pins googleapis/release-please-action to a commit" +lib=$(gh api "repos/googleapis/release-please-action/contents/package-lock.json?ref=$ref" --jq .content 2>/dev/null | base64 -d | + jq -r '.packages["node_modules/release-please"].version // empty') +[ -n "$lib" ] || incomplete "could not read the bundled release-please version at $ref" +gh api "repos/googleapis/release-please/contents/schemas/config.json?ref=v$lib" --jq .content 2>/dev/null | base64 -d >"$ARTIFACTS/schema.json" +jq -e . "$ARTIFACTS/schema.json" >/dev/null 2>&1 || incomplete "could not read schemas/config.json of release-please v$lib" +jq -e 'has(".")' "$WT/.release-please-manifest.json" >/dev/null || fail ".release-please-manifest.json has no root package" +cp "$WT/release-please-config.json" "$ARTIFACTS/config.json" +(cd "$ARTIFACTS" && run timeout 240 npx -y ajv-cli@5 validate --spec=draft7 --strict=false -s schema.json -d config.json) >"$ARTIFACTS/ajv.out" 2>&1 +rc=$? +cat "$ARTIFACTS/ajv.out" +detail "- action \`${ref:0:12}\` bundles release-please v$lib" +if grep -q 'config.json valid' "$ARTIFACTS/ajv.out"; then + pass "release-please-config.json is valid for release-please v$lib, which the pinned action bundles" +fi +grep -q 'config.json invalid' "$ARTIFACTS/ajv.out" || incomplete "ajv-cli did not run (exit $rc): $(tail -n1 "$ARTIFACTS/ajv.out")" +detail_block "$ARTIFACTS/ajv.out" 20 +fail "release-please-config.json is not valid for release-please v$lib" diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/generated/release-please-outputs-still-set.sh b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/generated/release-please-outputs-still-set.sh new file mode 100644 index 000000000..a9a3d465d --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-722/generated/release-please-outputs-still-set.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Network: reads the release-please-action source at the pinned commit through gh. +source "$VERIFY_ROOT/lib/check.sh" +command -v gh >/dev/null 2>&1 || incomplete "gh is not installed" +python3 -c 'import yaml' 2>/dev/null || incomplete "python3 with PyYAML is required" + +python3 - "$WT" >"$ARTIFACTS/usage.json" <<'PY' || incomplete "could not parse the workflows" +import glob, json, re, sys, yaml +wt = sys.argv[1] +out = [] +for p in sorted(glob.glob(f"{wt}/.github/workflows/*.y*ml")): + d = yaml.safe_load(open(p)) or {} + for jid, job in (d.get("jobs") or {}).items(): + text = json.dumps(job) + for st in job.get("steps") or []: + u = st.get("uses", "") + if u.split("@")[0] != "googleapis/release-please-action": + continue + sid = st.get("id") + read = sorted(set(re.findall(r"steps\.%s\.outputs\.([\w-]+)" % re.escape(sid), text))) if sid else [] + out.append({"file": p[len(wt) + 1:], "job": jid, "id": sid, "ref": u.split("@", 1)[1], "outputs": read}) +print(json.dumps(out)) +PY +cat "$ARTIFACTS/usage.json" +[ "$(jq 'length' "$ARTIFACTS/usage.json")" -gt 0 ] || skip "no workflow uses googleapis/release-please-action" + +missing=() +while IFS=$'\x1f' read -r ref outputs; do + src="$ARTIFACTS/index-$ref.ts" + if [ ! -s "$src" ]; then + gh api "repos/googleapis/release-please-action/contents/src/index.ts?ref=$ref" --jq .content 2>"$ARTIFACTS/gh.err" | base64 -d >"$src" || + incomplete "could not read src/index.ts at $ref: $(tail -n1 "$ARTIFACTS/gh.err")" + fi + for o in $outputs; do + if grep -qE "['\"\`]$o['\"\`]" "$src"; then + detail "- \`$o\` is set by the action at \`${ref:0:12}\` (\`$(grep -nE "['\"\`]$o['\"\`]" "$src" | head -n1 | sed 's/^\([0-9]*\):[[:space:]]*/src\/index.ts:\1: /' | cut -c1-110)\`)" + else + missing+=("$o@${ref:0:12}") + fi + done +done < <(jq -r '.[] | select(.outputs | length > 0) | [.ref, (.outputs | join(" "))] | join("\u001f")' "$ARTIFACTS/usage.json" | sort -u) +[ ${#missing[@]} -eq 0 ] || fail "release-please-action no longer sets output(s) that the workflows read: ${missing[*]}" +pass "the pinned release-please-action source sets every output that the workflows read ($(jq -r '[.[].outputs[]] | unique | join(", ")' "$ARTIFACTS/usage.json"))" diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/agent/impact.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/agent/impact.json new file mode 100644 index 000000000..eb386edf2 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/agent/impact.json @@ -0,0 +1,105 @@ +{ + "schema": 1, + "summary": "The title and body say that this grouped security update changes two indirect packages: dompurify 3.2.4 to 3.4.13 and uuid (removed). The Dependabot commit also changes four direct dependencies that the body does not name: @launchpad-ui/components 0.4.4 to 0.23.1, launchdarkly-js-client-sdk 3.4.0 to 3.9.5, react-router 7.18.2 to 8.3.0, and vite 6.4.3 to 8.2.1. Three of them break ldcli. react-router 8 and @launchpad-ui/components 0.23 need React 19 while the UI pins React 18.3.1, so npm ci fails. components 0.23.1 removes ToastContainer and ToastQueue, which ldcli uses. vite 8 is outside the peer ranges of @vitejs/plugin-react 4.5.0 and vite-plugin-singlefile 2.2.0. The PR also adds high advisories through a nested react-router 7.5.2.", + "tier": "high", + "changelog": [ + { + "package": "@launchpad-ui/components", + "range": "0.4.4..0.23.1", + "notes": "0.x minors with breaking changes. #1581 replaces ToastContainer/ToastQueue with ToastRegion/toastQueue (react-aria-components Toast). #1813 removes CopyToClipboard. #1712 makes react-aria and react-router peers. 0.23.x peers: react ^19, react-dom ^19, react-router ^7.15.1 || ^8.2.0, react-aria ^3.50, react-aria-components ^1.19, and @react-aria/* packages. The upstream-changes check found no notes. This summary comes from packages/components/CHANGELOG.md.", + "breaking": true, + "url": "https://github.com/launchdarkly/launchpad-ui/blob/main/packages/components/CHANGELOG.md" + }, + { + "package": "launchdarkly-js-client-sdk", + "range": "3.4.0..3.9.5", + "notes": "Minor and patch releases. js-sdk-common 5.8.x drops the uuid dependency (this removes the uuid advisory). The README adds a deprecation notice that points to the new JavaScript SDK. No API removals.", + "breaking": false, + "url": "https://github.com/launchdarkly/js-client-sdk/releases" + }, + { + "package": "react-router", + "range": "7.18.2..8.3.0", + "notes": "8.0.0: minimum React 19.2.7 (peer >=19.2.7), minimum Node 22.22.0, ESM only, react-router-dom removed, v8 future flags removed (now default), hasErrorBoundary and meta `data` removed. 8.0.1 to 8.3.0 are fixes and additions with no new removals in the declarative API. The upstream-changes check found no notes. This summary comes from the CHANGELOG.md in the npm tarball.", + "breaking": true, + "url": "https://github.com/remix-run/react-router/blob/main/packages/react-router/CHANGELOG.md" + }, + { + "package": "vite", + "range": "6.4.3..8.2.1", + "notes": "7.0: Node ^20.19 || >=22.12, default build target baseline-widely-available, Sass legacy API and splitVendorChunkPlugin removed. 8.0: Rolldown replaces esbuild and Rollup as the bundler, Oxc transforms, lightningcss is a dependency (MPL-2.0) for CSS. The bundled esbuild moves 0.25.5 to 0.28.1 as an optional peer.", + "breaking": true, + "url": "https://github.com/vitejs/vite/blob/v8.2.1/packages/vite/CHANGELOG.md" + } + ], + "usage": [ + "internal/dev_server/ui/src/main.tsx", + "internal/dev_server/ui/src/Sync.tsx", + "internal/dev_server/ui/src/App.tsx", + "internal/dev_server/ui/src/RouteSelector.tsx", + "internal/dev_server/ui/src/DebugSessionEventsPage.tsx", + "internal/dev_server/ui/src/util.ts", + "internal/dev_server/ui/src/Flags.tsx", + "internal/dev_server/ui/src/FlagsPage.tsx", + "internal/dev_server/ui/src/Flag.tsx", + "internal/dev_server/ui/vite.config.ts", + "internal/dev_server/ui/vitest.config.ts", + "internal/dev_server/ui/asset_handler.go" + ], + "behavior_changes_reachable": true, + "findings": [ + { + "severity": "block", + "text": "@launchpad-ui/components 0.23.1 no longer exports ToastContainer (src/main.tsx) or ToastQueue (src/Sync.tsx). The replacements are ToastRegion and toastQueue. The generated guard `launchpad-exports-guard` passes on base and fails on the PR, so the build breaks even with a working install.", + "evidence": [ + "generated/launchpad-exports-guard.sh", + "https://github.com/launchdarkly/launchpad-ui/blob/main/packages/components/CHANGELOG.md" + ] + }, + { + "severity": "block", + "text": "vite 8.2.1 is outside the peer ranges of @vitejs/plugin-react 4.5.0 (vite ^4.2 || ^5 || ^6) and vite-plugin-singlefile 2.2.0 (vite ^5.4.11 || ^6, and a rollup ^4.35 peer that vite 8 no longer uses). After the React conflict is fixed, npm ci still fails until these plugins are upgraded in the same change. vitest 3.2.7 keeps its own nested vite 7.3.6.", + "evidence": [ + "internal/dev_server/ui/package-lock.json (PR)", + "internal/dev_server/ui/vite.config.ts" + ] + }, + { + "severity": "info", + "text": "The ui-npm-ci block names @react-aria/focus ^3.22.1 from @launchpad-ui/components 0.23.1. That is the first conflict that npm reports. components 0.23.1 and react-router 8.3.0 also need react 19 (^19 and >=19.2.7), and the UI pins react 18.3.1. A fix needs React 19, @launchpad-ui/core 0.59.18 or later (core 0.49.22 needs react 18.3.1), and the core migration from #723.", + "evidence": [ + "logs/pr/ui-npm-ci.log" + ] + }, + { + "severity": "info", + "text": "The new high advisories (react-router, @launchpad-ui/navigation, turbo-stream) come from @launchpad-ui/navigation 0.12.82, which brings a nested react-router 7.5.2 (for example GHSA-49rj-9fvp-4h2h and GHSA-chx6-hx7r-mcp5) and turbo-stream 2.4.0 (GHSA-rxv8-25v2-qmq8). ldcli does not import Navigation, but the advisories are in the runtime tree.", + "evidence": [ + "state/checks/ui-npm-audit/pr/delta.json" + ] + }, + { + "severity": "info", + "text": "The two security fixes that the PR names need much less change. dompurify comes from isomorphic-dompurify (^3.1.6, through @launchpad-ui/core markdown), so a lockfile-only update to 3.4.13 fixes it. uuid comes from launchdarkly-js-sdk-common 5.3.0. launchdarkly-js-client-sdk 3.9.5 (js-sdk-common 5.8.3) removes it. The UI imports only the types LDFlagSet and LDFlagValue from the SDK, so the SDK update does not change the bundle. A focused PR with these two changes avoids the breaking updates.", + "evidence": [ + "internal/dev_server/ui/package-lock.json", + "internal/dev_server/ui/src/util.ts" + ] + }, + { + "severity": "info", + "text": "The lightningcss MPL-2.0 license decision is real, but lightningcss is a build-time dependency of vite 8 and does not go into dist/.", + "evidence": [ + "logs/pr/transitive-changes.log", + "logs/pr/license-changes.log" + ] + }, + { + "severity": "info", + "text": "No discriminating check can run until the tree installs. The guards ui-routing-guard and ui-singlefile-guard hold the routing and single-file dist behavior on base. Run them again after a coordinated upgrade.", + "evidence": [ + "generated/checks.json" + ] + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/app_routing_test.tsx.txt b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/app_routing_test.tsx.txt new file mode 100644 index 000000000..bdd1e9627 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/app_routing_test.tsx.txt @@ -0,0 +1,56 @@ +import React from 'react'; +import { render, screen, cleanup } from '@testing-library/react'; +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { BrowserRouter, useParams } from 'react-router'; +import App from '../App.tsx'; + +vi.mock('../FlagsPage.tsx', () => ({ default: () =>
stub:flags
})); +vi.mock('../EventsPage.tsx', () => ({ default: () =>
stub:events
})); +vi.mock('../DebugSessionsPage.tsx', () => ({ + default: () =>
stub:debug-sessions
, +})); +vi.mock('../DebugSessionEventsPage.tsx', () => ({ + default: function Stub() { + const { debugSessionKey } = useParams<{ debugSessionKey: string }>(); + return
stub:debug-session-events:{debugSessionKey}
; + }, +})); + +const renderAt = (path: string) => { + window.history.replaceState({}, '', path); + render( + + + , + ); +}; + +afterEach(() => cleanup()); + +describe('ldcli dev-server UI routing (App.tsx under BrowserRouter)', () => { + it.each([ + ['/', '/ui/flags', 'stub:flags', 'Flags'], + ['/ui', '/ui/flags', 'stub:flags', 'Flags'], + ['/ui/flags', '/ui/flags', 'stub:flags', 'Flags'], + ['/ui/events', '/ui/events', 'stub:events', 'Events'], + ['/ui/events/', '/ui/events/', 'stub:events', 'Select a view'], + ['/ui/debug-sessions', '/ui/debug-sessions', 'stub:debug-sessions', 'Debug Sessions'], + ])('%s renders %s', async (start, end, page, selector) => { + renderAt(start); + expect(await screen.findByText(page)).toBeTruthy(); + expect(window.location.pathname).toBe(end); + expect(screen.getAllByText(selector).length).toBeGreaterThan(0); + }); + + it('passes the :debugSessionKey route parameter', async () => { + renderAt('/ui/debug-sessions/sess%20one-42/events'); + expect( + await screen.findByText('stub:debug-session-events:sess one-42'), + ).toBeTruthy(); + }); + + it('renders no page for an unknown path', () => { + renderAt('/ui/nope'); + expect(screen.queryByText(/^stub:/)).toBeNull(); + }); +}); diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/checks.json b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/checks.json new file mode 100644 index 000000000..234eb19fe --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/checks.json @@ -0,0 +1,29 @@ +{ + "schema": 1, + "checks": [ + { + "id": "launchpad-exports-guard", + "title": "Every @launchpad-ui name that the UI imports is still exported at the locked versions", + "kind": "guard", + "rationale": "14 files in internal/dev_server/ui/src import 38 names from @launchpad-ui/components (including ToastContainer in src/main.tsx and ToastQueue in src/Sync.tsx) and 6 from @launchpad-ui/core. The check reads the versions from package-lock.json, downloads those tarballs with `npm pack` (network: npm registry), and checks each named import against the package type entry point. It does not need npm ci, so it runs even when the peer conflict stops the install.", + "script": "launchpad-exports-guard.sh", + "timeout": 300 + }, + { + "id": "ui-routing-guard", + "title": "Dev-server UI routes, redirects, and route params behave the same", + "kind": "guard", + "rationale": "react-router 7.18.2 to 8.3.0. ldcli uses react-router in declarative mode only: BrowserRouter (src/main.tsx), Routes/Route/Navigate (src/App.tsx), useLocation (src/RouteSelector.tsx), and useParams (src/DebugSessionEventsPage.tsx). The test renders the real App.tsx route table under BrowserRouter in jsdom and checks the / and /ui redirects to /ui/flags, each page route, a trailing slash, the :debugSessionKey param, and an unknown path.", + "script": "ui-routing-guard.sh", + "timeout": 600 + }, + { + "id": "ui-singlefile-guard", + "title": "The vite build still writes one self-contained dist/index.html", + "kind": "guard", + "rationale": "vite 6.4.3 to 8.2.1 replaces esbuild and Rollup with Rolldown and minifies CSS with lightningcss. internal/dev_server/ui/vite.config.ts uses vite-plugin-singlefile, and the Go binary embeds dist/ (internal/dev_server/ui/asset_handler.go). The check builds and asserts: only index.html and the favicon in dist/, no external script or stylesheet, an inline module script,
, and a size within 20% of base.", + "script": "ui-singlefile-guard.sh", + "timeout": 600 + } + ] +} diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/launchpad-exports-guard.sh b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/launchpad-exports-guard.sh new file mode 100644 index 000000000..8076a60f4 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/launchpad-exports-guard.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +dir="$WT/$UI_DIR_REL" +packs="$ARTIFACTS/packs" +mkdir -p "$packs" +pkgs=$(grep -rhoE "from ['\"]@launchpad-ui/[a-z0-9-]+['\"]" "$dir/src" | grep -oE '@launchpad-ui/[a-z0-9-]+' | sort -u) +[ -n "$pkgs" ] || skip "src/ imports no @launchpad-ui package" +for p in $pkgs; do + v=$(jq -r --arg k "node_modules/$p" '.packages[$k].version // empty' "$dir/package-lock.json") + [ -n "$v" ] || incomplete "$p is not in package-lock.json" + mkdir -p "$packs/$p" + tgz=$(cd "$packs/$p" && npm pack "$p@$v" --silent 2>>"$ARTIFACTS/pack.err" | tail -n1) + [ -n "$tgz" ] && [ -f "$packs/$p/$tgz" ] || incomplete "npm pack $p@$v failed (network?): $(tail -n1 "$ARTIFACTS/pack.err")" + tar xzf "$packs/$p/$tgz" -C "$packs/$p" || incomplete "could not unpack $p@$v" +done +cp "$(dirname "$0")/lp_exports.mjs.txt" "$ARTIFACTS/lp_exports.mjs" +node "$ARTIFACTS/lp_exports.mjs" "$dir" "$packs" >"$ARTIFACTS/exports.out" 2>&1 +rc=$? +cat "$ARTIFACTS/exports.out" +sum=$(grep -m1 '^SUMMARY' "$ARTIFACTS/exports.out") +[ -n "$sum" ] || incomplete "export scan did not finish: $(tail -n1 "$ARTIFACTS/exports.out")" +grep -v '^SUMMARY' "$ARTIFACTS/exports.out" >"$ARTIFACTS/exports.md" +detail_block "$ARTIFACTS/exports.md" 40 +n=$(grep -c '^ MISSING' "$ARTIFACTS/exports.out") +[ $rc -eq 0 ] && pass "every @launchpad-ui name that src/ imports is exported at the locked versions ($sum)" +recommend "Migrate the listed imports to their replacements (for example @launchpad-ui/components) in the same PR, or keep the old version." +fail "$n imported @launchpad-ui name(s) are no longer exported at the locked versions" diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/lp_exports.mjs.txt b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/lp_exports.mjs.txt new file mode 100644 index 000000000..5d898c89c --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/lp_exports.mjs.txt @@ -0,0 +1,60 @@ +// Usage: node lp_exports.mjs +// For each @launchpad-ui/* package that src/ imports by name, reads the +// package tarball unpacked in //package and checks that each +// imported name is exported by its type entry point. +import fs from 'node:fs'; +import path from 'node:path'; + +const [uiDir, packsDir] = process.argv.slice(2); +const srcFiles = []; +const walk = (d) => { + for (const e of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) { if (e.name !== '__verify__') walk(p); } + else if (/\.(tsx?|jsx?)$/.test(e.name)) srcFiles.push(p); + } +}; +walk(path.join(uiDir, 'src')); + +const wanted = new Map(); +const importRe = /import\s+(?:type\s+)?(?:[\w$]+\s*,\s*)?\{([^}]*)\}\s*from\s*['"](@launchpad-ui\/[\w-]+)['"]/g; +for (const f of srcFiles) { + const text = fs.readFileSync(f, 'utf8'); + for (const m of text.matchAll(importRe)) { + const names = m[1].split(',').map((s) => s.trim().replace(/^type\s+/, '').split(/\s+as\s+/)[0]).filter(Boolean); + const set = wanted.get(m[2]) ?? new Map(); + for (const n of names) set.set(n, [...(set.get(n) ?? []), path.relative(uiDir, f)]); + wanted.set(m[2], set); + } +} + +const exportsOf = (file, seen = new Set()) => { + const out = new Set(); + if (seen.has(file) || !fs.existsSync(file)) return out; + seen.add(file); + const text = fs.readFileSync(file, 'utf8'); + for (const m of text.matchAll(/export\s+(?:type\s+)?\{([^}]*)\}/g)) + for (const s of m[1].split(',')) { const n = s.trim().replace(/^type\s+/, '').split(/\s+as\s+/).pop(); if (n) out.add(n); } + for (const m of text.matchAll(/export\s+(?:declare\s+)?(?:const|let|var|function|class|type|interface|enum)\s+([\w$]+)/g)) out.add(m[1]); + for (const m of text.matchAll(/export\s+\*\s+from\s+['"](\.[^'"]+)['"]/g)) { + const base = path.resolve(path.dirname(file), m[1]); + for (const c of [base + '.d.ts', path.join(base, 'index.d.ts')]) for (const n of exportsOf(c, seen)) out.add(n); + } + return out; +}; + +let missing = 0, checked = 0; +for (const [pkg, names] of [...wanted].sort()) { + const root = path.join(packsDir, pkg, 'package'); + const pj = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')); + const dot = pj.exports?.['.']; + const types = (typeof dot === 'object' && dot?.types) || pj.types || pj.typings || 'dist/index.d.ts'; + const exp = exportsOf(path.resolve(root, types)); + const gone = [...names.keys()].filter((n) => !exp.has(n)).sort(); + checked += names.size; + missing += gone.length; + console.log(`${pkg}@${pj.version}: ${names.size} imported name(s), ${gone.length} missing`); + for (const n of gone) console.log(` MISSING ${pkg}@${pj.version} ${n} (used in ${[...new Set(names.get(n))].sort().join(', ')})`); +} +console.log(`SUMMARY checked=${checked} missing=${missing}`); +process.exit(missing ? 1 : 0); diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/ui-routing-guard.sh b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/ui-routing-guard.sh new file mode 100644 index 000000000..45c0c144f --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/ui-routing-guard.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +trap restore_tree EXIT +ensure_ui_deps || incomplete "npm ci failed, so the routing test did not run" +dir="$WT/$UI_DIR_REL" +mkdir -p "$dir/src/__verify__" +cp "$(dirname "$0")/app_routing_test.tsx.txt" "$dir/src/__verify__/zz_routing.test.tsx" +cd "$dir" || incomplete "UI directory missing" +run npx --no-install vitest run src/__verify__ >"$ARTIFACTS/vitest.out" 2>&1 +rc=$? +cat "$ARTIFACTS/vitest.out" +summary=$(grep -m1 -E '^ +Tests ' "$ARTIFACTS/vitest.out" | sed -E 's/ +/ /g; s/^ //') +[ -n "$summary" ] || incomplete "vitest did not report results: $(tail -n1 "$ARTIFACTS/vitest.out")" +detail "react-router $(jq -r .version node_modules/react-router/package.json), react $(jq -r .version node_modules/react/package.json): $summary" +[ $rc -eq 0 ] && pass "App.tsx routes, redirects, and params work ($summary)" +detail_block "$ARTIFACTS/vitest.out" 40 +fail "App.tsx routing changed ($summary)" diff --git a/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/ui-singlefile-guard.sh b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/ui-singlefile-guard.sh new file mode 100644 index 000000000..c55ea3106 --- /dev/null +++ b/scripts/dependency-pr/test/fixtures/agent-inputs/pr-779/generated/ui-singlefile-guard.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +source "$VERIFY_ROOT/lib/check.sh" +trap restore_tree EXIT +ensure_ui_deps || incomplete "npm ci failed, so the build did not run" +cd "$WT/$UI_DIR_REL" || incomplete "UI directory missing" +run npm run build >"$ARTIFACTS/build.out" 2>&1 || { tail -n 20 "$ARTIFACTS/build.out"; fail "npm run build fails"; } +html=dist/index.html +[ -f "$html" ] || fail "the build wrote no dist/index.html" +files=$(cd dist && find . -type f | sed 's#^\./##' | sort | paste -sd' ' -) +size=$(wc -c <"$html") +ref=$(git -C "$BASE_WT" show HEAD:"$UI_DIR_REL/dist/index.html" | wc -c) +ext=$(grep -oE ']*\ssrc=[^>]*>|]*rel="(stylesheet|modulepreload)"[^>]*>' "$html" | head -n3) +detail "dist files: \`$files\`; index.html $size bytes (base $ref bytes)" +problems=() +[ "$files" = "favicon-osmo-prod.svg index.html" ] || problems+=("dist/ has other files: $files") +[ -z "$ext" ] || problems+=("index.html loads external assets: $(printf '%s' "$ext" | paste -sd' ' -)") +grep -q '
' "$html" || problems+=("index.html has no
") +grep -q '