Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ on:
push:
branches: [main]
pull_request:
# Started by copilot-auto-merge.yml after it merges: a merge made with
# GITHUB_TOKEN starts no push run.
workflow_dispatch:

permissions:
contents: read
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ on:
push:
branches: [main]
pull_request:
# Started by copilot-auto-merge.yml after it merges: a merge made with
# GITHUB_TOKEN starts no push run.
workflow_dispatch:

permissions:
contents: read
Expand Down
140 changes: 140 additions & 0 deletions .github/workflows/copilot-auto-merge.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
# Pilot, this repo only: an owner PR that Copilot approved merges once every
# gate the ruleset requires passes (STANDARDS.md, Copilot auto-merge).
#
# It merges only the exact commit Copilot's latest review approved, and only
# while that commit is still the PR's head. Never a fork, Dependabot
# (auto-merge.yml handles those) or release-please: the release PR stays a
# human checkpoint before any caller picks a change up.
#
# It merges directly rather than switching GitHub's auto-merge on. Auto-merge
# stays on across pushes, so a push Copilot never saw could merge if its checks
# passed before anything switched it off. `gh pr merge --match-head-commit`
# makes GitHub refuse the merge once the head is not the approved commit, and
# the ruleset refuses it until that commit's gates pass, so there is no window.
#
# Triggers. Not `pull_request_review`: GitHub holds a run started by Copilot's
# review for manual approval before any job is read (action_required with no
# jobs, later failure), reportedly starts none for an approval without inline
# comments, and runs the PR's own copy of this file with a write token, so a
# PR could rewrite its own guards. Instead a scheduled sweep runs the copy on
# the default branch, so a PR never runs its own version. No
# `workflow_dispatch` either: a manual run can pick a PR branch's copy.
#
# A merge made with GITHUB_TOKEN starts no `push` workflow, so after each merge
# the sweep starts ci and checks on the default branch itself: a dispatch is one
# of the events that token may start.
#
# Nothing from a PR is checked out or run: the job reads PR metadata through
# the API and calls `gh pr merge`, and nothing else.
name: copilot-auto-merge

on:
schedule:
# Every 10 minutes: GitHub's schedule is best effort, so a run can lag.
- cron: "*/10 * * * *"

permissions: {}

# One sweep at a time; a queued one does the same thing, so none is cancelled.
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: false

jobs:
sweep:
name: merge owner PRs that Copilot approved
timeout-minutes: 10
runs-on: ubuntu-latest
permissions:
actions: write # starting ci and checks on the merge
contents: write # the squash merge
pull-requests: write # merging a pull request
steps:
- name: Merge what Copilot's latest review approved
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
OWNER: ${{ github.repository_owner }}
run: |
copilot='copilot-pull-request-reviewer[bot]'
failed=0
# Read from the API: a schedule event's payload has no repository.
default_branch=$(gh api "repos/$REPO" --jq .default_branch)
[ -n "$default_branch" ] || exit 1
# Empty fields print as "-": a tab is whitespace to `read`, so an
# empty field would shift the ones after it.
# Into the default branch only: the ruleset that makes a merge wait
# for the gates covers that branch alone.
prs=$(gh api --paginate \
"repos/$REPO/pulls?state=open&base=$default_branch&per_page=100" \
--jq '.[] | [.number, .user.login, (.head.repo.full_name // "-"),
.head.ref, .head.sha, .draft,
(.auto_merge.enabled_by.login // "-")] | @tsv')
while IFS=$'\t' read -r number author head_repo head_ref head_sha draft enabled_by <&3; do
[ -n "$number" ] || continue
# The owner's PRs from a branch in this repo, never a fork.
if [ "$author" != "$OWNER" ] || [ "$head_repo" != "$REPO" ]; then
continue
fi
case "$author" in dependabot\[bot\] | github-actions\[bot\]) continue ;; esac
case "$head_ref" in dependabot/* | release-please--*) continue ;; esac
# A draft is how the owner holds a PR back from this pilot.
[ "$draft" = false ] || continue
# Auto-merge the owner switched on is the owner's call.
[ "$enabled_by" = - ] || continue

# Per PR, so one failed call never stops the rest of the sweep.
# Reviews come oldest first, so the last line is Copilot's latest.
if ! reviews=$(gh api --paginate "repos/$REPO/pulls/$number/reviews?per_page=100" \
--jq '.[] | select(.user.login == "copilot-pull-request-reviewer[bot]")
| "\(.state) \(.commit_id)"'); then
echo "::warning::#$number: could not read its reviews"
failed=1
continue
fi
latest=$(printf '%s\n' "$reviews" | tail -n 1)
[ "$latest" = "APPROVED $head_sha" ] || continue

# Merge only once GitHub reports the PR mergeable with its required
# checks passed; until then the next sweep looks again.
if ! state=$(gh pr view "$number" --repo "$REPO" --json mergeStateStatus \
--jq .mergeStateStatus); then
echo "::warning::#$number: could not read its merge state"
failed=1
continue
fi
case "$state" in
CLEAN | UNSTABLE | HAS_HOOKS) ;;
*) continue ;;
esac
# GitHub refuses unless the head is still the approved commit.
if gh pr merge --squash --match-head-commit "$head_sha" \
--repo "$REPO" "$number"; then
Comment thread
atdr marked this conversation as resolved.
Resolved
echo "::notice::#$number: merged $head_sha, which $copilot approved"
for workflow in ci.yml checks.yml; do
if ! gh workflow run "$workflow" --ref "$default_branch" --repo "$REPO"; then
Comment thread
atdr marked this conversation as resolved.
Resolved
Comment thread
atdr marked this conversation as resolved.
Resolved
echo "::warning::#$number: merged, but could not start $workflow on $default_branch"
failed=1
fi
done
# A dispatch runs on the branch tip at that moment, so a merge
# landing before either one leaves this commit without that run.
# Checked after both: the tip only moves forward, so a tip still
# at the merge commit means both ran on it. Said loudly rather
# than threading a revision through every reusable workflow; the
# tip's runs still cover the combined result.
merged=$(gh pr view "$number" --repo "$REPO" --json mergeCommit \
--jq .mergeCommit.oid || true)
tip=$(gh api "repos/$REPO/commits/$default_branch" --jq .sha || true)
if [ -z "$merged" ] || [ "$merged" != "$tip" ]; then
echo "::warning::#$number: $default_branch moved past its merge commit ${merged:-(unknown)} before its ci and checks runs started, so they may test ${tip:-a later commit} instead"
failed=1
fi
else
echo "::warning::#$number: could not merge $head_sha"
failed=1
fi
done 3<<EOF
$prs
EOF
exit "$failed"
3 changes: 2 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ scripts/configure.sh <owner/repo> [--apply|--diff] # dry run by default
`conventions`, `dependabot-auto-merge`) run in callers' repos. **This repo's
own** (`ci`, `checks`, `pull-request`, `auto-merge`, `release-please`) call
the reusable ones locally (`uses: ./...`), so a change is exercised here
before any caller can pin it.
before any caller can pin it. `copilot-auto-merge` is a pilot for this repo
alone and calls nothing (STANDARDS.md, Copilot auto-merge).
- `template/core` is copied into every repo, then `template/<overlay>` on top;
an overlay file replaces the core file at the same path.
- `scripts/check-conventions.py` enforces the checkable rules in `STANDARDS.md`.
Expand Down
52 changes: 51 additions & 1 deletion STANDARDS.md
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,51 @@ repository variable, because a job-level `if:` can read `vars` but not
security group filtered a high advisory out of its own job and still reported
success, so the alert sat open with nothing to show for it.

## Copilot auto-merge (pilot)

In this repo only, and not yet a reusable workflow: an owner PR that Copilot
approved merges itself once every gate the ruleset requires passes.
`copilot-auto-merge.yml` merges it, and the ruleset refuses the merge until the
gates are green and every review thread is resolved, so a Copilot review with
comments blocks it. The ruleset asks Copilot for a review on each PR and on
every push to it.

- **Whose PRs.** Only the owner's, from a branch in this repo: never a fork,
never Dependabot (its own auto-merge above covers it), and never a
release-please PR, so the release stays a human checkpoint before any caller
picks a change up. A draft is left alone, which is how to hold one back.
- **Only the commit Copilot approved.** The merge goes ahead only while
Copilot's latest review is an approval of the PR's current head, and once
GitHub reports the PR mergeable. It names that commit (`--match-head-commit`),
so GitHub refuses it if a push has moved the head. Not GitHub's auto-merge:
that stays on across pushes, so a push Copilot never saw could merge if its
checks passed before anything switched it off.
- **No `pull_request_review` trigger.** GitHub holds a run started by Copilot's
review for manual approval before any job is read, reportedly starts none for
an approval without inline comments, and would run the PR's own copy of the
workflow with a write token. So a sweep every ten minutes merges what is
ready. It runs the copy on the default branch, so a PR, including the one that
changes the workflow, never runs its own version, and it has no
`workflow_dispatch`, since a manual run can pick a PR branch's copy. Nothing
from a PR is checked out or run.
- **CI on `main` still runs.** A merge made with `GITHUB_TOKEN` starts no `push`
workflow, so after each merge the sweep dispatches `ci` and `checks` on the
default branch, which that token may do. A dispatch runs on the branch tip, so
if another merge lands before either dispatch the sweep fails with a warning
naming the commit left without runs of its own; the tip's runs still cover the
combined result.
- **Risk accepted.** A change can be written by an agent, approved by Copilot
and merged with no human reading it. The release PR is then the only
checkpoint before the change reaches caller repos. On its first five PRs here
Copilot approved every one with no comments, so it is a light filter.

**Extend it** to other repos, as a reusable workflow, once it has merged at
least ten PRs here with no change reverted or fixed afterwards for something a
reviewer should have caught, and with no push after an approval ever seen to
merge. **End it** on the first change that reaches a release PR broken in a way
a reading would have caught, or if Copilot reviews stop arriving reliably:
delete the workflow and drop the ruleset's Copilot review rule.

## Repository settings

Tracked as data in `settings/` and applied by `scripts/configure.sh`;
Expand Down Expand Up @@ -281,7 +326,12 @@ workflow can open or approve a PR it was not built to.
The ruleset (`settings/ruleset-main.json`) blocks deletion and force pushes,
requires linear history, a PR with resolved conversations and squash merge, and
the gates. No bypass actors, so it binds admins too. Zero required approvals:
these are solo repos.
these are solo repos. Where the repo runs the Copilot auto-merge pilot, it also
asks Copilot to review each PR and every push to it, which reaches only authors
with Copilot code review, so not Dependabot or release-please. Elsewhere the
rule is dropped, since nothing acts on those reviews. A Copilot approval counts
toward required approvals only where a Copilot setting allows it, and even then
a count of one would leave those two waiting for an approval that never comes.

The exception is a repo whose owner commits straight to the default branch on
purpose, such as a link list where each change is one validated line and a PR
Expand Down
14 changes: 14 additions & 0 deletions scripts/configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,18 @@ if [ "$visibility" = public ]; then
run gh api -X PATCH "repos/$repo/code-scanning/default-setup" --silent \
-f state=configured

# The Copilot review rule only where the Copilot auto-merge pilot runs
# (scripts/ruleset.jq).
copilot_review=false
case "
$workflows
" in
*"
.github/workflows/copilot-auto-merge.yml
"*) copilot_review=true ;;
esac
ruleset=$(jq -c --arg gates "$gates" --arg admin_bypass "$admin_bypass" \
--arg copilot_review "$copilot_review" \
-f "$here/scripts/ruleset.jq" "$settings/ruleset-main.json")
if [ -n "$gates" ]; then
echo "requires: $(printf '%s' "$gates" | paste -sd, - | sed 's/,/, /g')"
Expand All @@ -265,6 +276,9 @@ if [ "$visibility" = public ]; then
if [ "$admin_bypass" = true ]; then
echo "admins may bypass the ruleset: the repo commits .github/admin-bypass"
fi
if [ "$copilot_review" = true ]; then
echo "requests a Copilot review on every push: the repo has copilot-auto-merge.yml"
fi
existing=$(gh api "repos/$repo/rulesets" --jq '.[] | select(.name == "main") | .id')
if [ -n "$existing" ]; then
run gh api -X PUT "repos/$repo/rulesets/$existing" --silent --input - <<<"$ruleset"
Expand Down
6 changes: 6 additions & 0 deletions scripts/ruleset.jq
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@
# $admin_bypass is "true" where the repo commits .github/admin-bypass: the
# repository admin role may then push to the default branch directly, while
# everyone else (Dependabot included) still goes through a gated PR.
#
# $copilot_review is "true" where the repo has copilot-auto-merge.yml. The
# Copilot review rule is kept only there: that workflow is what needs a fresh
# review on every push, and elsewhere the rule would spend Copilot credits on
# reviews nothing acts on.

($gates | split("\n") | map(select(. != ""))) as $present
| .rules |= map(
Expand All @@ -22,6 +27,7 @@
else . end)
| .rules |= map(select(.type != "required_status_checks"
or (.parameters.required_status_checks | length) > 0))
| .rules |= map(select(.type != "copilot_code_review" or $copilot_review == "true"))
| .bypass_actors = (if $admin_bypass == "true"
then [{"actor_id": 5, "actor_type": "RepositoryRole", "bypass_mode": "always"}]
else [] end)
7 changes: 7 additions & 0 deletions settings/ruleset-main.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,13 @@
{ "context": "checks gate", "integration_id": 15368 }
]
}
},
{
"type": "copilot_code_review",
"parameters": {
"review_on_push": true,
"review_draft_pull_requests": false
}
}
]
}
18 changes: 17 additions & 1 deletion tests/test_boilerplate.py
Original file line number Diff line number Diff line change
Expand Up @@ -307,10 +307,11 @@ def test_codecov_ships_only_with_the_overlays_that_report_coverage():
assert node.read_text() == python.read_text()


def ruleset(*gates: str, admin_bypass: bool = False) -> dict:
def ruleset(*gates: str, admin_bypass: bool = False, copilot_review: bool = False) -> dict:
result = subprocess.run(
["jq", "--arg", "gates", "".join(f"{g}\n" for g in gates),
"--arg", "admin_bypass", str(admin_bypass).lower(),
"--arg", "copilot_review", str(copilot_review).lower(),
"-f", str(ROOT / "scripts/ruleset.jq"), str(ROOT / "settings/ruleset-main.json")],
check=True, capture_output=True, text=True,
)
Expand Down Expand Up @@ -349,6 +350,21 @@ def test_ruleset_binds_admins_unless_the_repo_opts_out():
assert required_checks(ruleset("checks gate", admin_bypass=True)) == ["checks gate"]


@pytest.mark.parametrize("gates", [("checks gate",), ()])
def test_ruleset_requests_a_copilot_review_on_every_push(gates):
# A fresh review per push: copilot-auto-merge.yml acts only on an approval
# of the PR's current head.
rules = ruleset(*gates, copilot_review=True)["rules"]
(rule,) = [r for r in rules if r["type"] == "copilot_code_review"]
assert rule["parameters"] == {"review_on_push": True, "review_draft_pull_requests": False}


@pytest.mark.parametrize("gates", [("checks gate",), ()])
def test_ruleset_has_no_copilot_review_without_the_pilot(gates):
# Elsewhere the rule would spend Copilot credits on reviews nothing uses.
assert not [r for r in ruleset(*gates)["rules"] if r["type"] == "copilot_code_review"]


def test_template_claude_md_is_a_symlink():
assert os.readlink(ROOT / "template/core/CLAUDE.md") == "AGENTS.md"

Expand Down
Loading
Loading