diff --git a/.github/audit/expected-github-state.json b/.github/audit/expected-github-state.json index 3b076c2fc..65831cc6e 100644 --- a/.github/audit/expected-github-state.json +++ b/.github/audit/expected-github-state.json @@ -40,6 +40,10 @@ }, "botCollaborator": { "login": "dormouse-bot", "forbiddenRoles": ["admin", "maintain"] }, "environments": { + "blyg-publish": { + "policies": [{ "type": "branch", "name": "main" }, { "type": "tag", "name": "v*" }], + "secrets": ["BLYG_API_TOKEN"] + }, "hosted-preview": { "policies": [{ "type": "branch", "name": "main" }, { "type": "branch", "name": "refs/pull/*/merge" }], "requiredReviewers": true, diff --git a/.github/workflows/blyg-publish.yml b/.github/workflows/blyg-publish.yml new file mode 100644 index 000000000..3dcaf9a62 --- /dev/null +++ b/.github/workflows/blyg-publish.yml @@ -0,0 +1,40 @@ +name: Publish Dormouse Blyg + +on: + release: + types: [published] + push: + branches: [main] + paths: [CHANGELOG.md] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: blyg-publish + cancel-in-progress: false + +jobs: + publish: + if: github.repository == 'diffplug/dormouse' && (github.ref == 'refs/heads/main' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/v'))) + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: blyg-publish + steps: + # Release tags can predate this publisher. Always use reviewed main for + # publisher code and the canonical changelog, including later corrections. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: package.json + package-manager-cache: false + - run: node --test scripts/publish-blyg.test.mjs + - name: Publish and pin released changelog entries + run: node scripts/publish-blyg.mjs + env: + GH_TOKEN: ${{ github.token }} + BLYG_API_TOKEN: ${{ secrets.BLYG_API_TOKEN }} diff --git a/deploy/blyg/.gitignore b/deploy/blyg/.gitignore new file mode 100644 index 000000000..26f283e65 --- /dev/null +++ b/deploy/blyg/.gitignore @@ -0,0 +1,7 @@ +.bundle/ +.staging/ +.wrangler/ +.dev.vars* +backups/ +*.secret + diff --git a/deploy/blyg/README.md b/deploy/blyg/README.md new file mode 100644 index 000000000..97f75a28a --- /dev/null +++ b/deploy/blyg/README.md @@ -0,0 +1,65 @@ +# Dormouse Blyg + +[Public changelog](https://blyg.dormouse.sh/) · [Studio](https://blyg.dormouse.sh/studio) · [RSS](https://blyg.dormouse.sh/feed.xml) + +This directory deploys the official, unmodified Blygger Studio Worker to Cloudflare account `ned.twigg@diffplug.com`. `wrangler.jsonc` owns the domain and resource IDs. `release.json` pins the upstream archive and its SHA-256. The downloaded bundle, local credentials and backups are ignored; posts and private Studio state live in D1, and media live in R2. + +The publication contract is in `docs/specs/deploy.md` → “Blyg changelog”; CI credential placement is in `docs/specs/security-ci.md` → “Automated Maintainer (tend)”. This installation receives Webmentions even when nobody has Studio open. Use Studio to read and moderate responses; public display starts disabled. + +## Publishing + +`.github/workflows/blyg-publish.yml` synchronizes on GitHub release publication, changes to `CHANGELOG.md` on `main`, or manual dispatch. It checks out reviewed `main` and needs no package installation. `scripts/publish-blyg.mjs` supports the initial historical import and later corrections through the same path. Entries without a stable, published GitHub release are skipped, including historical extension-only versions without a GitHub release. Old entries retain their release dates in the text; their Blyg publication timestamps are the import time because stock Studio has no backdating API. + +Run these from the repository root: + +```sh +node scripts/publish-blyg.mjs --plan +node scripts/publish-blyg.mjs --dry-run +node scripts/publish-blyg.mjs +``` + +`--plan` checks public GitHub release eligibility without Studio credentials. `--dry-run` reads Studio to compare entries but makes no writes or pins. The other two commands need `BLYG_API_TOKEN` in their environment. Never paste credentials into command arguments or Git. The renewal helper below passes its token to a local synchronization without saving it. + +Make release corrections in `CHANGELOG.md`. Studio remains useful for responses, subscriptions, settings and independently authored posts. The publisher identifies its entries by their opening “Dormouse VERSION” heading and matching GitHub release link; retain both. Do not run local publication while the Actions publisher is running. After a failed or timed-out write, rerun the publisher: it reconciles existing drafts and versions before doing more work. A conflicting draft or deliberately withdrawn item stops the run for operator review. + +## Renew publishing access + +Stock Studio grants expire after 30 days; OAuth refresh does not extend that deadline. Renew before expiry or after a workflow reports HTTP 401: + +```sh +python3 deploy/blyg/authorize.py --publish +``` + +The helper prompts for the Studio owner password with terminal echo disabled, logs in over HTTPS, creates a REST token with read/draft/publish permissions, and pipes it directly to GitHub's `blyg-publish` environment secret `BLYG_API_TOKEN`. It revokes the previous grants named “Dormouse release publisher” after saving the replacement, then synchronizes pending releases. It never saves the password, session cookie or token to disk. Requires Python 3, Node, and `gh` authenticated with permission to set this environment's secrets. Run between publisher jobs so rotating the token does not interrupt an active run. + +Only for a new installation, `--initialize --publish` first sets the chosen owner password through Wrangler's stdin. It requires the separate random `COOKIE_SECRET` to already exist. Do not use `--initialize` during ordinary token renewal: it changes the owner password. No AI-provider credential is needed for release publication. + +## Deploy and update Studio + +Commands below start at the repository root. Run Wrangler from a temporary directory with an absolute config path, or under the repository's pinned Node runtime; ambient `npx` inside this monorepo may reject pnpm's `devEngines` setting. + +```sh +node deploy/blyg/install-bundle.mjs +pnpm exec npx --yes wrangler@4.141.0 deploy --dry-run --config deploy/blyg/wrangler.jsonc +pnpm exec npx --yes wrangler@4.141.0 deploy --config deploy/blyg/wrangler.jsonc +``` + +`install-bundle.mjs` downloads the exact official GitHub archive, verifies its pinned hash, checks archive paths and file types, and extracts the bundle without running upstream hooks. It does not deploy. The upstream MIT license and installation guide are retained in `.bundle/LICENSE` and `.bundle/README.md`. + +For an update, read every intervening [upstream changelog entry](https://github.com/blygger/blygger-studio/blob/main/CHANGELOG.md) and the target release instructions. Verify its published checksum, update `release.json`, and run the installer. Preserve this installation's configuration and secrets, incorporating only required config changes. Back up D1 and original R2 media before migrations; apply the official migrations with `d1 migrations apply DB --remote --config deploy/blyg/wrangler.jsonc`, then deploy. A Git revert does not reverse database migrations. Check the manifest's generator version, RSS, archive index, existing posts and pins, Studio login, and anonymous `/api/settings` returning 401. Commit the release pin and verified deployment record together. + +## Backups + +Git backs up configuration and publisher code, not posts, grants or media. D1 exports contain private data. Keep them outside Git with restricted permissions and store an encrypted copy off this computer. R2 originals need a separate backup; `media.r2_key` enumerates uploads in the current schema. The feed cache can be rebuilt. + +```sh +mkdir -p deploy/blyg/backups +chmod 700 deploy/blyg/backups +(umask 077; pnpm exec npx --yes wrangler@4.141.0 d1 export DB --remote --config deploy/blyg/wrangler.jsonc --output "deploy/blyg/backups/$(date -u +%Y%m%dT%H%M%SZ).sql") +``` + +No scheduled data-backup job is configured. Changing `COOKIE_SECRET` invalidates Studio sessions and delegated grants. Changing `OWNER_PASSWORD` invalidates owner sessions; revoke delegated grants in Studio separately if required. + +## Read from your personal Studio + +Subscribe to `https://blyg.dormouse.sh/` in `blyg.nedshed.dev`. Once imported, quote a release or selected passage through Studio's normal transclusion controls. Dormouse's receiver collects responses targeting Dormouse entries; the personal receiver collects responses targeting personal posts. diff --git a/deploy/blyg/authorize.py b/deploy/blyg/authorize.py new file mode 100644 index 000000000..a19f050bf --- /dev/null +++ b/deploy/blyg/authorize.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Renew Studio publishing access without writing credentials to disk or argv.""" +import argparse +import datetime +import getpass +import http.cookiejar +import json +import os +from pathlib import Path +import subprocess +import tempfile +import urllib.error +import urllib.parse +import urllib.request + +ORIGIN = "https://blyg.dormouse.sh" +NAME = "Dormouse release publisher" +ROOT = Path(__file__).resolve().parents[2] + + +class SameOriginRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if urllib.parse.urlsplit(newurl)[:2] != urllib.parse.urlsplit(ORIGIN)[:2]: + raise RuntimeError("Refusing cross-origin Studio redirect") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--initialize", action="store_true", help="Set a NEW installation's owner password in Cloudflare first") + parser.add_argument("--publish", action="store_true", help="Sync released changelog entries after renewing access") + args = parser.parse_args() + password = getpass.getpass("Choose the new Dormouse Studio owner password: " if args.initialize else "Dormouse Studio owner password: ") + if not password: + raise RuntimeError("Password must not be empty") + if args.initialize: + if len(password) < 16: + raise RuntimeError("Use an owner password with at least 16 characters") + if password != getpass.getpass("Confirm owner password: "): + raise RuntimeError("Passwords did not match") + release = json.loads((ROOT / "deploy/blyg/release.json").read_text()) + # Run outside the monorepo: npm does not understand pnpm's devEngines onFail=download. + with tempfile.TemporaryDirectory(prefix="dormouse-blyg-auth-") as cwd: + subprocess.run(["npx", "--yes", f"wrangler@{release['wranglerVersion']}", "secret", "put", "OWNER_PASSWORD", "--config", str(ROOT / "deploy/blyg/wrangler.jsonc")], input=password, text=True, cwd=cwd, check=True) + + jar = http.cookiejar.CookieJar() + client = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar), SameOriginRedirect()) + # Cloudflare Browser Integrity Check rejects urllib's generic default UA. + client.addheaders = [("User-Agent", "dormouse-blyg-publisher/1.0 (+https://dormouse.sh)")] + login = urllib.request.Request(ORIGIN + "/studio/login", data=urllib.parse.urlencode({"password": password}).encode(), headers={"Origin": ORIGIN}) + with client.open(login, timeout=60) as response: + response.read() + password = None + if not list(jar): + raise RuntimeError("Studio did not establish an owner session") + + def api(method, path, body=None): + request = urllib.request.Request(ORIGIN + "/api" + path, method=method, headers={"Origin": ORIGIN, "Content-Type": "application/json"}, data=None if body is None else json.dumps(body).encode()) + with client.open(request, timeout=60) as response: + return json.load(response) + + old = api("GET", "/authorizations")["items"] + grant = api("POST", "/authorizations", {"name": NAME, "resource": "api", "scope": ["owner:read", "owner:draft", "owner:publish"]}) + token = grant["access_token"] + try: + subprocess.run(["gh", "secret", "set", "BLYG_API_TOKEN", "--repo", "diffplug/dormouse", "--env", "blyg-publish"], input=token, text=True, check=True) + except Exception: + api("DELETE", "/authorizations/" + urllib.parse.quote(grant["authorization"]["id"], safe="")) + raise + # Only revoke the grants this dedicated publisher previously owned, after + # GitHub has accepted the replacement. Other clients' grants are untouched. + for prior in old: + if prior["name"] == NAME: + api("DELETE", "/authorizations/" + urllib.parse.quote(prior["id"], safe="")) + expires = datetime.datetime.fromtimestamp(grant["authorization"]["expiresAt"], datetime.timezone.utc) + print(f"Publishing token saved to GitHub environment blyg-publish; expires {expires.isoformat()}.") + if args.publish: + subprocess.run(["node", "scripts/publish-blyg.mjs"], cwd=ROOT, env={**os.environ, "BLYG_API_TOKEN": token}, check=True) + + +if __name__ == "__main__": + try: + main() + except urllib.error.HTTPError as error: + # Never print auth response bodies or request headers. + path = urllib.parse.urlsplit(error.url).path + raise SystemExit(f"HTTP {error.code} at {path}. Check Studio access and try again.") + except (RuntimeError, subprocess.CalledProcessError) as error: + raise SystemExit(str(error)) diff --git a/deploy/blyg/deployment.json b/deploy/blyg/deployment.json new file mode 100644 index 000000000..1a4647dcc --- /dev/null +++ b/deploy/blyg/deployment.json @@ -0,0 +1,17 @@ +{ + "deployedAt": "2026-10-08", + "studioVersion": "0.36.1", + "initialWorkerVersion": "a444cea0-f883-41a7-8b66-2b34bc8dd22c", + "publicUrl": "https://blyg.dormouse.sh/", + "verified": { + "publicPage": 200, + "manifest": 200, + "rss": 200, + "archiveIndex": 200, + "anonymousSettings": 401, + "webmentionAdvertised": true, + "ownerLogin": true, + "publishedReleases": 16, + "pinnedRevisions": 16 + } +} diff --git a/deploy/blyg/install-bundle.mjs b/deploy/blyg/install-bundle.mjs new file mode 100644 index 000000000..1c798420b --- /dev/null +++ b/deploy/blyg/install-bundle.mjs @@ -0,0 +1,36 @@ +#!/usr/bin/env node +// Download a checksum-pinned, unmodified upstream Worker; never run package hooks. +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { join } from 'node:path'; + +const root = fileURLToPath(new URL('.', import.meta.url)); +const release = JSON.parse(await readFile(join(root, 'release.json'), 'utf8')); +if (!/^\d+\.\d+\.\d+$/.test(release.version) || !/^[a-f0-9]{64}$/.test(release.archiveSha256)) { + throw new Error('Invalid pinned release'); +} +const name = `blygger-worker-${release.version}`; +const stage = join(root, '.staging', release.version); +await mkdir(stage, { recursive: true }); +const response = await fetch(`https://github.com/blygger/blygger-studio/releases/download/v${release.version}/${name}.tar.gz`); +if (!response.ok) throw new Error(`Bundle download failed: HTTP ${response.status}`); +const archive = Buffer.from(await response.arrayBuffer()); +if (createHash('sha256').update(archive).digest('hex') !== release.archiveSha256) { + throw new Error('Bundle checksum mismatch; nothing installed'); +} +const path = join(stage, `${name}.tar.gz`); +await writeFile(path, archive); +const entries = execFileSync('tar', ['-tzf', path], { encoding: 'utf8' }).trim().split('\n'); +if (entries.some(entry => !entry.startsWith(`${name}/`) || entry.split('/').includes('..'))) { + throw new Error('Unsafe archive path'); +} +const types = execFileSync('tar', ['-tvzf', path], { encoding: 'utf8' }).trim().split('\n'); +if (types.some(entry => !['-', 'd'].includes(entry[0]))) throw new Error('Archive contains a link or special file'); +await rm(join(stage, name), { recursive: true, force: true }); +execFileSync('tar', ['-xzf', path, '-C', stage]); +await readFile(join(stage, name, 'worker.js')); +await rm(join(root, '.bundle'), { recursive: true, force: true }); +await rename(join(stage, name), join(root, '.bundle')); +console.log(`Installed official Studio ${release.version} in deploy/blyg/.bundle (SHA-256 verified).`); diff --git a/deploy/blyg/release.json b/deploy/blyg/release.json new file mode 100644 index 000000000..45faa2550 --- /dev/null +++ b/deploy/blyg/release.json @@ -0,0 +1,5 @@ +{ + "version": "0.36.1", + "archiveSha256": "90b651ad40b6bc77615a29793165633b2d62c29571ed609696267664c0695f20", + "wranglerVersion": "4.141.0" +} diff --git a/deploy/blyg/wrangler.jsonc b/deploy/blyg/wrangler.jsonc new file mode 100644 index 000000000..fee95578b --- /dev/null +++ b/deploy/blyg/wrangler.jsonc @@ -0,0 +1,22 @@ +{ + "name": "blyg-dormouse-sh", + "account_id": "0a95e814ccf2b6a95d2dc3bea0a4a2b4", + "routes": [{ "pattern": "blyg.dormouse.sh", "custom_domain": true }], + "workers_dev": false, + "preview_urls": false, + "main": ".bundle/worker.js", + "no_bundle": true, + "compatibility_date": "2026-07-01", + "compatibility_flags": ["nodejs_compat"], + "cache": { "enabled": false }, + "exports": { "PublicHtml": { "type": "worker", "cache": { "enabled": true } } }, + "vars": { "MOUNT": "" }, + "triggers": { "crons": ["* * * * *", "*/15 * * * *", "0 0 * * *"] }, + "d1_databases": [{ + "binding": "DB", + "database_name": "blyg-dormouse-sh", + "database_id": "bd17595e-42e2-4dd9-82b8-ea2acbf94e78", + "migrations_dir": ".bundle/migrations" + }], + "r2_buckets": [{ "binding": "MEDIA", "bucket_name": "blyg-dormouse-sh-media" }] +} diff --git a/docs/specs/deploy.md b/docs/specs/deploy.md index 7812d6a75..366fe19b4 100644 --- a/docs/specs/deploy.md +++ b/docs/specs/deploy.md @@ -176,6 +176,18 @@ The website changelog page imports `website/src/data/changelog.json`, but **`CHA Source of truth: `create_release` in `scripts/sign-and-deploy.sh`; `website/scripts/generate-changelog.js`; lifecycle scripts in `website/package.json`. +### Blyg changelog + +**Must host `https://blyg.dormouse.sh/` with the checksum-pinned, unmodified Blygger Studio Worker.** Its Cloudflare D1 database and R2 bucket hold content; Git records deployment configuration and the upstream release checksum. Operator setup, updates and token renewal are in `deploy/blyg/README.md`. + +**Must publish only stable, public GitHub releases with a matching `CHANGELOG.md` entry**, oldest first when backfilling. `.github/workflows/blyg-publish.yml` runs on release publication, changelog changes on `main`, and manual dispatch; publisher code and changelog come from reviewed `main`. Versions without a public GitHub release remain unpublished. Studio records the Blyg publication time; the entry text retains the original release date. + +**Must retain one Studio item per release, pin every published revision, and leave unchanged content at its existing revision.** A correction uses the same item. An interrupted run resumes its matching draft or completes the current pin; conflicting drafts, duplicate release identities and withdrawn items require operator resolution. **Never edit release text in Studio**; change `CHANGELOG.md` and synchronize it. Avoid simultaneous manual and automated publication; Actions serializes its runs. + +**Must receive Webmentions independently of the publisher, with public response display off initially.** Studio owns verification and moderation. A publishing-token expiry stops synchronization without affecting the public site or receiver. Credential placement follows `docs/specs/security-ci.md` -> "Automated Maintainer (tend)". + +Source of truth: `syncPublications` in `scripts/publish-blyg.mjs`; `.github/workflows/blyg-publish.yml`; `deploy/blyg/wrangler.jsonc`. + ## Environment / secrets | Secret | Where | Purpose | diff --git a/docs/specs/security-ci.md b/docs/specs/security-ci.md index 27c98a656..558d75a6c 100644 --- a/docs/specs/security-ci.md +++ b/docs/specs/security-ci.md @@ -71,7 +71,7 @@ The diff window's evasions are `docs/specs/security.md` -> "Known gaps". - **FAIL IF** either admin-gating ruleset is missing or weakened. `Merge access` must target `~DEFAULT_BRANCH`, block exactly `creation`, `update`, and `deletion`, and carry admin (`RepositoryRole` actor `5`, `bypass_mode: exempt`) as its sole bypass actor; `Tag operations` must target `~ALL` tags except `refs/tags/hosted/**` ("Hosted Deployments"), block both `creation` and `update`, and carry the same admin-only bypass. Pinned by `scripts/github-state-check.mjs`. - **FAIL IF** `.config/tend.yaml` does not set `merge: restricted`, or any `tend-*.yaml` passes another `merge:`; tend's `yolo` mode makes the bot a `Merge access` bypass actor. Pinned by `scripts/workflow-lint.mjs`. - **FAIL IF** `dormouse-bot` holds `maintain` or `admin` on this repository. `GET /collaborators/dormouse-bot/permission` spells `push` as `write` in both `permission` and `role_name`, so check that neither of those two roles appears rather than string-comparing against `push`. Pinned by `scripts/github-state-check.mjs`. -- **FAIL IF** any GitHub environment except `hosted-preview` admits, by its deployment-branch policy, a ref that is not admin-gated by the `Tag operations` or `Merge access` rulesets; an environment's `can_admins_bypass` is not judged here. Hosted environments follow "Hosted Deployments" below. Today: `vscode-extension-publish` and `release-attest` (`v*` tag, admin-only via `Tag operations`); `security-audit` (`main` admin-only via `Merge access`, plus `v*` tag); `tend` (`main` only, admin-only via `Merge access`). An environment `.github/audit/expected-github-state.json` does not name fails until it is evaluated and named there (rationale). Pinned by `scripts/github-state-check.mjs`. +- **FAIL IF** any GitHub environment except `hosted-preview` admits, by its deployment-branch policy, a ref that is not admin-gated by the `Tag operations` or `Merge access` rulesets; an environment's `can_admins_bypass` is not judged here. Hosted environments follow "Hosted Deployments" below. Today: `vscode-extension-publish` and `release-attest` (`v*` tag, admin-only via `Tag operations`); `security-audit` and `blyg-publish` (`main` admin-only via `Merge access`, plus `v*` tag); `tend` (`main` only, admin-only via `Merge access`). An environment `.github/audit/expected-github-state.json` does not name fails until it is evaluated and named there (rationale). Pinned by `scripts/github-state-check.mjs`. - **FAIL IF** the secret inventory departs from this placement (rationale). It is exhaustive: a secret at repo, org, or environment level that no line names fails. One pass over `actions/secrets`, `actions/organization-secrets`, and each environment's secret listing answers every line: - `ARGOS_TOKEN` and `CHROMATIC_PROJECT_TOKEN` — repo level, the only secrets there; accepted with rotation (see "Reachable repo-level secrets"). No workflow reads `CHROMATIC_PROJECT_TOKEN` since Chromatic left CI; its absence is no violation. - `AUDIT_PAT` — in `security-audit`, absent at repo level. @@ -79,6 +79,7 @@ The diff window's evasions are `docs/specs/security.md` -> "Known gaps". - `TEND_BOT_TOKEN` — in `tend`, absent at repo level. - `CLAUDE_CODE_OAUTH_TOKEN` — in **both** `tend` and `security-audit`, absent at repo level. Environments do not inherit each other's secrets, so a rotation must set both. - `OVSX_PAT`, `VSCE_PAT` — in `vscode-extension-publish` only, absent at repo level. + - `BLYG_API_TOKEN` — in `blyg-publish` only, absent at repo level; Studio grants read, draft and publish access, expiring after 30 days. The environment admits only `main` and `v*` tags. - `CLOUDFLARE_API_TOKEN`, `NEON_API_KEY`, `PREVIEW_AUTH_SECRET` — in `hosted-preview`; `CLOUDFLARE_API_TOKEN`, `DATABASE_URL`, `BACKUP_AGE_IDENTITY` — in `hosted-production`; `HOSTED_TAG_APP_PRIVATE_KEY` — in `hosted-release-tag`. Each absent at repo level; the two `CLOUDFLARE_API_TOKEN`s are separate values. - `ANTHROPIC_API_KEY` — absent at repo *and* org level, for as long as `tend-*.yaml` passes `anthropic_api_key` to `max-sixty/tend/claude`. - `release-attest`'s own secret listing is empty **and** it declares no environment variables, so `id-token: write` stays the only credential `release.yml`'s two build jobs can reach. diff --git a/package.json b/package.json index a57e8e335..9e840f7be 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "build:hosted": "pnpm --filter dormouse-hosted build", "test:hosted": "pnpm --filter remote-lib-common build && pnpm --filter dormouse-hosted test", "build": "pnpm run build:vscode && pnpm --filter dormouse-lib build:pocket && pnpm --filter dormouse-website build && pnpm build:hosted", - "test": "node scripts/spec-lint.mjs && node scripts/md-unwrap.mjs && node --test scripts/md-unwrap.test.mjs scripts/md-merge.test.mjs && node --test scripts/prose-audit.test.mjs scripts/docs-surfaces.test.mjs scripts/entrypoint-check.test.mjs && node scripts/public-docs-lint.mjs && node scripts/xterm-lint.mjs && node --test scripts/xterm-bump.test.mjs && node scripts/loopback-lint.mjs && node scripts/outbound-lint.mjs && node scripts/deploy-lint.mjs && node scripts/installer-verify-test.mjs && node scripts/ps1-cmdlet-lint.mjs && node scripts/e2e-lint.mjs && node --test scripts/lint-selftest-isolation.test.mjs && node scripts/clamp-issue-body-selftest.mjs && node --test scripts/sign-and-deploy.test.mjs && node --test scripts/workflow-audit.test.mjs && node --test scripts/pairing-walkthrough/proc.test.mjs && node --test scripts/security-audit.test.mjs scripts/security-audit-canary.test.mjs && node scripts/workflow-lint.mjs && node --test scripts/workflow-lint.test.mjs scripts/github-state-check.test.mjs scripts/supply-chain-config.test.mjs && pnpm --filter dormouse-hosted test:deploy && pnpm --filter dormouse-hosted test:miniflare && pnpm -r --filter \"!dormouse-hosted\" run test", + "test": "node scripts/spec-lint.mjs && node scripts/md-unwrap.mjs && node --test scripts/md-unwrap.test.mjs scripts/md-merge.test.mjs && node --test scripts/prose-audit.test.mjs scripts/docs-surfaces.test.mjs scripts/entrypoint-check.test.mjs && node scripts/public-docs-lint.mjs && node scripts/xterm-lint.mjs && node --test scripts/xterm-bump.test.mjs && node scripts/loopback-lint.mjs && node scripts/outbound-lint.mjs && node scripts/deploy-lint.mjs && node scripts/installer-verify-test.mjs && node scripts/ps1-cmdlet-lint.mjs && node scripts/e2e-lint.mjs && node --test scripts/lint-selftest-isolation.test.mjs && node scripts/clamp-issue-body-selftest.mjs && node --test scripts/sign-and-deploy.test.mjs scripts/publish-blyg.test.mjs && node --test scripts/workflow-audit.test.mjs && node --test scripts/pairing-walkthrough/proc.test.mjs && node --test scripts/security-audit.test.mjs scripts/security-audit-canary.test.mjs && node scripts/workflow-lint.mjs && node --test scripts/workflow-lint.test.mjs scripts/github-state-check.test.mjs scripts/supply-chain-config.test.mjs && pnpm --filter dormouse-hosted test:deploy && pnpm --filter dormouse-hosted test:miniflare && pnpm -r --filter \"!dormouse-hosted\" run test", "lint:specs": "node scripts/spec-lint.mjs && node scripts/spec-lint-selftest.mjs", "lint:md": "node scripts/md-unwrap.mjs && node --test scripts/md-unwrap.test.mjs scripts/md-merge.test.mjs", "setup:git": "node scripts/setup-git.mjs", diff --git a/scripts/github-state-check.fixture.json b/scripts/github-state-check.fixture.json index bbb50c309..bbfc43566 100644 --- a/scripts/github-state-check.fixture.json +++ b/scripts/github-state-check.fixture.json @@ -225,8 +225,17 @@ "repos/diffplug/dormouse/environments": { "status": 200, "body": { - "total_count": 7, + "total_count": 8, "environments": [ + { + "name": "blyg-publish", + "protection_rules": [], + "can_admins_bypass": true, + "deployment_branch_policy": { + "protected_branches": false, + "custom_branch_policies": true + } + }, { "id": 21767729669, "name": "hosted-preview", @@ -942,6 +951,40 @@ } ] } + }, + "repos/diffplug/dormouse/environments/blyg-publish/deployment-branch-policies": { + "status": 200, + "body": { + "total_count": 2, + "branch_policies": [ + { + "name": "main", + "type": "branch" + }, + { + "name": "v*", + "type": "tag" + } + ] + } + }, + "repos/diffplug/dormouse/environments/blyg-publish/secrets": { + "status": 200, + "body": { + "total_count": 1, + "secrets": [ + { + "name": "BLYG_API_TOKEN" + } + ] + } + }, + "repos/diffplug/dormouse/environments/blyg-publish/variables": { + "status": 200, + "body": { + "total_count": 0, + "variables": [] + } } } } diff --git a/scripts/github-state-check.test.mjs b/scripts/github-state-check.test.mjs index 1e1e59a6f..dcf45e4a9 100644 --- a/scripts/github-state-check.test.mjs +++ b/scripts/github-state-check.test.mjs @@ -93,6 +93,8 @@ const violations = [ ['an unnamed environment admits every branch', (ctx) => addEnvironment(ctx, 'staging', { policy: { deployment_branch_policy: null } }), '`staging` admits only refs'], ['an unnamed environment holds a secret', (ctx) => addEnvironment(ctx, 'staging', { secrets: ['STAGING_TOKEN'] }), '`staging` holds exactly'], ['`tend` admits every branch', (ctx) => { env(ctx, 'tend').deployment_branch_policy = null; }, '`tend` admits only refs'], + ['`blyg-publish` admits a feature branch', (ctx) => { body(ctx, 'environments/blyg-publish/deployment-branch-policies').branch_policies.push({ name: 'feature/*', type: 'branch' }); }, '`blyg-publish` admits only refs'], + ['`blyg-publish` loses its publishing token', (ctx) => { body(ctx, 'environments/blyg-publish/secrets').secrets = []; }, '`blyg-publish` holds exactly the secrets'], ['`security-audit` admits protected branches', (ctx) => { env(ctx, 'security-audit').deployment_branch_policy = { protected_branches: true, custom_branch_policies: false }; }, '`security-audit` admits only refs'], ['`release-attest` admits a bot-pushable branch', (ctx) => { body(ctx, 'environments/release-attest/deployment-branch-policies').branch_policies.push({ name: 'feature/*', type: 'branch' }); }, '`release-attest` admits only refs'], ['`hosted-preview` admits another branch', (ctx) => { body(ctx, 'environments/hosted-preview/deployment-branch-policies').branch_policies.push({ name: 'dev', type: 'branch' }); }, '`hosted-preview` admits exactly'], diff --git a/scripts/publish-blyg.mjs b/scripts/publish-blyg.mjs new file mode 100644 index 000000000..4221b9a2c --- /dev/null +++ b/scripts/publish-blyg.mjs @@ -0,0 +1,165 @@ +#!/usr/bin/env node +import { readFile } from 'node:fs/promises'; +import { realpathSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; + +export const ORIGIN = 'https://blyg.dormouse.sh'; +const REPOSITORY = 'diffplug/dormouse'; +const TITLE = /^# Dormouse (\d+\.\d+\.\d+)\n/; + +function releaseVersion(markdown) { + const version = TITLE.exec(markdown)?.[1]; + return version && markdown.includes(`https://github.com/${REPOSITORY}/releases/tag/v${version})`) ? version : undefined; +} + +// Keep Markdown intact: nested bullets, links, and code are the release text. +export function changelogEntries(markdown) { + const entries = new Map(); + const normalized = markdown.replace(/\r\n?/g, '\n'); + const headings = [...normalized.matchAll(/^## (.+)$/gm)]; + for (let index = 0; index < headings.length; index++) { + const heading = headings[index]; + const match = /^\[(\d+\.\d+\.\d+)\] - (\d{4}-\d{2}-\d{2})\s*$/.exec(heading[1]); + if (!match) continue; + const [, version, date] = match; + if (entries.has(version)) throw new Error(`Duplicate changelog version ${version}`); + const body = normalized.slice(heading.index + heading[0].length, headings[index + 1]?.index).trim(); + if (!body) throw new Error(`Empty changelog for ${version}`); + entries.set(version, { version, date, body }); + } + return entries; +} + +export function publicationPlan(markdown, releases) { + const entries = changelogEntries(markdown); + return releases + .filter(release => !release.draft && !release.prerelease && release.published_at && /^v\d+\.\d+\.\d+$/.test(release.tag_name)) + .sort((a, b) => a.published_at.localeCompare(b.published_at)) + .flatMap(release => { + const entry = entries.get(release.tag_name.slice(1)); + if (!entry) throw new Error(`Published ${release.tag_name} is missing from CHANGELOG.md`); + const { version, date, body } = entry; + return [{ version, content_md: `# Dormouse ${version}\n\nReleased ${date}. [Download](https://github.com/${REPOSITORY}/releases/tag/v${version}).\n\n${body}\n\n---\n\n🔌 VS Code extension only · 🖥️ Desktop app only · Unmarked changes apply to both.\n` }]; + }); +} + +export function apiClient(token, fetcher = fetch) { + if (!token?.trim()) throw new Error('BLYG_API_TOKEN is missing. Run python3 deploy/blyg/authorize.py to renew publishing access.'); + return async (method, path, body) => { + // Callers construct only fixed API paths and validated item IDs. Never follow + // redirects with the publishing credential, including same-origin redirects. + if (!path.startsWith('/items')) throw new Error('Publisher API path rejected'); + for (let attempt = 0; ; attempt++) { + const response = await fetcher(`${ORIGIN}/api${path}`, { + method, redirect: 'error', signal: AbortSignal.timeout(60_000), + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + // 429 is a definite refusal, safe to retry. A timeout or 5xx on a mutation + // is ambiguous: fail and let the next run reconcile the stored draft/version. + if (response.status === 429 && attempt < 3) { + const seconds = Number(response.headers.get('retry-after') ?? 60); + await delay(Math.min(60, Math.max(1, Number.isFinite(seconds) ? seconds : 60)) * 1000); + continue; + } + if (!response.ok) { + const advice = response.status === 401 ? ' Renew the 30-day publishing token with deploy/blyg/authorize.py.' : ''; + throw new Error(`Studio ${method} ${path}: HTTP ${response.status}.${advice}`); + } + return response.json(); + } + }; +} + +function itemPath(item) { + if (!/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(item.id)) throw new Error('Invalid Studio item ID'); + return `/items/${item.id}`; +} + +export async function syncPublications(plan, api, { dryRun = false, log = console.log } = {}) { + const byVersion = new Map(); + for (let offset = 0; ; ) { + const page = await api('GET', `/items?limit=100&offset=${offset}`); + if (!Array.isArray(page.items) || !Number.isInteger(page.total)) throw new Error('Invalid Studio item listing'); + for (const item of page.items) { + const version = releaseVersion(item.content_md); + if (!version) continue; + if (byVersion.has(version)) throw new Error(`Multiple Studio items claim Dormouse ${version}; refusing to publish`); + byVersion.set(version, item); + } + offset += page.items.length; + if (offset >= page.total) break; + if (!page.items.length) throw new Error('Studio pagination stopped before total'); + } + for (const entry of plan) { + let item = byVersion.get(entry.version); + if (item) { + item = await api('GET', itemPath(item)); + if (item.status === 'withdrawn' || item.kind === 'withdrawn') throw new Error(`Dormouse ${entry.version} was withdrawn; refusing to restore it automatically`); + if (item.kind !== 'thread') throw new Error(`Dormouse ${entry.version} is not a thread`); + // An interrupted PATCH leaves our exact intended working copy. Resume it; + // anything else dirty may be a human's work and must not be overwritten. + if ((item.dirty || item.version === 0) && item.content_md !== entry.content_md) { + throw new Error(`Dormouse ${entry.version} has conflicting draft edits; resolve them in Studio first`); + } + if (item.version > 0 && !item.dirty && item.content_md === entry.content_md) { + if (!dryRun) await api('PUT', `${itemPath(item)}/versions/${item.version}/pin`); + log(`Unchanged Dormouse ${entry.version}: ${ORIGIN}/t/${item.id}/`); + continue; + } + } + if (dryRun) { + log(`${item ? 'Update' : 'Create'} Dormouse ${entry.version}`); + continue; + } + if (!item) { + item = await api('POST', '/items', { kind: 'thread', content_md: entry.content_md }); + byVersion.set(entry.version, item); + } else { + // Preserve the last live text before publishing a correction, even when a + // previous run reached publish but failed before its pin request completed. + if (item.version > 0) await api('PUT', `${itemPath(item)}/versions/${item.version}/pin`); + if (item.content_md !== entry.content_md) { + await api('PATCH', itemPath(item), { content_md: entry.content_md }); + } + } + const result = await api('POST', `${itemPath(item)}/publish`, { + note: item.version > 0 ? 'Updated from the Dormouse changelog.' : 'Published from the Dormouse changelog.', + }); + if (!Number.isInteger(result.version) || result.version < 1) throw new Error('Invalid published version'); + await api('PUT', `${itemPath(item)}/versions/${result.version}/pin`); + log(`Published Dormouse ${entry.version} (revision ${result.version}): ${ORIGIN}/t/${item.id}/`); + } +} + +async function githubReleases() { + const releases = []; + for (let page = 1; ; page++) { + const response = await fetch(`https://api.github.com/repos/${REPOSITORY}/releases?per_page=100&page=${page}`, { + redirect: 'error', signal: AbortSignal.timeout(30_000), + headers: { accept: 'application/vnd.github+json', 'user-agent': 'dormouse-blyg-publisher', + ...(process.env.GH_TOKEN ? { authorization: `Bearer ${process.env.GH_TOKEN}` } : {}) }, + }); + if (!response.ok) throw new Error(`GitHub release listing: HTTP ${response.status}`); + const batch = await response.json(); + releases.push(...batch); + if (batch.length < 100) return releases; + } +} + +async function main() { + const args = process.argv.slice(2); + if (args.some(arg => !['--dry-run', '--plan'].includes(arg))) throw new Error('Usage: node scripts/publish-blyg.mjs [--plan | --dry-run]'); + const markdown = await readFile(new URL('../CHANGELOG.md', import.meta.url), 'utf8'); + const plan = publicationPlan(markdown, await githubReleases()); + if (args.includes('--plan')) { + for (const entry of plan) console.log(`Eligible: Dormouse ${entry.version} (${entry.content_md.length} characters)`); + return; + } + await syncPublications(plan, apiClient(process.env.BLYG_API_TOKEN), { dryRun: args.includes('--dry-run') }); +} + +if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(error => { console.error(error.message); process.exitCode = 1; }); +} diff --git a/scripts/publish-blyg.test.mjs b/scripts/publish-blyg.test.mjs new file mode 100644 index 000000000..662579ef8 --- /dev/null +++ b/scripts/publish-blyg.test.mjs @@ -0,0 +1,113 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { apiClient, changelogEntries, publicationPlan, syncPublications } from './publish-blyg.mjs'; + +const markdown = '# Changelog\n\n## [Unreleased]\nSecret future\n\n## [1.0.0] - 2026-01-01\n### Added\n- Feature\n - Nested [link](https://example.com)\n\n## [0.9.0] - 2025-12-01\n- Older\n'; +const releases = [{ tag_name: 'v1.0.0', published_at: '2026-01-01T00:00:00Z', draft: false, prerelease: false }]; +const plan = publicationPlan(markdown, releases); +const id = '0123456789ABCDEFGHJKMNPQRS'; + +function studio(seed = []) { + const items = structuredClone(seed); + const calls = []; + const api = async (method, path, body) => { + calls.push({ method, path, body }); + if (method === 'GET' && path.startsWith('/items?')) return { items: structuredClone(items), total: items.length }; + let item = items.find(item => path.startsWith(`/items/${item.id}`)); + if (method === 'GET') return structuredClone(item); + if (method === 'POST' && path === '/items') { + item = { id, kind: 'thread', status: 'draft', version: 0, dirty: true, ...body }; + items.push(item); + return structuredClone(item); + } + if (method === 'PATCH') { Object.assign(item, body, { dirty: true }); return structuredClone(item); } + if (path.endsWith('/publish')) { + item.status = 'public'; item.version++; item.dirty = false; + return { ok: true, version: item.version }; + } + if (path.endsWith('/pin')) return { ok: true }; + throw new Error(`Unexpected ${method} ${path}`); + }; + return { api, items, calls }; +} +const options = { log: () => {} }; +const published = () => ({ id, kind: 'thread', status: 'public', version: 1, dirty: false, content_md: plan[0].content_md }); + +test('eligible releases exclude drafts, prereleases and unreleased; preserve Markdown', () => { + const result = publicationPlan(markdown, [...releases, { ...releases[0], tag_name: 'v0.9.0', draft: true }, { ...releases[0], tag_name: 'v0.9.0', prerelease: true }]); + assert.equal(result.length, 1); + assert.match(result[0].content_md, / - Nested \[link\]/); + assert.doesNotMatch(result[0].content_md, /Secret future|Older/); + assert.throws(() => publicationPlan(markdown, [{ ...releases[0], tag_name: 'v2.0.0' }]), /missing/); + assert.throws(() => changelogEntries(markdown + '\n## [1.0.0] - 2026-01-01\n- Duplicate'), /Duplicate/); +}); + +test('first run creates and pins; rerun does not create or publish a new revision', async () => { + const s = studio(); + await syncPublications(plan, s.api, options); + await syncPublications(plan, s.api, options); + assert.equal(s.items.length, 1); + assert.equal(s.items[0].version, 1); + assert.equal(s.calls.filter(call => call.path.endsWith('/publish')).length, 1); + assert.equal(s.calls.filter(call => call.path.endsWith('/pin')).length, 2); +}); + +test('correction pins prior revision before update and new revision after publish', async () => { + const s = studio([published()]); + await syncPublications([{ ...plan[0], content_md: plan[0].content_md.replace('- Feature', '- Corrected feature') }], s.api, options); + assert.deepEqual(s.calls.filter(call => call.method !== 'GET').map(call => [call.method, call.path]), [ + ['PUT', `/items/${id}/versions/1/pin`], ['PATCH', `/items/${id}`], ['POST', `/items/${id}/publish`], ['PUT', `/items/${id}/versions/2/pin`], + ]); +}); + +for (const stage of ['create', 'patch', 'publish']) { + test(`recovers when ${stage} succeeds but response is lost`, async () => { + const old = { ...published(), content_md: plan[0].content_md.replace('- Feature', '- Previous feature') }; + const s = studio(stage === 'create' ? [] : [old]); + const api = async (method, path, body) => { + const result = await s.api(method, path, body); + if ((stage === 'create' && method === 'POST' && path === '/items') || (stage === 'patch' && method === 'PATCH') || (stage === 'publish' && path.endsWith('/publish'))) { + throw new Error('Connection lost after mutation'); + } + return result; + }; + await assert.rejects(syncPublications(plan, api, options), /Connection lost/); + await syncPublications(plan, s.api, options); + assert.equal(s.items.length, 1); + assert.equal(s.items[0].version, stage === 'create' ? 1 : 2); + assert.equal(s.items[0].dirty, false); + }); +} + +test('refuses conflicting drafts, withdrawn entries, and duplicate ownership', async () => { + for (const items of [ + [{ ...published(), dirty: true, content_md: plan[0].content_md + 'Human draft' }], + [{ ...published(), kind: 'withdrawn' }], + [published(), { ...published(), id: '1123456789ABCDEFGHJKMNPQRS' }], + ]) { + const s = studio(items); + await assert.rejects(syncPublications(plan, s.api, options)); + assert.ok(s.calls.every(call => call.method === 'GET')); + } +}); + +test('dry run makes no writes, including pins', async () => { + for (const items of [[], [published()]]) { + const s = studio(items); + await syncPublications(plan, s.api, { ...options, dryRun: true }); + assert.ok(s.calls.every(call => call.method === 'GET')); + } +}); + +test('HTTP client refuses redirects, does not repeat ambiguous writes, and diagnoses expiry', async () => { + let count = 0; + const api = apiClient('test-token', async (url, init) => { + count++; + assert.equal(init.redirect, 'error'); + assert.ok(url.startsWith('https://blyg.dormouse.sh/api/')); + return new Response('{}', { status: 503 }); + }); + await assert.rejects(api('POST', '/items', {}), /503/); + assert.equal(count, 1); + await assert.rejects(apiClient('test-token', async () => new Response('{}', { status: 401 }))('GET', '/items'), /Renew the 30-day/); +}); diff --git a/scripts/spec-word-budgets.json b/scripts/spec-word-budgets.json index 606db2d0f..ed538b4d9 100644 --- a/scripts/spec-word-budgets.json +++ b/scripts/spec-word-budgets.json @@ -5,7 +5,7 @@ "docs/compatible-agents.md": 1850, "docs/specs/alert.md": 7100, "docs/specs/auto-update.md": 1100, - "docs/specs/deploy.md": 2000, + "docs/specs/deploy.md": 2200, "docs/specs/dor-browser.md": 5700, "docs/specs/dor-cli.md": 5400, "docs/specs/dor-tool.md": 6250, @@ -25,7 +25,7 @@ "docs/specs/remote-security-model.md": 5800, "docs/specs/reopen.md": 1300, "docs/specs/security-audit.md": 4000, - "docs/specs/security-ci.md": 3100, + "docs/specs/security-ci.md": 3150, "docs/specs/security-hosted.md": 2500, "docs/specs/security-local.md": 4800, "docs/specs/security-remote.md": 6000,