From ccac5046dd209a2c6d9ef1cfa0714a156d414d9f Mon Sep 17 00:00:00 2001 From: Jasper Mayone Date: Wed, 11 Nov 2026 22:56:52 -0500 Subject: [PATCH 1/2] feat: selvage on twill, a new app vm --- .gitattributes | 1 + README.md | 56 +++++++++ ad/TIERING.md | 4 +- ad/org.json | 6 +- ansible/inventory/hosts.yml | 5 +- bin/new-selvage-secrets | 164 +++++++++++++++++++++++++ flake.nix | 4 + hosts/alastor/edge.nix | 6 + hosts/central-db-schema.nix | 4 + hosts/twill/configuration.nix | 86 +++++++++++++ hosts/twill/selvage.nix | 89 ++++++++++++++ modules/selvage/default.nix | 213 +++++++++++++++++++++++++++++++++ secrets/secrets.nix | 42 +++++++ terraform/proxmox/README.md | 1 + terraform/proxmox/variables.tf | 8 ++ 15 files changed, 683 insertions(+), 6 deletions(-) create mode 100755 bin/new-selvage-secrets create mode 100644 hosts/twill/configuration.nix create mode 100644 hosts/twill/selvage.nix create mode 100644 modules/selvage/default.nix diff --git a/.gitattributes b/.gitattributes index e62d97f..a3eb979 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ # Run on Linux even from a Windows checkout (core.autocrlf would turn them CRLF). bin/new-krater-secrets text eol=lf bin/new-stash-secrets text eol=lf +bin/new-selvage-secrets text eol=lf diff --git a/README.md b/README.md index d138540..03f3594 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ Declarative infrastructure for Patchwork Labs: every Patchwork host, the NixOS m | `modules/stash/` | The Stash service module (web, worker, archived files on a host directory). | | `hosts/hank/stash.nix` | Stash production and staging on hank: instance settings and secrets. | | `bin/new-stash-secrets` | Writes the two secrets of Stash staging or production with a new random database password and `SECRET_KEY_BASE`. Prints no value. | +| `modules/selvage/` | The Selvage service module (web and worker, production only). | +| `hosts/twill/selvage.nix` | Selvage production on twill: instance settings and secrets. | +| `bin/new-selvage-secrets` | Writes Selvage's restricted database password and env file. Reads the existing central password with your age identity. Prints no value. | | `bin/set-env-secret` | Sets keys in an agenix env file, for example a Weave client id and secret. Asks for each value with the echo off. Prints no value. | | `hosts/weft/krater.nix` | Krater staging and production on weft: instance settings and secrets. Production is off until `hosts/weft/krater-production.nix` turns it on. | | `tests/` | NixOS VM tests. | @@ -51,6 +54,7 @@ The lab hosts below are NixOS VMs on pve01, VLAN 90 (`patchwork.internal`). alas | `batting` | 360 | 10.100.90.60 | 4 GB | Quilt prod + staging. nginx on `tailscale0:80` for alastor. | | `weft` | 361 | 10.100.90.61 | 4 GB | Krater staging, and Krater production (off until its switch). nginx on `tailscale0:80` for alastor. | | `hank` | 362 | 10.100.90.62 | 6 GB | Stash prod + staging (declared, not built yet). 80 GB disk for the archived files. nginx on `tailscale0:80` for alastor. | +| `twill` | 363 | 10.100.90.63 | 3 GB | Selvage prod (declared, not built yet). No staging yet. nginx on `tailscale0:80` for alastor, `/slack/` and `/up` only. | The Windows hosts are managed by Ansible (`ansible/`): @@ -75,6 +79,9 @@ flowchart LR weft -. "staging :5432" .-> hem hank -- "tailnet :5432" --> skein["skein: patchwork_central"] hank -. "staging :5432" .-> hem + alastor -- "tailnet, HTTP :80" --> twill["twill: Selvage (not built yet)"] + twill -- "tailnet :5432" --> skein + twill -- "tailnet :5432" --> seam ``` ## Login @@ -304,6 +311,55 @@ Do the steps in this order. To test top-ups on staging, add a sandbox `MERCURY_WRITE_TOKEN` with batting's public IP on its allowlist, and set `QUILT_TOPUPS` to `approval_only`. +## Selvage on twill + +[Selvage](https://github.com/patchworklabsorg/selvage), the Patchwork Slack bot, runs on twill as two Docker containers, like Stash on hank. `hosts/twill/selvage.nix` is the instance. `modules/selvage` is the service module. Production only: there is no Selvage staging instance yet. + +| Item | Value | +|---|---| +| URL | `https://selvage.patchworklabs.org` (router `selvage-twill` in `edge-alastor`). nginx on twill passes only `POST /slack/*` and `GET /up`. Every other path is 404. | +| Units | `selvage-web` (`./bin/rails server` on `127.0.0.1:3009`, migrates on start) and `selvage-worker` (`bundle exec good_job start`) | +| Image | `ghcr.io/patchworklabsorg/selvage:main` (option `image`). Pin it to `:` for a known build. | +| Databases | `patchwork_central` on skein and `patchwork_central_restricted` on seam. Role and schema `selvage` in each. | +| Secrets | `selvage-env.age`, `selvage-ghcr-token.age` (twill), `central-prod-selvage.age` (skein), `restricted-prod-selvage.age` (seam) | +| Tailnet | `tag:patchwork` and `tag:patchwork-selvage`. The second tag reaches skein:5432 and seam:5432 only. | + +```mermaid +flowchart LR + slack((Slack)) --> traefik["alastor: Traefik"] + traefik -- "tailnet :80" --> nginx["twill: nginx, /slack/ and /up only"] + nginx -- "127.0.0.1:3009" --> web[selvage-web] + worker[selvage-worker] -- "waits for /up" --> web + web -- "tailnet :5432" --> skein["skein: patchwork_central, schema selvage"] + web -- "tailnet :5432" --> seam["seam: patchwork_central_restricted, schema selvage"] + worker --> skein + worker --> seam +``` + +The env file has `CENTRAL_DATABASE_URL` and `RESTRICTED_DATABASE_URL`. Their passwords must be the values in `central-prod-selvage.age` and `restricted-prod-selvage.age`. skein and seam set those values on the role `selvage`. The role has no `CREATE` on either database, so the image must migrate with `db:migrate`, not `db:prepare`, and must not run `CREATE SCHEMA` or `enable_extension`. + +To deploy a new `main` image, run `sudo systemctl restart selvage-web` on twill. That restarts the worker too. + +### Go live + +Do the steps in this order. Steps 5 to 11 are on this repo's branch, before the merge. + +1. Merge the tailnet policy change in `jaspermayone/infra` (`tag:patchwork-selvage`, with grants to skein:5432 and seam:5432). +2. Merge the DNS record `selvage` (proxied to alastor) in `patchworklabsorg/dns`. +3. In `patchworklabsorg/selvage`, make sure that `image.yml` pushed `ghcr.io/patchworklabsorg/selvage:main`, and that `bin/docker-entrypoint` runs `db:migrate`. +4. Make sure that VMID 363 and `10.100.90.63` are free, and that pve01 has 3 GiB of memory to spare (`terraform/proxmox/README.md`, "Memory warning"). +5. Prepare AD for twill: run `.\converge.ps1` (it creates `twill-admins`), then `cd ansible && ansible-playbook ad.yml -t host-groups`. See "Login". +6. Create and install the VM: `bin/install-host twill`. +7. On twill, join the tailnet with both tags: `sudo tailscale up --advertise-tags=tag:patchwork,tag:patchwork-selvage`. +8. Get the host key: `ssh-keyscan -t ed25519 10.100.90.63`. Add it as `twill` in `secrets/secrets.nix`. Add `twill` to `selvage-env.age`, `selvage-ghcr-token.age` and `ad-join.age`. Remove the "not built yet" notes. +9. Write the secrets: `nix develop -c bin/new-selvage-secrets`. It writes `restricted-prod-selvage.age`, `selvage-env.age` and `selvage-ghcr-token.age`. It decrypts `central-prod-selvage.age` and `quilt-ghcr-token.age` with your age identity (`~/.ssh/id_ed25519`, or `AGE_IDENTITY`). +10. Re-key, so that twill can decrypt `ad-join.age`: `cd secrets && agenix -r`. +11. Fill in the Slack and Weave values: `bin/set-env-secret selvage-env.age SLACK_SIGNING_SECRET SLACK_BOT_TOKEN SLACK_ADMIN_TOKEN WEAVE_CLIENT_ID WEAVE_CLIENT_SECRET SELVAGE_AUDIT_CHANNEL`. Add `SLACK_BROWSER_TOKEN` and `SLACK_BROWSER_COOKIE` the same way when you need them. +12. Commit the `.age` files and `secrets.nix`, and merge after CI passes. The lab hosts follow `main`, so seam creates the role `selvage` with its password, and twill starts Selvage. To do it at once: `deploy .#seam`, then `deploy .#twill`. +13. On seam, make sure that `central-db-setup` succeeded: `journalctl -u central-db-setup`. On twill, check `systemctl status selvage-web selvage-worker` and `curl -i http://127.0.0.1:3009/up`. +14. Route the public name: in `jaspermayone/infra`, run `nix flake update patchwork-infra`, then `deploy .#alastor`. +15. Point the Slack app's request URLs at `https://selvage.patchworklabs.org/slack/events`, `/slack/commands` and `/slack/interactions`. + ## Commands ```bash diff --git a/ad/TIERING.md b/ad/TIERING.md index f267dcc..a817446 100644 --- a/ad/TIERING.md +++ b/ad/TIERING.md @@ -95,7 +95,7 @@ The rule: an account of tier N logs on only to assets of tier N. A higher tier c | Tier | Assets | Accounts | |---|---|---| | **0** | loom, spindle (DCs). bobbin (issuing CA). pattern (offline root CA). shuttle (NPS). The Proxmox hosts that run these VMs. The proposed Tier 0 jump host. remus, as long as it holds the `svc-ansible` password. | `adm_mayonej`, `svc-ansible`, `Administrator` (disabled), DSRM (LAPS-DSRM) | -| **1** | warp, batting, weft, hank, skein, seam, hem (NixOS). thimble (VyOS edge). App servers. Weave and the host that runs it (alastor). | `lnx--admins` members. `svc-lnx-join`, `svc-pve-ldap`, `svc-dhcp-dns`, `svc-weave-ldap`. No dedicated Tier 1 person account yet. | +| **1** | warp, batting, weft, hank, twill, skein, seam, hem (NixOS). thimble (VyOS edge). App servers. Weave and the host that runs it (alastor). | `lnx--admins` members. `svc-lnx-join`, `svc-pve-ldap`, `svc-dhcp-dns`, `svc-weave-ldap`. No dedicated Tier 1 person account yet. | | **2** | Staff workstations (`Computers` OU). VPN sessions. | Daily accounts: `mayonej`, staff, members | ```mermaid @@ -113,7 +113,7 @@ flowchart TB bg["Administrator (disabled, break-glass)"] end subgraph T1["Tier 1: servers"] - nix["warp, batting, weft, hank, skein, seam, hem"] + nix["warp, batting, weft, hank, twill, skein, seam, hem"] thimble["thimble VPN edge"] weave["Weave on alastor, svc-weave-ldap"] end diff --git a/ad/org.json b/ad/org.json index d4c941f..bbf5046 100644 --- a/ad/org.json +++ b/ad/org.json @@ -77,6 +77,7 @@ { "name": "ganymede-admins", "formerNames": ["GG-Ganymede-Admins"], "description": "Project Ganymede admins", "members": ["mayonej", "adam.sidwa"] }, { "name": "weft-admins", "description": "Root on weft (Krater): logon and sudo on weft only, through hostGroupMembers. Sudo and the docker group on weft give every Krater secret.", "members": ["mayonej", "adam.sidwa"] }, { "name": "hank-admins", "description": "Root on hank (Stash): logon and sudo on hank only, through hostGroupMembers. Sudo and the docker group on hank give every Stash secret and every archived file.", "members": ["mayonej"] }, + { "name": "twill-admins", "description": "Root on twill (Selvage): logon and sudo on twill only, through hostGroupMembers. Sudo and the docker group on twill give every Selvage secret, including its Slack admin token.", "members": ["mayonej"] }, { "name": "ganymede-users", "formerNames": ["GG-Ganymede-Users"], "description": "Project Ganymede users", "members": [], "manageMembers": false }, { "name": "hackboston-organizers", "formerNames": ["GG-HackBoston-Organizers"], "description": "HackBoston organizers", "members": ["cj", "daniel", "logan", "mayonej"] }, { "name": "hackathonhelp-organizers", "formerNames": ["GG-HackathonHelp-Organizers"], "description": "hackathon.help organizers", "members": ["wei"] }, @@ -94,7 +95,7 @@ "members": [] } ], - "linuxHosts": ["batting", "hank", "hem", "seam", "skein", "warp", "weft"], + "linuxHosts": ["batting", "hank", "hem", "seam", "skein", "twill", "warp", "weft"], "retiredHosts": ["sampler", "swatch"], "hostGroups": [ { "name": "lnx-{host}-users", "description": "Linux logon on {host}", "members": ["infra"] }, @@ -102,7 +103,8 @@ ], "hostGroupMembers": { "weft": { "users": ["weft-admins"], "admins": ["weft-admins"] }, - "hank": { "users": ["hank-admins"], "admins": ["hank-admins"] } + "hank": { "users": ["hank-admins"], "admins": ["hank-admins"] }, + "twill": { "users": ["twill-admins"], "admins": ["twill-admins"] } }, "resourceGroups": [ { "name": "password-reset", "formerNames": ["DL-Users-PasswordReset"], "description": "Reset password, unlock, change at next logon: Users/Staff, Board-Only, Sponsored-Projects", diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml index c8991eb..1905a93 100644 --- a/ansible/inventory/hosts.yml +++ b/ansible/inventory/hosts.yml @@ -2,7 +2,7 @@ # ============================================================================ # Patchwork hosts that Ansible manages: the Windows servers only. # --------------------------------------------------------------------------- -# The NixOS hosts (warp, batting, weft, hank, skein, seam, hem) are declared in ../flake.nix. They +# The NixOS hosts (warp, batting, weft, hank, twill, skein, seam, hem) are declared in ../flake.nix. They # join AD through modules/ad-join, not through Ansible. Do not add them here. # ad.yml -t host-groups reads their names from the flake. # @@ -49,6 +49,7 @@ all: # # warp, batting, NixOS, declared in ../flake.nix. Joined to AD by # weft, skein, modules/ad-join. - # seam, hem, hank + # seam, hem, hank, + # twill # pattern VMID 399. Offline root CA. Stopped. Never a target. # ======================================================================== diff --git a/bin/new-selvage-secrets b/bin/new-selvage-secrets new file mode 100755 index 0000000..06ad8c7 --- /dev/null +++ b/bin/new-selvage-secrets @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Write Selvage production's secrets: +# secrets/restricted-prod-selvage.age the password of the role selvage in +# patchwork_central_restricted on seam +# (password only, a new random value) +# secrets/selvage-env.age Selvage's env file +# secrets/selvage-ghcr-token.age the pwlbot read:packages token, copied +# from quilt-ghcr-token.age (only when +# that file is missing; skip it with +# NO_GHCR_TOKEN=1 for a public package) +# +# The env file gets: +# CENTRAL_DATABASE_URL with the password that already exists in +# central-prod-selvage.age (skein has set it on the +# role). The script decrypts that file with your +# age identity, into a shell variable only. +# RESTRICTED_DATABASE_URL with the new password above. +# SECRET_KEY_BASE a new random value. +# WEAVE_ISSUER https://weave.patchworklabs.org (not secret). +# The other keys empty: SLACK_SIGNING_SECRET, SLACK_BOT_TOKEN, +# SLACK_ADMIN_TOKEN, SLACK_BROWSER_TOKEN, SLACK_BROWSER_COOKIE, +# WEAVE_CLIENT_ID, WEAVE_CLIENT_SECRET, SELVAGE_AUDIT_CHANNEL. A key that is +# set in the environment is used instead, e.g. +# set -a; . ./selvage-third-party.env; set +a; bin/new-selvage-secrets +# Fill them in later with (asks for each value with the echo off) +# bin/set-env-secret selvage-env.age SLACK_SIGNING_SECRET SLACK_BOT_TOKEN ... +# or: cd secrets && agenix -e selvage-env.age +# +# Production only. There is no Selvage staging instance yet. +# +# Usage: bin/new-selvage-secrets +# Needs age, nix and openssl. The identity that decrypts +# central-prod-selvage.age (and quilt-ghcr-token.age) is $AGE_IDENTITY, or +# ~/.ssh/id_ed25519, as for bin/set-env-secret. The script never prints a +# value, and it stops if the password file or the env file exists. The +# recipients come from secrets/secrets.nix: the script encrypts with `age` +# itself, because `agenix -e` from a pipe fails on macOS. +set -euo pipefail +# No trace, even under `bash -x` or SHELLOPTS=xtrace: a trace shows the values. +set +x + +root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$root/secrets" + +db_file=restricted-prod-selvage.age +env_file=selvage-env.age +ghcr_file=selvage-ghcr-token.age +central_file=central-prod-selvage.age +quilt_ghcr_file=quilt-ghcr-token.age +central_host=skein.wildebeest-stargazer.ts.net +restricted_host=seam.wildebeest-stargazer.ts.net +identity="${AGE_IDENTITY:-$HOME/.ssh/id_ed25519}" + +for tool in age nix openssl; do + if ! command -v "$tool" >/dev/null; then + echo "$tool is not on PATH. Run this script in a shell that has it." >&2 + exit 1 + fi +done + +if [ ! -r "$identity" ]; then + echo "No age identity at $identity. Set AGE_IDENTITY to the key that decrypts $central_file." >&2 + exit 1 +fi + +# Encrypts stdin to the recipients that secrets.nix lists for the file. +encrypt() { + local file="$1" recipients + recipients="$(nix eval --raw -f secrets.nix "\"$file\".publicKeys" --apply 'keys: builtins.concatStringsSep "\n" keys')" + if [ -z "$recipients" ]; then + echo "secrets.nix lists no recipients for $file. Stop." >&2 + exit 1 + fi + age --encrypt --recipients-file <(printf '%s\n' "$recipients") --output "$file" + chmod 644 "$file" +} + +# Percent-encodes a URL userinfo part. Hex passwords pass through unchanged. +urlencode() { + # Bytes, not characters, so a multibyte character becomes one %XX per byte. + local LC_ALL=C + local s="$1" out="" c i + for ((i = 0; i < ${#s}; i++)); do + c="${s:i:1}" + case "$c" in + [A-Za-z0-9._~-]) out+="$c" ;; + *) out+="$(printf '%%%02X' "'$c")" ;; + esac + done + printf '%s' "$out" +} + +for file in "$db_file" "$env_file"; do + if [ -e "$file" ]; then + echo "secrets/$file exists. Stop. To change a value, use: bin/set-env-secret or agenix -e $file" >&2 + exit 1 + fi +done +if [ ! -e "$central_file" ]; then + echo "secrets/$central_file does not exist. Stop." >&2 + exit 1 +fi + +copy_ghcr=0 +if [ "${NO_GHCR_TOKEN:-}" != 1 ] && [ ! -e "$ghcr_file" ]; then + if [ ! -e "$quilt_ghcr_file" ]; then + echo "secrets/$quilt_ghcr_file does not exist. Set NO_GHCR_TOKEN=1 to skip the token. Stop." >&2 + exit 1 + fi + copy_ghcr=1 +fi + +umask 077 + +# The existing central password, in this variable only. $(...) drops the +# trailing newline. +central_password="$(age --decrypt -i "$identity" "$central_file")" +if [ -z "$central_password" ]; then + echo "secrets/$central_file decrypted to an empty value. Stop." >&2 + exit 1 +fi + +# Hex only, so the password needs no escaping in the URL. +restricted_password="$(openssl rand -hex 32)" +secret_key_base="$(openssl rand -hex 64)" + +# Write the env file first: when it fails, no new password file is left over. +# printf is a shell builtin, so the values are not in any process's arguments. +{ + printf '%s\n' "CENTRAL_DATABASE_URL=postgres://selvage:$(urlencode "$central_password")@${central_host}:5432/patchwork_central?sslmode=disable" + printf '%s\n' "RESTRICTED_DATABASE_URL=postgres://selvage:${restricted_password}@${restricted_host}:5432/patchwork_central_restricted?sslmode=disable" + printf '%s\n' "SECRET_KEY_BASE=${secret_key_base}" + printf '%s\n' "WEAVE_ISSUER=https://weave.patchworklabs.org" + for key in SLACK_SIGNING_SECRET SLACK_BOT_TOKEN SLACK_ADMIN_TOKEN SLACK_BROWSER_TOKEN \ + SLACK_BROWSER_COOKIE WEAVE_CLIENT_ID WEAVE_CLIENT_SECRET SELVAGE_AUDIT_CHANNEL; do + printf '%s=%s\n' "$key" "${!key:-}" + done +} | encrypt "$env_file" + +printf '%s' "$restricted_password" | encrypt "$db_file" + +unset central_password restricted_password secret_key_base + +written="secrets/$env_file and secrets/$db_file" +if [ "$copy_ghcr" = 1 ]; then + token="$(age --decrypt -i "$identity" "$quilt_ghcr_file")" + printf '%s' "$token" | encrypt "$ghcr_file" + unset token + written="$written and secrets/$ghcr_file" +fi + +echo "Wrote $written." +missing="" +for key in SLACK_SIGNING_SECRET SLACK_BOT_TOKEN SLACK_ADMIN_TOKEN WEAVE_CLIENT_ID \ + WEAVE_CLIENT_SECRET SELVAGE_AUDIT_CHANNEL; do + if [ -z "${!key:-}" ]; then + missing="$missing $key" + fi +done +if [ -n "$missing" ]; then + echo "Left empty:$missing" + echo "Fill them in with: bin/set-env-secret $env_file$missing" +fi +echo "Next: commit the files, then deploy seam so the selvage role there gets its password." diff --git a/flake.nix b/flake.nix index 16cc995..e2bbcac 100644 --- a/flake.nix +++ b/flake.nix @@ -43,6 +43,7 @@ "batting" # Quilt "weft" # Krater "hank" # Stash + "twill" # Selvage "skein" # patchwork_central, prod "seam" # patchwork_central_restricted, prod "hem" # both staging clusters @@ -83,6 +84,9 @@ # The Stash service module alone, with no instance settings. hank # imports it through hosts/hank/stash.nix. stash = ./modules/stash; + # The Selvage service module alone, with no instance settings. twill + # imports it through hosts/twill/selvage.nix. + selvage = ./modules/selvage; }; nixosConfigurations = nixpkgs.lib.genAttrs hosts mkHost; diff --git a/hosts/alastor/edge.nix b/hosts/alastor/edge.nix index b2e676d..d47290c 100644 --- a/hosts/alastor/edge.nix +++ b/hosts/alastor/edge.nix @@ -51,6 +51,12 @@ let backend = "hank"; hosts = [ "stash-staging.patchworklabs.org" ]; }; + # Selvage on twill (../twill/selvage.nix): production only. nginx on + # twill passes only /slack/ and /up. + selvage-twill = { + backend = "twill"; + hosts = [ "selvage.patchworklabs.org" ]; + }; # Every host in ../warp/redirects.nix. A new path on an existing host # needs no change here. redirects = { diff --git a/hosts/central-db-schema.nix b/hosts/central-db-schema.nix index 326aa26..f024e91 100644 --- a/hosts/central-db-schema.nix +++ b/hosts/central-db-schema.nix @@ -43,6 +43,10 @@ let apps = { # Quilt (funding and records). Its own schema only, no shared schemas. quilt = { }; + # Selvage (the Slack bot): moderation and reports. Its own schema only, no shared schemas. + # Production only for now: hem:5433 creates the role too, but with no + # restricted-staging-selvage.age it cannot log in there. + selvage = { }; }; sharedSchemas = { }; }; diff --git a/hosts/twill/configuration.nix b/hosts/twill/configuration.nix new file mode 100644 index 0000000..e6473d7 --- /dev/null +++ b/hosts/twill/configuration.nix @@ -0,0 +1,86 @@ +# twill — Selvage host (pve01 VMID 363, 10.100.90.63, 3 GB, 20 GB disk) +# +# One app per VM. Selvage holds a Slack admin token (xoxp) and reads the +# restricted database, so sudo and the docker group here are root over both. +# Only the break-glass admins (modules/common) and lnx-twill-admins get them. +# Only this host's key decrypts the Selvage env file. +# Public traffic arrives from alastor over the tailnet +# (nixosModules.edge-alastor) as plain HTTP to nginx on tailscale0:80. +# +# Tailnet tags: tag:patchwork (ssh from admins, port 80 from alastor) and +# tag:patchwork-selvage. The policy in jaspermayone/infra must let +# tag:patchwork-selvage reach skein:5432 and seam:5432 only. +# +# Not built yet. After the install, add its host key to secrets/secrets.nix +# and re-key its secrets (README.md, "Selvage on twill"). +{ config, ... }: +{ + imports = [ + ../proxmox-vm.nix + ../../modules/common + ./selvage.nix + ]; + + networking.hostName = "twill"; + patchwork.lanAddress = "10.100.90.63"; + patchwork.banner.role = "selvage host"; + + virtualisation.docker.enable = true; + + # The lab hosts do not use MagicDNS (modules/common, --accept-dns=false), + # so *.ts.net names do not resolve here. The Selvage env file connects to + # skein and seam by tailnet name. The containers use the host network, and + # Docker copies this /etc/hosts into each one at start. If skein or seam + # joins the tailnet again, its address changes: update it here (and the + # aliases in jaspermayone/infra tailscale/policy.hujson). + networking.hosts = { + "100.109.223.51" = [ "skein.wildebeest-stargazer.ts.net" ]; + "100.87.183.57" = [ "seam.wildebeest-stargazer.ts.net" ]; + }; + + zramSwap.enable = true; + + services.nginx = + let + upstream = "http://127.0.0.1:${toString config.patchwork.services.selvage.port}"; + in + { + enable = true; + # Only alastor reaches this, over the tailnet. alastor terminated TLS and + # set X-Forwarded-For and X-Forwarded-Proto (https); pass them on as they + # are. recommendedProxySettings would set X-Forwarded-Proto to http. + commonHttpConfig = '' + set_real_ip_from 100.64.0.0/10; + real_ip_header X-Forwarded-For; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; + proxy_set_header X-Forwarded-Host $host; + ''; + virtualHosts.selvage = { + serverName = "selvage.patchworklabs.org"; + listen = [ + { + addr = "0.0.0.0"; + port = 80; + } + ]; + # Slack calls the three request URLs under /slack/ (POST only), and + # the health check is GET /up. Nothing else of the app is public. + locations."/slack/" = { + proxyPass = upstream; + extraConfig = '' + limit_except POST { deny all; } + ''; + }; + locations."= /up".proxyPass = upstream; + locations."/".return = "404"; + }; + }; + # nginx is a host service, so the NixOS firewall applies to it (Docker's + # published ports would bypass it). Open only on the tailnet. + networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 80 ]; + + system.stateVersion = "26.05"; +} diff --git a/hosts/twill/selvage.nix b/hosts/twill/selvage.nix new file mode 100644 index 0000000..e672d9c --- /dev/null +++ b/hosts/twill/selvage.nix @@ -0,0 +1,89 @@ +# Selvage on twill: the instance settings and the agenix secrets. nginx on +# twill (./configuration.nix) fronts the port, and alastor routes the public +# name here over the tailnet (nixosModules.edge-alastor). +# +# Production only. There is no Selvage staging instance yet, so no +# selvage-staging-env.age and no route for a staging name. hem already has +# Selvage's central role (central-staging-selvage.age) for a later staging. +# +# Network: the containers use the host network, and twill reaches +# skein.wildebeest-stargazer.ts.net:5432 (patchwork_central) and +# seam.wildebeest-stargazer.ts.net:5432 (patchwork_central_restricted) over +# the tailnet, through the tag:patchwork-selvage grants in jaspermayone/infra +# tailscale/policy.hujson. +# +# Before Selvage runs (README.md, "Selvage on twill", has the full order): +# - secrets/restricted-prod-selvage.age (jsp + seam) and +# secrets/selvage-env.age (jsp + twill): `bin/new-selvage-secrets`, then +# deploy seam so the selvage role there gets its password. skein already +# has its password (central-prod-selvage.age). +# - secrets/selvage-ghcr-token.age (jsp + twill), while the image package +# is private. +# - The Slack and Weave values in the env file: +# `bin/set-env-secret selvage-env.age SLACK_SIGNING_SECRET ...`. +# - DNS for selvage.patchworklabs.org, proxied to alastor +# (patchworklabsorg/dns). +# +# Each secret is declared only when its .age file exists, so twill builds +# before the secrets are written. Until the env file decrypts on twill, the +# units do not start (ConditionPathExists in modules/selvage). +{ + config, + lib, + ... +}: +let + envFile = ../../secrets/selvage-env.age; + ghcrFile = ../../secrets/selvage-ghcr-token.age; + hasEnv = builtins.pathExists envFile; + hasGhcr = builtins.pathExists ghcrFile; +in +{ + imports = [ ../../modules/selvage ]; + + # Recipients are in ../../secrets/secrets.nix (jsp + twill's host key). + age.secrets = + lib.optionalAttrs hasEnv { + selvage-env = { + file = envFile; + mode = "400"; + }; + } + // lib.optionalAttrs hasGhcr { + # GitHub classic PAT with read:packages, for the private image. + selvage-ghcr-token = { + file = ghcrFile; + mode = "400"; + }; + }; + + # The containers read their env file only when they start, and the file's + # path never changes, so a new secret alone restarts nothing. Restart the + # units when the encrypted file changes, so an `agenix -e` reaches Selvage + # on the next switch. The trigger is the file's hash, not its path: the path + # is inside the flake's source, which changes on every commit. + systemd.services = lib.mkIf hasEnv ( + let + prod = builtins.hashFile "sha256" envFile; + in + { + selvage-web.restartTriggers = [ prod ]; + selvage-worker.restartTriggers = [ prod ]; + } + ); + + patchwork.services.selvage = { + enable = true; + # Pin to ghcr.io/patchworklabsorg/selvage: once a build is known good. + image = "ghcr.io/patchworklabsorg/selvage:main"; + # 3005 and 3006 are Quilt on batting, 3007 and 3008 Stash on hank. The + # ports are per host, so this is only for the humans. + port = 3009; + # agenix's path for the secret `selvage-env`. A string, so that it is the + # same before and after the file exists. + environmentFile = "${config.age.secretsDir}/selvage-env"; + registryCredentialsFile = if hasGhcr then config.age.secrets.selvage-ghcr-token.path else null; + # The pwlbot PAT, the same token as quilt-ghcr-token.age. + registryUsername = "pwlbot"; + }; +} diff --git a/modules/selvage/default.nix b/modules/selvage/default.nix new file mode 100644 index 0000000..7784098 --- /dev/null +++ b/modules/selvage/default.nix @@ -0,0 +1,213 @@ +# modules/selvage/default.nix +# Selvage: the Patchwork Labs Slack bot (Rails 8.1 API app, GoodJob). Two +# Docker containers from one image: web and worker. Exported by this flake as +# nixosModules.selvage; the host that runs it imports hosts/twill/selvage.nix, +# which sets the instance options below. Same shape as modules/stash, with +# production only. +# +# Databases: Selvage has no database of its own. It uses two central +# databases, and in each one the role `selvage` owns the schema `selvage` +# (hosts/central-db-schema.nix): +# CENTRAL_DATABASE_URL patchwork_central on skein (general data, polls, +# GoodJob; reads people.*) +# RESTRICTED_DATABASE_URL patchwork_central_restricted on seam +# (moderation and reports) +# The env file holds both URLs, passwords included: +# postgres://selvage:@skein.wildebeest-stargazer.ts.net:5432/patchwork_central?sslmode=disable +# postgres://selvage:@seam.wildebeest-stargazer.ts.net:5432/patchwork_central_restricted?sslmode=disable +# The passwords must be the ones in secrets/central-prod-selvage.age (skein) +# and secrets/restricted-prod-selvage.age (seam). The database hosts have no +# server TLS yet (modules/central-db); Tailscale encrypts the path, hence +# sslmode=disable. +# +# The role has no CREATE on either database, so the app must not run CREATE +# SCHEMA or enable_extension. The web container migrates both databases with +# the image's entrypoint before the server starts (`db:migrate`, as Stash +# does). The worker never migrates. +# +# Network: both containers use the host network (`--network host`), as Stash +# and Quilt do, so the tailnet names in the env file resolve through the +# host's /etc/hosts. The web server binds 127.0.0.1:, and nginx on the +# host proxies to it. +# +# Units: +# selvage-web pulls the image, runs `./bin/rails server`. The image's +# entrypoint migrates before the server starts. +# selvage-worker `bundle exec good_job start`. Waits for GET /up on the +# web server (so the migrations ran), and restarts on +# failure until the schema is there. partOf selvage-web, so +# `systemctl restart selvage-web` moves both to a new image. +# +# Image: `image` is a full ref. The default tag `main` follows the newest push +# to main. Pin it to `ghcr.io/patchworklabsorg/selvage:` for a known +# build. The repo is private, so its package may be private too: set +# `registryCredentialsFile` to a token with read:packages, as Quilt does. +{ + config, + lib, + pkgs, + ... +}: +let + dockerBin = "${config.virtualisation.docker.package}/bin/docker"; + cfg = config.patchwork.services.selvage; + name = "selvage"; + + # `--env 'KEY=value'` arguments for extraEnv. Each line ends with the + # two-space indent the next docker argument expects. + envArgs = lib.concatStrings ( + lib.mapAttrsToList (k: v: "--env ${lib.escapeShellArg "${k}=${v}"} \\\n ") cfg.extraEnv + ); + + # Log in to ghcr.io (when a token is set) and pull the image. + pull = pkgs.writeShellScript "${name}-pull" '' + set -e + ${lib.optionalString (cfg.registryCredentialsFile != null) '' + ${dockerBin} login ghcr.io \ + --username ${lib.escapeShellArg cfg.registryUsername} \ + --password-stdin < ${lib.escapeShellArg cfg.registryCredentialsFile} + ''} + ${dockerBin} pull ${lib.escapeShellArg cfg.image} + ''; + + common = unit: { + after = [ "docker.service" ]; + requires = [ "docker.service" ]; + wantedBy = [ "multi-user.target" ]; + # Until the env file decrypts on this host (its key must be a recipient), + # the units do not start, instead of restarting and pulling every 10 s. + unitConfig.ConditionPathExists = cfg.environmentFile; + # The worker restarts until the schema exists. Never stop retrying. + startLimitIntervalSec = 0; + serviceConfig = { + Type = "simple"; + TimeoutStartSec = "10min"; + ExecStop = "${dockerBin} stop ${name}-${unit}"; + Restart = "on-failure"; + RestartSec = "10s"; + StandardOutput = "journal"; + StandardError = "journal"; + SyslogIdentifier = "${name}-${unit}"; + }; + }; + + run = unit: '' + ${dockerBin} run \ + --name ${name}-${unit} \ + --network host \ + --env-file ${lib.escapeShellArg cfg.environmentFile} \ + --env RAILS_ENV=production \ + ${envArgs}--restart no \ + ${lib.escapeShellArg cfg.image}''; +in +{ + options.patchwork.services.selvage = { + enable = lib.mkEnableOption "Selvage, the Patchwork Labs Slack bot"; + + port = lib.mkOption { + type = lib.types.port; + default = 3009; + description = "Loopback port the web server binds to. The host's reverse proxy proxies to it."; + }; + + image = lib.mkOption { + type = lib.types.str; + default = "ghcr.io/patchworklabsorg/selvage:main"; + example = "ghcr.io/patchworklabsorg/selvage:0123abc"; + description = '' + Image ref for web and worker. Pin it to a commit sha for a known build. + Each start of selvage-web pulls it again. + ''; + }; + + environmentFile = lib.mkOption { + type = lib.types.str; + example = "/run/agenix/selvage-env"; + description = '' + Path on the host of the env file (an agenix secret): CENTRAL_DATABASE_URL, + RESTRICTED_DATABASE_URL, SECRET_KEY_BASE, SLACK_SIGNING_SECRET, + SLACK_BOT_TOKEN, SLACK_ADMIN_TOKEN, SLACK_BROWSER_TOKEN, + SLACK_BROWSER_COOKIE, WEAVE_ISSUER, WEAVE_CLIENT_ID, + WEAVE_CLIENT_SECRET, SELVAGE_AUDIT_CHANNEL. A string, not a path, so + that the host can name it before the secret exists. + ''; + }; + + registryCredentialsFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + File with a GitHub token that has read:packages, for a private image. + Null: pull without a login (a public package). + ''; + }; + + registryUsername = lib.mkOption { + type = lib.types.str; + default = "pwlbot"; + description = "GitHub user that owns the token in registryCredentialsFile."; + }; + + extraEnv = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + example = { + RAILS_LOG_LEVEL = "debug"; + }; + description = '' + Extra container env for web and worker. It overrides the env file, so + do not put a value here that you want to change in the env file. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + # ── Web (Rails) ───────────────────────────────────────────────────────── + systemd.services."${name}-web" = lib.recursiveUpdate (common "web") { + description = "Selvage web"; + serviceConfig = { + ExecStartPre = [ + pull + "-${dockerBin} stop ${name}-web" + "-${dockerBin} rm ${name}-web" + ]; + # Same command as the image's CMD, on the loopback port. The + # entrypoint matches `./bin/rails server` and migrates first. + ExecStart = pkgs.writeShellScript "${name}-web-start" '' + exec ${run "web"} \ + ./bin/rails server -b 127.0.0.1 -p ${toString cfg.port} + ''; + }; + }; + + # ── Worker (GoodJob) ──────────────────────────────────────────────────── + # Runs the jobs and the GoodJob cron. It never migrates, so it waits for + # the web server, which migrates before it answers /up. One worker only. + systemd.services."${name}-worker" = lib.recursiveUpdate (common "worker") { + description = "Selvage worker"; + after = [ + "docker.service" + "${name}-web.service" + ]; + partOf = [ "${name}-web.service" ]; + serviceConfig = { + ExecStartPre = [ + pull + "-${dockerBin} stop ${name}-worker" + "-${dockerBin} rm ${name}-worker" + (pkgs.writeShellScript "${name}-wait-for-web" '' + echo "Waiting for the web server (migrations)..." + until ${pkgs.curl}/bin/curl -fsS -o /dev/null \ + http://127.0.0.1:${toString cfg.port}/up; do + sleep 5 + done + '') + ]; + ExecStart = pkgs.writeShellScript "${name}-worker-start" '' + exec ${run "worker"} \ + bundle exec good_job start + ''; + }; + }; + }; +} diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 155fa6d..fc048f6 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -18,6 +18,10 @@ let batting = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEN68vw2ir+q4fMQV8WBJvH7WDG4D2uFjChwkkA93LJC"; weft = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAbzni2Be82ORFcHSPVr7OPpfs80Py04JtwyCKc4L28X"; hank = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICKNDAtCadjdzqG+5t6BncycydzqfqV3z02uvIejmO8S"; + # twill (Selvage) is not built yet. After its install, add + # twill = ""; + # here, add twill to selvage-env.age, selvage-ghcr-token.age and + # ad-join.age below, then `agenix -r`. # # Central DB passwords, one file per app and environment, password only: # central-prod-.age recipients: jsp, skein @@ -214,6 +218,9 @@ in jsp skein ]; + # Selvage's role password in patchwork_central (skein). Password only. + # twill does not get this file: the same password is in selvage-env.age, so + # a rotation must change both files. "central-prod-selvage.age".publicKeys = [ jsp skein @@ -227,6 +234,41 @@ in seam ]; + # Selvage's role password in patchwork_central_restricted (seam). Password + # only. bin/new-selvage-secrets writes it. twill does not get this file: the + # same password is in selvage-env.age, so a rotation must change both files. + "restricted-prod-selvage.age".publicKeys = [ + jsp + seam + ]; + + # Selvage (selvage.patchworklabs.org on twill) runtime env. + # bin/new-selvage-secrets writes it. Must contain: + # CENTRAL_DATABASE_URL=postgres://selvage:@skein.wildebeest-stargazer.ts.net:5432/patchwork_central?sslmode=disable + # ( is the value in central-prod-selvage.age) + # RESTRICTED_DATABASE_URL=postgres://selvage:@seam.wildebeest-stargazer.ts.net:5432/patchwork_central_restricted?sslmode=disable + # ( is the value in restricted-prod-selvage.age) + # SECRET_KEY_BASE=... + # SLACK_SIGNING_SECRET=... SLACK_BOT_TOKEN=xoxb-... + # SLACK_ADMIN_TOKEN=xoxp-... + # SLACK_BROWSER_TOKEN=xoxc-... SLACK_BROWSER_COOKIE=xoxd-... (optional) + # WEAVE_ISSUER=https://weave.patchworklabs.org + # WEAVE_CLIENT_ID=... WEAVE_CLIENT_SECRET=... (the Selvage app in Weave) + # SELVAGE_AUDIT_CHANNEL=... + # twill is not built yet: add its host key here after the install, then + # `agenix -r`. + "selvage-env.age".publicKeys = [ + jsp + ]; + + # GitHub classic PAT with read:packages only, for + # ghcr.io/patchworklabsorg/selvage while the package is private. The same + # pwlbot token as quilt-ghcr-token.age. Add twill after its install, as for + # selvage-env.age. + "selvage-ghcr-token.age".publicKeys = [ + jsp + ]; + # Quilt's role password in patchwork_central_restricted on hem:5433 (the # restricted staging cluster). Password only. hem decrypts it into the # restricted-staging container (hosts/hem/configuration.nix). alastor does diff --git a/terraform/proxmox/README.md b/terraform/proxmox/README.md index 19f2423..618c64c 100644 --- a/terraform/proxmox/README.md +++ b/terraform/proxmox/README.md @@ -13,6 +13,7 @@ This root manages the Patchwork NixOS VMs on the Proxmox node `pve01`. It uses t | `batting` | 360 | 10.100.90.60 | 4 GB | | `weft` | 361 | 10.100.90.61 | 4 GB | | `hank` | 362 | 10.100.90.62 | 6 GB | +| `twill` | 363 | 10.100.90.63 | 3 GB | hem, batting and weft are in the state (created with `qm create` and `tofu import`). To change one VM, target it, for example `tofu apply -target='proxmox_virtual_environment_vm.nixos["weft"]'`. A plan for an imported VM shows `agent.timeout` 15m to 2m. That value is only in the state. diff --git a/terraform/proxmox/variables.tf b/terraform/proxmox/variables.tf index e82273f..6cae320 100644 --- a/terraform/proxmox/variables.tf +++ b/terraform/proxmox/variables.tf @@ -103,6 +103,14 @@ variable "vms" { disk_gb = 80 ip = "10.100.90.62" } + # Selvage: the Slack bot. Web and worker containers, no files of its own. + twill = { + vmid = 363 + memory_mb = 3072 + cores = 2 + disk_gb = 20 + ip = "10.100.90.63" + } } } From 01b8e0dd194d7ebbf3b26de2eaf51563b9a6d2e3 Mon Sep 17 00:00:00 2001 From: Jasper Mayone Date: Wed, 11 Nov 2026 23:08:31 -0500 Subject: [PATCH 2/2] fix(selvage): review findings: required env keys, explicit migrate, worker on the web image, safe secrets script --- README.md | 19 ++++++--- bin/new-selvage-secrets | 68 +++++++++++++++++++----------- hosts/twill/selvage.nix | 3 +- modules/selvage/default.nix | 82 ++++++++++++++++++++++++++++++------- secrets/secrets.nix | 7 ++-- 5 files changed, 133 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 03f3594..d16b158 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ To test top-ups on staging, add a sandbox `MERCURY_WRITE_TOKEN` with batting's p | Item | Value | |---|---| | URL | `https://selvage.patchworklabs.org` (router `selvage-twill` in `edge-alastor`). nginx on twill passes only `POST /slack/*` and `GET /up`. Every other path is 404. | -| Units | `selvage-web` (`./bin/rails server` on `127.0.0.1:3009`, migrates on start) and `selvage-worker` (`bundle exec good_job start`) | +| Units | `selvage-web` (`./bin/rails db:migrate` in a one-off container, then `./bin/rails server` on `127.0.0.1:3009`) and `selvage-worker` (`bundle exec good_job start`, on the web container's image) | | Image | `ghcr.io/patchworklabsorg/selvage:main` (option `image`). Pin it to `:` for a known build. | | Databases | `patchwork_central` on skein and `patchwork_central_restricted` on seam. Role and schema `selvage` in each. | | Secrets | `selvage-env.age`, `selvage-ghcr-token.age` (twill), `central-prod-selvage.age` (skein), `restricted-prod-selvage.age` (seam) | @@ -336,7 +336,9 @@ flowchart LR worker --> seam ``` -The env file has `CENTRAL_DATABASE_URL` and `RESTRICTED_DATABASE_URL`. Their passwords must be the values in `central-prod-selvage.age` and `restricted-prod-selvage.age`. skein and seam set those values on the role `selvage`. The role has no `CREATE` on either database, so the image must migrate with `db:migrate`, not `db:prepare`, and must not run `CREATE SCHEMA` or `enable_extension`. +The env file has `CENTRAL_DATABASE_URL` and `RESTRICTED_DATABASE_URL`. Their passwords must be the values in `central-prod-selvage.age` and `restricted-prod-selvage.age`. skein and seam set those values on the role `selvage`. The role has no `CREATE` on either database. `selvage-web` runs `db:migrate`, never `db:prepare`, and the migrations must not run `CREATE SCHEMA` or `enable_extension`. + +The units do not start while `CENTRAL_DATABASE_URL`, `RESTRICTED_DATABASE_URL`, `SECRET_KEY_BASE`, `SLACK_SIGNING_SECRET` or `SLACK_BOT_TOKEN` is missing or empty in the env file (option `requiredEnv`). An empty signing secret would let anyone sign a Slack request. To deploy a new `main` image, run `sudo systemctl restart selvage-web` on twill. That restarts the worker too. @@ -346,15 +348,22 @@ Do the steps in this order. Steps 5 to 11 are on this repo's branch, before the 1. Merge the tailnet policy change in `jaspermayone/infra` (`tag:patchwork-selvage`, with grants to skein:5432 and seam:5432). 2. Merge the DNS record `selvage` (proxied to alastor) in `patchworklabsorg/dns`. -3. In `patchworklabsorg/selvage`, make sure that `image.yml` pushed `ghcr.io/patchworklabsorg/selvage:main`, and that `bin/docker-entrypoint` runs `db:migrate`. +3. In `patchworklabsorg/selvage`, merge the image workflow and make sure that `image.yml` pushed `ghcr.io/patchworklabsorg/selvage:main`. The migrations must not run `CREATE SCHEMA` or `enable_extension`. 4. Make sure that VMID 363 and `10.100.90.63` are free, and that pve01 has 3 GiB of memory to spare (`terraform/proxmox/README.md`, "Memory warning"). 5. Prepare AD for twill: run `.\converge.ps1` (it creates `twill-admins`), then `cd ansible && ansible-playbook ad.yml -t host-groups`. See "Login". 6. Create and install the VM: `bin/install-host twill`. 7. On twill, join the tailnet with both tags: `sudo tailscale up --advertise-tags=tag:patchwork,tag:patchwork-selvage`. 8. Get the host key: `ssh-keyscan -t ed25519 10.100.90.63`. Add it as `twill` in `secrets/secrets.nix`. Add `twill` to `selvage-env.age`, `selvage-ghcr-token.age` and `ad-join.age`. Remove the "not built yet" notes. 9. Write the secrets: `nix develop -c bin/new-selvage-secrets`. It writes `restricted-prod-selvage.age`, `selvage-env.age` and `selvage-ghcr-token.age`. It decrypts `central-prod-selvage.age` and `quilt-ghcr-token.age` with your age identity (`~/.ssh/id_ed25519`, or `AGE_IDENTITY`). -10. Re-key, so that twill can decrypt `ad-join.age`: `cd secrets && agenix -r`. -11. Fill in the Slack and Weave values: `bin/set-env-secret selvage-env.age SLACK_SIGNING_SECRET SLACK_BOT_TOKEN SLACK_ADMIN_TOKEN WEAVE_CLIENT_ID WEAVE_CLIENT_SECRET SELVAGE_AUDIT_CHANNEL`. Add `SLACK_BROWSER_TOKEN` and `SLACK_BROWSER_COOKIE` the same way when you need them. +10. Re-encrypt only `ad-join.age`, so that twill can decrypt it. Do not run `agenix -r`: it re-encrypts every file, and the hash-based restart triggers then restart Stash and Krater with no change. + + ```bash + cd secrets + age -d -i ~/.ssh/id_ed25519 ad-join.age | age -e -o ad-join.age.new \ + -R <(nix eval --raw -f secrets.nix '"ad-join.age".publicKeys' --apply 'k: builtins.concatStringsSep "\n" k') + mv ad-join.age.new ad-join.age + ``` +11. Fill in the Slack and Weave values. Selvage does not start without `SLACK_SIGNING_SECRET` and `SLACK_BOT_TOKEN`: `bin/set-env-secret selvage-env.age SLACK_SIGNING_SECRET SLACK_BOT_TOKEN SLACK_ADMIN_TOKEN WEAVE_CLIENT_ID WEAVE_CLIENT_SECRET SELVAGE_AUDIT_CHANNEL`. Add `SLACK_BROWSER_TOKEN` and `SLACK_BROWSER_COOKIE` the same way when you need them. 12. Commit the `.age` files and `secrets.nix`, and merge after CI passes. The lab hosts follow `main`, so seam creates the role `selvage` with its password, and twill starts Selvage. To do it at once: `deploy .#seam`, then `deploy .#twill`. 13. On seam, make sure that `central-db-setup` succeeded: `journalctl -u central-db-setup`. On twill, check `systemctl status selvage-web selvage-worker` and `curl -i http://127.0.0.1:3009/up`. 14. Route the public name: in `jaspermayone/infra`, run `nix flake update patchwork-infra`, then `deploy .#alastor`. diff --git a/bin/new-selvage-secrets b/bin/new-selvage-secrets index 06ad8c7..57a1b08 100755 --- a/bin/new-selvage-secrets +++ b/bin/new-selvage-secrets @@ -17,11 +17,14 @@ # RESTRICTED_DATABASE_URL with the new password above. # SECRET_KEY_BASE a new random value. # WEAVE_ISSUER https://weave.patchworklabs.org (not secret). -# The other keys empty: SLACK_SIGNING_SECRET, SLACK_BOT_TOKEN, -# SLACK_ADMIN_TOKEN, SLACK_BROWSER_TOKEN, SLACK_BROWSER_COOKIE, -# WEAVE_CLIENT_ID, WEAVE_CLIENT_SECRET, SELVAGE_AUDIT_CHANNEL. A key that is -# set in the environment is used instead, e.g. +# SLACK_SIGNING_SECRET, SLACK_BOT_TOKEN, SLACK_ADMIN_TOKEN, +# SLACK_BROWSER_TOKEN, SLACK_BROWSER_COOKIE, WEAVE_CLIENT_ID, +# WEAVE_CLIENT_SECRET and SELVAGE_AUDIT_CHANNEL only when they are set in +# the environment, e.g. # set -a; . ./selvage-third-party.env; set +a; bin/new-selvage-secrets +# A key that is not set is left out of the file, never written empty: an +# empty SLACK_SIGNING_SECRET would let anyone sign a Slack request. +# modules/selvage does not start Selvage until the Slack keys are set. # Fill them in later with (asks for each value with the echo off) # bin/set-env-secret selvage-env.age SLACK_SIGNING_SECRET SLACK_BOT_TOKEN ... # or: cd secrets && agenix -e selvage-env.age @@ -50,6 +53,8 @@ quilt_ghcr_file=quilt-ghcr-token.age central_host=skein.wildebeest-stargazer.ts.net restricted_host=seam.wildebeest-stargazer.ts.net identity="${AGE_IDENTITY:-$HOME/.ssh/id_ed25519}" +optional_keys=(SLACK_SIGNING_SECRET SLACK_BOT_TOKEN SLACK_ADMIN_TOKEN SLACK_BROWSER_TOKEN + SLACK_BROWSER_COOKIE WEAVE_CLIENT_ID WEAVE_CLIENT_SECRET SELVAGE_AUDIT_CHANNEL) for tool in age nix openssl; do if ! command -v "$tool" >/dev/null; then @@ -64,15 +69,16 @@ if [ ! -r "$identity" ]; then fi # Encrypts stdin to the recipients that secrets.nix lists for the file. +# $1 names the file in secrets.nix, $2 is where the ciphertext goes. encrypt() { - local file="$1" recipients + local file="$1" out="$2" recipients recipients="$(nix eval --raw -f secrets.nix "\"$file\".publicKeys" --apply 'keys: builtins.concatStringsSep "\n" keys')" if [ -z "$recipients" ]; then echo "secrets.nix lists no recipients for $file. Stop." >&2 exit 1 fi - age --encrypt --recipients-file <(printf '%s\n' "$recipients") --output "$file" - chmod 644 "$file" + age --encrypt --recipients-file <(printf '%s\n' "$recipients") --output "$out" + chmod 644 "$out" } # Percent-encodes a URL userinfo part. Hex passwords pass through unchanged. @@ -112,40 +118,55 @@ fi umask 077 -# The existing central password, in this variable only. $(...) drops the -# trailing newline. +# Read every input before anything is written, so a failed decrypt (a wrong +# passphrase) leaves no file behind. The plaintext lives in variables only. +# $(...) drops the trailing newline. central_password="$(age --decrypt -i "$identity" "$central_file")" if [ -z "$central_password" ]; then echo "secrets/$central_file decrypted to an empty value. Stop." >&2 exit 1 fi +token="" +if [ "$copy_ghcr" = 1 ]; then + token="$(age --decrypt -i "$identity" "$quilt_ghcr_file")" + if [ -z "$token" ]; then + echo "secrets/$quilt_ghcr_file decrypted to an empty value. Stop." >&2 + exit 1 + fi +fi # Hex only, so the password needs no escaping in the URL. restricted_password="$(openssl rand -hex 32)" secret_key_base="$(openssl rand -hex 64)" -# Write the env file first: when it fails, no new password file is left over. +# Encrypt to temporary names, then move all of them into place at the end. +tmpdir="$(mktemp -d "$PWD/.new-selvage-secrets.XXXXXX")" +trap 'rm -rf "$tmpdir"' EXIT + # printf is a shell builtin, so the values are not in any process's arguments. { printf '%s\n' "CENTRAL_DATABASE_URL=postgres://selvage:$(urlencode "$central_password")@${central_host}:5432/patchwork_central?sslmode=disable" printf '%s\n' "RESTRICTED_DATABASE_URL=postgres://selvage:${restricted_password}@${restricted_host}:5432/patchwork_central_restricted?sslmode=disable" printf '%s\n' "SECRET_KEY_BASE=${secret_key_base}" printf '%s\n' "WEAVE_ISSUER=https://weave.patchworklabs.org" - for key in SLACK_SIGNING_SECRET SLACK_BOT_TOKEN SLACK_ADMIN_TOKEN SLACK_BROWSER_TOKEN \ - SLACK_BROWSER_COOKIE WEAVE_CLIENT_ID WEAVE_CLIENT_SECRET SELVAGE_AUDIT_CHANNEL; do - printf '%s=%s\n' "$key" "${!key:-}" + for key in "${optional_keys[@]}"; do + if [ -n "${!key:-}" ]; then + printf '%s=%s\n' "$key" "${!key}" + fi done -} | encrypt "$env_file" - -printf '%s' "$restricted_password" | encrypt "$db_file" +} | encrypt "$env_file" "$tmpdir/$env_file" -unset central_password restricted_password secret_key_base +printf '%s' "$restricted_password" | encrypt "$db_file" "$tmpdir/$db_file" +if [ "$copy_ghcr" = 1 ]; then + printf '%s' "$token" | encrypt "$ghcr_file" "$tmpdir/$ghcr_file" +fi +unset central_password restricted_password secret_key_base token written="secrets/$env_file and secrets/$db_file" +mv "$tmpdir/$env_file" "$env_file" +mv "$tmpdir/$db_file" "$db_file" if [ "$copy_ghcr" = 1 ]; then - token="$(age --decrypt -i "$identity" "$quilt_ghcr_file")" - printf '%s' "$token" | encrypt "$ghcr_file" - unset token + mv "$tmpdir/$ghcr_file" "$ghcr_file" written="$written and secrets/$ghcr_file" fi @@ -158,7 +179,8 @@ for key in SLACK_SIGNING_SECRET SLACK_BOT_TOKEN SLACK_ADMIN_TOKEN WEAVE_CLIENT_I fi done if [ -n "$missing" ]; then - echo "Left empty:$missing" - echo "Fill them in with: bin/set-env-secret $env_file$missing" + echo "Not set:$missing" + echo "Set them before the merge, with: bin/set-env-secret $env_file$missing" + echo "Selvage does not start without SLACK_SIGNING_SECRET and SLACK_BOT_TOKEN." fi -echo "Next: commit the files, then deploy seam so the selvage role there gets its password." +echo "Next: commit the files. The merge deploys seam, which sets the selvage role's password." diff --git a/hosts/twill/selvage.nix b/hosts/twill/selvage.nix index e672d9c..e9d0ea5 100644 --- a/hosts/twill/selvage.nix +++ b/hosts/twill/selvage.nix @@ -20,7 +20,8 @@ # - secrets/selvage-ghcr-token.age (jsp + twill), while the image package # is private. # - The Slack and Weave values in the env file: -# `bin/set-env-secret selvage-env.age SLACK_SIGNING_SECRET ...`. +# `bin/set-env-secret selvage-env.age SLACK_SIGNING_SECRET ...`. The +# units do not start without SLACK_SIGNING_SECRET and SLACK_BOT_TOKEN. # - DNS for selvage.patchworklabs.org, proxied to alastor # (patchworklabsorg/dns). # diff --git a/modules/selvage/default.nix b/modules/selvage/default.nix index 7784098..3ea528e 100644 --- a/modules/selvage/default.nix +++ b/modules/selvage/default.nix @@ -21,9 +21,14 @@ # sslmode=disable. # # The role has no CREATE on either database, so the app must not run CREATE -# SCHEMA or enable_extension. The web container migrates both databases with -# the image's entrypoint before the server starts (`db:migrate`, as Stash -# does). The worker never migrates. +# SCHEMA or enable_extension. selvage-web runs `./bin/rails db:migrate` (both +# databases) in a one-off container before the server starts, so the +# migration does not depend on how the image's entrypoint matches its +# arguments. The worker never migrates. +# +# Required keys: the units refuse to start while a key in `requiredEnv` is +# missing or empty in the env file. An empty SLACK_SIGNING_SECRET would let +# anyone sign a Slack request with the empty key. # # Network: both containers use the host network (`--network host`), as Stash # and Quilt do, so the tailnet names in the env file resolve through the @@ -31,11 +36,12 @@ # host proxies to it. # # Units: -# selvage-web pulls the image, runs `./bin/rails server`. The image's -# entrypoint migrates before the server starts. +# selvage-web pulls the image, runs `./bin/rails db:migrate`, then +# `./bin/rails server`. # selvage-worker `bundle exec good_job start`. Waits for GET /up on the -# web server (so the migrations ran), and restarts on -# failure until the schema is there. partOf selvage-web, so +# web server (so the migrations ran), then runs the image +# that the web container runs, not a new pull, so it never +# runs code newer than the schema. partOf selvage-web, so # `systemctl restart selvage-web` moves both to a new image. # # Image: `image` is a full ref. The default tag `main` follows the newest push @@ -70,6 +76,22 @@ let ${dockerBin} pull ${lib.escapeShellArg cfg.image} ''; + # Stops the start when a required key is missing or empty in the env file. + checkEnv = pkgs.writeShellScript "${name}-check-env" '' + set -eu + missing="" + for key in ${lib.escapeShellArgs cfg.requiredEnv}; do + if ! ${pkgs.gnugrep}/bin/grep -Eq "^$key=.+" ${lib.escapeShellArg cfg.environmentFile}; then + missing="$missing $key" + fi + done + if [ -n "$missing" ]; then + echo "Empty or missing in ${cfg.environmentFile}:$missing" >&2 + echo "Set them with bin/set-env-secret selvage-env.age$missing" >&2 + exit 1 + fi + ''; + common = unit: { after = [ "docker.service" ]; requires = [ "docker.service" ]; @@ -91,14 +113,15 @@ let }; }; - run = unit: '' + # `image` is an argument: the worker runs the web container's image ID. + run = unit: image: '' ${dockerBin} run \ --name ${name}-${unit} \ --network host \ --env-file ${lib.escapeShellArg cfg.environmentFile} \ --env RAILS_ENV=production \ ${envArgs}--restart no \ - ${lib.escapeShellArg cfg.image}''; + ${image}''; in { options.patchwork.services.selvage = { @@ -148,6 +171,21 @@ in description = "GitHub user that owns the token in registryCredentialsFile."; }; + requiredEnv = lib.mkOption { + type = lib.types.listOf (lib.types.strMatching "[A-Z][A-Z0-9_]*"); + default = [ + "CENTRAL_DATABASE_URL" + "RESTRICTED_DATABASE_URL" + "SECRET_KEY_BASE" + "SLACK_SIGNING_SECRET" + "SLACK_BOT_TOKEN" + ]; + description = '' + Keys that must be set and not empty in the env file. The units do not + start without them. + ''; + }; + extraEnv = lib.mkOption { type = lib.types.attrsOf lib.types.str; default = { }; @@ -167,14 +205,26 @@ in description = "Selvage web"; serviceConfig = { ExecStartPre = [ + checkEnv pull "-${dockerBin} stop ${name}-web" "-${dockerBin} rm ${name}-web" + "-${dockerBin} rm -f ${name}-migrate" + # Both databases. The role cannot CREATE, so never db:prepare. + (pkgs.writeShellScript "${name}-migrate" '' + exec ${dockerBin} run --rm \ + --name ${name}-migrate \ + --network host \ + --env-file ${lib.escapeShellArg cfg.environmentFile} \ + --env RAILS_ENV=production \ + ${envArgs}--entrypoint "" \ + ${lib.escapeShellArg cfg.image} \ + ./bin/rails db:migrate + '') ]; - # Same command as the image's CMD, on the loopback port. The - # entrypoint matches `./bin/rails server` and migrates first. + # Same command as the image's CMD, on the loopback port. ExecStart = pkgs.writeShellScript "${name}-web-start" '' - exec ${run "web"} \ + exec ${run "web" (lib.escapeShellArg cfg.image)} \ ./bin/rails server -b 127.0.0.1 -p ${toString cfg.port} ''; }; @@ -192,7 +242,7 @@ in partOf = [ "${name}-web.service" ]; serviceConfig = { ExecStartPre = [ - pull + checkEnv "-${dockerBin} stop ${name}-worker" "-${dockerBin} rm ${name}-worker" (pkgs.writeShellScript "${name}-wait-for-web" '' @@ -203,8 +253,12 @@ in done '') ]; + # The image ID of the running web container, so web and worker run + # the same build even when `main` moved since the web unit started. ExecStart = pkgs.writeShellScript "${name}-worker-start" '' - exec ${run "worker"} \ + set -e + image="$(${dockerBin} inspect -f '{{.Image}}' ${name}-web)" + exec ${run "worker" "\"$image\""} \ bundle exec good_job start ''; }; diff --git a/secrets/secrets.nix b/secrets/secrets.nix index fc048f6..263d0e0 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -21,7 +21,8 @@ let # twill (Selvage) is not built yet. After its install, add # twill = ""; # here, add twill to selvage-env.age, selvage-ghcr-token.age and - # ad-join.age below, then `agenix -r`. + # ad-join.age below, then re-encrypt ad-join.age alone (README.md, + # "Selvage on twill", step 10). # # Central DB passwords, one file per app and environment, password only: # central-prod-.age recipients: jsp, skein @@ -255,8 +256,8 @@ in # WEAVE_ISSUER=https://weave.patchworklabs.org # WEAVE_CLIENT_ID=... WEAVE_CLIENT_SECRET=... (the Selvage app in Weave) # SELVAGE_AUDIT_CHANNEL=... - # twill is not built yet: add its host key here after the install, then - # `agenix -r`. + # twill is not built yet: add its host key here after the install, before + # bin/new-selvage-secrets writes this file. "selvage-env.age".publicKeys = [ jsp ];