diff --git a/.gitattributes b/.gitattributes index 52b3b6d..e62d97f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ # Run on Linux even from a Windows checkout (core.autocrlf would turn them CRLF). bin/new-krater-secrets text eol=lf +bin/new-stash-secrets text eol=lf diff --git a/README.md b/README.md index 51ead1b..abc9233 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ Declarative infrastructure for Patchwork Labs: every Patchwork host, the NixOS m | `modules/weave/` | The Weave service module. | | `modules/quilt/` | The Quilt service module. | | `modules/krater/` | The Krater service module (portal, worker, screenshot storage; Krater Compute off by default). Exported as `nixosModules.krater`. | +| `modules/stash/` | The Stash service module (web, worker, archived files on a host directory). | +| `hosts/hank/stash.nix` | Stash production and staging on hank: instance settings and secrets. | +| `bin/new-stash-secrets` | Writes the two secrets of Stash staging or production with a new random database password and `SECRET_KEY_BASE`. Prints no value. | | `hosts/weft/krater.nix` | Krater staging and production on weft: instance settings and secrets. Production is off until `hosts/weft/krater-production.nix` turns it on. | | `tests/` | NixOS VM tests. | | `secrets/` | agenix secrets. `secrets.nix` lists the recipients of each file. | @@ -46,6 +49,7 @@ The lab hosts below are NixOS VMs on pve01, VLAN 90 (`patchwork.internal`). alas | `hem` | 358 | 10.100.90.58 | 2 GB | Both staging clusters: central on 5432, restricted on 5433. Quilt staging uses the restricted one. | | `batting` | 360 | 10.100.90.60 | 4 GB | Quilt prod + staging. nginx on `tailscale0:80` for alastor. | | `weft` | 361 | 10.100.90.61 | 4 GB | Krater staging, and Krater production (off until its switch). nginx on `tailscale0:80` for alastor. | +| `hank` | 362 | 10.100.90.62 | 6 GB | Stash prod + staging (declared, not built yet). 80 GB disk for the archived files. nginx on `tailscale0:80` for alastor. | The Windows hosts are managed by Ansible (`ansible/`): @@ -64,10 +68,12 @@ flowchart LR alastor -- "tailnet, HTTP :80" --> warp["warp: misc"] alastor -- "tailnet, HTTP :80" --> batting["batting: Quilt"] alastor -- "tailnet, HTTP :80" --> weft["weft: Krater staging"] + alastor -- "tailnet, HTTP :80" --> hank["hank: Stash (not built yet)"] batting -- "tailnet :5432" --> seam["seam: patchwork_central_restricted"] batting -. "staging :5433" .-> hem["hem: :5432 / :5433"] weft -. "staging :5432" .-> hem - skein["skein: patchwork_central (no app connects yet)"] + hank -- "tailnet :5432" --> skein["skein: patchwork_central"] + hank -. "staging :5432" .-> hem ``` ## Login diff --git a/ad/TIERING.md b/ad/TIERING.md index e69fed5..f267dcc 100644 --- a/ad/TIERING.md +++ b/ad/TIERING.md @@ -95,7 +95,7 @@ The rule: an account of tier N logs on only to assets of tier N. A higher tier c | Tier | Assets | Accounts | |---|---|---| | **0** | loom, spindle (DCs). bobbin (issuing CA). pattern (offline root CA). shuttle (NPS). The Proxmox hosts that run these VMs. The proposed Tier 0 jump host. remus, as long as it holds the `svc-ansible` password. | `adm_mayonej`, `svc-ansible`, `Administrator` (disabled), DSRM (LAPS-DSRM) | -| **1** | warp, batting, weft, skein, seam, hem (NixOS). thimble (VyOS edge). App servers. Weave and the host that runs it (alastor). | `lnx--admins` members. `svc-lnx-join`, `svc-pve-ldap`, `svc-dhcp-dns`, `svc-weave-ldap`. No dedicated Tier 1 person account yet. | +| **1** | warp, batting, weft, hank, skein, seam, hem (NixOS). thimble (VyOS edge). App servers. Weave and the host that runs it (alastor). | `lnx--admins` members. `svc-lnx-join`, `svc-pve-ldap`, `svc-dhcp-dns`, `svc-weave-ldap`. No dedicated Tier 1 person account yet. | | **2** | Staff workstations (`Computers` OU). VPN sessions. | Daily accounts: `mayonej`, staff, members | ```mermaid @@ -113,7 +113,7 @@ flowchart TB bg["Administrator (disabled, break-glass)"] end subgraph T1["Tier 1: servers"] - nix["warp, batting, weft, skein, seam, hem"] + nix["warp, batting, weft, hank, skein, seam, hem"] thimble["thimble VPN edge"] weave["Weave on alastor, svc-weave-ldap"] end diff --git a/ad/org.json b/ad/org.json index 3c02b09..d4c941f 100644 --- a/ad/org.json +++ b/ad/org.json @@ -76,6 +76,7 @@ { "name": "secretary", "formerNames": ["GG-Secretary"], "description": "Secretary", "members": ["klukask"] }, { "name": "ganymede-admins", "formerNames": ["GG-Ganymede-Admins"], "description": "Project Ganymede admins", "members": ["mayonej", "adam.sidwa"] }, { "name": "weft-admins", "description": "Root on weft (Krater): logon and sudo on weft only, through hostGroupMembers. Sudo and the docker group on weft give every Krater secret.", "members": ["mayonej", "adam.sidwa"] }, + { "name": "hank-admins", "description": "Root on hank (Stash): logon and sudo on hank only, through hostGroupMembers. Sudo and the docker group on hank give every Stash secret and every archived file.", "members": ["mayonej"] }, { "name": "ganymede-users", "formerNames": ["GG-Ganymede-Users"], "description": "Project Ganymede users", "members": [], "manageMembers": false }, { "name": "hackboston-organizers", "formerNames": ["GG-HackBoston-Organizers"], "description": "HackBoston organizers", "members": ["cj", "daniel", "logan", "mayonej"] }, { "name": "hackathonhelp-organizers", "formerNames": ["GG-HackathonHelp-Organizers"], "description": "hackathon.help organizers", "members": ["wei"] }, @@ -93,14 +94,15 @@ "members": [] } ], - "linuxHosts": ["batting", "hem", "seam", "skein", "warp", "weft"], + "linuxHosts": ["batting", "hank", "hem", "seam", "skein", "warp", "weft"], "retiredHosts": ["sampler", "swatch"], "hostGroups": [ { "name": "lnx-{host}-users", "description": "Linux logon on {host}", "members": ["infra"] }, { "name": "lnx-{host}-admins", "description": "Linux sudo on {host}", "members": ["infra"] } ], "hostGroupMembers": { - "weft": { "users": ["weft-admins"], "admins": ["weft-admins"] } + "weft": { "users": ["weft-admins"], "admins": ["weft-admins"] }, + "hank": { "users": ["hank-admins"], "admins": ["hank-admins"] } }, "resourceGroups": [ { "name": "password-reset", "formerNames": ["DL-Users-PasswordReset"], "description": "Reset password, unlock, change at next logon: Users/Staff, Board-Only, Sponsored-Projects", diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml index ec6a0fc..c8991eb 100644 --- a/ansible/inventory/hosts.yml +++ b/ansible/inventory/hosts.yml @@ -2,7 +2,7 @@ # ============================================================================ # Patchwork hosts that Ansible manages: the Windows servers only. # --------------------------------------------------------------------------- -# The NixOS hosts (warp, batting, weft, skein, seam, hem) are declared in ../flake.nix. They +# The NixOS hosts (warp, batting, weft, hank, skein, seam, hem) are declared in ../flake.nix. They # join AD through modules/ad-join, not through Ansible. Do not add them here. # ad.yml -t host-groups reads their names from the flake. # @@ -49,6 +49,6 @@ all: # # warp, batting, NixOS, declared in ../flake.nix. Joined to AD by # weft, skein, modules/ad-join. - # seam, hem + # seam, hem, hank # pattern VMID 399. Offline root CA. Stopped. Never a target. # ======================================================================== diff --git a/bin/new-stash-secrets b/bin/new-stash-secrets new file mode 100755 index 0000000..e3fd3c7 --- /dev/null +++ b/bin/new-stash-secrets @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Write one Stash instance's two secrets with new random values: +# staging secrets/central-staging-stash.age the password of the role +# stash in patchwork_central +# on hem (password only) +# secrets/stash-staging-env.age Stash staging's env file +# production secrets/central-prod-stash.age the same on skein +# secrets/stash-env.age Stash production's env file +# Both files get the same password. The env file also gets a new +# SECRET_KEY_BASE. WEAVE_CLIENT_ID, WEAVE_CLIENT_SECRET and GITHUB_TOKEN come +# from the environment when they are set there, e.g. +# set -a; . ./stash-third-party.env; set +a; bin/new-stash-secrets staging +# and are left empty otherwise. Only the file's recipients (Jasper, hank) can +# fill them in later, with +# cd secrets && agenix -e stash-staging-env.age +# The issuer and the hostnames are not secret: hosts/hank/stash.nix sets +# staging's, and production's are in the env file as plain values. +# +# Usage: bin/new-stash-secrets staging|production [database host] +# The database host defaults to hem's (staging) or skein's (production) +# tailnet name, for a Stash host that reaches it over Tailscale (hank, with +# the name pinned in networking.hosts). +# Needs age, nix and openssl. The script never prints a value, and it stops if +# one of the files exists. The recipients come from secrets/secrets.nix: the +# script encrypts with `age` itself, because `agenix -e` from a pipe fails on +# macOS ("cp: skipping file '/dev/stdin'") and reports nothing. +set -euo pipefail + +case "${1:-}" in + staging) + db_file=central-staging-stash.age + env_file=stash-staging-env.age + db_host="${2:-hem.wildebeest-stargazer.ts.net}" + hosts=stash-staging.patchworklabs.org + issuer=https://weave-staging.patchworklabs.org + ;; + production) + db_file=central-prod-stash.age + env_file=stash-env.age + db_host="${2:-skein.wildebeest-stargazer.ts.net}" + hosts=stash.patchworklabs.org + issuer=https://weave.patchworklabs.org + ;; + *) + echo "Usage: $0 staging|production [database host]" >&2 + exit 1 + ;; +esac + +root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$root/secrets" + +for tool in age nix openssl; do + if ! command -v "$tool" >/dev/null; then + echo "$tool is not on PATH. Run this script in a shell that has it." >&2 + exit 1 + fi +done + +# Encrypts stdin to the recipients that secrets.nix lists for the file. +encrypt() { + local file="$1" recipients + recipients="$(nix eval --raw -f secrets.nix "\"$file\".publicKeys" --apply 'keys: builtins.concatStringsSep "\n" keys')" + if [ -z "$recipients" ]; then + echo "secrets.nix lists no recipients for $file. Stop." >&2 + exit 1 + fi + age --encrypt --recipients-file <(printf '%s\n' "$recipients") --output "$file" + chmod 644 "$file" +} + +for file in "$db_file" "$env_file"; do + if [ -e "$file" ]; then + echo "secrets/$file exists. Stop. To change a value, use: agenix -e $file" >&2 + exit 1 + fi +done + +umask 077 +# Hex only, so the password needs no escaping in the URL. +password="$(openssl rand -hex 32)" +secret_key_base="$(openssl rand -hex 64)" + +# printf is a shell builtin, so the values are not in any process's arguments. +printf '%s' "$password" | encrypt "$db_file" + +{ + printf '%s\n' "CENTRAL_DATABASE_URL=postgres://stash:${password}@${db_host}:5432/patchwork_central?sslmode=disable" + printf '%s\n' "SECRET_KEY_BASE=${secret_key_base}" + printf '%s\n' "STASH_HOSTS=${hosts}" + printf '%s\n' "WEAVE_ISSUER=${issuer}" + printf '%s\n' "WEAVE_CLIENT_ID=${WEAVE_CLIENT_ID:-}" + printf '%s\n' "WEAVE_CLIENT_SECRET=${WEAVE_CLIENT_SECRET:-}" + printf '%s\n' "GITHUB_TOKEN=${GITHUB_TOKEN:-}" +} | encrypt "$env_file" + +unset password secret_key_base + +echo "Wrote secrets/$db_file and secrets/$env_file." +missing="" +for key in WEAVE_CLIENT_ID WEAVE_CLIENT_SECRET; do + if [ -z "${!key:-}" ]; then + missing="$missing $key" + fi +done +if [ -n "$missing" ]; then + echo "Left empty:$missing" + echo "Jasper fills them in with: cd secrets && agenix -e $env_file" +fi diff --git a/flake.nix b/flake.nix index 39a8ea7..16cc995 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,7 @@ "warp" # misc: redirects and other low-risk shared things "batting" # Quilt "weft" # Krater + "hank" # Stash "skein" # patchwork_central, prod "seam" # patchwork_central_restricted, prod "hem" # both staging clusters @@ -79,6 +80,9 @@ # The Krater service module alone, with no instance settings. weft # imports it through hosts/weft/krater.nix. krater = ./modules/krater; + # The Stash service module alone, with no instance settings. hank + # imports it through hosts/hank/stash.nix. + stash = ./modules/stash; }; nixosConfigurations = nixpkgs.lib.genAttrs hosts mkHost; @@ -136,7 +140,12 @@ ansible-filters = pkgs.runCommand "ansible-filters" { - nativeBuildInputs = [ (pkgs.python3.withPackages (p: [ p.jinja2 p.pyyaml ])) ]; + nativeBuildInputs = [ + (pkgs.python3.withPackages (p: [ + p.jinja2 + p.pyyaml + ])) + ]; src = ./ansible; } '' @@ -193,6 +202,8 @@ # agenix -e / -r, and bin/new-quilt-staging-secrets. agenix.packages.${system}.default pkgs.openssl + # bin/new-stash-secrets encrypts with age itself. + pkgs.age ]; }; } diff --git a/hosts/alastor/edge.nix b/hosts/alastor/edge.nix index 4a1c5ff..b2e676d 100644 --- a/hosts/alastor/edge.nix +++ b/hosts/alastor/edge.nix @@ -42,6 +42,15 @@ let backend = "weft"; hosts = [ "krater-staging-storage.patchworklabs.org" ]; }; + # Stash on hank (../hank/stash.nix): production and staging. + stash-hank = { + backend = "hank"; + hosts = [ "stash.patchworklabs.org" ]; + }; + stash-staging-hank = { + backend = "hank"; + hosts = [ "stash-staging.patchworklabs.org" ]; + }; # Every host in ../warp/redirects.nix. A new path on an existing host # needs no change here. redirects = { diff --git a/hosts/central-db-schema.nix b/hosts/central-db-schema.nix index e2289b0..326aa26 100644 --- a/hosts/central-db-schema.nix +++ b/hosts/central-db-schema.nix @@ -22,6 +22,9 @@ let # Weave is the IDP, so it owns people.* and runs its migrations. weave = { }; selvage.read = [ "people" ]; + # Stash (the archive): archives, captures, artifacts, audit events, GoodJob, Active Storage + # rows. It stores requesters' Patchwork IDs only, so no shared schemas. + stash = { }; } // (if kraterProduction then { inherit krater; } else { }); # Apps on staging (hem) only, until they have a production instance. skein diff --git a/hosts/hank/configuration.nix b/hosts/hank/configuration.nix new file mode 100644 index 0000000..a9a7b31 --- /dev/null +++ b/hosts/hank/configuration.nix @@ -0,0 +1,114 @@ +# hank — Stash host (pve01 VMID 362, 10.100.90.62, 6 GB, 80 GB disk) +# +# One app per VM. Sudo and the docker group are root here, so only the +# break-glass admins (modules/common) and lnx-hank-admins get them. Only this +# host's key decrypts the Stash secrets. +# Public traffic arrives from alastor over the tailnet +# (nixosModules.edge-alastor) as plain HTTP to nginx on tailscale0:80. +# +# Tailnet tags: tag:patchwork (ssh from admins, port 80 from alastor) and +# tag:patchwork-stash. The policy in jaspermayone/infra must let +# tag:patchwork-stash reach skein:5432 (prod) and hem:5432 (staging) only. +# +# The disk holds the archive (/data/stash and /data/stash-staging): 80 GB, +# more than the other app VMs. Memory: Chromium and LibreOffice run in the +# worker container, with a 3 GB limit. +# +# Built on 2026-11-08 with nixos-anywhere from the installer ISO. Its host key +# is in secrets/secrets.nix. +{ config, ... }: +{ + imports = [ + ../proxmox-vm.nix + ../../modules/common + ./stash.nix + ]; + + networking.hostName = "hank"; + patchwork.lanAddress = "10.100.90.62"; + patchwork.banner.role = "stash host"; + + virtualisation.docker.enable = true; + + # The containers run as uid 1000 (`rails` in the image) and own /data/stash. + # Reserve that uid for a system account here, so NixOS gives the local login + # accounts (modules/common) 1001 and up, and no person owns the archive. + users.users.stash = { + uid = 1000; + group = "stash"; + isSystemUser = true; + description = "Stash containers (uid of the image's rails user)"; + }; + users.groups.stash.gid = 1000; + + # The lab hosts do not use MagicDNS (modules/common, --accept-dns=false), + # so *.ts.net names do not resolve here. The Stash env files connect to + # skein and hem by tailnet name. The containers use the host network, and + # Docker copies this /etc/hosts into each one at start. If skein or hem + # joins the tailnet again, its address changes: update it here (and the + # aliases in jaspermayone/infra tailscale/policy.hujson). Addresses from + # `tailscale status` on 2026-11-08. + networking.hosts = { + "100.109.223.51" = [ "skein.wildebeest-stargazer.ts.net" ]; + "100.67.124.123" = [ "hem.wildebeest-stargazer.ts.net" ]; + }; + + zramSwap.enable = true; + + services.nginx = { + enable = true; + # Only alastor reaches this, over the tailnet. alastor terminated TLS and + # set X-Forwarded-For and X-Forwarded-Proto (https); pass them on as they + # are. recommendedProxySettings would set X-Forwarded-Proto to http. + commonHttpConfig = '' + set_real_ip_from 100.64.0.0/10; + real_ip_header X-Forwarded-For; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; + proxy_set_header X-Forwarded-Host $host; + ''; + virtualHosts = { + stash = { + serverName = "stash.patchworklabs.org"; + listen = [ + { + addr = "0.0.0.0"; + port = 80; + } + ]; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString config.patchwork.services.stash.port}"; + # Artifact downloads redirect to a signed URL served by the same app: + # a git bundle or a video can be large, and slow. + extraConfig = '' + proxy_read_timeout 300s; + proxy_buffering off; + ''; + }; + }; + stash-staging = { + serverName = "stash-staging.patchworklabs.org"; + listen = [ + { + addr = "0.0.0.0"; + port = 80; + } + ]; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString config.patchwork.services.stash.staging.port}"; + extraConfig = '' + proxy_read_timeout 300s; + proxy_buffering off; + ''; + }; + }; + }; + }; + # nginx is a host service, so the NixOS firewall applies to it (Docker's + # published ports would bypass it). Open only on the tailnet. + networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 80 ]; + + system.stateVersion = "26.05"; +} diff --git a/hosts/hank/stash.nix b/hosts/hank/stash.nix new file mode 100644 index 0000000..7be6c7d --- /dev/null +++ b/hosts/hank/stash.nix @@ -0,0 +1,66 @@ +# Stash on hank: the instance settings and the agenix secrets. nginx on hank +# (./configuration.nix) fronts the ports, and alastor routes the public names +# here over the tailnet (nixosModules.edge-alastor). +# +# Network: the containers use the host network, and hank reaches +# skein.wildebeest-stargazer.ts.net:5432 (production) and +# hem.wildebeest-stargazer.ts.net:5432 (staging) over the tailnet, through +# the tag:patchwork-stash grants in jaspermayone/infra tailscale/policy.hujson. +# +# Before Stash runs: +# - secrets/stash-env.age (jsp + hank) and secrets/central-prod-stash.age +# (jsp + skein): `bin/new-stash-secrets production`, then deploy skein so +# the stash role gets its password. The same for staging with hem. +# - Stash's app in Weave (redirect URI https://stash.patchworklabs.org/auth/callback, +# scopes openid profile email roles, plus introspect; app roles viewer, +# archivist, admin). Fill WEAVE_CLIENT_ID and WEAVE_CLIENT_SECRET into the +# env file: `cd secrets && agenix -e stash-env.age`. +# - Weave has the `stash` scope, and Krater's app may request it. +# - DNS for the two hostnames, proxied to alastor (patchworklabsorg/dns). +{ config, ... }: +{ + imports = [ ../../modules/stash ]; + + # Recipients are in ../../secrets/secrets.nix (jsp + hank's host key). + age.secrets = { + stash-env = { + file = ../../secrets/stash-env.age; + mode = "400"; + }; + stash-staging-env = { + file = ../../secrets/stash-staging-env.age; + mode = "400"; + }; + }; + + patchwork.services.stash = { + enable = true; + # Pin to ghcr.io/patchworklabsorg/stash: once a build is known good. + image = "ghcr.io/patchworklabsorg/stash:main"; + # 3005 and 3006 are Quilt prod and staging on batting; 3007 and 3008 are + # Stash here. The ports are per host, so this is only for the humans. + port = 3007; + environmentFile = config.age.secrets.stash-env.path; + dataDir = "/data/stash"; + + # stash-staging.patchworklabs.org: RAILS_ENV=staging, its own data dir, + # patchwork_central on hem:5432, Weave staging. Deploy a new image with + # `sudo systemctl restart stash-staging-web`. + staging = { + enable = true; + image = "ghcr.io/patchworklabsorg/stash:main"; + port = 3008; + environmentFile = config.age.secrets.stash-staging-env.path; + dataDir = "/data/stash-staging"; + workerMemory = "2g"; + # Not secret, and extraEnv overrides the env file, so nobody can point + # staging at production Weave from the env file. + extraEnv = { + WEAVE_ISSUER = "https://weave-staging.patchworklabs.org"; + STASH_HOSTS = "stash-staging.patchworklabs.org"; + # One capture at a time on staging: production's two plus this on 6 GB. + GOOD_JOB_MAX_THREADS = "1"; + }; + }; + }; +} diff --git a/modules/stash/default.nix b/modules/stash/default.nix new file mode 100644 index 0000000..bc6782e --- /dev/null +++ b/modules/stash/default.nix @@ -0,0 +1,295 @@ +# modules/stash/default.nix +# Stash: the Patchwork Labs archive (Rails). Two Docker containers from one +# image: web and worker. Exported by this flake as nixosModules.stash; the +# host that runs it imports hosts/hank/stash.nix, which sets the instance +# options below. Same shape as modules/quilt. +# +# The contract with the app is docs/deploy.md in patchworklabsorg/stash. +# +# Database: Stash has no database of its own. It connects only to +# patchwork_central (skein, prod) as the role `stash`, which owns the schema +# `stash` (hosts/central-db-schema.nix). The env file holds the full +# CENTRAL_DATABASE_URL, password included: +# postgres://stash:@skein.wildebeest-stargazer.ts.net:5432/patchwork_central?sslmode=disable +# The password must be the one in secrets/central-prod-stash.age, which skein +# sets on the role. skein has no server TLS yet (modules/central-db); Tailscale +# encrypts the path, hence sslmode=disable. +# +# Files: the artifacts (MHTML, PDFs, git bundles, videos) live on the host in +# `dataDir`, bind-mounted into both containers at the same path, with +# STASH_STORAGE_SERVICE=disk and STASH_STORAGE_PATH set to it. The web +# container serves them through signed URLs; the worker writes them. An +# S3-compatible bucket is a later option: set STASH_STORAGE_SERVICE=s3 and the +# S3_* variables in the env file, and leave dataDir unused. +# +# Network: both containers use the host network (`--network host`), as Quilt +# does, so the tailnet names in the env file resolve through the host's +# /etc/hosts. The web server binds 127.0.0.1:, and nginx on the host +# proxies to it. +# +# The worker runs headless Chromium, git, yt-dlp, gallery-dl and LibreOffice +# inside the container. Chromium needs a large /dev/shm (`--shm-size`), and +# the worker gets a memory limit so a runaway capture cannot take the host. +# +# Units: +# stash-web pulls the image, runs `./bin/rails server`. The image's +# entrypoint runs `db:migrate` before the server starts. +# stash-worker `bundle exec good_job start`. Waits for GET /up on the web +# server (so the migrations ran), and restarts on failure +# until the schema is there. partOf stash-web, so +# `systemctl restart stash-web` moves both to a new image. +# +# Image: `image` is a full ref. The default tag `main` follows the newest push +# to main. Pin it to `ghcr.io/patchworklabsorg/stash:` for a known build. +# The package is public, so no registry login is needed. +# +# Staging (optional, `staging.enable`) is a second, separate instance from the +# same `mkInstance` function: units and containers stash-staging-web and +# stash-staging-worker, its own port, image, env file, data dir and extraEnv, +# with RAILS_ENV=staging (the construction border on every page). Its +# database is patchwork_central on hem:5432 (the central staging cluster). +{ + config, + lib, + pkgs, + ... +}: +let + dockerBin = "${config.virtualisation.docker.package}/bin/docker"; + cfg = config.patchwork.services.stash; + + # `--env 'KEY=value'` arguments for an instance's extraEnv. Each line ends + # with the two-space indent the next docker argument expects. + envArgs = + env: + lib.concatStrings (lib.mapAttrsToList (k: v: "--env ${lib.escapeShellArg "${k}=${v}"} \\\n ") env); + + # Every unit, container and script name of one Stash instance comes from + # `name`. Production is name = "stash". + mkInstance = + { + name, + label, + railsEnv, + port, + image, + environmentFile, + dataDir, + extraEnv, + workerMemory, + }: + let + pull = pkgs.writeShellScript "${name}-pull" '' + set -e + ${dockerBin} pull ${lib.escapeShellArg image} + ''; + + common = unit: { + after = [ "docker.service" ]; + requires = [ "docker.service" ]; + wantedBy = [ "multi-user.target" ]; + # The worker restarts until the schema exists. Never stop retrying. + startLimitIntervalSec = 0; + serviceConfig = { + Type = "simple"; + # The image carries Chromium and LibreOffice: a new layer takes a while. + TimeoutStartSec = "15min"; + ExecStop = "${dockerBin} stop ${name}-${unit}"; + Restart = "on-failure"; + RestartSec = "10s"; + StandardOutput = "journal"; + StandardError = "journal"; + SyslogIdentifier = "${name}-${unit}"; + }; + }; + + # The data dir is owned by uid 1000, the `rails` user inside the image. + # The host must reserve uid 1000 for a system account (hosts/hank gives + # it to `stash`), or the first local login account owns the archive. + prepareData = pkgs.writeShellScript "${name}-prepare-data" '' + set -e + mkdir -p ${lib.escapeShellArg dataDir} + chown 1000:1000 ${lib.escapeShellArg dataDir} + chmod 750 ${lib.escapeShellArg dataDir} + ''; + + run = unit: extraArgs: '' + ${dockerBin} run \ + --name ${name}-${unit} \ + --network host \ + --env-file ${environmentFile} \ + --env RAILS_ENV=${railsEnv} \ + --env STASH_STORAGE_SERVICE=disk \ + --env STASH_STORAGE_PATH=${lib.escapeShellArg dataDir} \ + --volume ${lib.escapeShellArg dataDir}:${lib.escapeShellArg dataDir} \ + ${envArgs extraEnv}${extraArgs}--restart no \ + ${lib.escapeShellArg image}''; + in + { + # ── Web (Rails) ───────────────────────────────────────────────────────── + systemd.services."${name}-web" = lib.recursiveUpdate (common "web") { + description = "${label} web"; + # Until the env file decrypts on this host (its key must be a recipient), + # the unit does not start, instead of restarting and pulling every 10 s. + unitConfig.ConditionPathExists = environmentFile; + serviceConfig = { + ExecStartPre = [ + prepareData + pull + "-${dockerBin} stop ${name}-web" + "-${dockerBin} rm ${name}-web" + ]; + # Same command as the image's CMD, on the loopback port. The + # entrypoint matches `./bin/rails server` and migrates first. + ExecStart = pkgs.writeShellScript "${name}-web-start" '' + exec ${run "web" ""} \ + ./bin/rails server -b 127.0.0.1 -p ${toString port} + ''; + }; + }; + + # ── Worker (GoodJob) ──────────────────────────────────────────────────── + # Runs the captures. It never migrates, so it waits for the web server, + # which migrates before it answers /up. + systemd.services."${name}-worker" = lib.recursiveUpdate (common "worker") { + description = "${label} worker"; + after = [ + "docker.service" + "${name}-web.service" + ]; + partOf = [ "${name}-web.service" ]; + serviceConfig = { + ExecStartPre = [ + pull + "-${dockerBin} stop ${name}-worker" + "-${dockerBin} rm ${name}-worker" + (pkgs.writeShellScript "${name}-wait-for-web" '' + echo "Waiting for the web server (migrations)..." + until ${pkgs.curl}/bin/curl -fsS -o /dev/null \ + http://127.0.0.1:${toString port}/up; do + sleep 5 + done + '') + ]; + ExecStart = pkgs.writeShellScript "${name}-worker-start" '' + exec ${run "worker" "--shm-size 1g --memory ${workerMemory} --memory-swap ${workerMemory} \\\n "} \ + bundle exec good_job start + ''; + }; + }; + }; + + instanceOptions = defaults: { + port = lib.mkOption { + type = lib.types.port; + default = defaults.port; + description = "Loopback port the web server binds to. The host's reverse proxy proxies to it."; + }; + + image = lib.mkOption { + type = lib.types.str; + default = "ghcr.io/patchworklabsorg/stash:main"; + example = "ghcr.io/patchworklabsorg/stash:0123abc"; + description = '' + Image ref for web and worker. Pin it to a commit sha for a known build. + Each start of the web unit pulls it again. + ''; + }; + + environmentFile = lib.mkOption { + type = lib.types.path; + description = '' + Path to the env file (an agenix secret) with the variables in + docs/deploy.md of patchworklabsorg/stash: CENTRAL_DATABASE_URL, + SECRET_KEY_BASE, STASH_HOSTS, WEAVE_ISSUER, WEAVE_CLIENT_ID, + WEAVE_CLIENT_SECRET, and optionally GITHUB_TOKEN. + ''; + }; + + dataDir = lib.mkOption { + type = lib.types.path; + default = defaults.dataDir; + description = '' + Directory on the host that holds the archived files (Active Storage + disk service). Bind-mounted into both containers at the same path. + Back it up: it is the archive. + ''; + }; + + workerMemory = lib.mkOption { + type = lib.types.str; + default = "3g"; + description = '' + Docker memory limit of the worker container (Chromium, LibreOffice, + yt-dlp). Swap is capped at the same value, so the limit is the whole + footprint and zram on the host does not stretch it. + ''; + }; + + extraEnv = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + example = { + RAILS_LOG_LEVEL = "debug"; + }; + description = '' + Extra container env for web and worker. It overrides the env file, so + do not put a value here that you want to change in the env file. + ''; + }; + }; +in +{ + options.patchwork.services.stash = { + enable = lib.mkEnableOption "Stash, the Patchwork Labs archive"; + } + // instanceOptions { + port = 3007; + dataDir = "/data/stash"; + } + // { + staging = { + enable = lib.mkEnableOption '' + a Stash staging instance (stash-staging-web and stash-staging-worker, + RAILS_ENV=staging, own port, image, env file and data dir) alongside + production + ''; + } + // instanceOptions { + port = 3008; + dataDir = "/data/stash-staging"; + }; + }; + + config = lib.mkIf cfg.enable ( + lib.mkMerge [ + (mkInstance { + name = "stash"; + label = "Stash"; + railsEnv = "production"; + inherit (cfg) + port + image + environmentFile + dataDir + extraEnv + workerMemory + ; + }) + + (lib.mkIf cfg.staging.enable (mkInstance { + name = "stash-staging"; + label = "Stash staging"; + railsEnv = "staging"; + inherit (cfg.staging) + port + image + environmentFile + dataDir + extraEnv + workerMemory + ; + })) + ] + ); +} diff --git a/secrets/ad-join.age b/secrets/ad-join.age index d6ea09b..760cacc 100644 --- a/secrets/ad-join.age +++ b/secrets/ad-join.age @@ -1,17 +1,19 @@ age-encryption.org/v1 --> ssh-ed25519 1uIO/w R94s+RqHxBUeOu6uRKQiaxDvnqYYRsS3ofBv7ezzDQY -qCq64bh7aeGNwWiVgQbMWkOMMKTWdfX4sBhI9wshLww --> ssh-ed25519 5jO4Pg lAtiMYBuxmp5+kDu61e1RjLnNXLFrc1AwGWwz4G9+hE -GuppA/26f5recxS6SecdT2/tBxPnnlGva/Za0mpj2Bg --> ssh-ed25519 zU9L3Q EkuHYeaYB0D8aimrEjZqUEjED3RF7StlmAl9Gw8LmiE -WXCwLz2MjjZJww23iNw7CIwcUjNNr1pTWkKnZMrQCcU --> ssh-ed25519 gW6fbQ FXR3OXskZQbI3HkdIiwldfJt7jn6Z/NvPtBRqmwKugU -RQe+wdABFfxHgOS1XoF0TEBHf1q0hfvn7bv4EfqvyC0 --> ssh-ed25519 YGz6dA dy1Rh4UGyCMwtgzaZQkivlzItNl8y/sF7xXL6F3tnTg -G3WsCoNtY2QJrZg3KTn1kTLtPGCjSsfL4sy9UjQMEVc --> ssh-ed25519 OeXtHA m8wSATmKirAzGikFF6LKEDIfLlRaBSCEZdD2gnQ8IjE -PxhKIFHPFf1XRMAKPhOhZh+srxx9cNMtMZOa63GH0XY --> ssh-ed25519 fqr1AA QK8ztGzPnbmGIzD225scDqYxTrfa4Na28YYZjZNYFDw -xzA48czCatDszXGR3anYGT4eog7LuLBJFKx4IYvDMro ---- 8tHDTQsO5xlfsYf7KS4L/GPw4XC/nc9eWwlmXFXi630 -æjM”]uqi4”J /m¥ik÷.¤D?&vÚK°€ÔЧ»¥�ˆ@GÊl®2'ØOqu¯>2ÓÕ·Ûj{HÊ€½2Ø2G. \ No newline at end of file +-> ssh-ed25519 1uIO/w 5c4UJhjUjYNP+AebmmqBhyzR1QaeHQP4wnYSdhcKlGU +2qOW/D/lRgEUU7K2h+VdcexRvLVr/0W82sBO67kcHDA +-> ssh-ed25519 5jO4Pg L0uNUgiy5cXDzvZ6JKFmdXUW0ejSAkvT2tIxwr2yiQA +6h3dyr8XOn71wrxFmRaPlTpydETptwrHa2z5g99nk70 +-> ssh-ed25519 zU9L3Q igipyKH9j0ujjtmecc3moNb5RM5QVz67y/xIebte920 +XvAtufkNRjINvix2GvaCVNKqUd8HRauHRKUeymvzIhc +-> ssh-ed25519 gW6fbQ aSbEdZz6SOCdYa6oFaBiZpJEoKnFTMu+G2Xiw2DezU8 +h0VM1vgI8yIScuGCQOuhc3xnqVsq3KU+g2bN8ibA3+4 +-> ssh-ed25519 YGz6dA ETosd4DDL6UWgSFhnXE6R5/fc2vHvebxmvINZilJJw8 +a2VGUBBy76pg89wAeEEKBFe2Fj0ydi+HHyZ9LcwtGto +-> ssh-ed25519 OeXtHA bTBch29B0pap4l2SDC2NtUpM2gu/zj5u/zJIwQaA9yg +9j/ZTTEa9Zq5Mcc7vcgFhiSoMt7/gzkSfbaWLOP/8uo +-> ssh-ed25519 fqr1AA uWOY6PRSZTJjuiYVlU2lwMPAQTIILF/2IkLK/oCNSjc +g5eI/XgTgJ6qVYVtDHsbb3/mpF1JKDGs85oypm/gHM4 +-> ssh-ed25519 ddGmUw FwFDn+u7SbVG0EUiY0AROIV0pcwPgckLrbq9ZKTwrVE +I7Kw/CVdnCzXN2PtGvDhlfVUc7Iu+H890rx6PqELvcc +--- 1WZsT+xinAY2CEBZWjqE9zHcs3nUm8WX9TA3NV7u2h0 +÷@¥G8•�"ã]öjÚ¬=þt§�n—xepïPÌ"Œ¶e�èþ¾¢�¦Ì&¡>�ÊÏçÎ5N_„Šö+3R©7áx­52 T \ No newline at end of file diff --git a/secrets/central-prod-stash.age b/secrets/central-prod-stash.age new file mode 100644 index 0000000..9db985a --- /dev/null +++ b/secrets/central-prod-stash.age @@ -0,0 +1,9 @@ +age-encryption.org/v1 +-> ssh-ed25519 1uIO/w RE1mTv/trdbxRSF508+FU6a/NLuXN0f2SBMdlCOFg0w +nohzEZllz6SskX2EmS5JMdI8Ccr/ad8geb212e0nl1s +-> ssh-ed25519 5jO4Pg j04S1FOo4u8ucq+cmZgEt9j84vu6dl+VnogvI7h8sDw +mzml9mgrO8vvwdYLDLIHA/cYZJWb1o9G3Jm2XgPGF7k +--- 8Tv49at11oXRQggiuXJ8JhfwG6SAvAzDNeHdTHCSnoQ +’Ψt}ŽÄ§, +r¿£­bãIýTÕðYp3Çž7„Mšhø8G¶–)ÏØ„«ÜKf“s±Èžñ(©.SbGÅVHÍ=:¸˜Ð„ßyÛ1$V"èkMû +ZöïJôn/[¹ \ No newline at end of file diff --git a/secrets/central-staging-stash.age b/secrets/central-staging-stash.age new file mode 100644 index 0000000..39de90f --- /dev/null +++ b/secrets/central-staging-stash.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> ssh-ed25519 1uIO/w d75BeXKjHnBrQTWE9Y5vl+mbFR+RBAo9WXEF7RSX+Vo +EKMHrDAxXU6BvfHWz5tj+nYzi8B8BFE2w9s1i415S3c +-> ssh-ed25519 YGz6dA Y01h5IQBYXtGl/uKh8VMWEEkGGzBEHUUMw+QNbmjZiM +3mYGZEorgVbla5qo+rbcIgPruxA9Lo4k+uT26zTbehM +--- 01Lq8o5ydBY3i0fIn0PloJdruRyA4/Sq1/jUGDcSBRA +æ¨À®ŒP5ïaÇâZ\f»šÅ}˜�ûºy5gš‹šYHÈAžœ0´‘&y³AïNÄ$eø�´R;w#†SöÉ¡Htµ`ú0RE¹Sg¸„×¼EÿÇ�¤µôÇ— ¼9 Ô¶:ÿê \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index a2d6617..155fa6d 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -17,6 +17,7 @@ let hem = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHFHZ/2Q8x+Z9toPli2yQZpQ4e8dBMOqDAFyAFaxgsIE"; batting = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEN68vw2ir+q4fMQV8WBJvH7WDG4D2uFjChwkkA93LJC"; weft = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAbzni2Be82ORFcHSPVr7OPpfs80Py04JtwyCKc4L28X"; + hank = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICKNDAtCadjdzqG+5t6BncycydzqfqV3z02uvIejmO8S"; # # Central DB passwords, one file per app and environment, password only: # central-prod-.age recipients: jsp, skein @@ -168,6 +169,44 @@ in batting ]; + # Stash (stash.patchworklabs.org on hank) runtime env. bin/new-stash-secrets + # writes it. Must contain the variables in docs/deploy.md of + # patchworklabsorg/stash: + # CENTRAL_DATABASE_URL=postgres://stash:@skein.wildebeest-stargazer.ts.net:5432/patchwork_central?sslmode=disable + # ( is the value in central-prod-stash.age) + # SECRET_KEY_BASE=... + # STASH_HOSTS=stash.patchworklabs.org + # WEAVE_ISSUER=https://weave.patchworklabs.org + # WEAVE_CLIENT_ID=... WEAVE_CLIENT_SECRET=... (the Stash app in Weave) + # GITHUB_TOKEN= (optional, read-only public access) + # hank is not built yet: add its host key here after the install, then + # `agenix -r`. + "stash-env.age".publicKeys = [ + jsp + hank + ]; + + # Stash staging (stash-staging.patchworklabs.org on hank, RAILS_ENV=staging) + # runtime env. Same variables, with staging values: the password in + # central-staging-stash.age, hem:5432, a Weave staging app. WEAVE_ISSUER + # and STASH_HOSTS are also set in hosts/hank/stash.nix and override this file. + "stash-staging-env.age".publicKeys = [ + jsp + hank + ]; + + # Stash's role password in patchwork_central (skein) and on hem:5432. + # Password only. hank does not get these files: the same password is in the + # env file, so a rotation must change both files. + "central-prod-stash.age".publicKeys = [ + jsp + skein + ]; + "central-staging-stash.age".publicKeys = [ + jsp + hem + ]; + # App role passwords in patchwork_central (skein). Password only. Add the # host that runs the app as a recipient when the app connects, and put the # same password in that app's env file. @@ -220,5 +259,6 @@ in hem batting weft + hank ]; } diff --git a/secrets/stash-env.age b/secrets/stash-env.age new file mode 100644 index 0000000..5e703bf --- /dev/null +++ b/secrets/stash-env.age @@ -0,0 +1,8 @@ +age-encryption.org/v1 +-> ssh-ed25519 1uIO/w bTqsC1Cu4xIZvsPb3l2Omyk2bc36mnxT45F0WwqcCHA +7T5kUEgAU1zjgwzO2/XnaID6gjTEVVX1oTRGuGePlGM +-> ssh-ed25519 ddGmUw puMnKJYAkP4O6bBH/uGS2ed6/P0ngrudDPYK4TWVSko +KzEA/qbrAixEGZd354/Yyf0UlH6nPuFd2PuS7CPVfvA +--- S04oJmbBcaYOQafCUb5k2oEhuHrblo3/sL6CJgzkals +Í‚ÝTì•%ìÝ  �“µ ,7°†FÑ“c‰.ê@ËÈR/1¿ðSë´÷t‘ ;>ž‘6ת”ÿÓµk$kâiUö\fü ²äˆÔv‡¶WÍr·�m¿Ýøf皢»Uo…Ðé2ˆ?q¹ ïØzÄ&Ž¡™â« ®Ñb˜Üi|ÙºéxþQzãôLLü�i홳?5”yرg5£bˬ~T `«6|w/À’ëR"Vcµ_*G¥´iµ +˜çgÌšk¸é £Z–Nw\QjisKdÖ™-ì$µ·Y}/Õ´‹)Kí›&ûóÍAÊyR«¦ �>°œêa…Z Â×è‚Óˆwû�£fÈ̵~ý5YtLÓÁ¤Â *�}<Ð�íÑY%¶ %ýM2!Ë<óÿ•k£>jÉ�&âñf*¨Œ¯'uÁáàÁ/F–ÒþB¢E¦)ŒIѸú’Ñëü–bl’­/’WÇ=`¥4Ã9€Æá¿° ’05§?ª§ÙØÎCû­Ýœµš(®ÎÀ˜1nŸEôDϺ#ðæù�Þ7pÙ¤ÉP?íj¬ï¦Y‰·p[v£ ^‚åYÛ"b^ïly¤¨Âá¬áRϯ]2 2øºÁqþkø•ÓD8ã9oT¡Ûâö„Ç%¾f5y+øñ \ No newline at end of file diff --git a/secrets/stash-staging-env.age b/secrets/stash-staging-env.age new file mode 100644 index 0000000..7cacd8b --- /dev/null +++ b/secrets/stash-staging-env.age @@ -0,0 +1,10 @@ +age-encryption.org/v1 +-> ssh-ed25519 1uIO/w ROYUr5T4HGODl/gQBc7103MDE8Mc2srgOC2pkD8+/lw +GOhz4axAfBKo/UmX0pZmq5gdx6ZFcFbcCs6yWpo+O4Q +-> ssh-ed25519 ddGmUw 6PGG1rGoQ6+Vmir75P+xVs1AQNDePIFKeWhMJnqt0DE +TlB1dDBSXZ8/29Gus71cs+LipctnhkGoGCfKs7L5zZM +--- R9PuYcS/220ou8LeZnyMPfvX2BZ1LUztO6rtikxlHrc +j3²¹õ$NìZDosËÓhán +Û§/�è—0<ø2æ'£¡Su—Îk¯ß—&‹L´Q³…iY[V›}Ð�®Õái«s6�Ó{†J”sø‚j�[ŽÅ:êZî«ûbà8+çD¿†NFà3X„“:ì;’Ž'ˆ§mQ2–÷cÕø¥À¬¸Ì?õ‡q$íxOƒó‡­í6Eg|` Á"½ü6¼ÒVêæ:ÓÀÇz)e†Ä¥+Äêá×ÃÔ›W(ç̸� H$WTfm]�Õ‡PULD'û\Ä~I~wðϺ[²€µP¤g)°ap)I¤Ñ¡Ã6,|± ½Ð–n *·�#J¢åvÊëXÊÂG{¶÷P"³ŸèTÙùHë +×/T�ºœñ=nAî –—–„›ûê|œ4™Ð +Á·U¡0/@ì!–c_P�EÌNžÔA÷‰Be-@"Åh¼—×J“@ Ú›´²áS4ƒˆê¡&×<á_öqBÛøŽ¶;þ£|€ŠoÓP­ Ý§ÃM®o‡Û›þÏ�¸¤e=Ê�}°Sh4¢=ˆ—?ÒLæ�IU6j3µ[Î÷œE?ÞÌfsâe[Þ fYæÇ�ù‰æESÁbqâDh=_t´=)ÇÄ›@­á|›˜¯¸ÊAº‚ŒÜ�–'rmj"ˆïÆ÷¤�ˆF6‰ \ No newline at end of file diff --git a/terraform/proxmox/README.md b/terraform/proxmox/README.md index 8d1b601..19f2423 100644 --- a/terraform/proxmox/README.md +++ b/terraform/proxmox/README.md @@ -12,6 +12,7 @@ This root manages the Patchwork NixOS VMs on the Proxmox node `pve01`. It uses t | `hem` | 358 | 10.100.90.58 | 2 GB | | `batting` | 360 | 10.100.90.60 | 4 GB | | `weft` | 361 | 10.100.90.61 | 4 GB | +| `hank` | 362 | 10.100.90.62 | 6 GB | hem, batting and weft are in the state (created with `qm create` and `tofu import`). To change one VM, target it, for example `tofu apply -target='proxmox_virtual_environment_vm.nixos["weft"]'`. A plan for an imported VM shows `agent.timeout` 15m to 2m. That value is only in the state. diff --git a/terraform/proxmox/variables.tf b/terraform/proxmox/variables.tf index f161cfe..4022779 100644 --- a/terraform/proxmox/variables.tf +++ b/terraform/proxmox/variables.tf @@ -93,6 +93,14 @@ variable "vms" { disk_gb = 40 ip = "10.100.90.61" } + # Stash: the archive. More disk than the others, because the archived files live on it. + hank = { + vmid = 362 + memory_mb = 6144 + cores = 2 + disk_gb = 80 + ip = "10.100.90.62" + } } }