From 184c77adcb276bd9c8c81796bcaa05e3ebb87a6e Mon Sep 17 00:00:00 2001 From: Adambomb210 Date: Thu, 19 Nov 2026 18:19:07 -0500 Subject: [PATCH] feat(alastor): post to Slack when Krater stops answering the edge ping krater-edge-ping counts each URL's failed minutes in its StateDirectory and, with a Slack incoming-webhook URL, posts after 3 in a row, every 60 while it lasts, and once when it answers again. Krater posts its own alerts, so it can't say it's down; this can. On once secrets/krater-edge-alert.age exists (recipients added to secrets.nix); until then nothing changes. The secret is declared only where agenix is, inside `config` so reading `options` can't recurse. Co-Authored-By: Claude Opus 5.5 --- hosts/alastor/edge.nix | 141 +++++++++++++++++++++++++++++++---------- secrets/secrets.nix | 9 +++ tests/krater-weft.nix | 10 +++ 3 files changed, 125 insertions(+), 35 deletions(-) diff --git a/hosts/alastor/edge.nix b/hosts/alastor/edge.nix index ddb9af4..40ae9f5 100644 --- a/hosts/alastor/edge.nix +++ b/hosts/alastor/edge.nix @@ -9,6 +9,7 @@ # same change in jaspermayone/infra. { config, + options, lib, pkgs, ... @@ -28,6 +29,19 @@ let ] ++ lib.optional kraterProduction "https://krater.patchworklabs.org/healthz/edge"; + # Slack when a ping keeps failing, and again when it answers. Krater posts + # its own alerts, so it can't say it's down: this is the one that can. A + # Slack incoming-webhook URL, alone on one line, in + # ../../secrets/krater-edge-alert.age; on as soon as that file exists. + alertSecret = ../../secrets/krater-edge-alert.age; + haveAlertSecret = builtins.pathExists alertSecret; + hasAgenix = options ? age; + webhookFile = config.patchwork.edge.kraterEdgeAlertWebhookFile; + # Minutes of failed pings before the alert (a deploy or a blip is shorter), + # and between reminders while it lasts. + alertAfter = 3; + remindEvery = 60; + # = { backend = ""; hosts = [ "" ... ]; }; routes = { # Not `quilt`/`quilt-staging`: those were the router names of the old @@ -109,45 +123,102 @@ in description = "Route krater.patchworklabs.org and krater-storage.patchworklabs.org to weft."; }; - config.systemd.services.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing { - description = "Tell Krater its runners' path to it works"; - path = [ pkgs.curl ]; - serviceConfig = { - Type = "oneshot"; - DynamicUser = true; - }; - # Every URL, even after one fails: staging and production are separate. - script = '' - status=0 - for url in ${lib.escapeShellArgs kraterPingUrls}; do - curl -fsS -o /dev/null --max-time 20 "$url" || { echo "no answer from $url" >&2; status=1; } - done - exit $status + options.patchwork.edge.kraterEdgeAlertWebhookFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = if haveAlertSecret && hasAgenix then config.age.secrets.krater-edge-alert.path else null; + description = '' + A file holding a Slack incoming-webhook URL. When set, krater-edge-ping + posts there after ${toString alertAfter} failed minutes in a row for a + URL, every ${toString remindEvery} minutes while it lasts, and once + when it answers again. Defaults to the agenix secret + krater-edge-alert.age once that file exists. ''; }; - config.systemd.timers.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing { - wantedBy = [ "timers.target" ]; - timerConfig = { - OnCalendar = "minutely"; - AccuracySec = "5s"; - }; - }; + # The secret only where agenix is: the edge test's bare system has no `age` + # option. Inside `config`, so that reading `options` here can't recurse. + config = lib.mkMerge [ + { + systemd.services.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing { + description = "Tell Krater its runners' path to it works"; + path = [ pkgs.curl ]; + serviceConfig = { + Type = "oneshot"; + DynamicUser = true; + # Consecutive failures per URL, kept between runs. + StateDirectory = "krater-edge-ping"; + } + // lib.optionalAttrs (webhookFile != null) { + LoadCredential = [ "webhook:${webhookFile}" ]; + }; + # Every URL, even after one fails: staging and production are separate. + # A failed Slack post doesn't fail the unit, and doesn't stop the count. + script = '' + status=0 + say() { + [ -n "''${CREDENTIALS_DIRECTORY:-}" ] && [ -r "$CREDENTIALS_DIRECTORY/webhook" ] || return 0 + curl -fsS -o /dev/null --max-time 10 -H 'Content-Type: application/json' \ + --data "{\"text\": \"$1\"}" "$(head -n1 "$CREDENTIALS_DIRECTORY/webhook")" \ + || echo "couldn't post to Slack" >&2 + } + for url in ${lib.escapeShellArgs kraterPingUrls}; do + host=''${url#https://} + host=''${host%%/*} + file="$STATE_DIRECTORY/$host" + failed=$(cat "$file" 2>/dev/null || echo 0) + if curl -fsS -o /dev/null --max-time 20 "$url"; then + if [ "$failed" -ge ${toString alertAfter} ]; then + say ":white_check_mark: alastor reaches $host again, after $failed minutes without an answer." + fi + echo 0 > "$file" + else + echo "no answer from $url" >&2 + status=1 + failed=$((failed + 1)) + echo "$failed" > "$file" + if [ "$failed" -eq ${toString alertAfter} ]; then + say ":rotating_light: alastor has had no answer from $host for $failed minutes (krater-edge-ping). Its runners likely can't reach it either. Check weft and the Krater service; the edge-ping runbook is in docs/guides/compute-runbook.md." + elif [ "$failed" -gt ${toString alertAfter} ] && [ $(( (failed - ${toString alertAfter}) % ${toString remindEvery} )) -eq 0 ]; then + say ":rotating_light: Still no answer from $host, for $failed minutes now." + fi + fi + done + exit $status + ''; + }; + + systemd.timers.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "minutely"; + AccuracySec = "5s"; + }; + }; - config.environment.etc."traefik/conf.d/patchwork.toml".source = - (pkgs.formats.toml { }).generate "patchwork.toml" - { - http = { - routers = lib.mapAttrs (name: r: { - rule = lib.concatMapStringsSep " || " (h: "Host(`${h}`)") r.hosts; - entryPoints = [ "websecure" ]; - tls.certResolver = "cloudflare"; - middlewares = [ "hsts" ]; - service = name; - }) routes; - services = lib.mapAttrs (_: r: { - loadBalancer.servers = [ { url = "http://${r.backend}.${tailnet}"; } ]; - }) routes; + environment.etc."traefik/conf.d/patchwork.toml".source = + (pkgs.formats.toml { }).generate "patchwork.toml" + { + http = { + routers = lib.mapAttrs (name: r: { + rule = lib.concatMapStringsSep " || " (h: "Host(`${h}`)") r.hosts; + entryPoints = [ "websecure" ]; + tls.certResolver = "cloudflare"; + middlewares = [ "hsts" ]; + service = name; + }) routes; + services = lib.mapAttrs (_: r: { + loadBalancer.servers = [ { url = "http://${r.backend}.${tailnet}"; } ]; + }) routes; + }; + }; + } + (lib.optionalAttrs hasAgenix { + age.secrets = lib.mkIf haveAlertSecret { + krater-edge-alert = { + file = alertSecret; + mode = "400"; }; }; + }) + ]; } diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 5f9014d..d0a6df7 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -333,4 +333,13 @@ in weft hank ]; + + # Slack incoming-webhook URL (alone on one line) for alastor's + # krater-edge-ping alert (hosts/alastor/edge.nix): posts when Krater stops + # answering alastor and when it's back. The alert is on once this file is + # committed. + "krater-edge-alert.age".publicKeys = [ + jsp + alastor + ]; } diff --git a/tests/krater-weft.nix b/tests/krater-weft.nix index 197e651..e4ff16f 100644 --- a/tests/krater-weft.nix +++ b/tests/krater-weft.nix @@ -59,6 +59,7 @@ let edgeOff = mkEdge { patchwork.edge.kraterProduction = false; }; edgeOffToml = edgeOff.environment.etc."traefik/conf.d/patchwork.toml".source; edgeProd = mkEdge { patchwork.edge.kraterProduction = true; }; + edgeAlert = mkEdge { patchwork.edge.kraterEdgeAlertWebhookFile = "/run/agenix/krater-edge-alert"; }; edgeProdToml = edgeProd.environment.etc."traefik/conf.d/patchwork.toml".source; units = on.systemd.units; pvc = prod.systemd.services; @@ -175,6 +176,15 @@ let edgeOff.systemd.timers.krater-edge-ping.timerConfig.OnCalendar == "minutely" && lib.hasInfix "krater-staging.patchworklabs.org/healthz/edge" edgeOff.systemd.services.krater-edge-ping.script && !(lib.hasInfix "//krater.patchworklabs.org" edgeOff.systemd.services.krater-edge-ping.script); + # Krater can't say it's down: alastor's ping does, once its webhook is set. + "edge alert off without its webhook" = + edgeOff.patchwork.edge.kraterEdgeAlertWebhookFile == null + && !(edgeOff.systemd.services.krater-edge-ping.serviceConfig ? LoadCredential); + "edge alert reads its webhook" = + edgeAlert.systemd.services.krater-edge-ping.serviceConfig.LoadCredential + == [ "webhook:/run/agenix/krater-edge-alert" ] + && edgeAlert.systemd.services.krater-edge-ping.serviceConfig.StateDirectory == "krater-edge-ping" + && lib.hasInfix "rotating_light" edgeAlert.systemd.services.krater-edge-ping.script; "edge pings production while its routes are on" = lib.hasInfix "https://krater.patchworklabs.org/healthz/edge" edgeProd.systemd.services.krater-edge-ping.script; "switch off: krater only on hem" =