diff --git a/hosts/weft/krater.nix b/hosts/weft/krater.nix index c09e197..2f310d2 100644 --- a/hosts/weft/krater.nix +++ b/hosts/weft/krater.nix @@ -338,6 +338,11 @@ in # archive. Stash takes them only while Krater's Weave app allows the # `stash` scope. Until then they wait in stash_archives and retry. KRATER_STASH_URL = "https://stash.patchworklabs.org"; + # alastor requests /healthz/edge every minute while production's routes + # are on (krater-edge-ping in ../alastor/edge.nix). With this on, a + # missing or stale ping holds what Krater would do about silent runners + # and alerts admins (patchworklabsorg/krater#84). + KRATER_EDGE_PING_EXPECTED = "true"; # Private channel for the worker's admin alerts. KRATER_SLACK_ALERTS_CHANNEL_ID = "C0CF352PB17"; }; diff --git a/tests/krater-weft.nix b/tests/krater-weft.nix index bcdd016..197e651 100644 --- a/tests/krater-weft.nix +++ b/tests/krater-weft.nix @@ -141,8 +141,8 @@ let !(prod.patchwork.services.krater-production.extraEnv ? KRATER_STAGING); "staging expects alastor's edge ping" = on.patchwork.services.krater.extraEnv.KRATER_EDGE_PING_EXPECTED == "true"; - "production does not expect the edge ping yet" = - !(prod.patchwork.services.krater-production.extraEnv ? KRATER_EDGE_PING_EXPECTED); + "production expects alastor's edge ping" = + prod.patchwork.services.krater-production.extraEnv.KRATER_EDGE_PING_EXPECTED == "true"; "staging label prefix" = on.patchwork.services.krater.compute.labelPrefix == "krater-staging-"; # Each deployment rents with its own Vast key, never the other's. "staging compute uses its own key" = @@ -162,6 +162,14 @@ let "production vhost follows the switch file" = (vhosts ? krater-prod) == switchFile; "production secret follows the switch file" = (on.age.secrets ? krater-env) == switchFile; "edge default follows the switch file" = edge.patchwork.edge.kraterProduction == switchFile; + # Both Krater deployments count a missing ping as their network being + # down, so alastor must send it. + "edge pings while krater expects it" = + edge.patchwork.edge.kraterEdgePing + || !( + on.patchwork.services.krater.extraEnv.KRATER_EDGE_PING_EXPECTED or "false" == "true" + || prod.patchwork.services.krater-production.extraEnv.KRATER_EDGE_PING_EXPECTED or "false" == "true" + ); # Krater holds actions on silent runners when this ping stops arriving. "edge pings krater every minute" = edgeOff.systemd.timers.krater-edge-ping.timerConfig.OnCalendar == "minutely" @@ -285,13 +293,10 @@ pkgs.runCommand "krater-weft" { } '' grep -F 'systemctl try-reload-or-restart nginx.service' "$converge" grep -F 'docker stop --time 30 "krater-web-$old"' "$converge" grep -F -- '--time 120 krater-worker' ${pkgs.writeText "worker-stop" svc.krater-worker.serviceConfig.ExecStop} - # The worker reads KRATER_EDGE_PING_EXPECTED. Staging's worker gets it; - # production's does not yet. + # The worker reads KRATER_EDGE_PING_EXPECTED. Both workers get it. # lib.escapeShellArg quotes the pair, because of the "=". grep -F -- "--env 'KRATER_EDGE_PING_EXPECTED=true'" ${svc.krater-worker.serviceConfig.ExecStart} - if grep -F 'KRATER_EDGE_PING_EXPECTED' ${pvc.krater-prod-worker.serviceConfig.ExecStart}; then - exit 1 - fi + grep -F -- "--env 'KRATER_EDGE_PING_EXPECTED=true'" ${pvc.krater-prod-worker.serviceConfig.ExecStart} # Production keeps its one container and its prepare script. for f in ${lib.concatStringsSep " " (lib.filter (lib.hasPrefix "/nix/store/") (lib.toList pvc.krater-prod-web.serviceConfig.ExecStartPre))}; do if grep -qF 'alembic upgrade head' "$f"; then