Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions hosts/weft/krater.nix
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,11 @@ in
# archive. Stash takes them only while Krater's Weave app allows the
# `stash` scope. Until then they wait in stash_archives and retry.
KRATER_STASH_URL = "https://stash.patchworklabs.org";
# alastor requests /healthz/edge every minute while production's routes
# are on (krater-edge-ping in ../alastor/edge.nix). With this on, a
# missing or stale ping holds what Krater would do about silent runners
# and alerts admins (patchworklabsorg/krater#84).
KRATER_EDGE_PING_EXPECTED = "true";
# Private channel for the worker's admin alerts.
KRATER_SLACK_ALERTS_CHANNEL_ID = "C0CF352PB17";
};
Expand Down
19 changes: 12 additions & 7 deletions tests/krater-weft.nix
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,8 @@ let
!(prod.patchwork.services.krater-production.extraEnv ? KRATER_STAGING);
"staging expects alastor's edge ping" =
on.patchwork.services.krater.extraEnv.KRATER_EDGE_PING_EXPECTED == "true";
"production does not expect the edge ping yet" =
!(prod.patchwork.services.krater-production.extraEnv ? KRATER_EDGE_PING_EXPECTED);
"production expects alastor's edge ping" =
prod.patchwork.services.krater-production.extraEnv.KRATER_EDGE_PING_EXPECTED == "true";
"staging label prefix" = on.patchwork.services.krater.compute.labelPrefix == "krater-staging-";
# Each deployment rents with its own Vast key, never the other's.
"staging compute uses its own key" =
Expand All @@ -162,6 +162,14 @@ let
"production vhost follows the switch file" = (vhosts ? krater-prod) == switchFile;
"production secret follows the switch file" = (on.age.secrets ? krater-env) == switchFile;
"edge default follows the switch file" = edge.patchwork.edge.kraterProduction == switchFile;
# Both Krater deployments count a missing ping as their network being
# down, so alastor must send it.
"edge pings while krater expects it" =
edge.patchwork.edge.kraterEdgePing
|| !(
on.patchwork.services.krater.extraEnv.KRATER_EDGE_PING_EXPECTED or "false" == "true"
|| prod.patchwork.services.krater-production.extraEnv.KRATER_EDGE_PING_EXPECTED or "false" == "true"
);
# Krater holds actions on silent runners when this ping stops arriving.
"edge pings krater every minute" =
edgeOff.systemd.timers.krater-edge-ping.timerConfig.OnCalendar == "minutely"
Expand Down Expand Up @@ -285,13 +293,10 @@ pkgs.runCommand "krater-weft" { } ''
grep -F 'systemctl try-reload-or-restart nginx.service' "$converge"
grep -F 'docker stop --time 30 "krater-web-$old"' "$converge"
grep -F -- '--time 120 krater-worker' ${pkgs.writeText "worker-stop" svc.krater-worker.serviceConfig.ExecStop}
# The worker reads KRATER_EDGE_PING_EXPECTED. Staging's worker gets it;
# production's does not yet.
# The worker reads KRATER_EDGE_PING_EXPECTED. Both workers get it.
# lib.escapeShellArg quotes the pair, because of the "=".
grep -F -- "--env 'KRATER_EDGE_PING_EXPECTED=true'" ${svc.krater-worker.serviceConfig.ExecStart}
if grep -F 'KRATER_EDGE_PING_EXPECTED' ${pvc.krater-prod-worker.serviceConfig.ExecStart}; then
exit 1
fi
grep -F -- "--env 'KRATER_EDGE_PING_EXPECTED=true'" ${pvc.krater-prod-worker.serviceConfig.ExecStart}
# Production keeps its one container and its prepare script.
for f in ${lib.concatStringsSep " " (lib.filter (lib.hasPrefix "/nix/store/") (lib.toList pvc.krater-prod-web.serviceConfig.ExecStartPre))}; do
if grep -qF 'alembic upgrade head' "$f"; then
Expand Down
Loading