Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 106 additions & 35 deletions hosts/alastor/edge.nix
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
# same change in jaspermayone/infra.
{
config,
options,
lib,
pkgs,
...
Expand All @@ -28,6 +29,19 @@ let
]
++ lib.optional kraterProduction "https://krater.patchworklabs.org/healthz/edge";

# Slack when a ping keeps failing, and again when it answers. Krater posts
# its own alerts, so it can't say it's down: this is the one that can. A
# Slack incoming-webhook URL, alone on one line, in
# ../../secrets/krater-edge-alert.age; on as soon as that file exists.
alertSecret = ../../secrets/krater-edge-alert.age;
haveAlertSecret = builtins.pathExists alertSecret;
hasAgenix = options ? age;
webhookFile = config.patchwork.edge.kraterEdgeAlertWebhookFile;
# Minutes of failed pings before the alert (a deploy or a blip is shorter),
# and between reminders while it lasts.
alertAfter = 3;
remindEvery = 60;

# <router name> = { backend = "<host>"; hosts = [ "<fqdn>" ... ]; };
routes = {
# Not `quilt`/`quilt-staging`: those were the router names of the old
Expand Down Expand Up @@ -109,45 +123,102 @@ in
description = "Route krater.patchworklabs.org and krater-storage.patchworklabs.org to weft.";
};

config.systemd.services.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing {
description = "Tell Krater its runners' path to it works";
path = [ pkgs.curl ];
serviceConfig = {
Type = "oneshot";
DynamicUser = true;
};
# Every URL, even after one fails: staging and production are separate.
script = ''
status=0
for url in ${lib.escapeShellArgs kraterPingUrls}; do
curl -fsS -o /dev/null --max-time 20 "$url" || { echo "no answer from $url" >&2; status=1; }
done
exit $status
options.patchwork.edge.kraterEdgeAlertWebhookFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = if haveAlertSecret && hasAgenix then config.age.secrets.krater-edge-alert.path else null;
description = ''
A file holding a Slack incoming-webhook URL. When set, krater-edge-ping
posts there after ${toString alertAfter} failed minutes in a row for a
URL, every ${toString remindEvery} minutes while it lasts, and once
when it answers again. Defaults to the agenix secret
krater-edge-alert.age once that file exists.
'';
};

config.systemd.timers.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "minutely";
AccuracySec = "5s";
};
};
# The secret only where agenix is: the edge test's bare system has no `age`
# option. Inside `config`, so that reading `options` here can't recurse.
config = lib.mkMerge [
{
systemd.services.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing {
description = "Tell Krater its runners' path to it works";
path = [ pkgs.curl ];
serviceConfig = {
Type = "oneshot";
DynamicUser = true;
# Consecutive failures per URL, kept between runs.
StateDirectory = "krater-edge-ping";
}
// lib.optionalAttrs (webhookFile != null) {
LoadCredential = [ "webhook:${webhookFile}" ];
};
# Every URL, even after one fails: staging and production are separate.
# A failed Slack post doesn't fail the unit, and doesn't stop the count.
script = ''
status=0
say() {
[ -n "''${CREDENTIALS_DIRECTORY:-}" ] && [ -r "$CREDENTIALS_DIRECTORY/webhook" ] || return 0
curl -fsS -o /dev/null --max-time 10 -H 'Content-Type: application/json' \
--data "{\"text\": \"$1\"}" "$(head -n1 "$CREDENTIALS_DIRECTORY/webhook")" \
|| echo "couldn't post to Slack" >&2
}
for url in ${lib.escapeShellArgs kraterPingUrls}; do
host=''${url#https://}
host=''${host%%/*}
file="$STATE_DIRECTORY/$host"
failed=$(cat "$file" 2>/dev/null || echo 0)
if curl -fsS -o /dev/null --max-time 20 "$url"; then
if [ "$failed" -ge ${toString alertAfter} ]; then
say ":white_check_mark: alastor reaches $host again, after $failed minutes without an answer."
fi
echo 0 > "$file"
else
echo "no answer from $url" >&2
status=1
failed=$((failed + 1))
echo "$failed" > "$file"
if [ "$failed" -eq ${toString alertAfter} ]; then
say ":rotating_light: alastor has had no answer from $host for $failed minutes (krater-edge-ping). Its runners likely can't reach it either. Check weft and the Krater service; the edge-ping runbook is in docs/guides/compute-runbook.md."
elif [ "$failed" -gt ${toString alertAfter} ] && [ $(( (failed - ${toString alertAfter}) % ${toString remindEvery} )) -eq 0 ]; then
say ":rotating_light: Still no answer from $host, for $failed minutes now."
fi
fi
done
exit $status
'';
};

systemd.timers.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "minutely";
AccuracySec = "5s";
};
};

config.environment.etc."traefik/conf.d/patchwork.toml".source =
(pkgs.formats.toml { }).generate "patchwork.toml"
{
http = {
routers = lib.mapAttrs (name: r: {
rule = lib.concatMapStringsSep " || " (h: "Host(`${h}`)") r.hosts;
entryPoints = [ "websecure" ];
tls.certResolver = "cloudflare";
middlewares = [ "hsts" ];
service = name;
}) routes;
services = lib.mapAttrs (_: r: {
loadBalancer.servers = [ { url = "http://${r.backend}.${tailnet}"; } ];
}) routes;
environment.etc."traefik/conf.d/patchwork.toml".source =
(pkgs.formats.toml { }).generate "patchwork.toml"
{
http = {
routers = lib.mapAttrs (name: r: {
rule = lib.concatMapStringsSep " || " (h: "Host(`${h}`)") r.hosts;
entryPoints = [ "websecure" ];
tls.certResolver = "cloudflare";
middlewares = [ "hsts" ];
service = name;
}) routes;
services = lib.mapAttrs (_: r: {
loadBalancer.servers = [ { url = "http://${r.backend}.${tailnet}"; } ];
}) routes;
};
};
}
(lib.optionalAttrs hasAgenix {
age.secrets = lib.mkIf haveAlertSecret {
krater-edge-alert = {
file = alertSecret;
mode = "400";
};
};
})
];
}
9 changes: 9 additions & 0 deletions secrets/secrets.nix
Original file line number Diff line number Diff line change
Expand Up @@ -333,4 +333,13 @@ in
weft
hank
];

# Slack incoming-webhook URL (alone on one line) for alastor's
# krater-edge-ping alert (hosts/alastor/edge.nix): posts when Krater stops
# answering alastor and when it's back. The alert is on once this file is
# committed.
"krater-edge-alert.age".publicKeys = [
jsp
alastor
];
}
10 changes: 10 additions & 0 deletions tests/krater-weft.nix
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ let
edgeOff = mkEdge { patchwork.edge.kraterProduction = false; };
edgeOffToml = edgeOff.environment.etc."traefik/conf.d/patchwork.toml".source;
edgeProd = mkEdge { patchwork.edge.kraterProduction = true; };
edgeAlert = mkEdge { patchwork.edge.kraterEdgeAlertWebhookFile = "/run/agenix/krater-edge-alert"; };
edgeProdToml = edgeProd.environment.etc."traefik/conf.d/patchwork.toml".source;
units = on.systemd.units;
pvc = prod.systemd.services;
Expand Down Expand Up @@ -175,6 +176,15 @@ let
edgeOff.systemd.timers.krater-edge-ping.timerConfig.OnCalendar == "minutely"
&& lib.hasInfix "krater-staging.patchworklabs.org/healthz/edge" edgeOff.systemd.services.krater-edge-ping.script
&& !(lib.hasInfix "//krater.patchworklabs.org" edgeOff.systemd.services.krater-edge-ping.script);
# Krater can't say it's down: alastor's ping does, once its webhook is set.
"edge alert off without its webhook" =
edgeOff.patchwork.edge.kraterEdgeAlertWebhookFile == null
&& !(edgeOff.systemd.services.krater-edge-ping.serviceConfig ? LoadCredential);
"edge alert reads its webhook" =
edgeAlert.systemd.services.krater-edge-ping.serviceConfig.LoadCredential
== [ "webhook:/run/agenix/krater-edge-alert" ]
&& edgeAlert.systemd.services.krater-edge-ping.serviceConfig.StateDirectory == "krater-edge-ping"
&& lib.hasInfix "rotating_light" edgeAlert.systemd.services.krater-edge-ping.script;
"edge pings production while its routes are on" =
lib.hasInfix "https://krater.patchworklabs.org/healthz/edge" edgeProd.systemd.services.krater-edge-ping.script;
"switch off: krater only on hem" =
Expand Down
Loading