From d91891e1c4060966c5d0ad7a30a3721859310d1d Mon Sep 17 00:00:00 2001 From: James Peru Date: Fri, 1 May 2026 03:08:35 +0300 Subject: [PATCH 1/8] KVM: make storage heartbeat fence action configurable The KVM agent's storage heartbeat scripts (kvmheartbeat.sh and kvmspheartbeat.sh) hard-code an immediate kernel-level reboot via 'echo b > /proc/sysrq-trigger' when a heartbeat write to primary storage times out. This bypasses all OS-level shutdown protections, drops every running VM on the host instantly, and triggers HA cascades onto surviving hosts. For NFS shared storage the binary "heartbeat-write-failed = host-is-dead" heuristic is reasonable. For LINSTOR/DRBD or other replicated local storage, the same disk serves application I/O, replication I/O and heartbeat I/O simultaneously - so a transient I/O contention spike can time out the heartbeat write without the host actually being unhealthy. The result is false-positive sysrq fencing. Adds a new agent.properties option: kvm.heartbeat.fence.action = reboot | graceful-reboot | restart-agent | log-only Default value is "reboot" so existing deployments keep their current behavior. Operators on replicated storage backends can choose a less destructive action: - graceful-reboot: 'systemctl reboot' instead of sysrq, allowing VMs a chance to shut down cleanly - restart-agent: restart cloudstack-agent only, preserving running VMs - log-only: log + alert, no automatic action The existing 'reboot.host.and.alert.management.on.heartbeat.timeout' boolean continues to function as a complete Java-side bypass. Refs: https://github.com/apache/cloudstack/issues/13089 --- agent/conf/agent.properties | 16 +++++++ .../agent/properties/AgentProperties.java | 19 +++++++++ scripts/vm/hypervisor/kvm/kvmheartbeat.sh | 42 ++++++++++++++++--- scripts/vm/hypervisor/kvm/kvmspheartbeat.sh | 42 ++++++++++++++++--- 4 files changed, 109 insertions(+), 10 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 4e36eff4d75f..3551abf5477b 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -318,6 +318,22 @@ iscsi.session.cleanup.enabled=false # This parameter specifies if the host must be rebooted when something goes wrong with the heartbeat. #reboot.host.and.alert.management.on.heartbeat.timeout=false +# Action taken by kvmheartbeat.sh / kvmspheartbeat.sh when a storage heartbeat +# write fails persistently. Supersedes the legacy binary +# 'reboot.host.and.alert.management.on.heartbeat.timeout' when set to a non-default value. +# +# Allowed values: +# reboot - immediate sysrq-trigger reboot (default; original behavior) +# graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly +# restart-agent - restart cloudstack-agent only; running VMs are preserved +# log-only - log + alert; take no automatic action (admin must investigate) +# +# The 'graceful-reboot', 'restart-agent', and 'log-only' actions are recommended +# for setups using LINSTOR/DRBD or any local storage with replication, where +# transient I/O contention can cause a heartbeat write to time out without the +# host actually being unhealthy. +#kvm.heartbeat.fence.action=reboot + # Enables manually setting CPU's topology on KVM's VM. #enable.manually.setting.cpu.topology.on.kvm.vm=true diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index e4775188d0ca..53c9387cfb04 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -622,6 +622,25 @@ public class AgentProperties{ public static final Property REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT = new Property<>("reboot.host.and.alert.management.on.heartbeat.timeout", false); + /** + * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh / kvmspheartbeat.sh) + * when a heartbeat write fails persistently. Allowed values: + * + * The non-default values are recommended for setups using LINSTOR/DRBD or other replicated + * local storage, where transient I/O contention can cause a heartbeat write to time out + * without the host actually being unhealthy.
+ * Read by the heartbeat shell scripts directly from agent.properties.
+ * Data type: String.
+ * Default value: {@code reboot} + */ + public static final Property KVM_HEARTBEAT_FENCE_ACTION + = new Property<>("kvm.heartbeat.fence.action", "reboot"); + /** * Enables manually setting CPU's topology on KVM's VM.
* Data type: Boolean.
diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh index 1fa49b807769..62b6e9e2be6f 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh @@ -156,11 +156,43 @@ then exit 0 elif [ "$cflag" == "1" ] then - /usr/bin/logger -t heartbeat "kvmheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? + # Read fence action from agent.properties (default: reboot for backward compatibility). + # Allowed values: reboot | graceful-reboot | restart-agent | log-only + AGENT_PROPS="/etc/cloudstack/agent/agent.properties" + FENCE_ACTION="reboot" + if [ -r "$AGENT_PROPS" ]; then + val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') + [ -n "$val" ] && FENCE_ACTION="$val" + fi + + case "$FENCE_ACTION" in + log-only) + /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." + exit 0 + ;; + restart-agent) + /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." + sync & + sleep 2 + systemctl restart cloudstack-agent + exit $? + ;; + graceful-reboot) + /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." + sync & + sleep 5 + systemctl reboot + exit $? + ;; + reboot|*) + # Original behavior: immediate kernel-level reboot via sysrq-trigger + /usr/bin/logger -t heartbeat "kvmheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." + sync & + sleep 5 + echo b > /proc/sysrq-trigger + exit $? + ;; + esac else write_hbLog exit $? diff --git a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh index 3cb459e3e854..bc0294cfdfb7 100755 --- a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh @@ -58,9 +58,41 @@ deleteVMs() { if [ "$cflag" == "1" ] then - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? + # Read fence action from agent.properties (default: reboot for backward compatibility). + # Allowed values: reboot | graceful-reboot | restart-agent | log-only + AGENT_PROPS="/etc/cloudstack/agent/agent.properties" + FENCE_ACTION="reboot" + if [ -r "$AGENT_PROPS" ]; then + val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') + [ -n "$val" ] && FENCE_ACTION="$val" + fi + + case "$FENCE_ACTION" in + log-only) + /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." + exit 0 + ;; + restart-agent) + /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." + sync & + sleep 2 + systemctl restart cloudstack-agent + exit $? + ;; + graceful-reboot) + /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." + sync & + sleep 5 + systemctl reboot + exit $? + ;; + reboot|*) + # Original behavior: immediate kernel-level reboot via sysrq-trigger + /usr/bin/logger -t heartbeat "kvmspheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." + sync & + sleep 5 + echo b > /proc/sysrq-trigger + exit $? + ;; + esac fi From 6c5938b098fe0d934826d4baf5d85ab651e17309 Mon Sep 17 00:00:00 2001 From: jmsperu Date: Sat, 9 May 2026 06:50:28 +0300 Subject: [PATCH 2/8] kvm-heartbeat: extract shared fence helper; add 'custom' action; rename to hard-reboot Per review on PR #13090: - Refactor common fence-action case into kvmha-fence.sh sourced by both kvmheartbeat.sh and kvmspheartbeat.sh (per @sureshanaparti). - Add 'custom' fence action that invokes an operator-supplied script (kvm.heartbeat.fence.custom.script, default /etc/cloudstack/agent/ heartbeat-fence-custom.sh) with the heartbeat script name as arg; falls back to hard-reboot if the script is missing/non-executable (per @NuxRo). - Rename canonical action 'reboot' -> 'hard-reboot' for clarity; keep 'reboot' accepted as alias so existing deployments don't break (per @DaanHoogland). Default behavior unchanged: sysrq-trigger reboot, required where a stale NFSv3 mount blocks systemctl reboot. --- agent/conf/agent.properties | 27 ++++-- .../agent/properties/AgentProperties.java | 31 +++++-- scripts/vm/hypervisor/kvm/kvmha-fence.sh | 85 +++++++++++++++++++ scripts/vm/hypervisor/kvm/kvmheartbeat.sh | 40 +-------- scripts/vm/hypervisor/kvm/kvmspheartbeat.sh | 40 +-------- 5 files changed, 134 insertions(+), 89 deletions(-) create mode 100755 scripts/vm/hypervisor/kvm/kvmha-fence.sh diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 3551abf5477b..f290f600325b 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -323,16 +323,25 @@ iscsi.session.cleanup.enabled=false # 'reboot.host.and.alert.management.on.heartbeat.timeout' when set to a non-default value. # # Allowed values: -# reboot - immediate sysrq-trigger reboot (default; original behavior) -# graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly -# restart-agent - restart cloudstack-agent only; running VMs are preserved -# log-only - log + alert; take no automatic action (admin must investigate) +# hard-reboot - immediate sysrq-trigger reboot (default; 'reboot' kept as alias). +# Required default for setups where a stale NFSv3 mount can prevent +# a graceful shutdown from completing. +# graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly. +# Use only if a stale storage mount cannot block shutdown. +# restart-agent - restart cloudstack-agent only; running VMs are preserved. +# log-only - log + alert; take no automatic action (admin must investigate). +# custom - invoke the script at 'kvm.heartbeat.fence.custom.script' (see below). +# Script is called with one positional arg: the heartbeat script name +# (e.g. 'kvmheartbeat.sh'). Falls back to hard-reboot if missing or +# not executable. # -# The 'graceful-reboot', 'restart-agent', and 'log-only' actions are recommended -# for setups using LINSTOR/DRBD or any local storage with replication, where -# transient I/O contention can cause a heartbeat write to time out without the -# host actually being unhealthy. -#kvm.heartbeat.fence.action=reboot +# The non-default values are recommended for setups using LINSTOR/DRBD or any local +# storage with replication, where transient I/O contention can cause a heartbeat +# write to time out without the host actually being unhealthy. +#kvm.heartbeat.fence.action=hard-reboot + +# Path to the operator-supplied script invoked when kvm.heartbeat.fence.action=custom. +#kvm.heartbeat.fence.custom.script=/etc/cloudstack/agent/heartbeat-fence-custom.sh # Enables manually setting CPU's topology on KVM's VM. #enable.manually.setting.cpu.topology.on.kvm.vm=true diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 53c9387cfb04..4a5314e44994 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -626,20 +626,39 @@ public class AgentProperties{ * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh / kvmspheartbeat.sh) * when a heartbeat write fails persistently. Allowed values: *
    - *
  • {@code reboot} (default) — immediate sysrq-trigger reboot; original behavior
    • - *
  • {@code graceful-reboot} — {@code systemctl reboot} instead of sysrq, lets VMs stop cleanly
    • - *
  • {@code restart-agent} — restart cloudstack-agent only; running VMs preserved
    • - *
  • {@code log-only} — log + alert, no automatic action
    • + *
  • {@code hard-reboot} (default; {@code reboot} accepted as alias) — immediate + * sysrq-trigger reboot. Required default for setups where a stale NFSv3 mount can + * prevent a graceful shutdown from completing.
    • + *
  • {@code graceful-reboot} — {@code systemctl reboot} instead of sysrq; allows VMs + * to stop cleanly. Use only if a stale storage mount cannot block shutdown.
    • + *
  • {@code restart-agent} — restart cloudstack-agent only; running VMs preserved.
    • + *
  • {@code log-only} — log + alert; take no automatic action (admin must investigate).
    • + *
  • {@code custom} — invoke the script at {@link #KVM_HEARTBEAT_FENCE_CUSTOM_SCRIPT} + * (default {@code /etc/cloudstack/agent/heartbeat-fence-custom.sh}). The script is + * called with one argument: the heartbeat script name (e.g. {@code kvmheartbeat.sh}). + * If the script is missing or not executable, falls back to {@code hard-reboot}.
    • *
* The non-default values are recommended for setups using LINSTOR/DRBD or other replicated * local storage, where transient I/O contention can cause a heartbeat write to time out * without the host actually being unhealthy.
* Read by the heartbeat shell scripts directly from agent.properties.
* Data type: String.
- * Default value: {@code reboot} + * Default value: {@code hard-reboot} */ public static final Property KVM_HEARTBEAT_FENCE_ACTION - = new Property<>("kvm.heartbeat.fence.action", "reboot"); + = new Property<>("kvm.heartbeat.fence.action", "hard-reboot"); + + /** + * Path to the operator-supplied script invoked when + * {@link #KVM_HEARTBEAT_FENCE_ACTION} is set to {@code custom}. The script must be + * executable and is called with a single positional argument: the heartbeat script name + * that triggered the fence (e.g. {@code kvmheartbeat.sh}). Read by the heartbeat shell + * scripts directly from agent.properties.
+ * Data type: String.
+ * Default value: {@code /etc/cloudstack/agent/heartbeat-fence-custom.sh} + */ + public static final Property KVM_HEARTBEAT_FENCE_CUSTOM_SCRIPT + = new Property<>("kvm.heartbeat.fence.custom.script", "/etc/cloudstack/agent/heartbeat-fence-custom.sh"); /** * Enables manually setting CPU's topology on KVM's VM.
diff --git a/scripts/vm/hypervisor/kvm/kvmha-fence.sh b/scripts/vm/hypervisor/kvm/kvmha-fence.sh new file mode 100755 index 000000000000..5f3006f71d7b --- /dev/null +++ b/scripts/vm/hypervisor/kvm/kvmha-fence.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Shared fence-action helper for kvmheartbeat.sh and kvmspheartbeat.sh. +# Sourced by both scripts; do not invoke directly. +# +# Usage from caller: +# source "$(dirname "$0")/kvmha-fence.sh" +# fence_action "kvmheartbeat.sh" # script name passed for log tagging + +AGENT_PROPS="${AGENT_PROPS:-/etc/cloudstack/agent/agent.properties}" + +fence_action() { + local source_script="${1:-kvmha}" + local FENCE_ACTION="hard-reboot" + local CUSTOM_SCRIPT="/etc/cloudstack/agent/heartbeat-fence-custom.sh" + + if [ -r "$AGENT_PROPS" ]; then + local val + val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') + [ -n "$val" ] && FENCE_ACTION="$val" + local cval + cval=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.custom\.script[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -n "$cval" ] && CUSTOM_SCRIPT="$cval" + fi + + case "$FENCE_ACTION" in + log-only) + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." + exit 0 + ;; + restart-agent) + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." + sync & + sleep 2 + systemctl restart cloudstack-agent + exit $? + ;; + graceful-reboot) + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." + sync & + sleep 5 + systemctl reboot + exit $? + ;; + custom) + if [ -x "$CUSTOM_SCRIPT" ]; then + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'custom' — running ${CUSTOM_SCRIPT}." + sync & + sleep 2 + "$CUSTOM_SCRIPT" "$source_script" + exit $? + else + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'custom' selected but ${CUSTOM_SCRIPT} is missing or not executable — falling back to hard-reboot." + sync & + sleep 5 + echo b > /proc/sysrq-trigger + exit $? + fi + ;; + hard-reboot|reboot|*) + # 'reboot' kept as alias for back-compat with pre-existing deployments. + /usr/bin/logger -t heartbeat "${source_script} will reboot system because it was unable to write the heartbeat to the storage." + sync & + sleep 5 + echo b > /proc/sysrq-trigger + exit $? + ;; + esac +} diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh index 62b6e9e2be6f..645d70dc9ad9 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh @@ -156,43 +156,9 @@ then exit 0 elif [ "$cflag" == "1" ] then - # Read fence action from agent.properties (default: reboot for backward compatibility). - # Allowed values: reboot | graceful-reboot | restart-agent | log-only - AGENT_PROPS="/etc/cloudstack/agent/agent.properties" - FENCE_ACTION="reboot" - if [ -r "$AGENT_PROPS" ]; then - val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') - [ -n "$val" ] && FENCE_ACTION="$val" - fi - - case "$FENCE_ACTION" in - log-only) - /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." - exit 0 - ;; - restart-agent) - /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." - sync & - sleep 2 - systemctl restart cloudstack-agent - exit $? - ;; - graceful-reboot) - /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." - sync & - sleep 5 - systemctl reboot - exit $? - ;; - reboot|*) - # Original behavior: immediate kernel-level reboot via sysrq-trigger - /usr/bin/logger -t heartbeat "kvmheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? - ;; - esac + # shellcheck disable=SC1091 + . "$(dirname "$0")/kvmha-fence.sh" + fence_action "kvmheartbeat.sh" else write_hbLog exit $? diff --git a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh index bc0294cfdfb7..1a5b2a24dd9f 100755 --- a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh @@ -58,41 +58,7 @@ deleteVMs() { if [ "$cflag" == "1" ] then - # Read fence action from agent.properties (default: reboot for backward compatibility). - # Allowed values: reboot | graceful-reboot | restart-agent | log-only - AGENT_PROPS="/etc/cloudstack/agent/agent.properties" - FENCE_ACTION="reboot" - if [ -r "$AGENT_PROPS" ]; then - val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') - [ -n "$val" ] && FENCE_ACTION="$val" - fi - - case "$FENCE_ACTION" in - log-only) - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." - exit 0 - ;; - restart-agent) - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." - sync & - sleep 2 - systemctl restart cloudstack-agent - exit $? - ;; - graceful-reboot) - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." - sync & - sleep 5 - systemctl reboot - exit $? - ;; - reboot|*) - # Original behavior: immediate kernel-level reboot via sysrq-trigger - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? - ;; - esac + # shellcheck disable=SC1091 + . "$(dirname "$0")/kvmha-fence.sh" + fence_action "kvmspheartbeat.sh" fi From c04098039eb9ff3b411c7c91b9afed480c785b3f Mon Sep 17 00:00:00 2001 From: jmsperu Date: Tue, 26 May 2026 21:14:43 +0300 Subject: [PATCH 3/8] kvmsmpheartbeat.sh: honour kvm.heartbeat.fence.action Per @weizhouapache's review on #13090, kvmsmpheartbeat.sh was still hard-coding the immediate-reboot fence action (echo b > /proc/sysrq-trigger) while kvmheartbeat.sh and kvmspheartbeat.sh delegate to kvmha-fence.sh. Source kvmha-fence.sh and call fence_action so this script picks up the same hard-reboot / graceful-reboot / restart-agent / log-only / custom behaviour from agent.properties. Default behaviour (no property set) remains hard-reboot via sysrq-trigger, so existing deployments are unaffected. --- scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh index b102a1a866bb..25dfb596e388 100755 --- a/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh @@ -207,11 +207,13 @@ then exit 0 elif [ "$cflag" == "1" ] then - /usr/bin/logger -t heartbeat "kvmsmpheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? + # Delegate the fence action to the shared helper so this script honours the + # kvm.heartbeat.fence.action property (hard-reboot / graceful-reboot / + # restart-agent / log-only / custom) like kvmheartbeat.sh and kvmspheartbeat.sh. + # Default (no property set) remains hard-reboot via sysrq-trigger. + # shellcheck disable=SC1091 + . "$(dirname "$0")/kvmha-fence.sh" + fence_action "kvmsmpheartbeat.sh" else write_hbLog exit $? From 8fdf6079c58baa20183ce74529d07dedc6d85176 Mon Sep 17 00:00:00 2001 From: James Peru Date: Mon, 24 Aug 2026 16:52:21 +0300 Subject: [PATCH 4/8] address review (#13090): drop the 'reboot' alias, simplify the agent.properties lookup, clarify precedence - kvm.heartbeat.fence.action accepts hard-reboot (default), graceful-reboot, restart-agent, log-only and custom; the misleading 'reboot' alias is gone. - kvmha-fence.sh reads the property with a plain grep on the key=value form. - agent.properties comment states that reboot.host.and.alert.management.on.heartbeat.timeout stays the global on/off switch and this setting only chooses the action. Signed-off-by: James Peru --- agent/conf/agent.properties | 7 ++++--- scripts/vm/hypervisor/kvm/kvmha-fence.sh | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index f290f600325b..87855138a740 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -319,11 +319,12 @@ iscsi.session.cleanup.enabled=false #reboot.host.and.alert.management.on.heartbeat.timeout=false # Action taken by kvmheartbeat.sh / kvmspheartbeat.sh when a storage heartbeat -# write fails persistently. Supersedes the legacy binary -# 'reboot.host.and.alert.management.on.heartbeat.timeout' when set to a non-default value. +# write fails persistently. Only consulted when fencing is enabled, i.e. +# 'reboot.host.and.alert.management.on.heartbeat.timeout' is true: that boolean remains +# the global on/off switch for fencing, this setting only chooses WHAT the fence does. # # Allowed values: -# hard-reboot - immediate sysrq-trigger reboot (default; 'reboot' kept as alias). +# hard-reboot - immediate sysrq-trigger reboot (default). # Required default for setups where a stale NFSv3 mount can prevent # a graceful shutdown from completing. # graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly. diff --git a/scripts/vm/hypervisor/kvm/kvmha-fence.sh b/scripts/vm/hypervisor/kvm/kvmha-fence.sh index 5f3006f71d7b..e4706ca040c6 100755 --- a/scripts/vm/hypervisor/kvm/kvmha-fence.sh +++ b/scripts/vm/hypervisor/kvm/kvmha-fence.sh @@ -32,10 +32,10 @@ fence_action() { if [ -r "$AGENT_PROPS" ]; then local val - val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') + val=$(grep "^kvm.heartbeat.fence.action=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') [ -n "$val" ] && FENCE_ACTION="$val" local cval - cval=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.custom\.script[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + cval=$(grep "^kvm.heartbeat.fence.custom.script=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') [ -n "$cval" ] && CUSTOM_SCRIPT="$cval" fi @@ -73,8 +73,8 @@ fence_action() { exit $? fi ;; - hard-reboot|reboot|*) - # 'reboot' kept as alias for back-compat with pre-existing deployments. + hard-reboot|*) + # Default, and the fallback for any unrecognised value: identical to the pre-existing behaviour. /usr/bin/logger -t heartbeat "${source_script} will reboot system because it was unable to write the heartbeat to the storage." sync & sleep 5 From 32818fb105de4fc335264780ebf7f0fbe70ef0b7 Mon Sep 17 00:00:00 2001 From: James Peru Date: Mon, 12 Oct 2026 23:48:55 +0300 Subject: [PATCH 5/8] address review (#13090): log-only only writes to syslog, drop the custom action - log-only is now described everywhere (agent.properties, AgentProperties Javadoc, kvmha-fence.sh) as writing a message to the host's syslog and taking no action: no reboot, no agent restart and no management server alert. The "log + alert" wording is gone. - The custom action and kvm.heartbeat.fence.custom.script are removed (agent.properties, AgentProperties.KVM_HEARTBEAT_FENCE_CUSTOM_SCRIPT, the custom branch and CUSTOM_SCRIPT handling in kvmha-fence.sh, the kvmsmpheartbeat.sh comment). Unknown values, including custom, still fall back to hard-reboot. - The AgentProperties Javadoc documents only hard-reboot (no reboot alias), names kvmsmpheartbeat.sh too, and states the precedence with reboot.host.and.alert.management.on.heartbeat.timeout. Claude-Session: https://claude.ai/code/session_01BhiE1Abhs315m3n3UPy6V7 --- agent/conf/agent.properties | 15 ++++---- .../agent/properties/AgentProperties.java | 35 ++++++------------- scripts/vm/hypervisor/kvm/kvmha-fence.sh | 34 ++++++------------ scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh | 2 +- 4 files changed, 28 insertions(+), 58 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 87855138a740..ef1a46f23f7f 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -318,7 +318,7 @@ iscsi.session.cleanup.enabled=false # This parameter specifies if the host must be rebooted when something goes wrong with the heartbeat. #reboot.host.and.alert.management.on.heartbeat.timeout=false -# Action taken by kvmheartbeat.sh / kvmspheartbeat.sh when a storage heartbeat +# Action taken by kvmheartbeat.sh / kvmspheartbeat.sh / kvmsmpheartbeat.sh when a storage heartbeat # write fails persistently. Only consulted when fencing is enabled, i.e. # 'reboot.host.and.alert.management.on.heartbeat.timeout' is true: that boolean remains # the global on/off switch for fencing, this setting only chooses WHAT the fence does. @@ -330,20 +330,17 @@ iscsi.session.cleanup.enabled=false # graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly. # Use only if a stale storage mount cannot block shutdown. # restart-agent - restart cloudstack-agent only; running VMs are preserved. -# log-only - log + alert; take no automatic action (admin must investigate). -# custom - invoke the script at 'kvm.heartbeat.fence.custom.script' (see below). -# Script is called with one positional arg: the heartbeat script name -# (e.g. 'kvmheartbeat.sh'). Falls back to hard-reboot if missing or -# not executable. +# log-only - only write a message to the host's syslog; take no action +# (no reboot, no agent restart, no management server alert). +# The operator must watch syslog and investigate. +# +# Any other value falls back to hard-reboot. # # The non-default values are recommended for setups using LINSTOR/DRBD or any local # storage with replication, where transient I/O contention can cause a heartbeat # write to time out without the host actually being unhealthy. #kvm.heartbeat.fence.action=hard-reboot -# Path to the operator-supplied script invoked when kvm.heartbeat.fence.action=custom. -#kvm.heartbeat.fence.custom.script=/etc/cloudstack/agent/heartbeat-fence-custom.sh - # Enables manually setting CPU's topology on KVM's VM. #enable.manually.setting.cpu.topology.on.kvm.vm=true diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 4a5314e44994..f3bc3e6f2460 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -623,21 +623,20 @@ public class AgentProperties{ = new Property<>("reboot.host.and.alert.management.on.heartbeat.timeout", false); /** - * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh / kvmspheartbeat.sh) - * when a heartbeat write fails persistently. Allowed values: + * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh, kvmspheartbeat.sh + * and kvmsmpheartbeat.sh) when a heartbeat write fails persistently. Only consulted when + * {@link #REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT} is true: that boolean remains the + * on/off switch for fencing, this setting only chooses what the fence does. Allowed values: *
    - *
  • {@code hard-reboot} (default; {@code reboot} accepted as alias) — immediate - * sysrq-trigger reboot. Required default for setups where a stale NFSv3 mount can - * prevent a graceful shutdown from completing.
    • - *
  • {@code graceful-reboot} — {@code systemctl reboot} instead of sysrq; allows VMs + *
  • {@code hard-reboot} (default): immediate sysrq-trigger reboot. Required default for setups + * where a stale NFSv3 mount can prevent a graceful shutdown from completing.
    • + *
  • {@code graceful-reboot}: {@code systemctl reboot} instead of sysrq; allows VMs * to stop cleanly. Use only if a stale storage mount cannot block shutdown.
    • - *
  • {@code restart-agent} — restart cloudstack-agent only; running VMs preserved.
    • - *
  • {@code log-only} — log + alert; take no automatic action (admin must investigate).
    • - *
  • {@code custom} — invoke the script at {@link #KVM_HEARTBEAT_FENCE_CUSTOM_SCRIPT} - * (default {@code /etc/cloudstack/agent/heartbeat-fence-custom.sh}). The script is - * called with one argument: the heartbeat script name (e.g. {@code kvmheartbeat.sh}). - * If the script is missing or not executable, falls back to {@code hard-reboot}.
    • + *
  • {@code restart-agent}: restart cloudstack-agent only; running VMs preserved.
    • + *
  • {@code log-only}: only write a message to the host's syslog and take no action (no reboot, + * no agent restart, no management server alert); the operator must investigate.
    • *
+ * Any other value falls back to {@code hard-reboot}.
* The non-default values are recommended for setups using LINSTOR/DRBD or other replicated * local storage, where transient I/O contention can cause a heartbeat write to time out * without the host actually being unhealthy.
@@ -648,18 +647,6 @@ public class AgentProperties{ public static final Property KVM_HEARTBEAT_FENCE_ACTION = new Property<>("kvm.heartbeat.fence.action", "hard-reboot"); - /** - * Path to the operator-supplied script invoked when - * {@link #KVM_HEARTBEAT_FENCE_ACTION} is set to {@code custom}. The script must be - * executable and is called with a single positional argument: the heartbeat script name - * that triggered the fence (e.g. {@code kvmheartbeat.sh}). Read by the heartbeat shell - * scripts directly from agent.properties.
- * Data type: String.
- * Default value: {@code /etc/cloudstack/agent/heartbeat-fence-custom.sh} - */ - public static final Property KVM_HEARTBEAT_FENCE_CUSTOM_SCRIPT - = new Property<>("kvm.heartbeat.fence.custom.script", "/etc/cloudstack/agent/heartbeat-fence-custom.sh"); - /** * Enables manually setting CPU's topology on KVM's VM.
* Data type: Boolean.
diff --git a/scripts/vm/hypervisor/kvm/kvmha-fence.sh b/scripts/vm/hypervisor/kvm/kvmha-fence.sh index e4706ca040c6..884e6e815736 100755 --- a/scripts/vm/hypervisor/kvm/kvmha-fence.sh +++ b/scripts/vm/hypervisor/kvm/kvmha-fence.sh @@ -16,8 +16,12 @@ # specific language governing permissions and limitations # under the License. # -# Shared fence-action helper for kvmheartbeat.sh and kvmspheartbeat.sh. -# Sourced by both scripts; do not invoke directly. +# Shared fence-action helper for kvmheartbeat.sh, kvmspheartbeat.sh and kvmsmpheartbeat.sh. +# Sourced by those scripts; do not invoke directly. +# +# kvm.heartbeat.fence.action in agent.properties selects the action: +# hard-reboot (default), graceful-reboot, restart-agent, or log-only (only writes to +# syslog and takes no action). Any other value falls back to hard-reboot. # # Usage from caller: # source "$(dirname "$0")/kvmha-fence.sh" @@ -28,51 +32,33 @@ AGENT_PROPS="${AGENT_PROPS:-/etc/cloudstack/agent/agent.properties}" fence_action() { local source_script="${1:-kvmha}" local FENCE_ACTION="hard-reboot" - local CUSTOM_SCRIPT="/etc/cloudstack/agent/heartbeat-fence-custom.sh" if [ -r "$AGENT_PROPS" ]; then local val val=$(grep "^kvm.heartbeat.fence.action=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') [ -n "$val" ] && FENCE_ACTION="$val" - local cval - cval=$(grep "^kvm.heartbeat.fence.custom.script=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') - [ -n "$cval" ] && CUSTOM_SCRIPT="$cval" fi case "$FENCE_ACTION" in log-only) - /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." + # Only writes to syslog; no reboot, no agent restart and no management server alert. + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'log-only' selected, taking no action. Operator must investigate." exit 0 ;; restart-agent) - /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'restart-agent', restarting cloudstack-agent (running VMs preserved)." sync & sleep 2 systemctl restart cloudstack-agent exit $? ;; graceful-reboot) - /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'graceful-reboot', rebooting via systemctl (allows running VMs to stop cleanly)." sync & sleep 5 systemctl reboot exit $? ;; - custom) - if [ -x "$CUSTOM_SCRIPT" ]; then - /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'custom' — running ${CUSTOM_SCRIPT}." - sync & - sleep 2 - "$CUSTOM_SCRIPT" "$source_script" - exit $? - else - /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'custom' selected but ${CUSTOM_SCRIPT} is missing or not executable — falling back to hard-reboot." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? - fi - ;; hard-reboot|*) # Default, and the fallback for any unrecognised value: identical to the pre-existing behaviour. /usr/bin/logger -t heartbeat "${source_script} will reboot system because it was unable to write the heartbeat to the storage." diff --git a/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh index 25dfb596e388..b5dddefe1a9c 100755 --- a/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh @@ -209,7 +209,7 @@ elif [ "$cflag" == "1" ] then # Delegate the fence action to the shared helper so this script honours the # kvm.heartbeat.fence.action property (hard-reboot / graceful-reboot / - # restart-agent / log-only / custom) like kvmheartbeat.sh and kvmspheartbeat.sh. + # restart-agent / log-only) like kvmheartbeat.sh and kvmspheartbeat.sh. # Default (no property set) remains hard-reboot via sysrq-trigger. # shellcheck disable=SC1091 . "$(dirname "$0")/kvmha-fence.sh" From 0b61bd3b2c78780598bf5a90141758946f4e38a5 Mon Sep 17 00:00:00 2001 From: James Peru Date: Wed, 21 Oct 2026 09:30:53 +0300 Subject: [PATCH 6/8] address review (#13090): restart-agent and log-only stop the host's VMs first With restart-agent or log-only the VMs kept running after the heartbeat went stale, so if the management server declared the host Down, HA could start the same VMs on another host and two copies would write to the same disks (raised by Damans227). - kvmha-fence.sh: restart-agent and log-only now stop every active VM on the host (virsh destroy) before acting; the host itself stays up. If libvirt cannot be queried or any VM is still active afterwards, the fence falls back to hard-reboot. - New agent property kvm.heartbeat.fence.keep.vms.running (default false) lets an operator keep the old behaviour; the fence then logs a WARNING and agent.properties / AgentProperties say it is only safe when no VM on the host can be restarted elsewhere by HA. - hard-reboot and graceful-reboot are unchanged. - The logger and sysrq trigger paths can be overridden through the environment (FENCE_LOGGER, FENCE_SYSRQ_TRIGGER), like AGENT_PROPS, so the new KvmHaFenceScriptTest runs the script with stand-ins for virsh, systemctl, logger and sysrq and never touches the machine running it. Agent module: 65 tests pass (6 new), bash -n clean on all heartbeat scripts. Claude-Session: https://claude.ai/code/session_01V82Ypx7SpAurn6ek1VXRAP --- agent/conf/agent.properties | 21 ++- .../agent/properties/AgentProperties.java | 22 ++- .../com/cloud/agent/KvmHaFenceScriptTest.java | 163 ++++++++++++++++++ scripts/vm/hypervisor/kvm/kvmha-fence.sh | 59 +++++-- 4 files changed, 246 insertions(+), 19 deletions(-) create mode 100644 agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index ef1a46f23f7f..beb69c0048d7 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -329,18 +329,31 @@ iscsi.session.cleanup.enabled=false # a graceful shutdown from completing. # graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly. # Use only if a stale storage mount cannot block shutdown. -# restart-agent - restart cloudstack-agent only; running VMs are preserved. -# log-only - only write a message to the host's syslog; take no action -# (no reboot, no agent restart, no management server alert). -# The operator must watch syslog and investigate. +# restart-agent - stop every VM on the host, then restart cloudstack-agent; the +# host itself stays up. +# log-only - stop every VM on the host, then only write a message to the +# host's syslog (no reboot, no agent restart, no management server +# alert). The operator must watch syslog and investigate. # # Any other value falls back to hard-reboot. # +# restart-agent and log-only stop the VMs because, once the heartbeat is stale, the +# management server may declare the host Down and HA may start the same VMs on another +# host; two copies would then write to the same disks. If any VM cannot be stopped, +# the fence falls back to hard-reboot. +# # The non-default values are recommended for setups using LINSTOR/DRBD or any local # storage with replication, where transient I/O contention can cause a heartbeat # write to time out without the host actually being unhealthy. #kvm.heartbeat.fence.action=hard-reboot +# Set to true to let restart-agent and log-only leave the VMs running. This is only +# safe when no VM on this host can be started on another host by HA (VM HA off for all +# of them and host HA not enabled). Otherwise a host the management server declares +# Down can end up with the same VM running twice and corrupting its disks. +# Has no effect on hard-reboot and graceful-reboot. +#kvm.heartbeat.fence.keep.vms.running=false + # Enables manually setting CPU's topology on KVM's VM. #enable.manually.setting.cpu.topology.on.kvm.vm=true diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index f3bc3e6f2460..2785555f5224 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -632,11 +632,15 @@ public class AgentProperties{ * where a stale NFSv3 mount can prevent a graceful shutdown from completing. *
  • {@code graceful-reboot}: {@code systemctl reboot} instead of sysrq; allows VMs * to stop cleanly. Use only if a stale storage mount cannot block shutdown.
    • - *
  • {@code restart-agent}: restart cloudstack-agent only; running VMs preserved.
    • - *
  • {@code log-only}: only write a message to the host's syslog and take no action (no reboot, - * no agent restart, no management server alert); the operator must investigate.
    • + *
  • {@code restart-agent}: stop every VM on the host, then restart cloudstack-agent; the host stays up.
    • + *
  • {@code log-only}: stop every VM on the host, then only write a message to the host's syslog (no + * reboot, no agent restart, no management server alert); the operator must investigate.
    • * * Any other value falls back to {@code hard-reboot}.
    + * {@code restart-agent} and {@code log-only} stop the VMs because, once the heartbeat is stale, the management + * server may declare the host Down and HA may start the same VMs on another host, where both copies would write + * to the same disks. If any VM cannot be stopped the fence falls back to {@code hard-reboot}. See + * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING}.
    * The non-default values are recommended for setups using LINSTOR/DRBD or other replicated * local storage, where transient I/O contention can cause a heartbeat write to time out * without the host actually being unhealthy.
    @@ -647,6 +651,18 @@ public class AgentProperties{ public static final Property KVM_HEARTBEAT_FENCE_ACTION = new Property<>("kvm.heartbeat.fence.action", "hard-reboot"); + /** + * When true, the {@code restart-agent} and {@code log-only} fence actions leave the host's VMs running instead of + * stopping them first. Only safe when no VM on the host can be started on another host by HA (VM HA off for all + * of them and host HA not enabled); otherwise a host the management server declares Down can end up with the + * same VM running twice, writing to the same disks. No effect on {@code hard-reboot} and {@code graceful-reboot}. + * Read by the heartbeat shell scripts directly from agent.properties.
    + * Data type: Boolean.
    + * Default value: false + */ + public static final Property KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING + = new Property<>("kvm.heartbeat.fence.keep.vms.running", false); + /** * Enables manually setting CPU's topology on KVM's VM.
    * Data type: Boolean.
    diff --git a/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java b/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java new file mode 100644 index 000000000000..860fe199b079 --- /dev/null +++ b/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +/** + * Runs scripts/vm/hypervisor/kvm/kvmha-fence.sh with stand-ins for virsh, systemctl, logger and the sysrq trigger, + * so nothing on the machine running the test is stopped, restarted or rebooted. + */ +public class KvmHaFenceScriptTest { + + private static final File SCRIPT = new File("../scripts/vm/hypervisor/kvm/kvmha-fence.sh"); + + private Path dir; + private Path running; + private Path calls; + private Path syslog; + private Path sysrq; + private Path props; + + @Before + public void setUp() throws IOException { + Assume.assumeTrue("needs bash", new File("/bin/bash").canExecute()); + Assume.assumeTrue("needs the fence script", SCRIPT.isFile()); + dir = Files.createTempDirectory("kvmha-fence"); + running = dir.resolve("running"); + calls = dir.resolve("calls"); + syslog = dir.resolve("syslog"); + sysrq = dir.resolve("sysrq"); + props = dir.resolve("agent.properties"); + Files.createFile(calls); + Files.createFile(syslog); + stub("virsh", "echo \"virsh $*\" >> '" + calls + "'\n" + + "case \" $* \" in\n" + + " *' list '*) cat '" + running + "' ;;\n" + + " *' destroy '*) vm=\"${@: -1}\"; grep -qx \"$vm\" '" + dir.resolve("stuck") + "' 2>/dev/null && exit 1;" + + " grep -vx \"$vm\" '" + running + "' > '" + running + ".new'; mv '" + running + ".new' '" + running + "' ;;\n" + + "esac\n"); + stub("systemctl", "echo \"systemctl $*\" >> '" + calls + "'\n"); + stub("logger", "echo \"$*\" >> '" + syslog + "'\n"); + stub("sleep", "exit 0\n"); + } + + @After + public void tearDown() throws IOException { + if (dir == null) { + return; + } + try (Stream paths = Files.walk(dir)) { + paths.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + + private void stub(final String name, final String body) throws IOException { + final Path file = dir.resolve(name); + Files.write(file, ("#!/bin/bash\n" + body).getBytes(StandardCharsets.UTF_8)); + Assert.assertTrue(file.toFile().setExecutable(true)); + } + + private int fence(final String properties, final String... runningVms) throws Exception { + Files.write(props, properties.getBytes(StandardCharsets.UTF_8)); + Files.write(running, (String.join("\n", runningVms) + "\n").getBytes(StandardCharsets.UTF_8)); + final ProcessBuilder pb = new ProcessBuilder("/bin/bash", "-c", "source \"$1\"; fence_action test.sh", "fence", + SCRIPT.getAbsolutePath()); + pb.environment().put("PATH", dir + File.pathSeparator + System.getenv("PATH")); + pb.environment().put("AGENT_PROPS", props.toString()); + pb.environment().put("FENCE_LOGGER", dir.resolve("logger").toString()); + pb.environment().put("FENCE_SYSRQ_TRIGGER", sysrq.toString()); + pb.redirectErrorStream(true); + pb.redirectOutput(dir.resolve("out").toFile()); + final Process process = pb.start(); + Assert.assertTrue("fence script did not finish", process.waitFor(60, TimeUnit.SECONDS)); + return process.exitValue(); + } + + private String read(final Path file) throws IOException { + return file.toFile().exists() ? new String(Files.readAllBytes(file), StandardCharsets.UTF_8) : ""; + } + + private boolean rebooted() throws IOException { + return read(sysrq).trim().equals("b"); + } + + @Test + public void logOnlyStopsEveryVmAndDoesNotReboot() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\n", "i-2-10-VM", "r-4-VM")); + Assert.assertTrue(read(calls).contains("destroy i-2-10-VM")); + Assert.assertTrue(read(calls).contains("destroy r-4-VM")); + Assert.assertEquals("", read(running).trim()); + Assert.assertFalse(read(calls).contains("systemctl")); + Assert.assertFalse(rebooted()); + } + + @Test + public void restartAgentStopsEveryVmBeforeRestartingTheAgent() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=restart-agent\n", "i-2-10-VM")); + final String log = read(calls); + Assert.assertTrue(log.contains("destroy i-2-10-VM")); + Assert.assertTrue(log.contains("systemctl restart cloudstack-agent")); + Assert.assertTrue("VMs must be stopped before the agent restarts", + log.indexOf("destroy i-2-10-VM") < log.indexOf("systemctl restart cloudstack-agent")); + Assert.assertFalse(rebooted()); + } + + @Test + public void aVmThatCannotBeStoppedFallsBackToHardReboot() throws Exception { + Files.write(dir.resolve("stuck"), "i-2-11-VM\n".getBytes(StandardCharsets.UTF_8)); + fence("kvm.heartbeat.fence.action=restart-agent\n", "i-2-10-VM", "i-2-11-VM"); + Assert.assertTrue("a VM still running must not be left behind", rebooted()); + Assert.assertFalse(read(calls).contains("systemctl restart cloudstack-agent")); + Assert.assertTrue(read(syslog).contains("falling back to hard-reboot")); + } + + @Test + public void keepVmsRunningIsHonouredWithAWarning() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.keep.vms.running=true\n", "i-2-10-VM")); + Assert.assertFalse(read(calls).contains("destroy")); + Assert.assertEquals("i-2-10-VM", read(running).trim()); + Assert.assertTrue(read(syslog).contains("WARNING")); + Assert.assertFalse(rebooted()); + } + + @Test + public void hardRebootIsTheDefaultAndDoesNotTouchVmsFirst() throws Exception { + fence("", "i-2-10-VM"); + Assert.assertTrue(rebooted()); + Assert.assertFalse(read(calls).contains("destroy")); + } + + @Test + public void unknownActionFallsBackToHardReboot() throws Exception { + fence("kvm.heartbeat.fence.action=custom\n", "i-2-10-VM"); + Assert.assertTrue(rebooted()); + } +} diff --git a/scripts/vm/hypervisor/kvm/kvmha-fence.sh b/scripts/vm/hypervisor/kvm/kvmha-fence.sh index 884e6e815736..e8bae7935969 100755 --- a/scripts/vm/hypervisor/kvm/kvmha-fence.sh +++ b/scripts/vm/hypervisor/kvm/kvmha-fence.sh @@ -21,39 +21,74 @@ # # kvm.heartbeat.fence.action in agent.properties selects the action: # hard-reboot (default), graceful-reboot, restart-agent, or log-only (only writes to -# syslog and takes no action). Any other value falls back to hard-reboot. +# syslog). Any other value falls back to hard-reboot. +# +# restart-agent and log-only leave the host up. Once the heartbeat is stale the management +# server may declare this host Down and HA may start its VMs on another host, so both +# actions first stop every VM still running here; if any VM cannot be stopped the fence +# falls back to hard-reboot. kvm.heartbeat.fence.keep.vms.running=true skips that step, +# which is only safe when no VM on this host can be restarted elsewhere by HA. # # Usage from caller: # source "$(dirname "$0")/kvmha-fence.sh" # fence_action "kvmheartbeat.sh" # script name passed for log tagging AGENT_PROPS="${AGENT_PROPS:-/etc/cloudstack/agent/agent.properties}" +FENCE_LOGGER="${FENCE_LOGGER:-/usr/bin/logger}" +FENCE_SYSRQ_TRIGGER="${FENCE_SYSRQ_TRIGGER:-/proc/sysrq-trigger}" + +read_agent_property() { + [ -r "$AGENT_PROPS" ] || return 0 + grep "^$1=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]' +} + +# Stops every active VM on this host. Fails if libvirt cannot be queried or any VM is still +# active afterwards, so the caller never assumes the VMs are stopped when they are not. +stop_all_vms() { + local source_script="$1" + local vms vm + vms=$(virsh -c qemu:///system list --name 2>/dev/null) || return 1 + for vm in $vms; do + "$FENCE_LOGGER" -t heartbeat "${source_script}: stopping VM ${vm} so it cannot run here while HA starts it on another host." + virsh -c qemu:///system destroy "$vm" > /dev/null 2>&1 + done + vms=$(virsh -c qemu:///system list --name 2>/dev/null) || return 1 + [ -z "$(echo "$vms" | tr -d '[:space:]')" ] +} fence_action() { local source_script="${1:-kvmha}" local FENCE_ACTION="hard-reboot" + local val + val=$(read_agent_property "kvm.heartbeat.fence.action") + [ -n "$val" ] && FENCE_ACTION="$val" - if [ -r "$AGENT_PROPS" ]; then - local val - val=$(grep "^kvm.heartbeat.fence.action=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') - [ -n "$val" ] && FENCE_ACTION="$val" - fi + case "$FENCE_ACTION" in + log-only|restart-agent) + if [ "$(read_agent_property "kvm.heartbeat.fence.keep.vms.running")" = "true" ]; then + "$FENCE_LOGGER" -t heartbeat "${source_script}: WARNING fence action '${FENCE_ACTION}' with kvm.heartbeat.fence.keep.vms.running=true leaves the VMs on this host running. If the management server marks this host Down, HA may start the same VMs on another host and both copies would write to the same disks." + elif ! stop_all_vms "$source_script"; then + "$FENCE_LOGGER" -t heartbeat "${source_script}: could not stop every VM on this host for fence action '${FENCE_ACTION}'; falling back to hard-reboot so no VM keeps running while HA may start it elsewhere." + FENCE_ACTION="hard-reboot" + fi + ;; + esac case "$FENCE_ACTION" in log-only) - # Only writes to syslog; no reboot, no agent restart and no management server alert. - /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'log-only' selected, taking no action. Operator must investigate." + # No reboot, no agent restart and no management server alert. + "$FENCE_LOGGER" -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'log-only' selected, taking no further action. Operator must investigate." exit 0 ;; restart-agent) - /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'restart-agent', restarting cloudstack-agent (running VMs preserved)." + "$FENCE_LOGGER" -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'restart-agent', restarting cloudstack-agent." sync & sleep 2 systemctl restart cloudstack-agent exit $? ;; graceful-reboot) - /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'graceful-reboot', rebooting via systemctl (allows running VMs to stop cleanly)." + "$FENCE_LOGGER" -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'graceful-reboot', rebooting via systemctl (allows running VMs to stop cleanly)." sync & sleep 5 systemctl reboot @@ -61,10 +96,10 @@ fence_action() { ;; hard-reboot|*) # Default, and the fallback for any unrecognised value: identical to the pre-existing behaviour. - /usr/bin/logger -t heartbeat "${source_script} will reboot system because it was unable to write the heartbeat to the storage." + "$FENCE_LOGGER" -t heartbeat "${source_script} will reboot system because it was unable to write the heartbeat to the storage." sync & sleep 5 - echo b > /proc/sysrq-trigger + echo b > "$FENCE_SYSRQ_TRIGGER" exit $? ;; esac From 50fce618fb9fae94133e0eb4eaeff94166415377 Mon Sep 17 00:00:00 2001 From: James Peru Date: Wed, 21 Oct 2026 10:40:40 +0300 Subject: [PATCH 7/8] address review (#13090): let the fence leave system VMs running New agent property kvm.heartbeat.fence.include.system.vms (default true). With true, restart-agent and log-only still stop every VM first. With false they stop only user VMs; system VMs (r-, s-, v- and b- names, as generated by VirtualMachineName) keep running and a running system VM no longer forces the hard-reboot fallback. A syslog warning notes that a virtual router could then run twice with the same IP and MAC. kvm.heartbeat.fence.keep.vms.running=true stays the master opt-out: it wins over include.system.vms and its warning says so. Agent module: 69 tests pass (4 new in KvmHaFenceScriptTest). Claude-Session: https://claude.ai/code/session_01V82Ypx7SpAurn6ek1VXRAP --- agent/conf/agent.properties | 15 +++++ .../agent/properties/AgentProperties.java | 17 ++++++ .../com/cloud/agent/KvmHaFenceScriptTest.java | 45 ++++++++++++++ scripts/vm/hypervisor/kvm/kvmha-fence.sh | 59 +++++++++++++++---- 4 files changed, 124 insertions(+), 12 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index beb69c0048d7..c333bace220c 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -352,8 +352,23 @@ iscsi.session.cleanup.enabled=false # of them and host HA not enabled). Otherwise a host the management server declares # Down can end up with the same VM running twice and corrupting its disks. # Has no effect on hard-reboot and graceful-reboot. +# This is the master opt-out: when true, nothing is stopped and +# kvm.heartbeat.fence.include.system.vms is ignored (the syslog warning says so). #kvm.heartbeat.fence.keep.vms.running=false +# Whether restart-agent and log-only also stop system VMs (virtual routers, console +# proxy, secondary storage and internal LB VMs, recognised by their r-, v-, s- and b- +# instance names). With true (default) every VM on the host is stopped. With false +# only user VMs are stopped; system VMs keep running and a system VM that is still +# running does not trigger the hard-reboot fallback. +# Risk of false: if the management server declares this host Down it restarts the +# system VMs on another host, so a virtual router could run twice with the same IP +# and MAC addresses on the guest network, and a second console proxy or secondary +# storage VM could run with the same addresses. Only set false if you accept that. +# Ignored when kvm.heartbeat.fence.keep.vms.running is true. Has no effect on +# hard-reboot and graceful-reboot. +#kvm.heartbeat.fence.include.system.vms=true + # Enables manually setting CPU's topology on KVM's VM. #enable.manually.setting.cpu.topology.on.kvm.vm=true diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 2785555f5224..5e0a24a19aa4 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -656,6 +656,8 @@ public class AgentProperties{ * stopping them first. Only safe when no VM on the host can be started on another host by HA (VM HA off for all * of them and host HA not enabled); otherwise a host the management server declares Down can end up with the * same VM running twice, writing to the same disks. No effect on {@code hard-reboot} and {@code graceful-reboot}. + * This is the master opt-out: when true nothing is stopped and {@link #KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS} + * is ignored.
    * Read by the heartbeat shell scripts directly from agent.properties.
    * Data type: Boolean.
    * Default value: false @@ -663,6 +665,21 @@ public class AgentProperties{ public static final Property KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING = new Property<>("kvm.heartbeat.fence.keep.vms.running", false); + /** + * Whether the {@code restart-agent} and {@code log-only} fence actions also stop system VMs (virtual routers, + * console proxy, secondary storage and internal LB VMs, recognised by their {@code r-}, {@code v-}, {@code s-} and + * {@code b-} instance names). When true every VM on the host is stopped. When false only user VMs are stopped; + * system VMs keep running and do not trigger the {@code hard-reboot} fallback. Risk of false: if the management + * server declares the host Down it restarts the system VMs on another host, so a virtual router could run twice + * with the same IP and MAC addresses. Ignored when {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true. No + * effect on {@code hard-reboot} and {@code graceful-reboot}. + * Read by the heartbeat shell scripts directly from agent.properties.
    + * Data type: Boolean.
    + * Default value: true + */ + public static final Property KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS + = new Property<>("kvm.heartbeat.fence.include.system.vms", true); + /** * Enables manually setting CPU's topology on KVM's VM.
    * Data type: Boolean.
    diff --git a/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java b/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java index 860fe199b079..7b9417241082 100644 --- a/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java +++ b/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java @@ -148,6 +148,51 @@ public void keepVmsRunningIsHonouredWithAWarning() throws Exception { Assert.assertFalse(rebooted()); } + @Test + public void includeSystemVmsTrueStopsSystemVmsToo() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.include.system.vms=true\n", + "i-2-10-VM", "r-4-VM", "s-1-VM", "v-2-VM", "b-5-VM")); + Assert.assertEquals("", read(running).trim()); + Assert.assertFalse(rebooted()); + } + + @Test + public void includeSystemVmsFalseStopsOnlyUserVms() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=restart-agent\nkvm.heartbeat.fence.include.system.vms=false\n", + "i-2-10-VM", "r-4-VM", "r-7-VM-1234", "s-1-VM", "v-2-VM", "b-5-VM")); + final String log = read(calls); + Assert.assertTrue(log.contains("destroy i-2-10-VM")); + for (final String systemVm : new String[] {"r-4-VM", "r-7-VM-1234", "s-1-VM", "v-2-VM", "b-5-VM"}) { + Assert.assertFalse(systemVm + " must keep running", log.contains("destroy " + systemVm)); + } + Assert.assertEquals("r-4-VM\nr-7-VM-1234\ns-1-VM\nv-2-VM\nb-5-VM", read(running).trim()); + Assert.assertTrue(log.contains("systemctl restart cloudstack-agent")); + Assert.assertTrue(read(syslog).contains("include.system.vms=false")); + Assert.assertFalse(rebooted()); + } + + @Test + public void includeSystemVmsFalseIgnoresAStuckSystemVmButNotAStuckUserVm() throws Exception { + Files.write(dir.resolve("stuck"), "r-4-VM\n".getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.include.system.vms=false\n", + "i-2-10-VM", "r-4-VM")); + Assert.assertFalse("a running system VM must not force a reboot", rebooted()); + + Files.write(dir.resolve("stuck"), "i-2-11-VM\n".getBytes(StandardCharsets.UTF_8)); + fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.include.system.vms=false\n", "i-2-11-VM", "r-4-VM"); + Assert.assertTrue("a user VM still running must still force a reboot", rebooted()); + } + + @Test + public void keepVmsRunningWinsOverIncludeSystemVms() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.keep.vms.running=true\n" + + "kvm.heartbeat.fence.include.system.vms=false\n", "i-2-10-VM", "r-4-VM")); + Assert.assertFalse(read(calls).contains("destroy")); + Assert.assertEquals("i-2-10-VM\nr-4-VM", read(running).trim()); + Assert.assertTrue(read(syslog).contains("overrides kvm.heartbeat.fence.include.system.vms")); + Assert.assertFalse(rebooted()); + } + @Test public void hardRebootIsTheDefaultAndDoesNotTouchVmsFirst() throws Exception { fence("", "i-2-10-VM"); diff --git a/scripts/vm/hypervisor/kvm/kvmha-fence.sh b/scripts/vm/hypervisor/kvm/kvmha-fence.sh index e8bae7935969..bd9bd7d93bf9 100755 --- a/scripts/vm/hypervisor/kvm/kvmha-fence.sh +++ b/scripts/vm/hypervisor/kvm/kvmha-fence.sh @@ -26,8 +26,11 @@ # restart-agent and log-only leave the host up. Once the heartbeat is stale the management # server may declare this host Down and HA may start its VMs on another host, so both # actions first stop every VM still running here; if any VM cannot be stopped the fence -# falls back to hard-reboot. kvm.heartbeat.fence.keep.vms.running=true skips that step, -# which is only safe when no VM on this host can be restarted elsewhere by HA. +# falls back to hard-reboot. kvm.heartbeat.fence.include.system.vms=false limits that step +# to user VMs: system VMs (virtual routers, console proxy, secondary storage and internal +# LB VMs) are left running and do not count towards the fallback. +# kvm.heartbeat.fence.keep.vms.running=true skips the step entirely and wins over +# include.system.vms; it is only safe when no VM on this host can be restarted elsewhere. # # Usage from caller: # source "$(dirname "$0")/kvmha-fence.sh" @@ -42,17 +45,40 @@ read_agent_property() { grep "^$1=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]' } -# Stops every active VM on this host. Fails if libvirt cannot be queried or any VM is still -# active afterwards, so the caller never assumes the VMs are stopped when they are not. -stop_all_vms() { - local source_script="$1" +# True when the libvirt domain name is a CloudStack system VM, using the names the management +# server gives them (com.cloud.vm.VirtualMachineName): r--[-] for virtual +# routers, and s-, v- and b-- for secondary storage, console proxy and internal +# LB VMs. User VMs are i--- and never match. +is_system_vm() { + [[ "$1" =~ ^r-[0-9]+-[^-]+(-[^-]+)?$ || "$1" =~ ^[svb]-[0-9]+-[^-]+$ ]] +} + +# Prints the active VMs the fence has to stop: every VM, or only user VMs when +# include_system_vms is not "true". Fails if libvirt cannot be queried. +vms_to_stop() { + local include_system_vms="$1" local vms vm vms=$(virsh -c qemu:///system list --name 2>/dev/null) || return 1 + for vm in $vms; do + if [ "$include_system_vms" = "true" ] || ! is_system_vm "$vm"; then + echo "$vm" + fi + done +} + +# Stops the active VMs selected by vms_to_stop. Fails if libvirt cannot be queried or any of +# them is still active afterwards, so the caller never assumes the VMs are stopped when they +# are not. System VMs left running on purpose do not make it fail. +stop_vms() { + local source_script="$1" + local include_system_vms="$2" + local vms vm + vms=$(vms_to_stop "$include_system_vms") || return 1 for vm in $vms; do "$FENCE_LOGGER" -t heartbeat "${source_script}: stopping VM ${vm} so it cannot run here while HA starts it on another host." virsh -c qemu:///system destroy "$vm" > /dev/null 2>&1 done - vms=$(virsh -c qemu:///system list --name 2>/dev/null) || return 1 + vms=$(vms_to_stop "$include_system_vms") || return 1 [ -z "$(echo "$vms" | tr -d '[:space:]')" ] } @@ -65,11 +91,20 @@ fence_action() { case "$FENCE_ACTION" in log-only|restart-agent) - if [ "$(read_agent_property "kvm.heartbeat.fence.keep.vms.running")" = "true" ]; then - "$FENCE_LOGGER" -t heartbeat "${source_script}: WARNING fence action '${FENCE_ACTION}' with kvm.heartbeat.fence.keep.vms.running=true leaves the VMs on this host running. If the management server marks this host Down, HA may start the same VMs on another host and both copies would write to the same disks." - elif ! stop_all_vms "$source_script"; then - "$FENCE_LOGGER" -t heartbeat "${source_script}: could not stop every VM on this host for fence action '${FENCE_ACTION}'; falling back to hard-reboot so no VM keeps running while HA may start it elsewhere." - FENCE_ACTION="hard-reboot" + local keep_running include_system_vms="true" + keep_running=$(read_agent_property "kvm.heartbeat.fence.keep.vms.running") + [ "$(read_agent_property "kvm.heartbeat.fence.include.system.vms")" = "false" ] && include_system_vms="false" + if [ "$keep_running" = "true" ]; then + # keep.vms.running is the master opt-out and wins over include.system.vms. + "$FENCE_LOGGER" -t heartbeat "${source_script}: WARNING fence action '${FENCE_ACTION}' with kvm.heartbeat.fence.keep.vms.running=true leaves every VM on this host running, system VMs included (this overrides kvm.heartbeat.fence.include.system.vms). If the management server marks this host Down, HA may start the same VMs on another host and both copies would write to the same disks." + else + if [ "$include_system_vms" = "false" ]; then + "$FENCE_LOGGER" -t heartbeat "${source_script}: WARNING fence action '${FENCE_ACTION}' with kvm.heartbeat.fence.include.system.vms=false stops only user VMs and leaves system VMs running. If the management server marks this host Down it may start those system VMs on another host, and a virtual router could then run twice with the same IP and MAC addresses." + fi + if ! stop_vms "$source_script" "$include_system_vms"; then + "$FENCE_LOGGER" -t heartbeat "${source_script}: could not stop every VM on this host that the fence must stop for action '${FENCE_ACTION}'; falling back to hard-reboot so no VM keeps running while HA may start it elsewhere." + FENCE_ACTION="hard-reboot" + fi fi ;; esac From 8fa1937d3f449f1d887d7ee492559e46edfe17a9 Mon Sep 17 00:00:00 2001 From: James Peru Date: Thu, 22 Oct 2026 13:33:06 +0300 Subject: [PATCH 8/8] address review (#13090): bound the VM stop phase and guarantee the hard-reboot fallback Before restart-agent or log-only stop any VM, the fence now arms a detached deadman (own session via setsid, else nohup) that hard-reboots the host unless every selected VM is confirmed stopped within kvm.heartbeat.fence.stop.vms.timeout seconds (default 30), so the fallback still happens when the agent kills the script on heartbeat.update.timeout. Each virsh call is also bounded by coreutils timeout (10 s, or the budget if lower). Claude-Session: https://claude.ai/code/session_011DLKuGySDaq3JQs6SAw3TC --- agent/conf/agent.properties | 12 ++ .../agent/properties/AgentProperties.java | 16 +++ .../com/cloud/agent/KvmHaFenceScriptTest.java | 119 +++++++++++++++- scripts/vm/hypervisor/kvm/kvmha-fence.sh | 128 +++++++++++++++++- 4 files changed, 267 insertions(+), 8 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index c333bace220c..781602e4fd9d 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -369,6 +369,18 @@ iscsi.session.cleanup.enabled=false # hard-reboot and graceful-reboot. #kvm.heartbeat.fence.include.system.vms=true +# Seconds restart-agent and log-only may spend stopping VMs before the host is +# hard-rebooted instead. Before stopping any VM the fence arms a detached deadman that +# keeps running even if the agent kills the heartbeat script when +# heartbeat.update.timeout runs out (stopping a VM can block for a long time on hung +# storage). If the VMs are not all confirmed stopped within this budget, the deadman +# hard-reboots the host. Each virsh call is also limited to 10 seconds (or this value +# if lower) when coreutils timeout is installed. Keep it below heartbeat.update.timeout +# (60 seconds by default). A value that is not a positive integer means 30. +# Has no effect on hard-reboot and graceful-reboot, or when +# kvm.heartbeat.fence.keep.vms.running is true. +#kvm.heartbeat.fence.stop.vms.timeout=30 + # Enables manually setting CPU's topology on KVM's VM. #enable.manually.setting.cpu.topology.on.kvm.vm=true diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 5e0a24a19aa4..30e7e54aebc0 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -680,6 +680,22 @@ public class AgentProperties{ public static final Property KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS = new Property<>("kvm.heartbeat.fence.include.system.vms", true); + /** + * Seconds the {@code restart-agent} and {@code log-only} fence actions may spend stopping VMs before the host is + * hard-rebooted instead. Before stopping any VM the fence arms a detached deadman that keeps running even if the + * agent kills the heartbeat script when {@link #HEARTBEAT_UPDATE_TIMEOUT} runs out (stopping a VM can block for a + * long time on hung storage). If the VMs are not all confirmed stopped within this budget, the deadman + * hard-reboots the host. Each virsh call is also limited to 10 seconds, or this value if lower, when coreutils + * {@code timeout} is installed. Keep it below {@link #HEARTBEAT_UPDATE_TIMEOUT}. A value that is not a positive + * integer falls back to the default. No effect on {@code hard-reboot} and {@code graceful-reboot}, or when + * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true.
    + * Read by the heartbeat shell scripts directly from agent.properties.
    + * Data type: Integer.
    + * Default value: 30 + */ + public static final Property KVM_HEARTBEAT_FENCE_STOP_VMS_TIMEOUT + = new Property<>("kvm.heartbeat.fence.stop.vms.timeout", 30); + /** * Enables manually setting CPU's topology on KVM's VM.
    * Data type: Boolean.
    diff --git a/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java b/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java index 7b9417241082..0cc3ff962daa 100644 --- a/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java +++ b/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java @@ -21,8 +21,12 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Comparator; +import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Stream; import org.junit.After; @@ -45,6 +49,8 @@ public class KvmHaFenceScriptTest { private Path syslog; private Path sysrq; private Path props; + private Path deadmanDir; + private Path hangPids; @Before public void setUp() throws IOException { @@ -56,17 +62,22 @@ public void setUp() throws IOException { syslog = dir.resolve("syslog"); sysrq = dir.resolve("sysrq"); props = dir.resolve("agent.properties"); + deadmanDir = dir.resolve("deadman"); + hangPids = dir.resolve("hang-pids"); Files.createFile(calls); Files.createFile(syslog); stub("virsh", "echo \"virsh $*\" >> '" + calls + "'\n" + "case \" $* \" in\n" + " *' list '*) cat '" + running + "' ;;\n" - + " *' destroy '*) vm=\"${@: -1}\"; grep -qx \"$vm\" '" + dir.resolve("stuck") + "' 2>/dev/null && exit 1;" + + " *' destroy '*) vm=\"${@: -1}\";" + + " if grep -qx \"$vm\" '" + dir.resolve("hang") + "' 2>/dev/null; then echo $$ >> '" + hangPids + "'; exec /bin/sleep 60; fi;" + + " grep -qx \"$vm\" '" + dir.resolve("stuck") + "' 2>/dev/null && exit 1;" + " grep -vx \"$vm\" '" + running + "' > '" + running + ".new'; mv '" + running + ".new' '" + running + "' ;;\n" + "esac\n"); stub("systemctl", "echo \"systemctl $*\" >> '" + calls + "'\n"); stub("logger", "echo \"$*\" >> '" + syslog + "'\n"); - stub("sleep", "exit 0\n"); + // Fence delays are skipped; the deadman's 1 s steps stay real so its budget is measured in real time. + stub("sleep", "if [ \"$1\" = 1 ]; then exec /bin/sleep 1; fi\nexit 0\n"); } @After @@ -74,6 +85,16 @@ public void tearDown() throws IOException { if (dir == null) { return; } + // Background processes the script may leave behind: armed deadmen and virsh stand-ins that hang. + final List pids = new ArrayList<>(deadmanPids()); + for (final String line : read(hangPids).split("\\s+")) { + if (!line.isEmpty()) { + pids.add(Long.parseLong(line)); + } + } + for (final long pid : pids) { + ProcessHandle.of(pid).ifPresent(ProcessHandle::destroyForcibly); + } try (Stream paths = Files.walk(dir)) { paths.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); } @@ -86,6 +107,12 @@ private void stub(final String name, final String body) throws IOException { } private int fence(final String properties, final String... runningVms) throws Exception { + final Process process = start(properties, runningVms); + Assert.assertTrue("fence script did not finish", process.waitFor(60, TimeUnit.SECONDS)); + return process.exitValue(); + } + + private Process start(final String properties, final String... runningVms) throws IOException { Files.write(props, properties.getBytes(StandardCharsets.UTF_8)); Files.write(running, (String.join("\n", runningVms) + "\n").getBytes(StandardCharsets.UTF_8)); final ProcessBuilder pb = new ProcessBuilder("/bin/bash", "-c", "source \"$1\"; fence_action test.sh", "fence", @@ -94,11 +121,35 @@ private int fence(final String properties, final String... runningVms) throws Ex pb.environment().put("AGENT_PROPS", props.toString()); pb.environment().put("FENCE_LOGGER", dir.resolve("logger").toString()); pb.environment().put("FENCE_SYSRQ_TRIGGER", sysrq.toString()); + pb.environment().put("FENCE_DEADMAN_DIR", deadmanDir.toString()); pb.redirectErrorStream(true); pb.redirectOutput(dir.resolve("out").toFile()); - final Process process = pb.start(); - Assert.assertTrue("fence script did not finish", process.waitFor(60, TimeUnit.SECONDS)); - return process.exitValue(); + return pb.start(); + } + + /** Pids of every deadman the script started, as each one logs its own pid when it starts. */ + private List deadmanPids() throws IOException { + final List pids = new ArrayList<>(); + final Matcher m = Pattern.compile("fence deadman \\(pid (\\d+)\\) armed").matcher(read(syslog)); + while (m.find()) { + pids.add(Long.parseLong(m.group(1))); + } + return pids; + } + + private boolean waitFor(final long millis, final Check check) throws Exception { + final long deadline = System.currentTimeMillis() + millis; + while (System.currentTimeMillis() < deadline) { + if (check.ok()) { + return true; + } + Thread.sleep(100); + } + return check.ok(); + } + + private interface Check { + boolean ok() throws Exception; } private String read(final Path file) throws IOException { @@ -205,4 +256,62 @@ public void unknownActionFallsBackToHardReboot() throws Exception { fence("kvm.heartbeat.fence.action=custom\n", "i-2-10-VM"); Assert.assertTrue(rebooted()); } + + @Test + public void deadmanRebootsWhenTheAgentKillsTheScriptWhileAVmStopHangs() throws Exception { + Files.write(dir.resolve("hang"), "i-2-12-VM\n".getBytes(StandardCharsets.UTF_8)); + final Process process = start("kvm.heartbeat.fence.action=restart-agent\nkvm.heartbeat.fence.stop.vms.timeout=2\n", + "i-2-12-VM"); + Assert.assertTrue("virsh destroy never started", waitFor(5000, () -> !read(hangPids).trim().isEmpty())); + Thread.sleep(500); + // What the agent does when heartbeat.update.timeout runs out (Script.execute). + process.destroyForcibly(); + Assert.assertTrue(process.waitFor(5, TimeUnit.SECONDS)); + Assert.assertFalse("the script itself must not have reached its fallback yet", rebooted()); + Assert.assertTrue("the deadman must hard-reboot the host after the script was killed", + waitFor(6000, this::rebooted)); + Assert.assertTrue(read(syslog).contains("fence deadman is hard-rebooting the host")); + Assert.assertFalse(read(calls).contains("systemctl restart cloudstack-agent")); + } + + @Test + public void deadmanIsDisarmedWhenEveryVmStopsWithinTheBudget() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.stop.vms.timeout=2\n", + "i-2-10-VM", "r-4-VM")); + Assert.assertEquals("", read(running).trim()); + Assert.assertTrue("the deadman must have been armed", deadmanDir.toFile().isDirectory()); + Assert.assertTrue("arm and pid files must be gone", waitFor(3000, () -> { + final String[] left = deadmanDir.toFile().list(); + return left != null && left.length == 0; + })); + for (final long pid : deadmanPids()) { + Assert.assertTrue("deadman " + pid + " must not be left running", + waitFor(3000, () -> !ProcessHandle.of(pid).map(ProcessHandle::isAlive).orElse(false))); + } + // Past the 2 s budget nothing may reboot the host. + Thread.sleep(3000); + Assert.assertFalse(rebooted()); + } + + @Test + public void invalidStopBudgetFallsBackToTheDefault() throws Exception { + for (final String value : new String[] {"abc", "0", "-5", "1.5"}) { + Files.write(syslog, new byte[0]); + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.stop.vms.timeout=" + value + "\n", + "i-2-10-VM")); + final String log = read(syslog); + Assert.assertTrue(value + " must be reported", log.contains("kvm.heartbeat.fence.stop.vms.timeout='" + value + "'")); + Assert.assertTrue(value + " must give the default budget", log.contains("with a budget of 30 s")); + Assert.assertTrue(log.contains("at most 10 s per virsh call")); + Assert.assertFalse(rebooted()); + } + } + + @Test + public void unsetStopBudgetUsesTheDefaultWithoutAWarning() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\n", "i-2-10-VM")); + final String log = read(syslog); + Assert.assertTrue(log.contains("with a budget of 30 s")); + Assert.assertFalse(log.contains("is not a positive number")); + } } diff --git a/scripts/vm/hypervisor/kvm/kvmha-fence.sh b/scripts/vm/hypervisor/kvm/kvmha-fence.sh index bd9bd7d93bf9..77c582eb5fdb 100755 --- a/scripts/vm/hypervisor/kvm/kvmha-fence.sh +++ b/scripts/vm/hypervisor/kvm/kvmha-fence.sh @@ -32,6 +32,23 @@ # kvm.heartbeat.fence.keep.vms.running=true skips the step entirely and wins over # include.system.vms; it is only safe when no VM on this host can be restarted elsewhere. # +# The agent kills this script once heartbeat.update.timeout (60 s by default) runs out, and +# on hung storage stopping a VM can block for much longer than that. So before stopping any +# VM the fence arms a detached deadman that survives the agent killing this script: unless +# every VM the fence must stop is confirmed gone within kvm.heartbeat.fence.stop.vms.timeout +# seconds (default 30, keep it below heartbeat.update.timeout), the deadman hard-reboots the +# host. Each virsh call is also bounded by coreutils timeout, when present, to at most 10 s +# (or the budget if lower) so one stuck domain cannot use up the whole budget. +# +# Properties read from agent.properties: +# kvm.heartbeat.fence.action hard-reboot (default), graceful-reboot, +# restart-agent or log-only +# kvm.heartbeat.fence.keep.vms.running false (default); true stops nothing +# kvm.heartbeat.fence.include.system.vms true (default); false stops only user VMs +# kvm.heartbeat.fence.stop.vms.timeout 30 (default); seconds the VM stop phase may take +# before the deadman hard-reboots the host. A value +# that is not a positive integer means 30. +# # Usage from caller: # source "$(dirname "$0")/kvmha-fence.sh" # fence_action "kvmheartbeat.sh" # script name passed for log tagging @@ -39,12 +56,106 @@ AGENT_PROPS="${AGENT_PROPS:-/etc/cloudstack/agent/agent.properties}" FENCE_LOGGER="${FENCE_LOGGER:-/usr/bin/logger}" FENCE_SYSRQ_TRIGGER="${FENCE_SYSRQ_TRIGGER:-/proc/sysrq-trigger}" +# Where the deadman keeps its arm file; /run is a local tmpfs, so hung storage cannot block it. +FENCE_DEADMAN_DIR="${FENCE_DEADMAN_DIR:-/run/cloudstack-agent/kvmha-fence}" +FENCE_STOP_VMS_TIMEOUT_DEFAULT=30 +FENCE_VIRSH_TIMEOUT_MAX=10 +# Per-call virsh timeout in seconds; empty means unbounded. Set by fence_action. +FENCE_VIRSH_TIMEOUT="" +FENCE_DEADMAN_ARM="" read_agent_property() { [ -r "$AGENT_PROPS" ] || return 0 grep "^$1=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]' } +# Prints the VM stop budget in seconds from kvm.heartbeat.fence.stop.vms.timeout. Unset gives +# the default; anything else that is not a positive integer (up to 6 digits) gives the default +# with a warning in syslog. +stop_vms_budget() { + local source_script="$1" val + val=$(read_agent_property "kvm.heartbeat.fence.stop.vms.timeout") + if [[ "$val" =~ ^[0-9]{1,6}$ ]] && [ $((10#$val)) -gt 0 ]; then + echo $((10#$val)) + return 0 + fi + if [ -n "$val" ]; then + "$FENCE_LOGGER" -t heartbeat "${source_script}: WARNING kvm.heartbeat.fence.stop.vms.timeout='${val}' is not a positive number of seconds; using the default of ${FENCE_STOP_VMS_TIMEOUT_DEFAULT} s." + fi + echo "$FENCE_STOP_VMS_TIMEOUT_DEFAULT" +} + +# Runs virsh against the system libvirt, bounded by FENCE_VIRSH_TIMEOUT seconds when coreutils +# timeout is available. Without timeout the call is unbounded and only the deadman covers it. +fence_virsh() { + if [ -n "$FENCE_VIRSH_TIMEOUT" ] && command -v timeout > /dev/null 2>&1; then + timeout -k 2 "$FENCE_VIRSH_TIMEOUT" virsh -c qemu:///system "$@" + else + virsh -c qemu:///system "$@" + fi +} + +# Body of the deadman; runs in its own detached bash (see arm_deadman), never in the fence +# script itself. Waits up to budget seconds in 1 s steps and exits quietly as soon as the arm +# file is removed. If the arm file is still there when the budget runs out, the VM stop phase +# did not finish in time (or the agent killed the fence script), so it hard-reboots the host. +fence_deadman() { + local arm="$1" budget="$2" source_script="$3" + local deadline=$((SECONDS + budget)) + # Survive the agent or a service stop sending TERM or HUP; only disarm_deadman stops it early. + trap '' HUP INT TERM + echo "$$" > "${arm}.pid" + trap 'rm -f "${arm}.pid"' EXIT + "$FENCE_LOGGER" -t heartbeat "${source_script}: fence deadman (pid $$) armed; the host will be hard-rebooted in ${budget} s unless every VM the fence must stop is confirmed stopped." + while [ "$SECONDS" -lt "$deadline" ]; do + [ -e "$arm" ] || exit 0 + sleep 1 + done + [ -e "$arm" ] || exit 0 + "$FENCE_LOGGER" -t heartbeat "${source_script}: VMs were not confirmed stopped within kvm.heartbeat.fence.stop.vms.timeout (${budget} s); fence deadman is hard-rebooting the host so no VM keeps running while HA may start it elsewhere." + sync & + sleep 5 + echo b > "$FENCE_SYSRQ_TRIGGER" +} + +# Starts the deadman for a budget of $2 seconds, detached from this script (its own session +# when setsid exists) so it keeps running if the agent kills the fence script on timeout. +# Fails if it cannot be armed. +arm_deadman() { + local source_script="$1" budget="$2" arm + # The deadman runs a new bash, so it gets the values it needs passed in explicitly. + # shellcheck disable=SC2016 + local body='source "$1"; fence_deadman "$2" "$3" "$4"' + mkdir -p "$FENCE_DEADMAN_DIR" 2> /dev/null || return 1 + arm="${FENCE_DEADMAN_DIR}/armed.$$" + : > "$arm" 2> /dev/null || return 1 + FENCE_DEADMAN_ARM="$arm" + if command -v setsid > /dev/null 2>&1; then + FENCE_LOGGER="$FENCE_LOGGER" FENCE_SYSRQ_TRIGGER="$FENCE_SYSRQ_TRIGGER" \ + setsid nohup /bin/bash -c "$body" fence-deadman "${BASH_SOURCE[0]}" "$arm" "$budget" "$source_script" \ + < /dev/null > /dev/null 2>&1 & + else + FENCE_LOGGER="$FENCE_LOGGER" FENCE_SYSRQ_TRIGGER="$FENCE_SYSRQ_TRIGGER" \ + nohup /bin/bash -c "$body" fence-deadman "${BASH_SOURCE[0]}" "$arm" "$budget" "$source_script" \ + < /dev/null > /dev/null 2>&1 & + fi + disown 2> /dev/null || true +} + +# Stops the deadman: removing the arm file is enough on its own (it checks every second), +# killing it just makes it go away at once. +disarm_deadman() { + local arm="$FENCE_DEADMAN_ARM" pid + [ -n "$arm" ] || return 0 + rm -f "$arm" + pid=$(cat "${arm}.pid" 2> /dev/null) + if [[ "$pid" =~ ^[0-9]+$ ]]; then + kill -KILL "$pid" 2> /dev/null + fi + rm -f "${arm}.pid" + FENCE_DEADMAN_ARM="" +} + # True when the libvirt domain name is a CloudStack system VM, using the names the management # server gives them (com.cloud.vm.VirtualMachineName): r--[-] for virtual # routers, and s-, v- and b-- for secondary storage, console proxy and internal @@ -58,7 +169,7 @@ is_system_vm() { vms_to_stop() { local include_system_vms="$1" local vms vm - vms=$(virsh -c qemu:///system list --name 2>/dev/null) || return 1 + vms=$(fence_virsh list --name 2>/dev/null) || return 1 for vm in $vms; do if [ "$include_system_vms" = "true" ] || ! is_system_vm "$vm"; then echo "$vm" @@ -76,7 +187,7 @@ stop_vms() { vms=$(vms_to_stop "$include_system_vms") || return 1 for vm in $vms; do "$FENCE_LOGGER" -t heartbeat "${source_script}: stopping VM ${vm} so it cannot run here while HA starts it on another host." - virsh -c qemu:///system destroy "$vm" > /dev/null 2>&1 + fence_virsh destroy "$vm" > /dev/null 2>&1 done vms=$(vms_to_stop "$include_system_vms") || return 1 [ -z "$(echo "$vms" | tr -d '[:space:]')" ] @@ -101,9 +212,20 @@ fence_action() { if [ "$include_system_vms" = "false" ]; then "$FENCE_LOGGER" -t heartbeat "${source_script}: WARNING fence action '${FENCE_ACTION}' with kvm.heartbeat.fence.include.system.vms=false stops only user VMs and leaves system VMs running. If the management server marks this host Down it may start those system VMs on another host, and a virtual router could then run twice with the same IP and MAC addresses." fi - if ! stop_vms "$source_script" "$include_system_vms"; then + local budget + budget=$(stop_vms_budget "$source_script") + FENCE_VIRSH_TIMEOUT=$(( budget < FENCE_VIRSH_TIMEOUT_MAX ? budget : FENCE_VIRSH_TIMEOUT_MAX )) + "$FENCE_LOGGER" -t heartbeat "${source_script}: stopping VMs for fence action '${FENCE_ACTION}' with a budget of ${budget} s (kvm.heartbeat.fence.stop.vms.timeout) and at most ${FENCE_VIRSH_TIMEOUT} s per virsh call." + if ! arm_deadman "$source_script" "$budget"; then + "$FENCE_LOGGER" -t heartbeat "${source_script}: could not arm the fence deadman in ${FENCE_DEADMAN_DIR}; falling back to hard-reboot because the VM stop phase could not be bounded." + FENCE_ACTION="hard-reboot" + elif ! stop_vms "$source_script" "$include_system_vms"; then + # The deadman stays armed: if this script is killed before it reboots, the deadman still does. "$FENCE_LOGGER" -t heartbeat "${source_script}: could not stop every VM on this host that the fence must stop for action '${FENCE_ACTION}'; falling back to hard-reboot so no VM keeps running while HA may start it elsewhere." FENCE_ACTION="hard-reboot" + else + # Every selected VM is confirmed gone, so the host may stay up. + disarm_deadman fi fi ;;