diff --git a/deploy/cutover-fleet-chain22.sh b/deploy/cutover-fleet-chain22.sh new file mode 100755 index 00000000..26b3cbad --- /dev/null +++ b/deploy/cutover-fleet-chain22.sh @@ -0,0 +1,1140 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016,SC2028 # single-quoted remote / bash -c scripts expand on the far side, and the +# remote text's backslashes are for the far shell — both on purpose +# Cut the fleet over from chain 21 to chain 22 — all-stop, then a quorum of validators started +# together, then the archives — over the 25 hosts of the inventory that chain 22 keeps (nothing here +# creates a host). archive2 is DROPPED (owner, 2026-11-19): it is not a chain-22 validator, it is +# unreachable on a locked account, and this script never contacts it (it is filtered out of the +# inventory as it is read, and `on`/`cp_to` refuse its name and address besides). +# +# DERIVED from deploy/cutover-fleet-chain21.sh (chain 21 = v0.8.0, live 2026-10-12, on v0.8.6 since +# 2026-11-09; read it first). Its phases, its stop gate (chain 21 keeps running through stage, the +# cut, the second operator's hash and push; `stop` is gated on every host being ready), its datadir +# handling (nodes.env-style units: suffix - → -, volumes behind +# symlinks kept; guardian hosts: chain21.conf/data-21 → chain22.conf/data-22, a stale data-22 moved +# aside), the sha-checked backups, the marker that licenses `start`, the chain-16 trap rules and the +# rollback are chain 21's. Against chain 21's script: +# +# * THE BUILD IS v0.8.7, CONSTRAINT SET 11. A set-11 rand-node REFUSES every chain-14–21 genesis +# (constraint_set::build_genesis): there is no running both chains on one binary. So `switch` +# keeps v0.8.6 as /root/rand-node.pre-c22 and /root/rand.pre-c22 AND chain 21's msauth blob as +# /root/msauth-vk-production.bin.pre-c22 (v0.8.6 on chain 21 READS it — chain 21 names +# hc_auth_multisig — and the set-11 blob is another file), and `rollback` restores all three. +# * SEVEN SIGNED FILES, NOT THREE: rand-node, rand, msauth-vk-production.bin (set 11: 168d79a8…71b3, +# 23 851 bytes) and the four perps state-proof keys perp-vk-production-t18-*.bin (CON-601; the +# pins of docs/deploy.md "The perps state-proof verifier keys"). A chain-22 node refuses to start +# without every one beside its binary. Every pin is read from the SIGNED SHA256SUMS and must equal +# the decided value; each host checks every staged and every installed file by sha256 and by the +# binary's own `keygen-msauth-vk` / `keygen-perp-vk` pins, before and after the install, and again +# in the same ssh call as its `systemctl start` (chain 21's msauth gate, widened). +# * THE BEACON (#189). Each validator host needs its own beacon.seed.json, made ON THAT HOST: +# - `beacon-init` runs `rand-node beacon init --chain-id 22` (the STAGED v0.8.7 binary: v0.8.6 +# has no `beacon`) on each of the 25 validators into a staging dir that survives a re-cut +# (/root/beacon-c22, or /var/lib/randnode/beacon-c22 owned by randnode on a guardian host), +# and prints each public commitment. A host that already has a seed keeps it (never +# overwritten). THE SEED NEVER LEAVES ITS HOST: no phase reads, prints or copies it off. +# - `beacon-collect` (read-only toward the fleet) reads each host's commitment back with +# `rand-node beacon commitment`, checks the file names chain 22 and that host's own key +# address, and writes BEACON_TSV — " <64 hex>", the format cut-chain22-genesis.sh reads. +# - `stop` refuses unless deploy/check-beacon-seed.sh passes on every validator against the +# PUSHED genesis with the staging dir (so a bad seed is caught while chain 21 still runs); +# `switch` copies the seed into the new datadir on the same host (0600; never between hosts) +# and runs the check there; `start` runs it on every validator before any start and again +# in the same ssh call as each start; `wait` runs it with --rpc. +# The script ships deploy/check-beacon-seed.sh to each host (`stage`, sha-checked) — the hosts +# have no checkout. +# * THE CONSTRAINT SET (issue #166). `push` and `switch` run deploy/check-constraint-set.sh +# --print-row with LOCAL_NODE's `constraint-set` marker (the binary, this tree and the vendored +# zkVM must all be set 11), every host's probe and new datadir must carry the marker `11`, and +# `verify` runs the full row check (CHAIN_CONSTRAINT_SETS must hold `(22, "", 11),`), as +# docs/deploy.md asks of this script's verify. `check-limits` (the cut script's) makes the same +# call; neither passes until the row is committed — commit it while chain 21 still runs. +# * 25 VALIDATORS, NOT 26: the validator set is the cut script's GENESIS_NAMES plus BONDED (read +# from deploy/cut-chain22-genesis.sh, not restated here); the inventory's validator rows minus +# DROPPED_VALIDATORS must be exactly that set. Quorum 2·25/3+1 = 17. Archives (role "archive": +# e, obs1) and observers (no "validator" in the role) start AFTER the first group and after +# check-limits; the first group is every other validator (23 ≥ 17), bootstraps C and D first. +# * THE HALT GUARD: `stop` refuses — before any host is contacted — unless +# CONFIRM_HALT_CHAIN21=halt-chain-21-for-

, P the first 8 hex of the chain-22 genesis hash +# being pushed. A typed phrase bound to this genesis: a confirmation left in the environment +# from an earlier attempt does not stop the fleet for another genesis. +# * THE INVENTORY is configuration/fleet/inventory.tsv in the private configuration repository +# (INVENTORY; columns by header name: name ip role ssh_key key_path datadir address …), not +# deploy/nodes.env + GUARDIAN_HOSTS; each host's ssh key, node key and datadir are read from it. +# No address is written here, and the roll record names hosts by name only. +# * PRUNED_SCAN_OK=1 is part of the cut: e and obs1 report prune_floor 187556 and no unpruned +# chain-21 node exists. `snapshot`, `balances` and `cut` wrap deploy/cut-chain22-genesis.sh with +# it set (and `cut` with the collected BEACON_TSV); the cut record must carry +# `scan-floor: 187556` (the cut script checks). +# * THE BRIDGE: `bridge-stop` stops rand-guardian on the six guardian hosts (and rand-relayer on +# RELAYER_HOST) before the snapshot; `bridge-start` starts them again once chain 22 serves — +# only with BRIDGE_ON_CHAIN22=yes: the daemons' configs (chain id, cursor) are the bridge +# session's to switch first. +# * A ROLL RECORD (ROLL_RECORD, default deploy/roll-records/v0.8.7-chain22-cutover.record) is +# appended by every phase that changes something: names, hashes and times, never addresses. +# +# INVENTORY=…/configuration/fleet/inventory.tsv is needed by every phase. In order: +# +# RELEASE_SUMS=rel/SHA256SUMS OLD_RELEASE_SUMS=old/SHA256SUMS deploy/cutover-fleet-chain22.sh preflight +# # READ-ONLY: 25 hosts reachable, v0.8.6, chain 21, +# # disk, keys; the signed v0.8.7 sums and files +# RELEASE_SUMS=… deploy/cutover-fleet-chain22.sh stage # v0.8.7 + 5 key files + the seed checker → /root/stage-c22 +# deploy/cutover-fleet-chain22.sh beacon-init # each validator makes ITS seed; prints the 25 commitments +# (each validator operator: an offline, encrypted backup of its beacon-c22/beacon.seed.json) +# deploy/cutover-fleet-chain22.sh beacon-collect # read-only: the 25 commitments → BEACON_TSV +# [RELAYER_HOST=…] deploy/cutover-fleet-chain22.sh bridge-stop # guardians (+ relayer) down +# CHAIN21_RPC= deploy/cutover-fleet-chain22.sh snapshot

+# CHAIN21_RPC=… WALLET= deploy/cutover-fleet-chain22.sh balances +# CUT_RECORD=… CHAIN21_SNAPSHOT= NODE=… WALLET=… … deploy/cutover-fleet-chain22.sh cut +# (the second operator re-derives the hash and writes second-hash into the record; the +# CHAIN_CONSTRAINT_SETS row the cut printed is committed) +# LOCAL_NODE= RELEASE_SUMS=… deploy/cutover-fleet-chain22.sh push +# # chain 21 STILL RUNNING +# ── the owner's go, typed in the executing session, before anything below ── +# CONFIRM_HALT_CHAIN21=halt-chain-21-for-

LOCAL_NODE=… RELEASE_SUMS=… OLD_RELEASE_SUMS=… \ +# deploy/cutover-fleet-chain22.sh stop # GATED; stops all 25 +# LOCAL_NODE=… RELEASE_SUMS=… OLD_RELEASE_SUMS=… deploy/cutover-fleet-chain22.sh switch +# deploy/cutover-fleet-chain22.sh start # every gate on all 25; C, D, then the first group (23) +# LOCAL_NODE=… deploy/cutover-fleet-chain22.sh check-limits # cut script's, against C +# deploy/cutover-fleet-chain22.sh start-archives # e, obs1 (and any observer) +# LOCAL_NODE=… deploy/cutover-fleet-chain22.sh wait # all 25 on chain 22, keys loaded, beacon --rpc +# BRIDGE_ON_CHAIN22=yes [RELAYER_HOST=…] deploy/cutover-fleet-chain22.sh bridge-start +# LOCAL_NODE=… deploy/cutover-fleet-chain22.sh verify # the CHAIN_CONSTRAINT_SETS row; the record +# deploy/cutover-fleet-chain22.sh status # read-only +# Rollback (bridge stopped first): +# deploy/cutover-fleet-chain22.sh quiet # read-only: nothing burned or minted on chain 22? +# deploy/cutover-fleet-chain22.sh halt # stop a fleet that does not run chain 21 (refuses one that does) +# OLD_RELEASE_SUMS=… deploy/cutover-fleet-chain22.sh rollback # chain 21's units, v0.8.6 binaries, chain 21's msauth blob +# deploy/cutover-fleet-chain22.sh start # the rollback marker licenses it: all 25, C and D first +# deploy/cutover-fleet-chain22.sh wait21 # until all 25 serve chain 21 and C's height moves +# A re-cut after a switch: halt, rollback, start, wait21, then push / stop / switch the new genesis. +# The beacon seeds stay in each host's staging dir and are reused (they are bound to chain 22 and +# the validator, not to a genesis hash); a re-cut keeps the same BEACON_TSV. +# +# THE TRAP FROM CHAIN 16: run each phase ALONE, never piped or chained; read `echo $?` first. +# `push`, `stop`, `switch`, `check-limits`, `wait` and `verify` refuse without LOCAL_NODE; `start` +# refuses without exactly one marker from a successful `switch` (or `rollback`). +# +# Env: INVENTORY (required), DROPPED_VALIDATORS (from the cut script), TAG (v0.8.7), STAGE_FROM +# (release|url|path) / RELEASE_URL / STAGE_URL / STAGE_DIR, RELEASE_SUMS / RELEASE_SUMS_SIG, +# WANT_SHA / WANT_SHA_WALLET (optional, must agree with the signed file), OLD_SHA / OLD_SHA_WALLET or +# OLD_RELEASE_SUMS / OLD_RELEASE_SUMS_SIG (v0.8.6), CUT_RECORD, BEACON_TSV, LOCAL_NODE, BOOT_NAMES +# ("c d"), MIN_FREE_GB (5), CHECK_LIMITS_RPC (else a tunnel to the first bootstrap), RELAYER_HOST / +# RELAYER_SSH_KEY, GUARDIAN_KNOWN_HOSTS, MARKER_DIR, ROLL_RECORD, CONFIRM_HALT_CHAIN21, DRY_RUN, SELFTEST. +# +# DRY_RUN=1 every phase prints every remote command and local write; no ssh/scp, no marker, no +# record line, no halt. Missing inputs become placeholders, loudly. +# SELFTEST=1 exec deploy/rehearse-cutover-fleet-chain22.sh: the whole cutover against 26 FAKE hosts +# (archive2 among them, never reachable) behind a fake ssh/scp/curl and fake rand-node +# stubs. No binary needed, no network. +# +# Nothing here touches node keys or reads a beacon seed's contents. Rollback by hand (if `rollback` +# itself cannot run): restore /root/rand-node.pre-c22, /root/rand.pre-c22 and +# /root/msauth-vk-production.bin.pre-c22 into /usr/local/bin; on a node host restore +# /root/rand-node.service.$OLD.bak over the unit; on a guardian host move /root/chain21.conf.c21 +# back into /etc/systemd/system/rand-node.service.d/ and delete chain22.conf; daemon-reload; start. +# The four perp-vk files may stay (chain 21's genesis pins no state-proof key). Chain 21 resumes +# from the height it stopped at: its datadirs are never touched here. +set -euo pipefail +cd "$(dirname "$0")/.." +if [ "${SELFTEST:-}" = 1 ]; then exec bash deploy/rehearse-cutover-fleet-chain22.sh "$@"; fi +# shellcheck disable=SC1091 # sourced from the repo root (the cd above) +. deploy/lib/cut-policy.sh +# shellcheck disable=SC1091 +. deploy/lib/verify-release.sh + +PHASE=${1:?preflight|stage|beacon-init|beacon-collect|bridge-stop|snapshot|balances|cut|push|stop|switch|start|check-limits|start-archives|wait|bridge-start|verify|status|quiet|halt|rollback|wait21} +DRY=${DRY_RUN:-0} +TAG=${TAG:-v0.8.7} +WANT_VERSION=${WANT_VERSION:-0.8.7} +STAGE_FROM=${STAGE_FROM:-release} +RELEASE_URL=${RELEASE_URL:-https://github.com/randprotocol/fullnode/releases/download/$TAG} +STAGE_URL=${STAGE_URL:-} +STAGE_DIR=${STAGE_DIR:-} +RELEASE_SUMS=${RELEASE_SUMS:-} +RELEASE_SUMS_SIG=${RELEASE_SUMS_SIG:-${RELEASE_SUMS:+$RELEASE_SUMS.sig}} +WANT_SHA=${WANT_SHA:-} +WANT_SHA_WALLET=${WANT_SHA_WALLET:-} +WANT_SHA_VK= +NEED_FLAGS=${NEED_FLAGS:---testnet --binding-domain --proof-window-blocks --max-gas-price --gas-byte-load --program-state-cell-fee --fees --multisig --auth-multisig-guest --program-state-refuse-panics --tokens-incremental-root --program-state-reference-block-window --beacon-commitment} +CHAIN_ID=22 +# ── the decided values (docs/deploy.md; deploy/cut-chain22-genesis.sh). Not overridable: another +# blob, key, guest or limit edits this script, the cut script and docs/deploy.md together. +VK=msauth-vk-production.bin +MSAUTH_VK_SHA=168d79a880b13b6d3e5b1f2a0f50561590712179c63189b52038cb2172a971b3 # constraint set 11 +MSAUTH_VK_BYTES=23851 +OLD_VK_SHA=68facb06c7f321aaf1d4d541acf6b5c0e9d7f17788ed1c9add6c5a8a8f7acd30 # chain 21's (set 9), what v0.8.6 reads +PVK_PINS="perp-vk-production-t18-pg14-in16-pub10-p2h20.bin:ae3d0dc276061fd2a87b2e91b239f6dcccb746b49dedeecfdb2aab977c95c370 +perp-vk-production-t18-pg14-in16-pub7-p2h20.bin:4dc1f18bdd6d9104ba1622710cf2c146f71759676dbdcb70dad036152242e8b4 +perp-vk-production-t18-pg14-in16-pub8-p2h20.bin:6392e5e9b7348f3729ba2031e0e3ec9788d6ed62a14521526482c92615f55106 +perp-vk-production-t18-pg14-in16-pub9-p2h20.bin:c31636aa281004163e6fe0debd81854f11d7bca6c8062a91b1501c7e593e5bf0" +PVK_PINS=$(printf '%s\n' "$PVK_PINS" | LC_ALL=C sort | tr '\n' ' ') # one sorted line: ": …" +PVK_FILES=$(for fp in $PVK_PINS; do printf '%s ' "${fp%%:*}"; done) +REL_FILES="rand-node rand $VK $PVK_FILES" +HC_AUTH_MULTISIG=41db73c507ab1f24ebe76d9c425b42a08ed9a02b4b1284d410502729ce6b9cd1 # MSAUTH_HC, set 11 +MAX_PROOF_BYTES=6291456 +MAX_BLOCK_BYTES=19922944 +WANT_CONSTRAINT_SET=11 +NODE_RS=${NODE_RS:-crates/randprotocol-node/src/node.rs} +TREE_SET=$(sed -n 's/^pub const BUILD_CONSTRAINT_SET: u32 = \([0-9][0-9]*\);.*$/\1/p' "$NODE_RS") +[ "$TREE_SET" = "$WANT_CONSTRAINT_SET" ] || { echo "this tree's BUILD_CONSTRAINT_SET ($NODE_RS) is '${TREE_SET:-?}', not $WANT_CONSTRAINT_SET — chain 22 is cut and launched from a set-11 tree" >&2; exit 1; } +# ── the build chain 21 runs: v0.8.6 (tag 6591c884). Its binary pins have no default. +OLD_SHA=${OLD_SHA:-} +OLD_SHA_WALLET=${OLD_SHA_WALLET:-} +OLD_RELEASE_SUMS=${OLD_RELEASE_SUMS:-} +OLD_RELEASE_SUMS_SIG=${OLD_RELEASE_SUMS_SIG:-${OLD_RELEASE_SUMS:+$OLD_RELEASE_SUMS.sig}} +OLD_VERSION=${OLD_VERSION:-0.8.6} +OLD_GIT_SHA=${OLD_GIT_SHA:-6591c884187126f118460c940d15715032bd124f} +[[ "$OLD_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "$OLD_GIT_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "OLD_VERSION / OLD_GIT_SHA are not a version and a 40-hex commit" >&2; exit 1; } +OLD=ca76c276 # chain 21's genesis prefix: the datadir suffix +OLD_HASH=ca76c2766ed6090ac6ba5dad86b724dd5d2b80c0edfaed287cd8b129e663ca9a +CUT_SCRIPT=deploy/cut-chain22-genesis.sh +CUT_RECORD=${CUT_RECORD:-$HOME/.rand-chain22/cut-record.txt} +BEACON_TSV=${BEACON_TSV:-$HOME/.rand-chain22/beacon-commitments.tsv} +MARKER_DIR=${MARKER_DIR:-/tmp/chain22-cutover} +ROLL_RECORD=${ROLL_RECORD:-deploy/roll-records/v0.8.7-chain22-cutover.record} +MIN_FREE_GB=${MIN_FREE_GB:-5} +WAIT_TRIES=${WAIT_TRIES:-90} +GUARDIAN_KNOWN_HOSTS=${GUARDIAN_KNOWN_HOSTS:-$HOME/.ssh/rand_guardian_known_hosts} +RELAYER_HOST=${RELAYER_HOST:-} +RELAYER_SSH_KEY=${RELAYER_SSH_KEY:-$HOME/.ssh/rand_guardian_ed25519} +S=/root/stage-c22 # the staging dir on every host +for v in RELEASE_URL STAGE_URL WANT_VERSION; do + case "${!v}" in *[\'\;\ \"\`\$\\\|\<\>\(\)\{\}]*) echo "$v contains a quote, ';', a space or a shell metacharacter — refused" >&2; exit 1;; esac +done +for f in $NEED_FLAGS; do [[ "$f" =~ ^--[a-z0-9-]+$ ]] || { echo "NEED_FLAGS: '$f' is not a --flag — refused" >&2; exit 1; }; done +[[ "$MIN_FREE_GB" =~ ^[0-9]{1,4}$ ]] || { echo "MIN_FREE_GB=$MIN_FREE_GB is not a number of GB" >&2; exit 1; } +[[ "$WAIT_TRIES" =~ ^[1-9][0-9]{0,3}$ ]] || { echo "WAIT_TRIES=$WAIT_TRIES is not a count" >&2; exit 1; } +[ -z "$RELAYER_HOST" ] || [[ "$RELAYER_HOST" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]] || { echo "RELAYER_HOST=$RELAYER_HOST is not an IPv4 address" >&2; exit 1; } + +# ── the validator set, from the cut script (one source) ──────────────────────────────────────── +cut_default() { sed -n "s/^$1=\${$1:-\(.*\)}\$/\1/p" "$CUT_SCRIPT" | head -1; } +GENESIS_NAMES=$(cut_default GENESIS_NAMES) +BONDED=$(cut_default BONDED) +DROPPED_VALIDATORS=$(sed -n 's/^DROPPED_VALIDATORS=\([a-z0-9 ]*\)$/\1/p' "$CUT_SCRIPT" | head -1) +[ -n "$GENESIS_NAMES" ] && [ -n "$BONDED" ] && [ -n "$DROPPED_VALIDATORS" ] \ + || { echo "could not read GENESIS_NAMES / BONDED / DROPPED_VALIDATORS from $CUT_SCRIPT — an anchor moved; update this script" >&2; exit 1; } +# shellcheck disable=SC2086 # word-split on purpose: name lists +VALSET=$(printf '%s\n' $GENESIS_NAMES $BONDED | sort | tr '\n' ' ') +NVALSET=$(wc -w <<<"$VALSET" | tr -d ' ') +QUORUM=$(( 2 * NVALSET / 3 + 1 )) +[ "$NVALSET" = 25 ] || { echo "$CUT_SCRIPT names $NVALSET chain-22 validators, not 25" >&2; exit 1; } + +# ── the inventory ────────────────────────────────────────────────────────────────────────────── +INVENTORY=${INVENTORY:-} +[ -n "$INVENTORY" ] || { echo "INVENTORY is unset — point it at configuration/fleet/inventory.tsv (the private configuration repository)" >&2; exit 1; } +[ -f "$INVENTORY" ] || { echo "INVENTORY=$INVENTORY is not a file" >&2; exit 1; } +if grep -qE '(^|[[:space:]])203\.0\.113\.' "$INVENTORY"; then echo "$INVENTORY holds placeholder hosts (203.0.113.x) — point INVENTORY at the operator's file" >&2; exit 1; fi +INV_ROWS=$(awk -F'\t' ' + NR == 1 { for (i = 1; i <= NF; i++) col[$i] = i + n = split("name ip role ssh_key key_path datadir address", w, " ") + for (i = 1; i <= n; i++) if (!(w[i] in col)) { print "MISSING " w[i]; exit } + next } + /^[[:space:]]*(#|$)/ { next } + { a = $col["address"]; if (a == "") a = "-" + print $col["name"], $col["ip"], $col["role"], $col["ssh_key"], $col["key_path"], $col["datadir"], a }' "$INVENTORY") +case "$INV_ROWS" in MISSING*) echo "$INVENTORY has no '${INV_ROWS#MISSING }' column" >&2; exit 1;; esac +ALL_N=""; VAL_N=""; FIRST_N=""; LATE_N=""; GUARD_N=""; DROPPED_IPS=""; SEEN="" +while read -r n ip role key kp dd addr; do + [ -n "$n" ] || continue + [[ "$n" =~ ^[a-z][a-z0-9]{0,15}$ ]] || { echo "$INVENTORY: name '$n' is not [a-z][a-z0-9]*" >&2; exit 1; } + case " $SEEN " in *" $n "*) echo "$INVENTORY: $n is listed twice" >&2; exit 1;; esac; SEEN="$SEEN $n" + [[ "$ip" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]] || { echo "$INVENTORY: $n's ip '$ip' is not an IPv4 address" >&2; exit 1; } + case " $DROPPED_VALIDATORS " in *" $n "*) DROPPED_IPS="$DROPPED_IPS $ip"; continue;; esac # never contacted + for v in "$kp" "$dd"; do [[ "$v" =~ ^/[A-Za-z0-9._/-]+$ ]] || { echo "$INVENTORY: $n: '$v' is not a plain absolute path" >&2; exit 1; }; done + [[ "$role" =~ ^[a-z+-]+$ ]] || { echo "$INVENTORY: $n's role '$role' is not a role" >&2; exit 1; } + key=${key/#\~/$HOME} + [[ "$key" =~ ^[A-Za-z0-9._/-]+$ ]] || { echo "$INVENTORY: $n's ssh_key '$key' is not a plain path" >&2; exit 1; } + printf -v "INV_ip_$n" '%s' "$ip"; printf -v "INV_key_$n" '%s' "$kp"; printf -v "INV_dd_$n" '%s' "$dd" + printf -v "INV_ssh_$n" '%s' "$key"; printf -v "INV_addr_$n" '%s' "$addr" + case "$role" in *guardian-host*) printf -v "INV_kind_$n" guardian; GUARD_N="$GUARD_N $n";; *) printf -v "INV_kind_$n" node;; esac + case "$role" in *validator*) printf -v "INV_val_$n" 1; VAL_N="$VAL_N $n";; *) printf -v "INV_val_$n" 0;; esac + case "$role" in *archive*|observer*) LATE_N="$LATE_N $n";; *validator*) FIRST_N="$FIRST_N $n";; *) LATE_N="$LATE_N $n";; esac + ALL_N="$ALL_N $n" +done <<<"$INV_ROWS" +ALL_N=${ALL_N# }; VAL_N=${VAL_N# }; FIRST_N=${FIRST_N# }; LATE_N=${LATE_N# }; GUARD_N=${GUARD_N# } +for d in $DROPPED_VALIDATORS; do case " $SEEN " in *" $d "*) ;; *) echo "$INVENTORY has no row for the dropped $d — is this the fleet's inventory?" >&2; exit 1;; esac; done +# shellcheck disable=SC2086 +HAVE=$(printf '%s\n' $VAL_N | sort | tr '\n' ' ') +[ "$HAVE" = "$VALSET" ] || { echo "the inventory's validators without $DROPPED_VALIDATORS ($HAVE) are not chain 22's 25 (GENESIS_NAMES + BONDED: $VALSET)" >&2; exit 1; } +NALL=$(wc -w <<<"$ALL_N" | tr -d ' '); NFIRST=$(wc -w <<<"$FIRST_N" | tr -d ' ') +[ "$NFIRST" -ge "$QUORUM" ] || { echo "the first group ($FIRST_N) has $NFIRST validators, below the quorum $QUORUM — chain 22 would not commit before the archives" >&2; exit 1; } +[ "$(wc -w <<<"$GUARD_N" | tr -d ' ')" = 6 ] || { echo "the inventory has $(wc -w <<<"$GUARD_N" | tr -d ' ') guardian hosts, expected 6" >&2; exit 1; } +BOOT_NAMES=${BOOT_NAMES:-c d} +for b in $BOOT_NAMES; do case " $FIRST_N " in *" $b "*) ;; *) echo "bootstrap $b is not a first-group validator of the inventory" >&2; exit 1;; esac; done +BOOT1=${BOOT_NAMES%% *} +iget() { local v="INV_${2}_$1"; printf '%s' "${!v:-}"; } + +dry() { [ "$DRY" = 1 ]; } +missing() { # + if dry; then echo "== DRY RUN: $1 — using placeholder $3" >&2; printf -v "$2" '%s' "$3" + else echo "$PHASE: $1 — nothing done" >&2; exit 1; fi +} +refuse_dropped() { # : archive2 (any dropped validator) is never contacted + local d + for d in $DROPPED_VALIDATORS; do [ "$1" != "$d" ] || { echo "REFUSED: $d is dropped from chain 22 and is never contacted" >&2; return 1; }; done + case " $DROPPED_IPS " in *" $2 "*) echo "REFUSED: $2 is a dropped validator's address and is never contacted" >&2; return 1;; esac + [ -n "$2" ] || { echo "REFUSED: $1 is not an inventory host" >&2; return 1; } +} +sshopts() { # → the ssh options for that host (its inventory key; the guardians' pinned known_hosts) + if [ "$(iget "$1" kind)" = guardian ]; then + printf '%s\n' -i "$(iget "$1" ssh)" -o IdentitiesOnly=yes -o "UserKnownHostsFile=$GUARDIAN_KNOWN_HOSTS" -o StrictHostKeyChecking=yes -o ConnectTimeout=20 -o BatchMode=yes + else + printf '%s\n' -i "$(iget "$1" ssh)" -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -o BatchMode=yes + fi +} +on() { # + local n=$1 ip; ip=$(iget "$1" ip); shift + refuse_dropped "$n" "$ip" || return 1 + if dry; then echo "[dry-run] ssh root@$n($ip) $*"; return 0; fi + local o=(); while IFS= read -r l; do o+=("$l"); done < <(sshopts "$n") + ssh -n "${o[@]}" "root@$ip" "$@" +} +cp_to() { # + local n=$1 ip dst=$2; ip=$(iget "$1" ip); shift 2 + refuse_dropped "$n" "$ip" || return 1 + if dry; then echo "[dry-run] scp $* root@$n($ip):$dst"; return 0; fi + local o=(); while IFS= read -r l; do o+=("$l"); done < <(sshopts "$n") + scp -q "${o[@]}" "$@" "root@$ip:$dst" +} +# Per-host values for a remote script: its name, node key, chain-21 datadir, validator flag, kind. +hvars() { printf "H_NAME='%s'; H_KEY='%s'; H_DD='%s'; H_VAL=%s; H_KIND=%s; " "$1" "$(iget "$1" key)" "$(iget "$1" dd)" "$(iget "$1" val)" "$(iget "$1" kind)"; } +EACH_OUT="" +each() { # : in parallel, the last line of each; fail if any failed. + # EACH_OUT, when set, names a directory that keeps each host's full output as . + local hosts=$1 script=$2 + if dry; then + printf '[dry-run] on each of %s hosts (%s) in parallel, over ssh as root, H_NAME/H_KEY/H_DD/H_VAL/H_KIND from the inventory:\n' "$(wc -w <<<"$hosts" | tr -d ' ')" "$hosts" + printf '%s\n' "$script" | sed 's/^/[dry-run] | /' + return 0 + fi + local n dir; dir=$(mktemp -d) + for n in $hosts; do ( if on "$n" "$(hvars "$n")$script" > "$dir/$n" 2>&1; then echo ok > "$dir/$n.rc"; else echo FAIL > "$dir/$n.rc"; fi ) & done; wait + local bad=0 + for n in $hosts; do printf ' %-8s %s %s\n' "$n" "$(cat "$dir/$n.rc")" "$(tail -1 "$dir/$n")"; [ "$(cat "$dir/$n.rc")" = ok ] || bad=1; done + if [ -n "$EACH_OUT" ]; then for n in $hosts; do cp "$dir/$n" "$EACH_OUT/$n"; done; fi + rm -rf "$dir"; return $bad +} +rpc() { dry && { on "$1" "curl -s -m5 -X POST 127.0.0.1:8545 … $2" >&2; return 0; } + on "$1" "curl -s -m5 -X POST 127.0.0.1:8545 -H 'content-type: application/json' -d '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"$2\",\"params\":[]}'" 2>/dev/null || true; } +local_write() { local what=$1; shift; if dry; then echo "[dry-run] local: $what"; else "$@"; fi; } +mark() { local_write "write $MARKER_DIR/$1" bash -c 'mkdir -p "$1" && echo "$2 $(date -u +%FT%TZ)" > "$1/$3"' _ "$MARKER_DIR" "${2:-}" "$1"; } +unmark() { local_write "rm -f $MARKER_DIR/$1" bash -c 'rm -f "$1"/$2' _ "$MARKER_DIR" "$1"; } +clear_markers() { local_write "rm -f $MARKER_DIR/{switched,first,limits,rollback}-*" bash -c 'rm -f "$1"/switched-* "$1"/first-* "$1"/limits-* "$1"/rollback-*' _ "$MARKER_DIR"; } +# shellcheck disable=SC2086 # $1 is a glob on purpose +markers() { ls "$MARKER_DIR"/$1 2>/dev/null || true; } +# The roll record: appended, never written over; names and hashes, no addresses. +rec() { + dry && { echo "[dry-run] local: append to $ROLL_RECORD: $PHASE: $*"; return 0; } + if [ ! -f "$ROLL_RECORD" ]; then + mkdir -p "$(dirname "$ROLL_RECORD")" + { echo "# Roll record — chain 21 → chain 22 on $TAG (a chain cut: all-stop, then a quorum, then the archives)." + echo "#" + echo "# Written by deploy/cutover-fleet-chain22.sh as the phases ran; one line per phase, appended, never" + echo "# written over. Names only: the fleet's addresses are the operator's (configuration/fleet)." + echo + echo "release: $TAG (v$WANT_VERSION, constraint set $WANT_CONSTRAINT_SET); chain 21 ran v$OLD_VERSION ($OLD_GIT_SHA)" + echo "hosts: $NALL — validators $NVALSET ($VAL_N), first group $NFIRST ($FIRST_N), late ${LATE_N:-none}; quorum $QUORUM" + echo "skipped: $DROPPED_VALIDATORS (dropped from chain 22; never contacted)" + echo "tool: deploy/cutover-fleet-chain22.sh ($(git rev-parse --short HEAD 2>/dev/null || echo 'not a git checkout'))" + } > "$ROLL_RECORD" + fi + printf '%s: %s %s\n' "$PHASE" "$(date -u +%FT%TZ)" "$*" >> "$ROLL_RECORD" +} + +# ── the v0.8.7 pins, from the signed SHA256SUMS only ───────────────────────────────────────────── +PINS="" # ": …" for the seven release files, set by need_new_shas +need_new_shas() { + local got name var fp + if [ -z "$RELEASE_SUMS" ]; then + missing "RELEASE_SUMS is unset — the $TAG release's SHA256SUMS (signed: SHA256SUMS.sig beside it) is required" \ + WANT_SHA "" + WANT_SHA_WALLET=""; WANT_SHA_VK=$MSAUTH_VK_SHA + PINS="rand-node:$WANT_SHA rand:$WANT_SHA_WALLET $VK:$WANT_SHA_VK $PVK_PINS"; return 0 + fi + verify_release_sums "$RELEASE_SUMS" "$RELEASE_SUMS_SIG" || { echo "$PHASE: the $TAG SHA256SUMS signature does not verify — nothing done" >&2; exit 1; } + for name in rand-node:WANT_SHA rand:WANT_SHA_WALLET "$VK:WANT_SHA_VK"; do + var=${name#*:}; name=${name%%:*} + got=$(release_sha "$RELEASE_SUMS" "$name") || { echo "$PHASE: the signed SHA256SUMS lists no $name — nothing done" >&2; exit 1; } + [ -z "${!var}" ] || [ "${!var}" = "$got" ] || { echo "$PHASE: $var=${!var} is not what the signed SHA256SUMS lists for $name ($got) — nothing done" >&2; exit 1; } + printf -v "$var" '%s' "$got" + done + [ "$WANT_SHA_VK" = "$MSAUTH_VK_SHA" ] || { echo "$PHASE: the signed SHA256SUMS lists $VK at $WANT_SHA_VK, not constraint set 11's decided blob $MSAUTH_VK_SHA — nothing done" >&2; exit 1; } + for fp in $PVK_PINS; do + got=$(release_sha "$RELEASE_SUMS" "${fp%%:*}") || { echo "$PHASE: the signed SHA256SUMS lists no ${fp%%:*} (CON-601: a chain-22 node does not start without it) — nothing done" >&2; exit 1; } + [ "$got" = "${fp#*:}" ] || { echo "$PHASE: the signed SHA256SUMS lists ${fp%%:*} at $got, not the decided ${fp#*:} — nothing done" >&2; exit 1; } + done + [ -z "$OLD_SHA" ] || [ "$WANT_SHA" != "$OLD_SHA" ] || { echo "$PHASE: the signed rand-node is chain 21's v$OLD_VERSION — chain 22 needs v$WANT_VERSION; nothing done" >&2; exit 1; } + PINS="rand-node:$WANT_SHA rand:$WANT_SHA_WALLET $VK:$WANT_SHA_VK $PVK_PINS" + echo "== the signed SHA256SUMS ($RELEASE_SUMS) pins rand-node ${WANT_SHA:0:12}…, rand ${WANT_SHA_WALLET:0:12}…, $VK ${WANT_SHA_VK:0:12}… and the four perps keys" +} +need_old_shas() { + local got + if [ -z "$OLD_SHA" ] && [ -n "$OLD_RELEASE_SUMS" ]; then + verify_release_sums "$OLD_RELEASE_SUMS" "$OLD_RELEASE_SUMS_SIG" || { echo "$PHASE: the v$OLD_VERSION SHA256SUMS signature does not verify — nothing done" >&2; exit 1; } + OLD_SHA=$(release_sha "$OLD_RELEASE_SUMS" rand-node) || exit 1 + got=$(release_sha "$OLD_RELEASE_SUMS" rand) || exit 1 + [ -z "$OLD_SHA_WALLET" ] || [ "$OLD_SHA_WALLET" = "$got" ] || { echo "$PHASE: OLD_SHA_WALLET is not what v$OLD_VERSION's signed SHA256SUMS lists" >&2; exit 1; } + OLD_SHA_WALLET=$got + fi + if [[ ! "$OLD_SHA" =~ ^[0-9a-f]{64}$ || ! "$OLD_SHA_WALLET" =~ ^[0-9a-f]{64}$ ]]; then + missing "OLD_SHA / OLD_SHA_WALLET (or OLD_RELEASE_SUMS) must give the sha256 of chain 21's v$OLD_VERSION Linux rand-node / rand" \ + OLD_SHA "" + OLD_SHA_WALLET="" + fi +} +LOCAL_SET="" +need_local_node() { + local f help d + if [ -z "${LOCAL_NODE:-}" ]; then + dry && { echo "== DRY RUN: LOCAL_NODE is unset — the real run refuses (the chain-16 trap); hash checks become placeholders" >&2; LOCAL_NODE=; return 0; } + echo "$PHASE: LOCAL_NODE is unset — point it at a v$WANT_VERSION rand-node on this machine; nothing done (the chain-16 trap)" >&2; exit 1 + fi + [ -x "$LOCAL_NODE" ] || { echo "$PHASE: LOCAL_NODE=$LOCAL_NODE is not executable — nothing done" >&2; exit 1; } + help=$("$LOCAL_NODE" genesis --help) + for f in $NEED_FLAGS; do grep -q -- "$f" <<<"$help" || { echo "$PHASE: $LOCAL_NODE has no $f (not the v$WANT_VERSION build) — nothing done" >&2; exit 1; }; done + case "$("$LOCAL_NODE" --version)" in "rand-node $WANT_VERSION"*) ;; *) echo "$PHASE: $LOCAL_NODE is $("$LOCAL_NODE" --version), not rand-node $WANT_VERSION — nothing done" >&2; exit 1;; esac +} +genesis_hash() { # → " " + local d h c=- + if [ -z "${LOCAL_NODE:-}" ]; then echo " -"; return 0; fi + d=$(mktemp -d) + h=$("$LOCAL_NODE" init --datadir "$d/p" --genesis "$1" 2>/dev/null | grep -o 'genesis [0-9a-f]*' | cut -d' ' -f2 || true) + [ ! -f "$d/p/constraint-set" ] || c=$(tr -d '[:space:]' < "$d/p/constraint-set") + rm -rf "$d"; echo "${h:--} ${c:--}" +} +# The file must BE chain 22's genesis: chain id, the set-11 msauth guest, the decided limits, the four +# pinned state-proof keys, a beacon with one commitment per validator — and, when BEACON_TSV exists, +# exactly the commitments the fleet's hosts gave (beacon-collect). +need_chain22_genesis() { + if [ ! -f "$1" ]; then + dry && { echo "== DRY RUN: $1 is not a file (the cut has not run) — genesis checks skipped" >&2; return 0; } + echo "$PHASE: $1 is not a file — nothing done" >&2; exit 1 + fi + local why + why=$(python3 - "$1" "$HC_AUTH_MULTISIG" "$MAX_PROOF_BYTES" "$MAX_BLOCK_BYTES" "$PVK_PINS" "$NVALSET" "$BEACON_TSV" <<'PY' +import json, os, sys +gf, hc, mpb, mbb, pins, nval, tsv = sys.argv[1:8] +try: + g = json.load(open(gf)) +except Exception as e: + print(f"is not JSON ({e})"); sys.exit(0) +def no(m): print(m); sys.exit(0) +if g.get("chain_id") != 22: no(f"has chain_id {g.get('chain_id')}, not 22") +if g.get("hc_auth_multisig") != hc: no(f"has hc_auth_multisig {g.get('hc_auth_multisig')}, not constraint set 11's {hc}") +if g.get("max_proof_bytes") != int(mpb): no(f"has max_proof_bytes {g.get('max_proof_bytes')}, not {mpb}") +if g.get("max_block_bytes") != int(mbb): no(f"has max_block_bytes {g.get('max_block_bytes')}, not {mbb}") +if len(g.get("validators", [])) != int(nval): no(f"names {len(g.get('validators', []))} validators, not {nval}") +want = sorted(p.split(":", 1)[1] for p in pins.split()) +got = sorted(k.get("sha256", "") for k in (g.get("perps") or {}).get("state_proof_keys", [])) +if got != want: no(f"pins perps.state_proof_keys {[x[:12] for x in got]}, not the four decided keys {[x[:12] for x in want]} (CON-601)") +b = g.get("beacon") +if not b: no("has no beacon section (#189)") +cs = [c.get("commitment") for c in b.get("commitments", [])] +if len(cs) != int(nval) or len(set(cs)) != len(cs): no(f"has {len(cs)} beacon commitments ({len(set(cs))} distinct), not {nval}") +if os.path.isfile(tsv): + t = [l.split() for l in open(tsv) if l.strip() and not l.lstrip().startswith("#")] + if sorted(x[1] for x in t if len(x) == 2) != sorted(cs): + no(f"has beacon commitments that are not the {len(t)} in {tsv} (beacon-collect) — a genesis cut from other commitments; its validators would miss their slots") +PY +) + [ -z "$why" ] || { echo "$PHASE: $1 $why — nothing done" >&2; exit 1; } +} +file_sha() { if [ -f "$1" ]; then shasum -a 256 "$1" | cut -d' ' -f1; else echo ""; fi; } +NEW=""; P=""; SHA=""; GENESIS="" +need_genesis() { + GENESIS=$1 + need_chain22_genesis "$GENESIS" + SHA=$(file_sha "$GENESIS") + read -r NEW LOCAL_SET <<<"$(genesis_hash "$GENESIS")" + [ "$LOCAL_SET" != - ] || LOCAL_SET="" + if [ ${#NEW} -eq 64 ]; then P=${NEW:0:8} + elif dry; then P="

" + else echo "$PHASE: $LOCAL_NODE derived no genesis hash from $GENESIS — nothing done" >&2; exit 1; fi + [ "$P" != "$OLD" ] || { echo "$PHASE: $GENESIS is chain 21's genesis ($NEW) — nothing done" >&2; exit 1; } +} +# deploy/check-constraint-set.sh for this genesis: the binary (LOCAL_NODE's init marker), this tree +# and the vendored zkVM are one constraint set (--print-row: the table is not consulted — the row +# cannot predate the cut). With "row", the full CHAIN_CONSTRAINT_SETS row check (verify). +constraint_set_ok() { + if dry && [ ${#NEW} -ne 64 ]; then echo "[dry-run] local: deploy/check-constraint-set.sh --chain $CHAIN_ID --genesis-hash --binary-set ${1:+(the row) }"; return 0; fi + [ -n "$LOCAL_SET" ] || { echo "$PHASE: LOCAL_NODE's init wrote no constraint-set marker — not a v$WANT_VERSION build with the guard; nothing done" >&2; exit 1; } + if [ "${1:-}" = row ]; then + deploy/check-constraint-set.sh --chain "$CHAIN_ID" --genesis-hash "$NEW" --binary-set "$LOCAL_SET"; return + fi + local row + row=$(deploy/check-constraint-set.sh --chain "$CHAIN_ID" --genesis-hash "$NEW" --binary-set "$LOCAL_SET" --print-row) \ + || { echo "$PHASE: the constraint-set check refused (above) — nothing done" >&2; exit 1; } + echo "== constraint set: LOCAL_NODE $LOCAL_SET = this tree = the vendored zkVM; chain 22's row is $row" + if deploy/check-constraint-set.sh --chain "$CHAIN_ID" --genesis-hash "$NEW" --binary-set "$LOCAL_SET" >/dev/null 2>&1; then + echo "== CHAIN_CONSTRAINT_SETS already holds it" + else + echo "== ⚠ CHAIN_CONSTRAINT_SETS ($NODE_RS's crate) does not hold $row yet: check-limits and verify refuse until it is committed — add it now, while chain 21 still runs" + fi +} +second_rebuild_ok() { + if dry && [ ${#NEW} -ne 64 ]; then + echo "[dry-run] local: require_second_rebuild $CUT_RECORD — the record's second-hash: must equal it"; return 0; fi + if grep -q "^second-rebuild-waived: .*genesis $NEW" "$CUT_RECORD" 2>/dev/null; then + echo "== ⚠ the cut record carries a second-rebuild WAIVER for genesis $NEW (no second operator's hash)"; return 0; fi + if require_second_rebuild "$CUT_RECORD" "$NEW"; then return 0; fi + if dry; then echo "== DRY RUN: the real $PHASE REFUSES here (above); the dry run goes on to show the rest"; return 0; fi + echo "$PHASE: nothing done" >&2; exit 1 +} + +# ── remote snippets ───────────────────────────────────────────────────────────────────────────── +# Per host: BDIR (its beacon staging dir), and — once P is known — NEWDIR (its chain-22 datadir) and +# HG (the genesis its node reads). Literal paths on purpose (the rehearsal maps them per fake host). +r_paths() { + echo "if [ \"\$H_KIND\" = guardian ]; then BDIR=/var/lib/randnode/beacon-c22; NEWDIR=/var/lib/randnode/data-22; HG=/var/lib/randnode/genesis-chain22.json; OWN=randnode + else BDIR=/root/beacon-c22; NEWDIR=\${H_DD%-$OLD}-$P; HG=$S/genesis-chain22.json; OWN=root; fi" +} +staged_ok() { + local fp out="" + for fp in $PINS; do out="${out}[ \"\$(sha256sum $S/${fp%%:*} 2>/dev/null | cut -d' ' -f1)\" = '${fp#*:}' ] || { echo 'staged ${fp%%:*} is not the pinned $TAG file ($S)'; exit 1; } + "; done + echo "${out}[ \"\$(sha256sum $S/check-beacon-seed.sh 2>/dev/null | cut -d' ' -f1)\" = '$CHECKER_SHA' ] || { echo 'staged check-beacon-seed.sh is not this tree'\''s'; exit 1; }" +} +vk_check() { # : docs/deploy.md's msauth check + echo "vk_got=\$($1 keygen-msauth-vk --check $2) && vk_pin=\$($1 keygen-msauth-vk --print-pinned production) && [ \"\$vk_got\" = \"\$vk_pin\" ] && [ \"\$vk_pin\" = '$MSAUTH_VK_SHA' ] && [ \"\$(wc -c < $2 | tr -d ' ')\" = '$MSAUTH_VK_BYTES' ] || { echo \"msauth key NOT verified: $2 is \${vk_got:-unreadable}, $1 pins \${vk_pin:-nothing}, decided $MSAUTH_VK_SHA\"; exit 1; }" +} +pvk_check() { #

: the four perps keys (CON-601) — the binary's pins are the decided four, each file holds to its pin + echo "pv=\$($1 keygen-perp-vk --print-pinned production | awk '{print \$1\":\"\$2}' | LC_ALL=C sort | tr '\\n' ' ') && [ \"\$pv\" = '$PVK_PINS' ] || { echo \"perps keys NOT verified: $1 pins \${pv:-nothing}, not the decided four\"; exit 1; } + for fp in $PVK_PINS; do f=\${fp%%:*}; p=\${fp#*:} + [ \"\$(sha256sum $2/\$f 2>/dev/null | cut -d' ' -f1)\" = \"\$p\" ] && [ \"\$($1 keygen-perp-vk --check $2/\$f 2>/dev/null)\" = \"\$p\" ] || { echo \"perps key NOT verified: $2/\$f is not \${p:0:12}…\"; exit 1; } + done" +} +beacon_check() { # []: validators only; deploy/check-beacon-seed.sh on the host + echo "BS=skipped; if [ \"\$H_VAL\" = 1 ]; then BS=\$(bash $S/check-beacon-seed.sh --node $1 --genesis $2 --datadir $3 --key \$H_KEY${4:+ --rpc $4} 2>&1) || { echo \"\$H_NAME: beacon seed NOT verified — \$(printf '%s\\n' \"\$BS\" | tail -1)\"; exit 1; }; BS=ok; fi" +} +cs_check() { # : the constraint-set marker init wrote + echo "[ \"\$(tr -d '[:space:]' < $1/constraint-set 2>/dev/null)\" = '$WANT_CONSTRAINT_SET' ] || { echo \"$1/constraint-set says '\$(cat $1/constraint-set 2>/dev/null)', not constraint set $WANT_CONSTRAINT_SET\"; exit 1; }" +} +staged_self() { + echo "$S/rand-node --version | grep -q 'rand-node $WANT_VERSION' || { echo \"staged rand-node is \$($S/rand-node --version 2>&1 | head -1), not $WANT_VERSION\"; exit 1; } + GH=\$($S/rand-node genesis --help); for f in $NEED_FLAGS; do printf '%s' \"\$GH\" | grep -q -- \"\$f\" || { echo \"staged rand-node has no \$f\"; exit 1; }; done + $(vk_check $S/rand-node $S/$VK) + $(pvk_check $S/rand-node $S)" +} +pushed_ok() { + echo "[ \"\$(sha256sum $S/genesis-chain22.json 2>/dev/null | cut -d' ' -f1)\" = '$SHA' ] || { echo 'the chain-22 genesis is not pushed here (sha256 differs or no file)'; exit 1; } + rm -rf $S/probe; GH=\$($S/rand-node init --datadir $S/probe --genesis $S/genesis-chain22.json | grep -o 'genesis [0-9a-f]*' | cut -d' ' -f2) + [ \"\$GH\" = '$NEW' ] || { rm -rf $S/probe; echo \"genesis hash \$GH, expected $NEW\"; exit 1; } + $(cs_check $S/probe) + rm -rf $S/probe" +} +installed_gate() { # what every chain-22 host must hold before its node starts + echo "$(r_paths) + $(vk_check /usr/local/bin/rand-node /usr/local/bin/$VK) + $(pvk_check /usr/local/bin/rand-node /usr/local/bin) + $(cs_check \$NEWDIR) + $(beacon_check /usr/local/bin/rand-node \$HG \$NEWDIR)" +} +stop_all() { + each "$ALL_N" 'systemctl stop rand-node; if systemctl is-active --quiet rand-node; then echo "$(hostname) STILL ACTIVE"; exit 1; fi; echo "$(hostname) stopped"' \ + || { echo "$PHASE: some hosts did not stop (above) — stop them by hand; switch and rollback refuse a running node" >&2; exit 1; } +} +CHECKER_SHA=$(file_sha deploy/check-beacon-seed.sh) + +case "$PHASE" in +preflight) + need_old_shas + need_new_shas + if [ "$STAGE_FROM" = path ] && [ -n "$STAGE_DIR" ] && ! dry; then + for f in $REL_FILES; do verify_binary "$STAGE_DIR/$f" "$RELEASE_SUMS" "$f" || { echo "preflight: $STAGE_DIR/$f is not the signed $f" >&2; exit 1; }; done + echo "== the seven $TAG files in $STAGE_DIR are the signed ones" + else + echo "== the $TAG files at the release URL, as $BOOT1 fetches them (HEAD only):" + BASE=$RELEASE_URL; [ "$STAGE_FROM" != url ] || BASE=${STAGE_URL%%\?*} + on "$BOOT1" "for f in $REL_FILES; do curl -fsSIL -o /dev/null --retry 3 '$BASE/'\$f || { echo \"\$f is not at the release\"; exit 1; }; done; echo 'all seven present'" \ + || { echo "preflight: the release does not hold every file (above)" >&2; exit 1; } + fi + echo "== preflight (read-only) on $NALL hosts ($DROPPED_VALIDATORS never contacted): v$OLD_VERSION rand-node ${OLD_SHA:0:12}…, rand ${OLD_SHA_WALLET:0:12}…, chain 21's msauth blob, chain-21 datadir -$OLD / data-21, ≥ $MIN_FREE_GB GB free $(date -u +%T)" + each "$ALL_N" "set -e + [ \"\$(sha256sum /usr/local/bin/rand-node | cut -d' ' -f1)\" = '$OLD_SHA' ] || { echo \"installed rand-node is \$(/usr/local/bin/rand-node --version 2>&1 | head -1) sha256 \$(sha256sum /usr/local/bin/rand-node | cut -c1-12)…, not chain 21's v$OLD_VERSION ${OLD_SHA:0:12}…\"; exit 1; } + [ \"\$(sha256sum /usr/local/bin/rand | cut -d' ' -f1)\" = '$OLD_SHA_WALLET' ] || { echo \"installed rand is not chain 21's v$OLD_VERSION\"; exit 1; } + [ \"\$(sha256sum /usr/local/bin/$VK 2>/dev/null | cut -d' ' -f1)\" = '$OLD_VK_SHA' ] || { echo 'installed $VK is not chain 21'\''s blob (${OLD_VK_SHA:0:12}…) — the rollback needs it'; exit 1; } + for b in rand-node rand $VK; do [ ! -e /root/\$b.pre-c22 ] || { echo \"a /root/\$b.pre-c22 backup already exists (an earlier switch?)\"; exit 1; }; done + [ -f \"\$H_KEY\" ] || { echo \"no node key at \$H_KEY\"; exit 1; } + command -v python3 >/dev/null || { echo 'no python3 (check-beacon-seed.sh needs it)'; exit 1; } + D=/etc/systemd/system/rand-node.service.d + if [ \"\$H_KIND\" = guardian ]; then + [ -f \$D/chain21.conf ] && [ ! -f \$D/chain22.conf ] || { echo 'a guardian host without chain21.conf, or with a chain22.conf already'; exit 1; } + [ \"\$H_DD\" = /var/lib/randnode/data-21 ] && grep -q -- '--datadir /var/lib/randnode/data-21 ' \$D/chain21.conf || { echo 'chain21.conf has no data-21 datadir, or the inventory names another'; exit 1; } + [ ! -e /var/lib/randnode/data-22 ] || { echo '/var/lib/randnode/data-22 already exists — move it aside'; exit 1; } + [ ! -e /root/chain21.conf.c21 ] || { echo '/root/chain21.conf.c21 already exists (an earlier switch?)'; exit 1; } + else + [ ! -f \$D/chain21.conf ] || { echo 'the inventory says node, the host has a guardian drop-in'; exit 1; } + case \"\$H_DD\" in *-$OLD) ;; *) echo \"the inventory's datadir \$H_DD has no -$OLD suffix\"; exit 1;; esac + grep -q -- \"--datadir \$H_DD \" /etc/systemd/system/rand-node.service || { echo \"the unit does not run the inventory's datadir \$H_DD\"; exit 1; } + fi + [ -d \"\$H_DD/db\" ] || { echo \"\$H_DD has no db\"; exit 1; } + FREE=\$(df -Pk \"\$H_DD/\" | awk 'NR==2 {print \$4}'); [ \"\$FREE\" -ge $((MIN_FREE_GB * 1048576)) ] || { echo \"only \$((FREE / 1048576)) GB free under \$H_DD (want $MIN_FREE_GB)\"; exit 1; } + G=\$(curl -s -m5 -X POST 127.0.0.1:8545 -H 'content-type: application/json' -d '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"rand_getGenesisHash\",\"params\":[]}' | grep -oE '\"result\":\"[0-9a-f]+\"' | cut -d'\"' -f4) + [ \"\$G\" = '$OLD_HASH' ] || { echo \"serves genesis \${G:-nothing}, not chain 21 ($OLD…)\"; exit 1; } + curl -s -m5 -X POST 127.0.0.1:8545 -H 'content-type: application/json' -d '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"rand_getHealth\",\"params\":[]}' | grep -q '\"status\":\"ok\"' || { echo 'rand_getHealth is not ok'; exit 1; } + VER=\$(curl -s -m5 -X POST 127.0.0.1:8545 -H 'content-type: application/json' -d '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"rand_getVersion\",\"params\":[]}') + printf '%s' \"\$VER\" | grep -q '\"version\":\"$OLD_VERSION\"' && printf '%s' \"\$VER\" | grep -q '\"git_sha\":\"$OLD_GIT_SHA\"' \\ + || { echo \"rand_getVersion answers \$(printf '%s' \"\$VER\" | grep -oE '\"(version|git_sha)\":\"[^\"]*\"' | tr '\\n' ' '), not $OLD_VERSION / ${OLD_GIT_SHA:0:8}\"; exit 1; } + $(r_paths) + ST=\$( [ -e $S/rand-node ] && echo ', staged' || echo ', nothing staged') + BC=\$( [ \"\$H_VAL\" = 1 ] && { [ -f \$BDIR/beacon.seed.json ] && echo ', beacon seed made' || echo ', no beacon seed yet'; } || echo ', not a validator') + echo \"\$H_NAME \$H_KIND ready: v$OLD_VERSION, chain 21, healthy, \$((FREE / 1048576)) GB free\$ST\$BC\"" \ + || { echo "preflight: not every host is ready — fix those before staging or stopping anything" >&2; exit 1; } + echo "PREFLIGHT OK — nothing was changed ($NALL hosts; $NVALSET validators; first group $NFIRST ≥ quorum $QUORUM; late: ${LATE_N:-none})" + ;; +stage) + clear_markers + need_new_shas + [ -f deploy/check-beacon-seed.sh ] || { echo "stage: no deploy/check-beacon-seed.sh in this tree" >&2; exit 1; } + FETCH="" + case "$STAGE_FROM" in + release|url) + if [ "$STAGE_FROM" = url ]; then + [ -n "$STAGE_URL" ] || { echo "stage: STAGE_FROM=url needs STAGE_URL (a private base URL holding the seven files) — nothing staged" >&2; exit 1; } + BASE=$STAGE_URL; SHOWN="${STAGE_URL%%\?*}"; SHOWN="${SHOWN%/*}/…" + else BASE=$RELEASE_URL; SHOWN=$RELEASE_URL; fi + case "$BASE" in https://*) ;; *) echo "stage: $SHOWN is not https:// — nothing staged" >&2; exit 1;; esac + Q=""; case "$BASE" in *\?*) Q="?${BASE#*\?}"; BASE=${BASE%%\?*};; esac + echo "== stage $TAG (seven files) + check-beacon-seed.sh on $NALL hosts from $SHOWN $(date -u +%T)" + FETCH="for f in $REL_FILES; do curl -fsSL --retry 10 --retry-all-errors --retry-delay 5 -o $S/\$f.part '$BASE/'\$f'$Q'; done" + ;; + path) + [ -n "$STAGE_DIR" ] || { echo "stage: STAGE_FROM=path needs STAGE_DIR holding the seven $TAG files — nothing staged" >&2; exit 1; } + for f in $REL_FILES; do + [ -f "$STAGE_DIR/$f" ] || { echo "stage: STAGE_DIR has no $f — nothing staged" >&2; exit 1; } + dry || verify_binary "$STAGE_DIR/$f" "$RELEASE_SUMS" "$f" || { echo "stage: $STAGE_DIR/$f is not the signed $f — nothing staged" >&2; exit 1; } + done + if [ "${STAGE_ALLOW_NON_ELF:-}" != 1 ]; then + for b in rand-node rand; do [ "$(head -c 4 "$STAGE_DIR/$b" | od -An -c | tr -d ' ')" = '177ELF' ] || { echo "stage: $STAGE_DIR/$b is not a Linux ELF binary (a macOS build?) — nothing staged" >&2; exit 1; }; done + fi + echo "== stage $TAG from $STAGE_DIR (laptop scp; signed sha256s checked here first) $(date -u +%T)" + ;; + *) echo "stage: STAGE_FROM=$STAGE_FROM — release, url or path" >&2; exit 1;; + esac + # The checker comes from this tree (the hosts have no checkout); path mode copies the seven too. + for n in $ALL_N; do ( on "$n" "mkdir -p $S && rm -f $S/*.part" && cp_to "$n" "$S/check-beacon-seed.sh.part" deploy/check-beacon-seed.sh \ + && if [ "$STAGE_FROM" = path ]; then for f in $REL_FILES; do cp_to "$n" "$S/$f.part" "$STAGE_DIR/$f" || exit 1; done; fi ) & done; wait + CHECKS=""; for fp in $PINS; do CHECKS="$CHECKS [ \"\$(sha256sum $S/${fp%%:*}.part 2>/dev/null | cut -d' ' -f1)\" = '${fp#*:}' ] || BAD=\"\$BAD ${fp%%:*}\";"; done + each "$ALL_N" "set -e; cd $S + $FETCH + BAD=''; $CHECKS [ \"\$(sha256sum $S/check-beacon-seed.sh.part 2>/dev/null | cut -d' ' -f1)\" = '$CHECKER_SHA' ] || BAD=\"\$BAD check-beacon-seed.sh\" + if [ -n \"\$BAD\" ]; then rm -f $S/*.part; echo \"SHA MISMATCH (\$BAD) — removed\"; exit 1; fi + for f in $REL_FILES check-beacon-seed.sh; do mv $S/\$f.part $S/\$f; done + chmod 755 $S/rand-node $S/rand; chmod 644 $S/$VK $S/perp-vk-production-*.bin $S/check-beacon-seed.sh + $(staged_self) + echo \"\$H_NAME staged \$($S/rand-node --version), msauth key \${vk_pin:0:12}… and the four perps keys verified\"" \ + || { echo "stage: not every host staged — fix those before going on" >&2; exit 1; } + rec "staged $TAG on $NALL hosts: rand-node ${WANT_SHA:0:16}, rand ${WANT_SHA_WALLET:0:16}, $VK ${WANT_SHA_VK:0:16}, four perps keys, check-beacon-seed.sh ${CHECKER_SHA:0:16}" + echo "STAGED $(date -u +%T) — nothing was installed; the fleet still runs chain 21 on v$OLD_VERSION. Next: beacon-init" + ;; +beacon-init) + need_new_shas + echo "== beacon-init on the $NVALSET validators: each makes ITS OWN seed with the staged v$WANT_VERSION (chain id $CHAIN_ID); a seed already there is kept $(date -u +%T)" + each "$VAL_N" "set -e + $(staged_ok) + $(r_paths) + A=\$($S/rand-node address --key \$H_KEY | sed -n 's/^address: \\(.*\\)\$/\\1/p' | head -1); [ -n \"\$A\" ] || { echo 'no address from the node key'; exit 1; } + if [ -f \$BDIR/beacon.seed.json ]; then WHAT=kept + else + mkdir -p \$BDIR; chmod 700 \$BDIR + $S/rand-node beacon init --datadir \$BDIR --key \$H_KEY --chain-id $CHAIN_ID > \$BDIR/.init.out 2>/dev/null || { rm -f \$BDIR/.init.out; echo 'rand-node beacon init failed'; exit 1; } + rm -f \$BDIR/.init.out; chmod 600 \$BDIR/beacon.seed.json; chown -R \$OWN:\$OWN \$BDIR; WHAT=made + fi + C=\$($S/rand-node beacon commitment --datadir \$BDIR 2>\$BDIR/.c.err) || { tail -1 \$BDIR/.c.err; rm -f \$BDIR/.c.err; echo 'the seed file does not re-derive'; exit 1; } + W=\$(sed -n 's/^chain \\([0-9][0-9]*\\), validator \\([^,]*\\), .*/\\1 \\2/p' \$BDIR/.c.err | head -1); rm -f \$BDIR/.c.err + [ \"\$W\" = \"$CHAIN_ID \$A\" ] || { echo \"the seed file names '\$W', not chain $CHAIN_ID and this key's address \$A\"; exit 1; } + echo \"beacon \$H_NAME \$C (\$WHAT; validator \$A)\"" \ + || { echo "beacon-init: not every validator has a seed (above) — fix those and re-run (a seed already made is kept)" >&2; exit 1; } + rec "beacon seeds present on the $NVALSET validators (made on each host, never copied; commitments only leave the hosts)" + echo "BEACON-INIT DONE — each operator now keeps an OFFLINE, ENCRYPTED backup of its host's beacon-c22/beacon.seed.json; next: beacon-collect" + ;; +beacon-collect) + need_new_shas + OUT=$(mktemp -d); trap 'rm -rf "$OUT"' EXIT + echo "== beacon-collect (read-only toward the fleet): the $NVALSET public commitments → $BEACON_TSV $(date -u +%T)" + EACH_OUT=$OUT each "$VAL_N" "set -e + $(staged_ok) + $(r_paths) + [ -f \$BDIR/beacon.seed.json ] || { echo 'no beacon seed here — run beacon-init'; exit 1; } + A=\$($S/rand-node address --key \$H_KEY | sed -n 's/^address: \\(.*\\)\$/\\1/p' | head -1) + E=\$(mktemp); C=\$($S/rand-node beacon commitment --datadir \$BDIR 2>\$E) || { tail -1 \$E; rm -f \$E; echo 'the seed file does not re-derive'; exit 1; } + W=\$(sed -n 's/^chain \\([0-9][0-9]*\\), validator \\([^,]*\\), .*/\\1 \\2/p' \$E | head -1); rm -f \$E + [ \"\$W\" = \"$CHAIN_ID \$A\" ] || { echo \"the seed file names '\$W', not chain $CHAIN_ID and this key's address \$A\"; exit 1; } + M=\$(stat -c %a \$BDIR/beacon.seed.json 2>/dev/null || stat -f %Lp \$BDIR/beacon.seed.json); [ \"\$M\" = 600 ] || [ \"\$M\" = 400 ] || { echo \"the seed file is mode \$M, not 0600\"; exit 1; } + echo \"beacon \$H_NAME \$C \$A\"" \ + || { echo "beacon-collect: not every validator answered (above) — nothing written" >&2; exit 1; } + if dry; then echo "[dry-run] local: write $BEACON_TSV — \" <64 hex>\" for the $NVALSET validators, in GENESIS_NAMES + BONDED order"; exit 0; fi + TMPTSV=$(mktemp) + # shellcheck disable=SC2094 # the rm in the refusal is of the file being written, then exit + { echo "# chain-22 beacon commitments, one per validator: what each host's own \`rand-node beacon init --chain-id 22\` made." + echo "# Collected by deploy/cutover-fleet-chain22.sh beacon-collect $(date -u +%FT%TZ). Public: commitments only, never a seed." + for n in $GENESIS_NAMES $BONDED; do + l=$(grep '^beacon ' "$OUT/$n" | tail -1) + # shellcheck disable=SC2086 # "beacon
": split on purpose + set -- $l + [ "${2:-}" = "$n" ] && [[ "${3:-}" =~ ^[0-9a-f]{64}$ ]] && [[ ! "$3" =~ ^0+$ ]] || { echo "beacon-collect: $n answered '$l', not a commitment — nothing written" >&2; rm -f "$TMPTSV"; exit 1; } + a=$(iget "$n" addr); [ "$a" = - ] || [ "$a" = "$4" ] || echo "beacon-collect: ⚠ $n's key derives $4 and the inventory says $a — the inventory's address column is stale; the cut and check-beacon-seed.sh use the key's" >&2 + echo "$n $3" + done + } > "$TMPTSV" + [ "$(grep -v '^#' "$TMPTSV" | awk '{print $2}' | sort -u | wc -l | tr -d ' ')" = "$NVALSET" ] || { echo "beacon-collect: two validators gave the same commitment — nothing written" >&2; rm -f "$TMPTSV"; exit 1; } + if [ -f "$BEACON_TSV" ]; then + if diff -q <(grep -v '^#' "$BEACON_TSV") <(grep -v '^#' "$TMPTSV") >/dev/null; then echo "== $BEACON_TSV already holds these $NVALSET commitments — unchanged"; rm -f "$TMPTSV" + else + [ "${BEACON_TSV_REPLACE:-}" = 1 ] || { echo "beacon-collect: $BEACON_TSV exists with OTHER commitments — a genesis cut from it would not match these seeds; BEACON_TSV_REPLACE=1 moves it aside" >&2; rm -f "$TMPTSV"; exit 1; } + mv "$BEACON_TSV" "$BEACON_TSV.replaced-$(date -u +%Y%m%dT%H%M%SZ)"; mv "$TMPTSV" "$BEACON_TSV"; echo "== the old $BEACON_TSV moved aside" + fi + else mkdir -p "$(dirname "$BEACON_TSV")"; mv "$TMPTSV" "$BEACON_TSV"; fi + rec "collected $NVALSET beacon commitments into BEACON_TSV (sha256 $(file_sha "$BEACON_TSV" | cut -c1-16))" + echo "COLLECTED — $BEACON_TSV holds $NVALSET commitments; next: bridge-stop, then snapshot" + ;; +bridge-stop|bridge-start) + if [ "$PHASE" = bridge-start ]; then + [ "${BRIDGE_ON_CHAIN22:-}" = yes ] || dry || { echo "bridge-start: BRIDGE_ON_CHAIN22=yes is required — the guardian and relayer configs (chain id 22, the Rand cursor) are the bridge session's to switch first; nothing started" >&2; exit 1; } + VERB=start; WANT=active + else VERB=stop; WANT=inactive; fi + if [ -z "$RELAYER_HOST" ] && [ "${RELAYER_HANDLED:-}" != yes ] && ! dry; then + echo "$PHASE: RELAYER_HOST is unset — give the relayer's address (rand-relayer-1; not in the inventory), or RELAYER_HANDLED=yes if the bridge session ${VERB}s it by hand; nothing done" >&2; exit 1; fi + # Before any host is contacted: the relayer address must not be a dropped validator's (archive2). + if [ -n "$RELAYER_HOST" ]; then + [[ "$RELAYER_HOST" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]] || { echo "$PHASE: RELAYER_HOST=$RELAYER_HOST is not an IPv4 address — refused" >&2; exit 1; } + case " $DROPPED_IPS " in *" $RELAYER_HOST "*) echo "$PHASE: RELAYER_HOST is a dropped validator's address — refused; nothing done" >&2; exit 1;; esac + fi + echo "== $PHASE: rand-guardian on the six guardian hosts${RELAYER_HOST:+, rand-relayer on the relayer} $(date -u +%T)" + each "$GUARD_N" "systemctl $VERB rand-guardian; sleep 1; s=\$(systemctl is-active rand-guardian || true); [ \"\$s\" = $WANT ] || { echo \"\$H_NAME rand-guardian is \$s\"; exit 1; }; echo \"\$H_NAME rand-guardian \$s\"" \ + || { echo "$PHASE: not every guardian is $WANT (above)" >&2; exit 1; } + if [ -n "$RELAYER_HOST" ]; then + if dry; then echo "[dry-run] ssh root@relayer($RELAYER_HOST) systemctl $VERB rand-relayer" + else + ssh -n -i "$RELAYER_SSH_KEY" -o IdentitiesOnly=yes -o "UserKnownHostsFile=$GUARDIAN_KNOWN_HOSTS" -o StrictHostKeyChecking=yes -o ConnectTimeout=20 -o BatchMode=yes "root@$RELAYER_HOST" \ + "systemctl $VERB rand-relayer; sleep 1; s=\$(systemctl is-active rand-relayer || true); echo \"relayer \$s\"; [ \"\$s\" = $WANT ]" || { echo "$PHASE: the relayer is not $WANT" >&2; exit 1; } + fi + fi + rec "rand-guardian $WANT on the six guardian hosts; relayer: $(if [ -n "$RELAYER_HOST" ]; then echo "$WANT"; else echo "by hand (RELAYER_HANDLED)"; fi)" + echo "$(tr '[:lower:]' '[:upper:]' <<<"$PHASE") DONE — guardians 7/8 on the laptop are the bridge session's (deploy/chain21-bridge-steps.md's lineage)" + ;; +snapshot|balances|cut) + # deploy/cut-chain22-genesis.sh, with PRUNED_SCAN_OK=1 (owner, 2026-11-19: e and obs1 prune below + # 187556 and no unpruned chain-21 node exists) and, for the cut, the collected BEACON_TSV. + ARGS=() + case "$PHASE" in + snapshot|balances) + DIR=${2:?$PHASE }; ARGS=("$PHASE" "$DIR") + [ -n "${CHAIN21_RPC:-}" ] || dry || { echo "$PHASE: CHAIN21_RPC is unset — open a tunnel to an archive (e or obs1): ssh -N -L 8545:127.0.0.1:8545 root@, then CHAIN21_RPC=http://127.0.0.1:8545" >&2; exit 1; } + # The bridge must be down before the state the cut carries is read. + echo "== the bridge daemons must be stopped before the $PHASE (read-only check on the guardian hosts)" + each "$GUARD_N" "if systemctl is-active --quiet rand-guardian; then echo \"\$H_NAME rand-guardian STILL ACTIVE\"; exit 1; fi; echo \"\$H_NAME rand-guardian inactive\"" \ + || { echo "$PHASE: a guardian runs — bridge-stop first; nothing read" >&2; exit 1; } + ;; + cut) + [ -f "$BEACON_TSV" ] || dry || { echo "cut: $BEACON_TSV is missing — beacon-collect first" >&2; exit 1; } + if [ -f "$BEACON_TSV" ]; then + [ "$(grep -v '^#' "$BEACON_TSV" | awk 'NF {print $1}' | sort | tr '\n' ' ')" = "$VALSET" ] \ + || { echo "cut: $BEACON_TSV does not name exactly chain 22's $NVALSET validators — re-run beacon-collect" >&2; exit 1; } + fi + ;; + esac + if dry; then echo "[dry-run] local: PRUNED_SCAN_OK=1 BEACON_TSV=$BEACON_TSV $CUT_SCRIPT ${ARGS[*]}"; exit 0; fi + env -u DRY_RUN -u SELFTEST PRUNED_SCAN_OK=1 BEACON_TSV="$BEACON_TSV" CUT_RECORD="$CUT_RECORD" bash "$CUT_SCRIPT" ${ARGS[@]+"${ARGS[@]}"} \ + || { echo "$PHASE: $CUT_SCRIPT refused (above)" >&2; exit 1; } + if [ "$PHASE" = cut ]; then rec "the cut ($CUT_SCRIPT) with PRUNED_SCAN_OK=1 and BEACON_TSV sha256 $(file_sha "$BEACON_TSV" | cut -c1-16)" + else rec "$CUT_SCRIPT $PHASE with PRUNED_SCAN_OK=1"; fi + ;; +push) + need_local_node + need_new_shas + need_genesis "${2:?push }" + clear_markers + constraint_set_ok + if [ "${SECOND_REBUILD_WAIVED:-}" = 1 ]; then + require_cut_record "$CUT_RECORD" || { echo "push: a waiver is recorded in the cut record, and $CUT_RECORD is not a filled one — nothing pushed" >&2; exit 1; } + WHY=${SECOND_REBUILD_WAIVED_REASON:-one operator holds every validator key and every host; no second operator exists to rebuild the genesis for this cut} + echo "== ⚠⚠ SECOND_REBUILD_WAIVED=1 — pushing genesis $NEW WITHOUT a second operator's rebuild ($WHY); recorded in $CUT_RECORD" + if ! grep -q "^second-rebuild-waived: .*genesis $NEW" "$CUT_RECORD"; then + LINE=$(printf 'second-rebuild-waived: %s genesis %s by %s@%s: %s' "$(date -u +%FT%TZ)" "$NEW" "$(id -un)" "$(hostname -s)" "$WHY") + local_write "append to $CUT_RECORD: $LINE" bash -c 'printf "%s\n" "$1" >> "$2"' _ "$LINE" "$CUT_RECORD" + fi + else + second_rebuild_ok + fi + echo "== push $GENESIS (sha256 $SHA, genesis $NEW) to $NALL hosts $(date -u +%T)" + for n in $ALL_N; do ( cp_to "$n" "$S/genesis-chain22.json" "$GENESIS" ) & done; wait + each "$ALL_N" "set -e + $(staged_ok) + $(pushed_ok) + echo \"\$H_NAME genesis $NEW verified by the staged v$WANT_VERSION binary, constraint set $WANT_CONSTRAINT_SET\"" \ + || { echo "push: not every host holds the genesis — do NOT stop chain 21; re-run push" >&2; exit 1; } + rec "pushed genesis $NEW (file sha256 $SHA) to $NALL hosts; chain 21 still running" + echo "PUSHED — chain 21 still runs; next: the owner's go, then CONFIRM_HALT_CHAIN21=halt-chain-21-for-$P … stop $GENESIS (alone; check \$?)" + ;; +stop) + GENESIS=${2:?stop — stop is gated on push (halt stops a fleet that does not run chain 21)} + need_local_node + need_genesis "$GENESIS" + # THE HALT GUARD, before any host is contacted and before any marker is touched. + PHRASE="halt-chain-21-for-$P" + if [ "${CONFIRM_HALT_CHAIN21:-}" != "$PHRASE" ]; then + if dry; then echo "== DRY RUN: the real stop REFUSES without CONFIRM_HALT_CHAIN21=$PHRASE; the dry run shows the rest and stops nothing" + else + echo "stop: REFUSED — this halts chain 21 on all $NALL hosts. Give the owner's go as CONFIRM_HALT_CHAIN21=$PHRASE (bound to this genesis, $NEW); got '${CONFIRM_HALT_CHAIN21:-nothing}'. Nothing contacted, nothing stopped" >&2; exit 1 + fi + fi + clear_markers + need_new_shas + need_old_shas + second_rebuild_ok + echo "== stop gate on $NALL hosts: the staged files at their pins, the pushed genesis $NEW, v$OLD_VERSION + chain 21's blob installed, no rand-guardian running, each validator's beacon seed opening its genesis commitment $(date -u +%T)" + each "$ALL_N" "set -e + $(staged_ok) + $(staged_self) + $(pushed_ok) + for b in rand-node:$OLD_SHA rand:$OLD_SHA_WALLET $VK:$OLD_VK_SHA; do n=\${b%%:*}; s=\${b#*:} + [ -e /root/\$n.pre-c22 ] || [ \"\$(sha256sum /usr/local/bin/\$n | cut -d' ' -f1)\" = \"\$s\" ] || { echo \"installed \$n is not chain 21's (switch would refuse to back it up)\"; exit 1; } + done + if [ \"\$H_KIND\" = guardian ]; then systemctl is-active --quiet rand-guardian && { echo 'rand-guardian still active — bridge-stop first'; exit 1; }; fi + $(r_paths) + $(beacon_check $S/rand-node $S/genesis-chain22.json \$BDIR) + echo \"\$H_NAME ready: staged v$WANT_VERSION, keys verified, genesis $NEW pushed, beacon \$BS\"" \ + || { echo "stop: not every host is ready (above) — NOTHING STOPPED; chain 21 keeps running" >&2; exit 1; } + if dry; then echo "[dry-run] stop all $NALL (the real run, with the phrase)"; stop_all; exit 0; fi + H0=$(rpc "$BOOT1" rand_status | grep -oE '"height":[0-9]+' | head -1 | cut -d: -f2 || true) + echo "== stop all $NALL $(date -u +%T) — chain 21 head at $BOOT1: ${H0:-?}" + stop_all + rec "stopped chain 21 (v$OLD_VERSION) on $NALL hosts at height ${H0:-?} at $BOOT1, for genesis $NEW (CONFIRM_HALT_CHAIN21 given)" + echo "STOPPED $(date -u +%T) — next: switch $GENESIS (alone; check \$?)" + ;; +halt) + clear_markers + echo "== halt: no host may be running chain 21 (its datadir -$OLD / data-21) $(date -u +%T)" + each "$ALL_N" "D=\$(systemctl cat rand-node 2>/dev/null | grep -o -- '--datadir [^ ]*' | tail -1 | cut -d' ' -f2) + if systemctl is-active --quiet rand-node; then + [ -n \"\$D\" ] || { echo \"\$H_NAME is active and its unit names no --datadir — cannot tell its chain; refusing\"; exit 1; } + case \"\$D\" in *-$OLD|*/data-21) echo \"\$H_NAME RUNS CHAIN 21 (\$D) — use stop \"; exit 1;; esac + G=\$(curl -s -m5 -X POST 127.0.0.1:8545 -H 'content-type: application/json' -d '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"rand_getGenesisHash\",\"params\":[]}' | grep -oE '\"result\":\"[0-9a-f]+\"' | cut -d'\"' -f4 || true) + [ \"\$G\" != '$OLD_HASH' ] || { echo \"\$H_NAME SERVES CHAIN 21's genesis — use stop \"; exit 1; } + fi + echo \"\$H_NAME may halt (\$(systemctl is-active rand-node || true), \$D)\"" \ + || { echo "halt: a host runs chain 21 — nothing halted; chain 21 is stopped only by \`stop \` with CONFIRM_HALT_CHAIN21" >&2; exit 1; } + stop_all + rec "halted (no host ran chain 21)" + echo "HALTED $(date -u +%T) — next: rollback (or push + switch for a re-cut)" + ;; +switch) + need_local_node + need_new_shas + need_old_shas + need_genesis "${2:?switch }" + clear_markers + second_rebuild_ok + constraint_set_ok + echo "== switch to chain 22, genesis $NEW (datadir suffix -$P / data-22), installing v$WANT_VERSION + the five key files $(date -u +%T)" + # Order on each host: refuse while running; everything staged and pushed at its pins; the keys + # verified by the STAGED binary — all before any change; then the backups (v$OLD_VERSION binaries + # and chain 21's blob, once, only if they are chain 21's), the install, the installed keys re- + # verified by the installed binary, the datadir init (its constraint-set marker), the beacon seed + # copied in ON THE HOST and checked, the unit / drop-in rewrite. + each "$ALL_N" "set -e + systemctl is-active --quiet rand-node && { echo 'STILL RUNNING — refusing'; exit 1; } + $(staged_ok) + [ \"\$(sha256sum $S/genesis-chain22.json | cut -d' ' -f1)\" = '$SHA' ] || { echo 'genesis not pushed'; exit 1; } + $(r_paths) + D=/etc/systemd/system/rand-node.service.d + if [ \"\$H_KIND\" = guardian ]; then + systemctl is-active --quiet rand-guardian && { echo 'rand-guardian still active — bridge-stop first'; exit 1; } + if [ -f \$D/chain22.conf ]; then + [ \"\$(cat /var/lib/randnode/data-22.genesis 2>/dev/null)\" = '$NEW' ] || { echo 'chain22.conf already runs another chain-22 genesis — a re-cut over a switched fleet needs rollback first; nothing switched'; exit 1; } + else + grep -q -- '--datadir /var/lib/randnode/data-21 ' \$D/chain21.conf || { echo 'chain21.conf has no data-21 datadir — nothing switched'; exit 1; } + fi + else + grep -q -- '-$P ' /etc/systemd/system/rand-node.service || grep -q -- \"--datadir \$H_DD \" /etc/systemd/system/rand-node.service \\ + || { echo 'unit names neither the inventory datadir (chain 21) nor -$P (this genesis) — a re-cut over a switched fleet needs rollback first; nothing switched'; exit 1; } + fi + if [ \"\$H_VAL\" = 1 ]; then [ -f \$BDIR/beacon.seed.json ] || { echo 'no beacon seed in the staging dir — beacon-init; nothing switched'; exit 1; }; fi + $(vk_check $S/rand-node $S/$VK) + $(pvk_check $S/rand-node $S) + for b in rand-node:$OLD_SHA rand:$OLD_SHA_WALLET $VK:$OLD_VK_SHA; do n=\${b%%:*}; s=\${b#*:} + if [ ! -e /root/\$n.pre-c22 ]; then + [ \"\$(sha256sum /usr/local/bin/\$n | cut -d' ' -f1)\" = \"\$s\" ] || { echo \"installed \$n is not chain 21's — no backup made, nothing switched\"; exit 1; } + cp -a /usr/local/bin/\$n /root/\$n.pre-c22 + fi + done + install -m 755 $S/rand-node /usr/local/bin/rand-node; install -m 755 $S/rand /usr/local/bin/rand + install -m 644 $S/$VK /usr/local/bin/$VK + for f in $PVK_FILES; do install -m 644 $S/\$f /usr/local/bin/\$f; done + $(vk_check /usr/local/bin/rand-node /usr/local/bin/$VK) + $(pvk_check /usr/local/bin/rand-node /usr/local/bin) + if [ \"\$H_KIND\" = guardian ]; then + V=/var/lib/randnode + if [ -d \$V/data-22/db ] && [ \"\$(cat \$V/data-22.genesis 2>/dev/null)\" != '$NEW' ]; then + STALE=\$V/data-22.stale-\$(cut -c1-8 \$V/data-22.genesis 2>/dev/null || echo unknown) + [ ! -e \$STALE ] || { echo \"\$V/data-22 is stale and \$STALE already exists — sort them out by hand\"; exit 1; } + mv \$V/data-22 \$STALE; rm -f \$V/data-22.genesis + fi + install -d -o randnode -g randnode \$V/data-22 + install -m 644 -o randnode -g randnode $S/genesis-chain22.json \$HG + [ -d \$V/data-22/db ] || runuser -u randnode -- /usr/local/bin/rand-node init --datadir \$V/data-22 --genesis \$HG >/dev/null + echo '$NEW' > \$V/data-22.genesis + else + if [ -L \"\$H_DD\" ]; then T=\$(dirname \$(readlink \"\$H_DD\"))/\$(basename \$NEWDIR); mkdir -p \$T; ln -sfn \$T \$NEWDIR; fi + [ -d \$NEWDIR/db ] || /usr/local/bin/rand-node init --datadir \$NEWDIR --genesis \$HG >/dev/null + fi + $(cs_check \$NEWDIR) + if [ \"\$H_VAL\" = 1 ]; then + if [ -f \$NEWDIR/beacon.seed.json ]; then cmp -s \$BDIR/beacon.seed.json \$NEWDIR/beacon.seed.json || { echo \"\$NEWDIR/beacon.seed.json differs from the staging seed — sort it out by hand\"; exit 1; } + else install -m 600 -o \$OWN -g \$OWN \$BDIR/beacon.seed.json \$NEWDIR/beacon.seed.json; fi + fi + $(beacon_check /usr/local/bin/rand-node \$HG \$NEWDIR) + if [ \"\$H_KIND\" = guardian ]; then + if [ -f \$D/chain21.conf ]; then + sed 's#--datadir /var/lib/randnode/data-21 #--datadir /var/lib/randnode/data-22 #' \$D/chain21.conf > \$D/chain22.conf + mv \$D/chain21.conf /root/chain21.conf.c21 + fi + grep -q -- '--datadir /var/lib/randnode/data-22 ' \$D/chain22.conf + else + U=/etc/systemd/system/rand-node.service + if ! grep -q -- '-$P ' \$U; then + [ -e /root/rand-node.service.$OLD.bak ] || cp -a \$U /root/rand-node.service.$OLD.bak + sed -i \"s#--datadir \$H_DD #--datadir \$NEWDIR #\" \$U + fi + grep -q -- \"--datadir \$NEWDIR \" \$U + fi + systemctl daemon-reload + echo \"\$H_NAME ready: \$(/usr/local/bin/rand-node --version), keys verified, set $WANT_CONSTRAINT_SET, beacon \$BS, \$(systemctl cat rand-node | grep -o -- '--datadir [^ ]*' | tail -1)\"" \ + || { echo "switch: some hosts are not ready — fix them and re-run switch; do NOT start. Or roll back (the fleet is stopped): rollback → start → wait21, then the bridge on chain 21" >&2; exit 1; } + mark "switched-$P" "$NEW" + rec "switched $NALL hosts to genesis $NEW: v$WANT_VERSION installed, *.pre-c22 kept, constraint-set marker $WANT_CONSTRAINT_SET, beacon seeds checked on $NVALSET validators" + echo "SWITCHED — marker $MARKER_DIR/switched-$P written; next: start (alone; check \$?)" + ;; +rollback) + clear_markers + need_old_shas + echo "== rollback to chain 21 (datadir -$OLD / data-21, v$OLD_VERSION binaries, chain 21's msauth blob) $(date -u +%T)" + each "$ALL_N" "set -e + systemctl is-active --quiet rand-node && { echo 'STILL RUNNING — refusing'; exit 1; } + D=/etc/systemd/system/rand-node.service.d + if [ \"\$H_KIND\" = guardian ]; then systemctl is-active --quiet rand-guardian && { echo 'rand-guardian still active — stop the bridge first'; exit 1; }; fi + for b in rand-node:$OLD_SHA rand:$OLD_SHA_WALLET $VK:$OLD_VK_SHA; do n=\${b%%:*}; s=\${b#*:} + if [ \"\$(sha256sum /usr/local/bin/\$n 2>/dev/null | cut -d' ' -f1)\" != \"\$s\" ]; then + [ -f /root/\$n.pre-c22 ] || { echo \"no /root/\$n.pre-c22 to restore\"; exit 1; } + [ \"\$(sha256sum /root/\$n.pre-c22 | cut -d' ' -f1)\" = \"\$s\" ] || { echo \"/root/\$n.pre-c22 is not chain 21's\"; exit 1; } + m=755; [ \$n != $VK ] || m=644; install -m \$m /root/\$n.pre-c22 /usr/local/bin/\$n + fi + done + if [ \"\$H_KIND\" = guardian ]; then + if [ -f \$D/chain22.conf ]; then + [ -f /root/chain21.conf.c21 ] || { echo 'no /root/chain21.conf.c21 to restore'; exit 1; } + mv /root/chain21.conf.c21 \$D/chain21.conf; rm -f \$D/chain22.conf + fi + grep -q -- '--datadir /var/lib/randnode/data-21 ' \$D/chain21.conf + [ -d /var/lib/randnode/data-21/db ] || { echo 'data-21 has no db'; exit 1; } + else + U=/etc/systemd/system/rand-node.service + if ! grep -q -- \"--datadir \$H_DD \" \$U; then + [ -f /root/rand-node.service.$OLD.bak ] || { echo 'no /root/rand-node.service.$OLD.bak to restore'; exit 1; } + grep -q -- \"--datadir \$H_DD \" /root/rand-node.service.$OLD.bak || { echo 'the backup unit does not run the inventory datadir'; exit 1; } + cp -a /root/rand-node.service.$OLD.bak \$U + fi + [ -d \"\$H_DD/db\" ] || { echo \"\$H_DD has no db\"; exit 1; } + fi + systemctl daemon-reload + echo \"\$H_NAME on chain 21: \$(/usr/local/bin/rand-node --version), \$(systemctl cat rand-node | grep -o -- '--datadir [^ ]*' | tail -1)\"" \ + || { echo "rollback: some hosts are not back on chain 21 — fix them and re-run rollback; do NOT start" >&2; exit 1; } + mark "rollback-$OLD" "ROLLBACK to chain 21 ($OLD)" + rec "rolled $NALL hosts back to chain 21: v$OLD_VERSION binaries and chain 21's msauth blob restored from *.pre-c22, units / drop-ins restored" + echo "ROLLED BACK — marker $MARKER_DIR/rollback-$OLD written; next: start (alone; check \$?), then the bridge on chain 21" + ;; +start|start-archives) + if dry; then + echo "[dry-run] local: $PHASE needs $( [ "$PHASE" = start ] && echo "exactly one switched-* or rollback-* marker" || echo "first-

and limits-

markers") in $MARKER_DIR" + KIND=switch; P="

"; MARK="" + elif [ "$PHASE" = start ]; then + # shellcheck disable=SC2046 # marker paths have no spaces + set -- $(markers 'switched-*') $(markers 'rollback-*') + [ $# -ge 1 ] || { echo "start: no switched-

/ rollback-* marker in $MARKER_DIR — a successful switch (or rollback) writes it; refusing" >&2; exit 1; } + [ $# -eq 1 ] || { echo "start: $# markers ($*) — ambiguous; remove the stale ones by hand; refusing" >&2; exit 1; } + MARK=$1; case "$MARK" in */rollback-*) KIND=rollback;; *) KIND=switch; P=${MARK##*/switched-};; esac + echo "== start on the $KIND recorded in $MARK: $(cat "$MARK")" + else + # shellcheck disable=SC2046 + set -- $(markers 'first-*') + [ $# -eq 1 ] || { echo "start-archives: no single first-

marker in $MARKER_DIR — start (the first group) writes it; refusing" >&2; exit 1; } + MARK=$1; KIND=switch; P=${MARK##*/first-} + [ -f "$MARKER_DIR/limits-$P" ] || { echo "start-archives: no limits-$P marker — run check-limits against $BOOT1 first; refusing" >&2; exit 1; } + fi + GATE="" + if [ "$KIND" = switch ]; then + if [ "$PHASE" = start ]; then GROUP=$FIRST_N; GHOSTS=$ALL_N; else GROUP=$LATE_N; GHOSTS=$LATE_N; fi + echo "== the start gate on $(wc -w <<<"$GHOSTS" | tr -d ' ') hosts: msauth + perps keys by the installed rand-node, constraint set $WANT_CONSTRAINT_SET, each validator's beacon seed $(date -u +%T)" + each "$GHOSTS" "set -e + $(installed_gate) + echo \"\$H_NAME keys verified, set $WANT_CONSTRAINT_SET, beacon \$BS\"" \ + || { echo "$PHASE: the gate failed on a host (above) — nothing started (re-run switch on those hosts)" >&2; exit 1; } + GATE="set -e; $(installed_gate);" + else GROUP=$ALL_N; fi + [ -n "$GROUP" ] || { echo "$PHASE: no host in this group"; exit 0; } + ONE="$GATE systemctl start rand-node; sleep 2; systemctl is-active --quiet rand-node || { echo \"\$H_NAME NOT ACTIVE after start\"; exit 1; }; echo \"\$H_NAME active\"" + if dry; then each "$GROUP" "$ONE"; exit 0; fi + RES=$(mktemp -d) + start_one() { if on "$1" "$(hvars "$1")$ONE" > "$RES/$1" 2>&1; then echo ok > "$RES/$1.rc"; else echo FAIL > "$RES/$1.rc"; fi; } + FAILED=""; OKN=0 + if [ "$PHASE" = start ]; then + echo "== start the bootstraps ($BOOT_NAMES) $(date -u +%T)" + for n in $BOOT_NAMES; do start_one "$n"; printf ' %-8s %s %s\n' "$n" "$(cat "$RES/$n.rc")" "$(tail -1 "$RES/$n")"; done + fi + echo "== start $( [ "$PHASE" = start ] && echo "the rest of the $( [ "$KIND" = switch ] && echo first group || echo fleet)" || echo "the archives / observers ($GROUP)") together $(date -u +%T)" + for n in $GROUP; do case " $BOOT_NAMES " in *" $n "*) [ "$PHASE" = start ] && continue;; esac; start_one "$n" & done; wait + for n in $GROUP; do + case " $BOOT_NAMES " in *" $n "*) [ "$PHASE" = start ] || printf ' %-8s %s %s\n' "$n" "$(cat "$RES/$n.rc")" "$(tail -1 "$RES/$n")";; *) printf ' %-8s %s %s\n' "$n" "$(cat "$RES/$n.rc")" "$(tail -1 "$RES/$n")";; esac + if [ "$(cat "$RES/$n.rc")" = ok ]; then OKN=$((OKN+1)); else FAILED="$FAILED $n"; fi + done + rm -rf "$RES" + if [ -n "$FAILED" ]; then + rec "NOT STARTED on:$FAILED ($OKN of $(wc -w <<<"$GROUP" | tr -d ' ') active)" + echo "$PHASE: NOT STARTED on:$FAILED — $OKN of $(wc -w <<<"$GROUP" | tr -d ' ') active$( [ "$PHASE" = start ] && [ "$KIND" = switch ] && { [ "$OKN" -ge "$QUORUM" ] && echo " (≥ quorum $QUORUM: chain 22 can commit)" || echo " (BELOW quorum $QUORUM: chain 22 cannot commit)"; }); the marker $MARK is kept; fix those hosts and re-run $PHASE (or halt + rollback)" >&2; exit 1 + fi + if [ "$KIND" = rollback ]; then + local_write "rm -f $MARK" rm -f "$MARK" + rec "started chain 21 again on $OKN hosts (rollback)" + echo "START-DONE $(date -u +%T) — all $NALL active on chain 21; next: wait21" + elif [ "$PHASE" = start ]; then + local_write "rm -f $MARK" rm -f "$MARK"; mark "first-$P" "first group started" + rec "started the first group on chain 22: $OKN validators ($FIRST_N), quorum $QUORUM; bootstraps $BOOT_NAMES first" + echo "START-DONE $(date -u +%T) — $OKN of the first group active (≥ quorum $QUORUM); next: check-limits , then start-archives" + else + unmark "first-$P"; unmark "limits-$P" + rec "started the archives / observers on chain 22: $GROUP" + echo "START-ARCHIVES-DONE $(date -u +%T) — next: wait " + fi + ;; +check-limits) + GENESIS=${2:?check-limits } + need_local_node + need_genesis "$GENESIS" + RPC=${CHECK_LIMITS_RPC:-} + TUN="" + if [ -z "$RPC" ]; then + PORT=${CHECK_LIMITS_PORT:-18545} + if dry; then echo "[dry-run] local: ssh -N -L 127.0.0.1:$PORT:127.0.0.1:8545 root@$BOOT1 (a tunnel to the first node), then" + else + ip=$(iget "$BOOT1" ip); o=(); while IFS= read -r l; do o+=("$l"); done < <(sshopts "$BOOT1") + ssh -N -o ExitOnForwardFailure=yes "${o[@]}" -L "127.0.0.1:$PORT:127.0.0.1:8545" "root@$ip" & TUN=$! + trap '[ -z "$TUN" ] || kill "$TUN" 2>/dev/null || true' EXIT + for _ in $(seq 1 15); do curl -s -m2 -o /dev/null "http://127.0.0.1:$PORT" && break; sleep 1; done + fi + RPC=http://127.0.0.1:$PORT + fi + if dry; then echo "[dry-run] local: NODE=$LOCAL_NODE $CUT_SCRIPT check-limits $RPC $GENESIS; on success write $MARKER_DIR/limits-$P"; exit 0; fi + echo "== check-limits against $BOOT1 ($RPC) $(date -u +%T)" + if env -u DRY_RUN -u SELFTEST NODE="$LOCAL_NODE" bash "$CUT_SCRIPT" check-limits "$RPC" "$GENESIS"; then + mark "limits-$P" "$NEW" + rec "check-limits passed against $BOOT1 for genesis $NEW" + echo "CHECK-LIMITS OK — next: start-archives" + else + rec "check-limits FAILED against $BOOT1 for genesis $NEW" + echo "check-limits: the cut script refused (above) — the archives are not started until it passes" >&2; exit 1 + fi + ;; +wait) + GENESIS=${2:?wait } + need_local_node + read -r NEW _ <<<"$(genesis_hash "$GENESIS")"; P=${NEW:0:8} + if dry; then + echo "[dry-run] every ${WAIT_SLEEP:-20} s, up to $WAIT_TRIES times: on each of the $NALL hosts rand_getGenesisHash == $NEW, rand_getHealth ok, rand_status.msauth_key loaded, perps_keys loaded (4); $BOOT1's height > 2; then check-beacon-seed.sh --rpc on the $NVALSET validators" + rpc "$BOOT1" rand_status; each "$VAL_N" "$(r_paths); $(beacon_check /usr/local/bin/rand-node \$HG \$NEWDIR 127.0.0.1:8545)"; exit 0 + fi + [ ${#NEW} -eq 64 ] || { echo "wait: could not derive the genesis hash" >&2; exit 1; } + for _ in $(seq 1 "$WAIT_TRIES"); do + sleep "${WAIT_SLEEP:-20}"; ok=0; late="" + for n in $ALL_N; do + g=$(rpc "$n" rand_getGenesisHash | grep -oE '"result":"[0-9a-f]+"' | cut -d'"' -f4 || true) + s=$(rpc "$n" rand_getHealth | grep -oE '"status":"[a-z_]+"' | cut -d'"' -f4 || true) + st=$(rpc "$n" rand_status) + k=$(grep -oE '"msauth_key":"[a-z]+"' <<<"$st" | cut -d'"' -f4 || true) + pk=$(tr -d ' ' <<<"$st" | grep -oE '"perps_keys":\{[^}]*\}' || true) + if [ "$g" = "$NEW" ] && [ "$s" = ok ] && [ "$k" = loaded ] && grep -q '"source":"loaded"' <<<"$pk" && grep -q '"keys":4' <<<"$pk"; then ok=$((ok+1)); else late="$late $n"; fi + done + H=$(rpc "$BOOT1" rand_status | grep -oE '"height":[0-9]+' | head -1 | cut -d: -f2 || true) + echo " $(date -u +%T) $ok/$NALL on chain 22, healthy, msauth + perps keys loaded; $BOOT1 height ${H:-?}${late:+; not yet:$late}" + if [ "$ok" = "$NALL" ] && [ -n "$H" ] && [ "$H" -gt 2 ]; then + echo "== every validator's beacon seed against its running node (check-beacon-seed.sh --rpc)" + each "$VAL_N" "set -e; $(r_paths); $(beacon_check /usr/local/bin/rand-node \$HG \$NEWDIR 127.0.0.1:8545); echo \"\$H_NAME beacon seed opens the commitment in force\"" \ + || { rec "all $NALL on $NEW at height $H, but a validator's beacon seed does not open its commitment in force"; echo "wait: a validator's beacon seed does not open its commitment in force (above) — it misses its slots" >&2; exit 1; } + rec "CHAIN 22 LIVE: all $NALL hosts on $NEW, healthy, msauth + perps keys loaded, $BOOT1 at height $H; beacon seeds open on $NVALSET validators" + echo "=== $(date -u +%T) CUTOVER DONE: all $NALL on $NEW, committing at $H — next: bridge-start, verify"; exit 0 + fi + done + echo "=== not all $NALL healthy after $WAIT_TRIES polls — check the laggards"; exit 1 + ;; +verify) + GENESIS=${2:?verify } + need_local_node + need_genesis "$GENESIS" + echo "== verify: chain 22's CHAIN_CONSTRAINT_SETS row in this tree (issue #166) $(date -u +%T)" + if constraint_set_ok row; then + rec "verified: CHAIN_CONSTRAINT_SETS holds (22, $NEW, $WANT_CONSTRAINT_SET); a release for chain 22 may be tagged from this tree" + echo "VERIFIED — the row is there; status follows" + else + rec "verify: CHAIN_CONSTRAINT_SETS does NOT hold chain 22's row yet" + echo "verify: add the row the cut printed (cut record constraint-set-row:), commit, and re-run" >&2; exit 1 + fi + for n in $ALL_N; do ( + v=$(rpc "$n" rand_getVersion | grep -oE '"version":"[^"]+"' | tr '\n' ' ' || true) + h=$(rpc "$n" rand_status | grep -oE '"height":[0-9]+' | head -1 || true) + printf ' %-8s %s %s\n' "$n" "$v" "${h:-no answer}" ) & done; wait + ;; +wait21) + if dry; then echo "[dry-run] every ${WAIT_SLEEP:-20} s, up to $WAIT_TRIES times: each of the $NALL hosts serves genesis $OLD…, healthy; $BOOT1's height strictly above the previous poll's"; rpc "$BOOT1" rand_status; exit 0; fi + PREV="" + for _ in $(seq 1 "$WAIT_TRIES"); do + sleep "${WAIT_SLEEP:-20}"; ok=0; late="" + for n in $ALL_N; do + g=$(rpc "$n" rand_getGenesisHash | grep -oE '"result":"[0-9a-f]+"' | cut -d'"' -f4 || true) + s=$(rpc "$n" rand_getHealth | grep -oE '"status":"[a-z_]+"' | cut -d'"' -f4 || true) + if [ "$g" = "$OLD_HASH" ] && [ "$s" = ok ]; then ok=$((ok+1)); else late="$late $n"; fi + done + H=$(rpc "$BOOT1" rand_status | grep -oE '"height":[0-9]+' | head -1 | cut -d: -f2 || true) + echo " $(date -u +%T) $ok/$NALL on chain 21 and healthy; $BOOT1 height ${H:-?} (previous ${PREV:-?})${late:+; not yet:$late}" + if [ "$ok" = "$NALL" ] && [ -n "$H" ] && [ -n "$PREV" ] && [ "$H" -gt "$PREV" ]; then + rec "chain 21 committing again on all $NALL ($BOOT1 at $H)" + echo "=== $(date -u +%T) CHAIN 21 COMMITTING AGAIN: all $NALL on $OLD…, $BOOT1 at $H (was $PREV)"; exit 0; fi + PREV=$H + done + echo "=== chain 21 is not committing on all $NALL after 30 min"; exit 1 + ;; +quiet) + GENESIS=${2:?quiet } + if [ ! -f "$GENESIS" ]; then + dry && { echo "== DRY RUN: $GENESIS is not a file — quiet would compare the bootstraps' rand_getBridgeState with its burn_sequence and Σ locked"; rpc "$BOOT1" rand_getBridgeState; exit 0; } + echo "quiet: $GENESIS is not a file — the chain-22 genesis the fleet runs" >&2; exit 1 + fi + WANT=$(python3 -c 'import json,sys +g=json.load(open(sys.argv[1])); b=g["bridge"] +locked=sum(int(k.get("locked",0)) for t in g["tokens"]["tokens"] for k in t["backings"]) +print(b["burn_sequence"], locked)' "$GENESIS") + echo "== genesis: burn_sequence ${WANT% *}, Σ locked ${WANT#* }" + if dry; then rpc "$BOOT1" rand_getBridgeState; exit 0; fi + bad=0 + for n in $BOOT_NAMES; do + GOT=$(rpc "$n" rand_getBridgeState | python3 -c 'import json,sys +try: + r=json.load(sys.stdin)["result"] + print(r["burn_sequence"], sum(int(a.get("locked") or 0) for a in r.get("assets",[]))) +except Exception as e: + print("unreadable", e)' || true) + if [ "$GOT" = "$WANT" ]; then echo " $n: burn_sequence ${GOT% *}, Σ locked ${GOT#* } — as at genesis" + else echo " $n: $GOT — NOT the genesis values: something was burned or minted on chain 22"; bad=1; fi + done + [ $bad = 0 ] || { echo "quiet: chain 22 has bridge activity — a rollback loses it; the owner decides" >&2; exit 1; } + echo "QUIET — nothing burned or minted on chain 22 (as the bootstraps report)" + ;; +status) + for n in $ALL_N; do ( + v=$(rpc "$n" rand_getVersion | grep -oE '"version":"[^"]+"|"git_sha":"[a-z0-9]+"|"chain_id":[0-9]+' | tr '\n' ' ' || true) + h=$(rpc "$n" rand_status | grep -oE '"height":[0-9]+|"msauth_key":"[a-z]+"' | head -2 | tr '\n' ' ' || true) + printf ' %-8s %s %s\n' "$n" "$v" "${h:-no answer}" ) & done; wait + echo " ($DROPPED_VALIDATORS: dropped, not contacted)" + ;; +*) echo "unknown phase $PHASE" >&2; exit 1 ;; +esac diff --git a/deploy/rehearse-cutover-fleet-chain22.sh b/deploy/rehearse-cutover-fleet-chain22.sh new file mode 100755 index 00000000..26ba97f1 --- /dev/null +++ b/deploy/rehearse-cutover-fleet-chain22.sh @@ -0,0 +1,637 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016,SC2030,SC2031,SC2034,SC2012,SC2086 # the chk bodies are single-quoted and eval'd: they expand later, and read the variables set here +# Rehearse deploy/cutover-fleet-chain22.sh end to end against 26 FAKE hosts — local directories +# behind a fake `ssh`/`scp`, addressed by TEST-NET-1 addresses (192.0.2.x, RFC 5737: unroutable). +# No network, no real host, NO BINARY: the script under test is COPIED into a temp tree whose +# inventory names only the fake hosts and whose deploy/release-signers holds only a throwaway key +# made here; the fake `ssh`/`scp` refuse any host outside 192.0.2.0/24 and log every host they are +# asked for; the fake `curl` serves downloads only from a local directory for the made-up +# https://rehearsal.invalid/ base; and the run aborts unless the fakes are first on PATH. +# +# deploy/rehearse-cutover-fleet-chain22.sh +# SELFTEST=1 deploy/cutover-fleet-chain22.sh # the same +# +# DERIVED from deploy/rehearse-cutover-fleet-chain21.sh. Where chain 21's wrapped a real rand-node, +# this one uses STUBS (python): "v0.8.6" (chain 21's build: init accepts only chain 21, pins chain +# 21's msauth blob, has no `beacon` and no `keygen-perp-vk`) and "v0.8.7" (constraint set 11: init +# REFUSES every chain but 22 and writes the `constraint-set` marker 11; pins the set-11 msauth blob +# and the four perps keys; `beacon init / commitment / status` keep a seed file whose commitments +# re-derive from its seeds). The key FILES are the real ones: the committed set-11 msauth blob and +# four perp-vk files, and chain 21's msauth blob read from the v0.8.6 tag (`git show`). The REAL +# deploy/check-beacon-seed.sh and deploy/check-constraint-set.sh run, the first on the fake hosts. +# The cut script is a stub that logs its arguments and environment (its own SELFTEST covers it). +# +# The fleet: the inventory's 26 rows — the 18 genesis validators, obs1, archive2 and g1..g6 — with +# e and obs1 archives on symlinked volumes. archive2 has NO fake host: any ssh or scp to it fails, +# and the run checks at the end that the fake ssh/scp were never asked for its address. +# +# What it does NOT prove: the remote commands run under this machine's tools behind small shims +# (`systemctl`, `runuser`, `sha256sum`, `install -o`, GNU `sed -i`, `chown`, the node's RPC, a +# download), not on Ubuntu under systemd; the binaries are stubs, so nothing here derives a real +# genesis hash, loads a real key or opens a real beacon chain (the node's own tests do that). +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +unset SELFTEST DRY_RUN +REPO=$PWD +BLOB=$PWD/crates/randprotocol-zkvm/keys/msauth-vk-production.bin +TESTBLOB=$PWD/crates/randprotocol-zkvm/keys/msauth-vk-test.bin +KEYS=$PWD/crates/randprotocol-zkvm/keys +for f in "$BLOB" "$TESTBLOB" "$KEYS"/perp-vk-production-t18-pg14-in16-pub{7,8,9,10}-p2h20.bin deploy/check-beacon-seed.sh deploy/check-constraint-set.sh; do + [ -f "$f" ] || { echo "rehearse: $f is missing" >&2; exit 1; } +done +command -v python3 >/dev/null || { echo "rehearse: needs python3" >&2; exit 1; } +BIN=$(mktemp -d); T=$(mktemp -d) +if [ -n "${REHEARSE_KEEP:-}" ]; then echo "rehearse: keeping $T and $BIN"; else trap 'rm -rf "$T" "$BIN"' EXIT; fi +OLDBLOB=$BIN/old-msauth-vk-production.bin +git show v0.8.6:crates/randprotocol-zkvm/keys/msauth-vk-production.bin > "$OLDBLOB" 2>/dev/null \ + || git show 6591c884187126f118460c940d15715032bd124f:crates/randprotocol-zkvm/keys/msauth-vk-production.bin > "$OLDBLOB" 2>/dev/null \ + || { echo "rehearse: cannot read chain 21's msauth blob from the v0.8.6 tag (git show v0.8.6:…) — fetch the tags" >&2; exit 1; } +sha() { shasum -a 256 "$1" | cut -d' ' -f1; } +OLD_VK=68facb06c7f321aaf1d4d541acf6b5c0e9d7f17788ed1c9add6c5a8a8f7acd30 +[ "$(sha "$OLDBLOB")" = "$OLD_VK" ] || { echo "rehearse: v0.8.6's msauth blob is not chain 21's $OLD_VK" >&2; exit 1; } +PVK7=perp-vk-production-t18-pg14-in16-pub7-p2h20.bin; PVK8=perp-vk-production-t18-pg14-in16-pub8-p2h20.bin +PVK10=perp-vk-production-t18-pg14-in16-pub10-p2h20.bin +mkdir -p "$BIN"/{old,new,badrel,badpvk,badvk} "$T/repo/deploy/lib" "$T/repo/deploy/cut-records" "$T/shim" "$T/rshim" "$T/hosts" "$T/markers" "$T/key" "$T/cs" + +# ── the rand-node stubs ─────────────────────────────────────────────────────────────────────── +cat > "$T/stub.py" <<'STUB' +import hashlib, json, os, secrets, shutil, sys +V = "@VERSION@"; NEW = V == "0.8.7" +E = os.environ; host = E.get("FAKE_HOST", "") +a = sys.argv[1:] +def opt(n): return a[a.index(n) + 1] +def sha(p): return hashlib.sha256(open(p, "rb").read()).hexdigest() +def die(m, rc=1): print(f"error: {m} (rehearsal stub v{V})", file=sys.stderr); sys.exit(rc) +def here(var): return E.get(var) and E.get(var + "_HOST", host) == host +PVK = {"perp-vk-production-t18-pg14-in16-pub7-p2h20.bin": "4dc1f18bdd6d9104ba1622710cf2c146f71759676dbdcb70dad036152242e8b4", + "perp-vk-production-t18-pg14-in16-pub8-p2h20.bin": "6392e5e9b7348f3729ba2031e0e3ec9788d6ed62a14521526482c92615f55106", + "perp-vk-production-t18-pg14-in16-pub9-p2h20.bin": "c31636aa281004163e6fe0debd81854f11d7bca6c8062a91b1501c7e593e5bf0", + "perp-vk-production-t18-pg14-in16-pub10-p2h20.bin": "ae3d0dc276061fd2a87b2e91b239f6dcccb746b49dedeecfdb2aab977c95c370"} +FLAGS21 = "--testnet --binding-domain --proof-window-blocks --max-gas-price --gas-byte-load --program-state-cell-fee --fees --multisig --auth-multisig-guest" +FLAGS22 = FLAGS21 + " --program-state-refuse-panics --tokens-incremental-root --program-state-reference-block-window --beacon-commitment" +def commit(seed, addr, cid): return hashlib.sha256(f"{seed}|{addr}|{cid}".encode()).hexdigest() +if a[:1] == ["--version"]: + print(f"rand-node {V} (rehearsal stub)") +elif a[:2] == ["genesis", "--help"]: + print("usage: rand-node genesis " + (FLAGS22 if NEW else FLAGS21)) +elif a[:1] == ["init"]: + d, g = opt("--datadir"), opt("--genesis"); cid = json.load(open(g)).get("chain_id") + if NEW and cid != 22: die(f"constraint set 11 cannot run chain {cid} (constraint_set::build_genesis)") + if not NEW and cid != 21: die(f"v{V} runs chain 21, not {cid}") + h = sha(g); os.makedirs(d + "/db", exist_ok=True); shutil.copy(g, d + "/genesis.json") + open(d + "/genesis.hash", "w").write(h) + open(d + "/constraint-set", "w").write((E["REHEARSE_SET"] if here("REHEARSE_SET") else ("11" if NEW else "9")) + "\n") + print(f"initialised {d}: chain {cid}, genesis {h}") +elif a[:1] == ["keygen-msauth-vk"]: + if a[1] == "--check": print(sha(a[2])) + elif a[1] == "--print-pinned": print(E["REHEARSE_PIN"] if here("REHEARSE_PIN") else + ("168d79a880b13b6d3e5b1f2a0f50561590712179c63189b52038cb2172a971b3" if NEW else "68facb06c7f321aaf1d4d541acf6b5c0e9d7f17788ed1c9add6c5a8a8f7acd30")) + else: die("keygen-msauth-vk: --check or --print-pinned") +elif a[:1] == ["keygen-perp-vk"] and NEW: + if a[1] == "--check": print(sha(a[2])) + elif a[1] == "--print-pinned": + for i, (f, p) in enumerate(PVK.items()): print(f, E["REHEARSE_PVK_PIN"] if i == 0 and here("REHEARSE_PVK_PIN") else p) + else: die("keygen-perp-vk: --check or --print-pinned") +elif a[:1] == ["address"]: + print("address: " + json.load(open(opt("--key")))["address"]) +elif a[:2] == ["beacon", "init"] and NEW: + d, k, cid = opt("--datadir"), opt("--key"), int(opt("--chain-id")); p = d + "/beacon.seed.json" + if os.path.exists(p): die(f"{p} exists — refusing to overwrite") + addr = json.load(open(k))["address"]; seed = secrets.token_hex(32); c = commit(seed, addr, cid) + os.makedirs(d, exist_ok=True); fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.write(fd, json.dumps({"chain_id": cid, "validator": addr, "seeds": [{"seed": seed, "commitment": c}]}).encode()); os.close(fd) + print(f"wrote {p} (secret; chain {cid}, validator {addr}). Never copy it to another host. Commitment:", file=sys.stderr); print(c) +elif a[:2] in (["beacon", "commitment"], ["beacon", "status"]) and NEW: + p = opt("--datadir") + "/beacon.seed.json" + try: f = json.load(open(p)) + except Exception as e: die(f"cannot read {p}: {e}") + for s in f["seeds"]: + if commit(s["seed"], f["validator"], f["chain_id"]) != s["commitment"]: die(f"{p}: a chain does not re-derive from its seed") + c = f["seeds"][-1]["commitment"] + if a[1] == "commitment": + print(f"chain {f['chain_id']}, validator {f['validator']}, {len(f['seeds'])} chain(s); newest commitment:", file=sys.stderr); print(c) + else: + print(f"validator {f['validator']} on chain {f['chain_id']} at height 0; beacon value {'0' * 64}") + print(f"commitment in force {c}: the seed file opens it; remaining reveals: 1000") +else: + die(f"unrecognized subcommand {' '.join(a[:2])}", 2) +STUB +mkstub() { { echo '#!/usr/bin/env python3'; sed "s/@VERSION@/$2/" "$T/stub.py"; } > "$1"; chmod 755 "$1"; } +mkstub "$BIN/old/rand-node" 0.8.6; mkstub "$BIN/new/rand-node" 0.8.7 +printf '#!/usr/bin/env bash\necho "rand 0.8.6 (rehearsal stub)"\n' > "$BIN/old/rand" +printf '#!/usr/bin/env bash\necho "rand 0.8.7 (rehearsal stub)"\n' > "$BIN/new/rand" +cp "$OLDBLOB" "$BIN/old/msauth-vk-production.bin" +cp "$BLOB" "$KEYS"/perp-vk-production-*.bin "$BIN/new/" +cp "$BIN"/new/* "$BIN/badrel/"; printf '#!/usr/bin/env bash\n# a tampered release\nexit 1\n' > "$BIN/badrel/rand-node" +cp "$BIN"/new/* "$BIN/badpvk/"; cp "$KEYS/$PVK8" "$BIN/badpvk/$PVK7" # one perps key's bytes wrong +cp "$BIN"/new/* "$BIN/badvk/"; cp "$TESTBLOB" "$BIN/badvk/msauth-vk-production.bin" +chmod 755 "$BIN"/*/rand* + +# ── the throwaway release key and the signed SHA256SUMS of each fake release ─────────────────── +ssh-keygen -q -t ed25519 -N '' -C rehearsal -f "$T/key/rel" >/dev/null +echo "rehearsal@randprotocol namespaces=\"rand-release\" $(cut -d' ' -f1,2 "$T/key/rel.pub")" > "$T/repo/deploy/release-signers" +sums() { (cd "$1" && shasum -a 256 rand-node rand msauth-vk-production.bin perp-vk-production-*.bin | sed 's/ / /') > "$2"; } +sign() { ssh-keygen -q -Y sign -n rand-release -f "$T/key/rel" "$1" >/dev/null 2>&1; } +sums "$BIN/new" "$T/SUMS"; sign "$T/SUMS" +sed "s/^$(sha "$BLOB") /$(sha "$TESTBLOB") /" "$T/SUMS" > "$T/SUMS-otherblob"; sign "$T/SUMS-otherblob" +sed "s/^$(sha "$KEYS/$PVK7") /$(sha "$KEYS/$PVK8") /" "$T/SUMS" > "$T/SUMS-otherpvk"; sign "$T/SUMS-otherpvk" +grep -v " $PVK10\$" "$T/SUMS" > "$T/SUMS-nopvk"; sign "$T/SUMS-nopvk" +cp "$T/SUMS" "$T/SUMS-tampered"; cp "$T/SUMS.sig" "$T/SUMS-tampered.sig"; echo "0000000000000000000000000000000000000000000000000000000000000000 extra" >> "$T/SUMS-tampered" +cp "$T/SUMS" "$T/SUMS-unsigned" +(cd "$BIN/old" && shasum -a 256 rand-node rand | sed 's/ / /') > "$T/OLD-SUMS"; sign "$T/OLD-SUMS" + +# ── the fake ssh / scp (the laptop side): log every host asked for ───────────────────────────── +cat > "$T/shim/ssh" <<'FAKE_SSH' +#!/usr/bin/env bash +host=""; cmd="" +while [ $# -gt 0 ]; do + case "$1" in + -n|-N|-q) shift;; -o|-i|-L) shift 2;; + root@*) host=${1#root@}; shift; cmd="$*"; break;; + *) shift;; + esac +done +echo "$host" >> "$FAKE_SSH_LOG" +case "$host" in 192.0.2.*) ;; *) echo "FAKE ssh: refusing non-TEST-NET host '$host'" >&2; exit 255;; esac +R=$FAKE_HOSTS/$host +[ -d "$R" ] || { echo "FAKE ssh: ssh: connect to host $host port 22: Connection timed out" >&2; exit 255; } +cmd=$(printf '%s' "$cmd" | /usr/bin/sed -e "s#/etc/systemd/system#$R/etc/systemd/system#g" -e "s#/usr/local/bin#$R/usr/local/bin#g" -e "s#/var/lib/randnode#$R/var/lib/randnode#g" -e "s#/root#$R/root#g") +FAKE_ROOT=$R FAKE_HOST=$host PATH="$FAKE_RSHIM:$PATH" bash -c "$cmd" +FAKE_SSH +cat > "$T/shim/scp" <<'FAKE_SCP' +#!/usr/bin/env bash +srcs=(); dst="" +while [ $# -gt 0 ]; do case "$1" in -q) shift;; -o|-i) shift 2;; root@*) dst=$1; shift;; *) srcs+=("$1"); shift;; esac; done +host=${dst#root@}; path=${host#*:}; host=${host%%:*} +echo "$host" >> "$FAKE_SSH_LOG" +case "$host" in 192.0.2.*) ;; *) echo "FAKE scp: refusing $host" >&2; exit 255;; esac +[ -d "$FAKE_HOSTS/$host" ] || { echo "FAKE scp: connect to $host: Connection timed out" >&2; exit 255; } +cp "${srcs[@]}" "$FAKE_HOSTS/$host$path" +FAKE_SCP +# ── what a fake host's shell finds first on PATH (the host side) ─────────────────────────────── +cat > "$T/rshim/systemctl" <<'FAKE_SYSTEMCTL' +#!/usr/bin/env bash +st=$FAKE_ROOT/state; mkdir -p $st +case "$1" in + is-active) q=0; [ "$2" = --quiet ] && { q=1; shift; }; s=$(cat $st/$2 2>/dev/null || echo inactive); [ $q = 1 ] || echo $s; [ "$s" = active ];; + stop) echo inactive > $st/$2;; + start) [ -e $st/refuse-start ] || echo active > $st/$2;; + daemon-reload) :;; + cat) cat $FAKE_ROOT/etc/systemd/system/$2.service $FAKE_ROOT/etc/systemd/system/$2.service.d/*.conf 2>/dev/null;; + *) echo "fake systemctl: $*" >&2; exit 1;; +esac +FAKE_SYSTEMCTL +printf '#!/usr/bin/env bash\nexec shasum -a 256 "$@"\n' > "$T/rshim/sha256sum" +printf '#!/usr/bin/env bash\ncat $FAKE_ROOT/hostname\n' > "$T/rshim/hostname" +printf '#!/usr/bin/env bash\n# runuser -u randnode -- cmd...\nshift 3; exec "$@"\n' > "$T/rshim/runuser" +printf '#!/usr/bin/env bash\n# chown: no randnode user here\nexit 0\n' > "$T/rshim/chown" +cat > "$T/rshim/install" <<'FAKE_INSTALL' +#!/usr/bin/env bash +a=(); while [ $# -gt 0 ]; do case "$1" in -o|-g) shift 2;; *) a+=("$1"); shift;; esac; done +exec /usr/bin/install "${a[@]}" +FAKE_INSTALL +cat > "$T/rshim/sed" <<'FAKE_SED' +#!/usr/bin/env bash +if [ "$1" = -i ]; then shift; exec /usr/bin/sed -i '' "$@"; fi +exec /usr/bin/sed "$@" +FAKE_SED +cat > "$T/rshim/curl" <<'FAKE_CURL' +#!/usr/bin/env bash +# FAKE curl. A download (-o FILE URL) or a HEAD (-I) is served from $FAKE_RELEASE_DIR for +# https://rehearsal.invalid/ only. Otherwise the node RPC on 127.0.0.1:8545, answered only while +# rand-node is "active", for the chain of the datadir the unit / drop-in names. +out=""; url=""; prev=""; head=0 +for a in "$@"; do [ "$prev" = -o ] && out=$a; case "$a" in https://*) url=$a;; -fsSIL|-I) head=1;; esac; prev=$a; done +if [ -n "$url" ]; then + case "$url" in https://rehearsal.invalid/*) ;; *) echo "FAKE curl: refusing $url" >&2; exit 22;; esac + f=${url%%\?*}; f=${f##*/}; [ -f "$FAKE_RELEASE_DIR/$f" ] || exit 22 + [ $head = 1 ] && exit 0 + cp "$FAKE_RELEASE_DIR/$f" "$out"; exit 0 +fi +[ "$(cat $FAKE_ROOT/state/rand-node 2>/dev/null)" = active ] || exit 7 +body="$*" +dd=$(cat $FAKE_ROOT/etc/systemd/system/rand-node.service $FAKE_ROOT/etc/systemd/system/rand-node.service.d/*.conf 2>/dev/null | grep -o -- '--datadir [^ ]*' | tail -1 | cut -d' ' -f2) +B=$FAKE_ROOT/usr/local/bin +OLDH=ca76c2766ed6090ac6ba5dad86b724dd5d2b80c0edfaed287cd8b129e663ca9a +case "$body" in + *rand_getGenesisHash*) + case "$dd" in *-ca76c276|*/data-21) h=$OLDH;; *) [ -f "$dd/genesis.hash" ] || exit 7; h=$(cat "$dd/genesis.hash");; esac + [ "${REHEARSE_SERVE_OLD:-}" = "$FAKE_HOST" ] && h=$OLDH + echo "{\"id\":1,\"jsonrpc\":\"2.0\",\"result\":\"$h\"}";; + *rand_getHealth*) echo '{"id":1,"jsonrpc":"2.0","result":{"status":"ok"}}';; + *rand_getBridgeState*) + python3 - "$FAKE_ROOT/root/stage-c22/genesis-chain22.json" "${REHEARSE_MINTED:-0}" <<'PY' +import json, sys +g = json.load(open(sys.argv[1])); b = g["bridge"] +assets = [{"locked": str(k.get("locked", 0))} for t in g["tokens"]["tokens"] for k in t["backings"]] +assets[0]["locked"] = str(int(assets[0]["locked"]) + int(sys.argv[2])) +print(json.dumps({"id": 1, "jsonrpc": "2.0", "result": {"enabled": True, "burn_sequence": b["burn_sequence"], "assets": assets}})) +PY + ;; + *rand_status*) + n=$(( $(cat $FAKE_ROOT/state/height 2>/dev/null || echo 4) + 1 )); echo $n > $FAKE_ROOT/state/height + k=absent; g=$($B/rand-node keygen-msauth-vk --check $B/msauth-vk-production.bin 2>/dev/null); p=$($B/rand-node keygen-msauth-vk --print-pinned production 2>/dev/null) + [ -n "$g" ] && [ "$g" = "$p" ] && k=loaded + pk='{"source":"absent","keys":0}'; pins=$($B/rand-node keygen-perp-vk --print-pinned production 2>/dev/null) + if [ -n "$pins" ]; then c=0; while read -r f s; do [ "$(shasum -a 256 $B/$f 2>/dev/null | cut -d' ' -f1)" = "$s" ] && c=$((c+1)); done <<<"$pins"; [ $c = 4 ] && pk='{"source":"loaded","keys":4}'; fi + echo "{\"id\":1,\"jsonrpc\":\"2.0\",\"result\":{\"height\":$n,\"view\":6,\"msauth_key\":\"$k\",\"perps_keys\":$pk}}";; + *rand_getVersion*) + case "$($B/rand-node --version 2>/dev/null)" in + "rand-node 0.8.6"*) v=0.8.6; gs=6591c884187126f118460c940d15715032bd124f;; + *) v=0.8.7; gs=fake;; + esac + echo "{\"id\":1,\"jsonrpc\":\"2.0\",\"result\":{\"chain_id\":21,\"git_sha\":\"$gs\",\"version\":\"$v\"}}";; +esac +FAKE_CURL +# ── the 26 inventory rows; 25 fake hosts (archive2 has none), on chain 21, on "v0.8.6" ───────── +NAMES18="a b c d e f lon1 sfo3 tor1 blr1 syd1 atl1 ams3 nyc1 nyc2 sfo2 mkc1 mem1" +cat > "$T/mkfleet.sh" <<'MKFLEET' +#!/usr/bin/env bash +set -euo pipefail +T=$1; OLDB=$2; NAMES18=$3; H=$T/hosts; rm -rf $H; mkdir -p $H +INV=$T/inventory.tsv +printf 'name\tip\thostname\trole\tssh_key\tkey_path\tdatadir\taddress\tpeer_id\tbootstraps\textra_flags\tufw\tother_units\n' > $INV +mk() { # ip name — the installed files are COPIES (switch's `install` writes over them) + local R=$H/$1; mkdir -p $R/root/keys $R/etc/systemd/system $R/usr/local/bin $R/var/lib/randnode $R/state + echo $2 > $R/hostname; cp $OLDB/rand-node $OLDB/rand $OLDB/msauth-vk-production.bin $R/usr/local/bin/; echo active > $R/state/rand-node +} +i=0 +for n in $NAMES18 obs1 archive2; do + i=$((i+1)); ip=192.0.2.$i; role=validator; dd=/root/data-rand-node-$n-ca76c276 + case $n in e) role=validator+archive;; obs1) role=validator+archive; dd=/root/data-obs1-ca76c276;; archive2) role=validator+archive+prover-member; dd=/root/data-archive2-ca76c276;; esac + printf '%s\t%s\trand-node-%s\t%s\t~/.ssh/id_ed25519\t/root/keys/node-%s.key.json\t%s\taddr-%s\t12D3Fake%s\t\t--validator\tactive\t\n' $n $ip $n $role $n $dd $n $n >> $INV + [ $n != archive2 ] || continue # archive2: dropped and unreachable — no fake host at all + mk $ip rand-node-$n; R=$H/$ip + echo "{\"address\": \"addr-$n\"}" > $R/root/keys/node-$n.key.json + if [ $n = e ] || [ $n = obs1 ]; then mkdir -p $R/mnt/volume$dd/db; ln -s $R/mnt/volume$dd $R$dd; else mkdir -p $R$dd/db; fi + printf '[Service]\nExecStart=%s run --datadir %s --key %s --validator\n' "$R/usr/local/bin/rand-node" "$R$dd" "$R/root/keys/node-$n.key.json" > $R/etc/systemd/system/rand-node.service +done +for g in 1 2 3 4 5 6; do + ip=192.0.2.10$g; mk $ip rand-guardian-$g; R=$H/$ip + printf 'g%s\t%s\trand-guardian-%s\tvalidator+guardian-host\t~/.ssh/rand_guardian_ed25519\t/var/lib/randnode/node.key.json\t/var/lib/randnode/data-21\taddr-g%s\t12D3FakeG%s\t\t--validator\tactive\trand-guardian.service\n' $g $ip $g $g $g >> $INV + echo "{\"address\": \"addr-g$g\"}" > $R/var/lib/randnode/node.key.json + mkdir -p $R/etc/systemd/system/rand-node.service.d $R/var/lib/randnode/data-21/db + printf '[Service]\nUser=randnode\n' > $R/etc/systemd/system/rand-node.service + printf '[Service]\nExecStart=\nExecStart=%s run --datadir %s --key %s --validator\n' "$R/usr/local/bin/rand-node" "$R/var/lib/randnode/data-21" "$R/var/lib/randnode/node.key.json" > $R/etc/systemd/system/rand-node.service.d/chain21.conf + echo active > $R/state/rand-guardian +done +mkdir -p $H/192.0.2.150/state; echo rand-relayer-1 > $H/192.0.2.150/hostname; echo active > $H/192.0.2.150/state/rand-relayer # the relayer +MKFLEET +chmod +x "$T"/shim/* "$T"/rshim/* "$T/mkfleet.sh" +# ── the temp tree: the script under test, the real helpers, a stub cut script ───────────────── +cp deploy/cutover-fleet-chain22.sh deploy/check-beacon-seed.sh deploy/check-constraint-set.sh deploy/cut-record.template "$T/repo/deploy/" +cp deploy/lib/cut-policy.sh deploy/lib/verify-release.sh deploy/lib/fleet-env.sh "$T/repo/deploy/lib/" +{ echo '#!/usr/bin/env bash' + echo '# rehearsal stub of deploy/cut-chain22-genesis.sh: logs its arguments and environment; the validator lists are the real ones' + grep -E '^(GENESIS_NAMES|BONDED|DROPPED_VALIDATORS)=' deploy/cut-chain22-genesis.sh + echo 'echo "STUB $* PRUNED_SCAN_OK=${PRUNED_SCAN_OK:-} BEACON_TSV=${BEACON_TSV:-} NODE=${NODE:-} DRY_RUN=${DRY_RUN:-}" >> "$STUB_LOG"' + echo 'echo "cut-chain22-genesis.sh (stub) ran: ${1:-cut}"' + echo 'if [ "${1:-}" = check-limits ]; then exit "${STUB_LIMITS_RC:-0}"; fi; exit "${STUB_RC:-0}"' +} > "$T/repo/deploy/cut-chain22-genesis.sh" +cp crates/randprotocol-node/src/constraint_set.rs "$T/cs/no-row.rs" +export NODE_RS=$REPO/crates/randprotocol-node/src/node.rs VENDORED_MACHINE_RS=$REPO/crates/randprotocol-zkvm/src/machine.rs CONSTRAINT_SET_RS=$T/cs/no-row.rs +"$T/mkfleet.sh" "$T" "$BIN/old" "$NAMES18" +export PATH="$T/shim:$PATH" FAKE_HOSTS=$T/hosts FAKE_RSHIM=$T/rshim FAKE_RELEASE_DIR=$BIN/new FAKE_SSH_LOG=$T/ssh.log STUB_LOG=$T/stub.log +[ "$(command -v ssh)" = "$T/shim/ssh" ] && [ "$(command -v scp)" = "$T/shim/scp" ] || { echo "rehearse: the fake ssh/scp are not first on PATH — abort" >&2; exit 99; } +! cut -f2 "$T/inventory.tsv" | tail -n +2 | grep -qv '^192\.0\.2\.' || { echo "rehearse: the fake inventory names a non-TEST-NET host — abort" >&2; exit 99; } +: > "$T/ssh.log"; : > "$T/stub.log" +ARCHIVE2_IP=$(awk -F'\t' '$1 == "archive2" {print $2}' "$T/inventory.tsv") +OLD_SHA=$(sha "$BIN/old/rand-node"); OLD_SHA_WALLET=$(sha "$BIN/old/rand") +NEW_SHA=$(sha "$BIN/new/rand-node"); NEW_SHA_WALLET=$(sha "$BIN/new/rand"); VK_SHA=$(sha "$BLOB") +export INVENTORY=$T/inventory.tsv MARKER_DIR=$T/markers WAIT_SLEEP=0 MIN_FREE_GB=0 \ + RELEASE_URL=https://rehearsal.invalid/v0.8.7 CUT_RECORD=$T/cut-record.txt BEACON_TSV=$T/beacon.tsv \ + ROLL_RECORD=$T/roll.record GUARDIAN_KNOWN_HOSTS=$T/known_hosts CHECK_LIMITS_RPC=http://127.0.0.1:1/rehearsal +unset RELEASE_SUMS RELEASE_SUMS_SIG OLD_RELEASE_SUMS WANT_SHA WANT_SHA_WALLET LOCAL_NODE CONFIRM_HALT_CHAIN21 RELAYER_HOST RELAYER_HANDLED BRIDGE_ON_CHAIN22 PRUNED_SCAN_OK +NEWENV=("RELEASE_SUMS=$T/SUMS") +OLDENV=("OLD_RELEASE_SUMS=$T/OLD-SUMS") +LN=("LOCAL_NODE=$BIN/new/rand-node") +F=$T/repo/deploy/cutover-fleet-chain22.sh +hash_of() { sha "$1"; } # the stub's init: the genesis hash is the file's sha256 +record() { sed -e 's/<[^>]*>/filled (rehearsal)/' "$REPO/deploy/cut-record.template" | sed "s/^second-hash:.*/second-hash: $1/" > "$CUT_RECORD"; } +pass=0; fail=0 +t() { # cmd… + local what=$1 want=$2 phrase=$3; shift 3 + "$@" > "$T/out.log" 2>&1; local rc=$? + if { [ "$want" = 0 ] && [ $rc = 0 ]; } || { [ "$want" != 0 ] && [ $rc != 0 ]; }; then + if grep -qF -- "$phrase" "$T/out.log"; then echo "ok — $what (rc $rc): $(grep -F -- "$phrase" "$T/out.log" | tail -1 | cut -c1-140)"; pass=$((pass+1)) + else echo "FAIL — $what: rc $rc as wanted but no \"$phrase\": $(tail -3 "$T/out.log" | cut -c1-240)"; fail=$((fail+1)); cp "$T/out.log" "$T/fail-$fail.log"; fi + else echo "FAIL — $what: rc $rc, wanted $want: $(tail -4 "$T/out.log" | cut -c1-300)"; fail=$((fail+1)); cp "$T/out.log" "$T/fail-$fail.log"; fi +} +chk() { if eval "$2"; then echo "ok — $1"; pass=$((pass+1)); else echo "FAIL — $1"; fail=$((fail+1)); fi; } +has() { if grep -qF -- "$2" "$T/out.log"; then echo "ok — $1"; pass=$((pass+1)); else echo "FAIL — $1: no \"$2\""; fail=$((fail+1)); fi; } +hasnt() { if ! grep -qF -- "$2" "$T/out.log"; then echo "ok — $1"; pass=$((pass+1)); else echo "FAIL — $1: found \"$2\""; fail=$((fail+1)); fi; } +H=$T/hosts +ip_of() { awk -F'\t' -v n="$1" '$1 == n {print $2}' "$T/inventory.tsv"; } +nmark() { ls "$T/markers" | wc -l | tr -d ' '; } +units() { for h in "$H"/192.0.2.{1..19} "$H"/192.0.2.10{1..6}; do cat $h/etc/systemd/system/rand-node.service $h/etc/systemd/system/rand-node.service.d/*.conf 2>/dev/null; done | shasum | cut -c1-16; } +tree() { find "$H" -type f -o -type l | sort | xargs shasum 2>/dev/null | shasum | cut -c1-16; } +active() { grep -l '^active$' "$H"/192.0.2.{1..19}/state/rand-node "$H"/192.0.2.10{1..6}/state/rand-node 2>/dev/null | wc -l | tr -d ' '; } +installed() { for h in "$H"/192.0.2.{1..19} "$H"/192.0.2.10{1..6}; do if [ -f $h/usr/local/bin/$1 ]; then sha $h/usr/local/bin/$1; else echo none; fi; done | sort | uniq -c | awk '{print $1":"$2}' | tr '\n' ' '; } +staged() { ls "$H"/*/root/stage-c22/rand-node 2>/dev/null | wc -l | tr -d ' '; } +bdir() { case "$1" in 192.0.2.10?) echo "$H/$1/var/lib/randnode/beacon-c22";; *) echo "$H/$1/root/beacon-c22";; esac; } +sshn() { wc -l < "$T/ssh.log" | tr -d ' '; } +U0=$(units) + +# ── DRY_RUN: prints, touches nothing, contacts no host ── +TR0=$(tree) +for ph in preflight stage beacon-init beacon-collect bridge-stop status halt rollback start start-archives wait21; do + t "DRY_RUN=1 $ph with no inputs prints and touches nothing" 0 "[dry-run]" env DRY_RUN=1 "$F" $ph +done +t "DRY_RUN=1 snapshot shows the cut script with PRUNED_SCAN_OK=1" 0 "PRUNED_SCAN_OK=1" env DRY_RUN=1 "$F" snapshot "$T/snap" +t "DRY_RUN=1 cut shows BEACON_TSV" 0 "BEACON_TSV=$T/beacon.tsv" env DRY_RUN=1 "$F" cut +t "DRY_RUN=1 push prints the push" 0 "[dry-run] scp" env DRY_RUN=1 "$F" push "$T/no-genesis.json" +t "DRY_RUN=1 stop without the phrase says the real stop refuses, and stops nothing" 0 "REFUSES without CONFIRM_HALT_CHAIN21=halt-chain-21-for-" env DRY_RUN=1 "$F" stop "$T/no-genesis.json" +t "DRY_RUN=1 switch prints the perps key check" 0 "keygen-perp-vk --print-pinned production" env DRY_RUN=1 "$F" switch "$T/no-genesis.json" +t "DRY_RUN=1 check-limits prints the tunnel and the cut script" 0 "check-limits" env -u CHECK_LIMITS_RPC DRY_RUN=1 "$F" check-limits "$T/no-genesis.json" +t "DRY_RUN=1 wait prints check-beacon-seed.sh --rpc" 0 "--rpc 127.0.0.1:8545" env DRY_RUN=1 "$F" wait "$T/no-genesis.json" +t "DRY_RUN=1 bridge-start" 0 "systemctl start rand-guardian" env DRY_RUN=1 "$F" bridge-start +chk "…the dry runs changed no host, contacted none, wrote no marker, record, stub call or BEACON_TSV" '[ "$(tree)" = "$TR0" ] && [ "$(sshn)" = 0 ] && [ "$(active)" = 25 ] && [ "$(nmark)" = 0 ] && [ ! -e $ROLL_RECORD ] && [ ! -s $T/stub.log ] && [ ! -e $BEACON_TSV ]' +# ── the inventory and input hygiene ── +t "no INVENTORY" 1 "INVENTORY is unset" env -u INVENTORY "$F" status +sed 's/192\.0\.2\.7\t/203.0.113.7\t/' "$T/inventory.tsv" > "$T/inv-ph.tsv" +t "an inventory with a placeholder host" 1 "placeholder hosts" env INVENTORY="$T/inv-ph.tsv" "$F" status +grep -v '^archive2' "$T/inventory.tsv" > "$T/inv-noa2.tsv" +t "an inventory without archive2's row (not the fleet's)" 1 "no row for the dropped archive2" env INVENTORY="$T/inv-noa2.tsv" "$F" status +grep -v '^nyc2' "$T/inventory.tsv" > "$T/inv-24.tsv" +t "an inventory missing a validator (24)" 1 "are not chain 22's 25" env INVENTORY="$T/inv-24.tsv" "$F" status +sed 's/^g3\t192\.0\.2\.103/g3\t192.0.2.103;touch/' "$T/inventory.tsv" > "$T/inv-bad.tsv" +t "an address that is not IPv4" 1 "is not an IPv4 address" env INVENTORY="$T/inv-bad.tsv" "$F" status +sed 's#/root/keys/node-b.key.json#/root/keys/$(id).json#' "$T/inventory.tsv" > "$T/inv-meta.tsv" +t "a key path with shell metacharacters" 1 "is not a plain absolute path" env INVENTORY="$T/inv-meta.tsv" "$F" status +t "a WANT_VERSION with a quote and ';'" 1 "shell metacharacter" env "WANT_VERSION=0.8.7'; touch /tmp/x" "$F" status +t "archive2 as a bootstrap is refused" 1 "is not a first-group validator" env BOOT_NAMES="archive2 c" "$F" status +t "the relayer at archive2's address is refused" 1 "dropped validator's address" env RELAYER_HOST="$ARCHIVE2_IP" "$F" bridge-stop +chk "…none of these contacted a host" '[ "$(sshn)" = 0 ]' +# ── preflight ── +t "preflight without v0.8.6's pins" 1 "OLD_SHA / OLD_SHA_WALLET" env "${NEWENV[@]}" "$F" preflight +t "preflight without the signed v0.8.7 sums" 1 "RELEASE_SUMS is unset" env "${OLDENV[@]}" "$F" preflight +t "preflight with a v0.8.7 SHA256SUMS whose signature does not verify" 1 "signature does not verify" env "${OLDENV[@]}" RELEASE_SUMS="$T/SUMS-unsigned" "$F" preflight +t "preflight on the healthy chain-21 fleet: 25 hosts, the seven release files present" 0 "PREFLIGHT OK" env "${OLDENV[@]}" "${NEWENV[@]}" "$F" preflight +has "…25 validators, first group 23 ≥ quorum 17, archives late" "25 validators; first group 23 ≥ quorum 17; late: e obs1" +has "…the release holds all seven files" "all seven present" +chk "preflight changed nothing" '[ "$(units)" = "$U0" ] && [ "$(active)" = 25 ] && [ "$(nmark)" = 0 ]' +t "preflight with too little disk (MIN_FREE_GB=9999)" 1 "GB free under" env "${OLDENV[@]}" "${NEWENV[@]}" MIN_FREE_GB=9999 "$F" preflight +cp "$BIN/new/rand-node" "$H/192.0.2.7/usr/local/bin/rand-node" +t "preflight with one host on another rand-node" 1 "not chain 21's v0.8.6" env "${OLDENV[@]}" "${NEWENV[@]}" "$F" preflight +cp "$BIN/old/rand-node" "$H/192.0.2.7/usr/local/bin/rand-node" +cp "$BLOB" "$H/192.0.2.8/usr/local/bin/msauth-vk-production.bin" +t "preflight with one host not holding chain 21's msauth blob" 1 "the rollback needs it" env "${OLDENV[@]}" "${NEWENV[@]}" "$F" preflight +cp "$OLDBLOB" "$H/192.0.2.8/usr/local/bin/msauth-vk-production.bin" +sed 's#/root/data-rand-node-f-ca76c276#/root/data-rand-node-f-elsewhere-ca76c276#' "$T/inventory.tsv" > "$T/inv-dd.tsv" +t "preflight when the inventory's datadir is not the unit's" 1 "does not run the inventory's datadir" env INVENTORY="$T/inv-dd.tsv" "${OLDENV[@]}" "${NEWENV[@]}" "$F" preflight +t "halt while chain 21 runs is refused" 1 "RUNS CHAIN 21" "$F" halt +# ── stage ── +t "beacon-init before stage (no staged v0.8.7)" 1 "not every validator has a seed" env "${NEWENV[@]}" "$F" beacon-init +t "stage without RELEASE_SUMS" 1 "RELEASE_SUMS is unset" "$F" stage +t "stage with an unsigned SHA256SUMS" 1 "signature does not verify" env RELEASE_SUMS="$T/SUMS-unsigned" "$F" stage +t "stage with a SHA256SUMS changed after signing" 1 "signature does not verify" env RELEASE_SUMS="$T/SUMS-tampered" "$F" stage +t "stage with a signed SHA256SUMS whose msauth blob is not set 11's" 1 "not constraint set 11's decided blob" env RELEASE_SUMS="$T/SUMS-otherblob" "$F" stage +t "stage with a signed SHA256SUMS whose perps key is not the decided one" 1 "not the decided" env RELEASE_SUMS="$T/SUMS-otherpvk" "$F" stage +t "stage with a signed SHA256SUMS missing a perps key" 1 "a chain-22 node does not start without it" env RELEASE_SUMS="$T/SUMS-nopvk" "$F" stage +t "stage with a WANT_SHA that disagrees with the signed file" 1 "is not what the signed SHA256SUMS lists" env "${NEWENV[@]}" WANT_SHA="$OLD_SHA" "$F" stage +t "stage from a path holding a non-ELF build" 1 "not a Linux ELF" env "${NEWENV[@]}" STAGE_FROM=path STAGE_DIR="$BIN/new" "$F" stage +t "stage from a path whose perps key is not the signed one" 1 "is not the signed" env "${NEWENV[@]}" STAGE_FROM=path STAGE_DIR="$BIN/badpvk" STAGE_ALLOW_NON_ELF=1 "$F" stage +chk "…nothing staged" '[ "$(staged)" = 0 ]' +t "stage from a release whose rand-node does not match" 1 "not every host staged" env "${NEWENV[@]}" FAKE_RELEASE_DIR="$BIN/badrel" "$F" stage +has "…each host said SHA MISMATCH and removed it" "SHA MISMATCH ( rand-node) — removed" +t "stage from a release whose perps key bytes do not match" 1 "not every host staged" env "${NEWENV[@]}" FAKE_RELEASE_DIR="$BIN/badpvk" "$F" stage +has "…naming the key file" "SHA MISMATCH ( $PVK7) — removed" +chk "…no staged or partial file left anywhere" '[ "$(staged)" = 0 ] && ! ls $H/*/root/stage-c22/*.part >/dev/null 2>&1' +t "stage with a staged binary that pins another msauth key" 1 "msauth key NOT verified" env "${NEWENV[@]}" REHEARSE_PIN="$(sha "$TESTBLOB")" "$F" stage +t "stage with a staged binary that pins another perps key" 1 "perps keys NOT verified" env "${NEWENV[@]}" REHEARSE_PVK_PIN="$(sha "$TESTBLOB")" "$F" stage +t "stage from a plain-http url" 1 "is not https://" env "${NEWENV[@]}" STAGE_FROM=url STAGE_URL=http://rehearsal.invalid/x "$F" stage +t "stage from the release" 0 "STAGED" env "${NEWENV[@]}" "$F" stage +has "…each host verified the msauth key and the four perps keys with the staged binary" "msauth key ${VK_SHA:0:12}… and the four perps keys verified" +chk "25 hosts hold the seven files + the seed checker at their pins; installed still v0.8.6; 25 active" '[ "$(staged)" = 25 ] && [ "$(installed rand-node)" = "25:$OLD_SHA " ] && [ "$(active)" = 25 ] && [ "$(cat $H/*/root/stage-c22/check-beacon-seed.sh | shasum | cut -c1-40)" = "$(for i in $(seq 25); do cat deploy/check-beacon-seed.sh; done | shasum | cut -c1-40)" ] && [ "$(ls $H/*/root/stage-c22/perp-vk-production-*.bin | wc -l | tr -d " ")" = 100 ]' +t "stage again from a local path (STAGE_ALLOW_NON_ELF=1: the stubs are scripts)" 0 "STAGED" env "${NEWENV[@]}" STAGE_FROM=path STAGE_DIR="$BIN/new" STAGE_ALLOW_NON_ELF=1 "$F" stage +# ── the beacon: init on each host, collect the commitments ── +t "beacon-init on the 25 validators" 0 "BEACON-INIT DONE" env "${NEWENV[@]}" "$F" beacon-init +chk "25 seed files, each in its own host's staging dir (guardians under /var/lib/randnode), mode 0600, each naming its host's key" '[ "$(for i in $(seq 1 19) 101 102 103 104 105 106; do b=$(bdir 192.0.2.$i); python3 -c "import json,sys,os,stat; f=json.load(open(sys.argv[1])); k=json.load(open(sys.argv[2])); assert f[\"validator\"]==k[\"address\"] and f[\"chain_id\"]==22 and stat.S_IMODE(os.stat(sys.argv[1]).st_mode)==0o600; print(\"ok\")" $b/beacon.seed.json $( [ $i -gt 100 ] && echo $H/192.0.2.$i/var/lib/randnode/node.key.json || echo $H/192.0.2.$i/root/keys/*.key.json); done | grep -c ok)" = 25 ]' +SEEDS=$(for f in "$H"/*/root/beacon-c22/beacon.seed.json "$H"/*/var/lib/randnode/beacon-c22/beacon.seed.json; do python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["seeds"][0]["seed"])' "$f"; done) +chk "…no seed value appears in the phase's output" '! grep -qF -f <(printf "%s\n" $SEEDS) $T/out.log' +chk "…no seed file anywhere outside its host (none on the laptop side)" '[ -z "$(find $T $BIN -name beacon.seed.json -not -path "$H/*" 2>/dev/null)" ]' +S0=$(cat "$H"/*/root/beacon-c22/beacon.seed.json "$H"/*/var/lib/randnode/beacon-c22/beacon.seed.json | shasum) +t "beacon-init again keeps every seed" 0 "(kept; validator addr-" env "${NEWENV[@]}" "$F" beacon-init +chk "…byte for byte" '[ "$(cat $H/*/root/beacon-c22/beacon.seed.json $H/*/var/lib/randnode/beacon-c22/beacon.seed.json | shasum)" = "$S0" ]' +cp "$(bdir 192.0.2.1)/beacon.seed.json" "$T/a-seed.keep"; cp "$(bdir 192.0.2.101)/beacon.seed.json" "$(bdir 192.0.2.1)/beacon.seed.json" +t "beacon-collect when a seed file was copied from another host" 1 "not chain 22 and this key's address addr-a" env "${NEWENV[@]}" "$F" beacon-collect +cp "$T/a-seed.keep" "$(bdir 192.0.2.1)/beacon.seed.json"; chmod 600 "$(bdir 192.0.2.1)/beacon.seed.json" +chmod 644 "$(bdir 192.0.2.104)/beacon.seed.json" +t "beacon-collect when a seed file is mode 0644" 1 "not 0600" env "${NEWENV[@]}" "$F" beacon-collect +chmod 600 "$(bdir 192.0.2.104)/beacon.seed.json" +chk "…no BEACON_TSV written by the refusals" '[ ! -e $BEACON_TSV ]' +t "beacon-collect" 0 "COLLECTED" env "${NEWENV[@]}" "$F" beacon-collect +chk "BEACON_TSV: the 25 names of GENESIS_NAMES + BONDED, each with its host's own commitment" '[ "$(grep -v "^#" $BEACON_TSV | awk "{print \$1}" | tr "\n" " ")" = "$NAMES18 obs1 g1 g2 g3 g4 g5 g6 " ] && for n in a lon1 obs1 g3; do i=$(ip_of $n); c=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))[\"seeds\"][-1][\"commitment\"])" $(bdir $i)/beacon.seed.json); grep -qx "$n $c" $BEACON_TSV || exit 1; done' +chk "…and no seed in it" '! grep -qF -f <(printf "%s\n" $SEEDS) $BEACON_TSV' +t "beacon-collect again: unchanged" 0 "already holds these 25 commitments" env "${NEWENV[@]}" "$F" beacon-collect +cp "$BEACON_TSV" "$T/beacon.keep"; sed -i '' '3s/ [0-9a-f]*$/ 1111111111111111111111111111111111111111111111111111111111111111/' "$BEACON_TSV" +t "beacon-collect over a BEACON_TSV holding other commitments" 1 "exists with OTHER commitments" env "${NEWENV[@]}" "$F" beacon-collect +t "…replaced on BEACON_TSV_REPLACE=1, the old one moved aside" 0 "moved aside" env "${NEWENV[@]}" BEACON_TSV_REPLACE=1 "$F" beacon-collect +chk "…the new file is the hosts' again" 'diff -q <(grep -v "^#" $T/beacon.keep) <(grep -v "^#" $BEACON_TSV) >/dev/null && ls $BEACON_TSV.replaced-* >/dev/null' +# ── the bridge, the snapshot and the cut (the cut script is a stub that logs) ── +t "snapshot while the guardians run" 1 "bridge-stop first" env CHAIN21_RPC=http://127.0.0.1:8545 "$F" snapshot "$T/snap" +t "bridge-stop without RELAYER_HOST" 1 "RELAYER_HOST is unset" "$F" bridge-stop +t "bridge-stop (guardians + the relayer)" 0 "BRIDGE-STOP DONE" env RELAYER_HOST=192.0.2.150 RELAYER_SSH_KEY="$T/key/rel" "$F" bridge-stop +chk "…six guardians and the relayer inactive" '[ "$(cat $H/192.0.2.10{1..6}/state/rand-guardian | sort -u)" = inactive ] && [ "$(cat $H/192.0.2.150/state/rand-relayer)" = inactive ]' +t "snapshot without CHAIN21_RPC" 1 "CHAIN21_RPC is unset" "$F" snapshot "$T/snap" +t "snapshot" 0 "rand-guardian inactive" env CHAIN21_RPC=http://127.0.0.1:8545 "$F" snapshot "$T/snap" +chk "…the cut script ran snapshot with PRUNED_SCAN_OK=1 and no DRY_RUN" 'grep -q "^STUB snapshot $T/snap PRUNED_SCAN_OK=1 .* DRY_RUN=$" $T/stub.log' +t "balances" 0 "inactive" env CHAIN21_RPC=http://127.0.0.1:8545 WALLET=/x/rand "$F" balances "$T/snap" +chk "…with PRUNED_SCAN_OK=1" 'grep -q "^STUB balances $T/snap PRUNED_SCAN_OK=1" $T/stub.log' +t "a cut the cut script refuses" 1 "refused" env STUB_RC=1 "$F" cut +grep -v '^nyc2 ' "$BEACON_TSV" > "$T/beacon-24.tsv" +t "the cut with a BEACON_TSV missing a validator" 1 "does not name exactly chain 22's 25" env BEACON_TSV="$T/beacon-24.tsv" "$F" cut +t "the cut" 0 "(stub) ran: cut" "$F" cut +chk "…the cut script ran with PRUNED_SCAN_OK=1 and the collected BEACON_TSV" 'tail -1 $T/stub.log | grep -q "^STUB PRUNED_SCAN_OK=1 BEACON_TSV=$BEACON_TSV NODE=.* DRY_RUN=$"' +# The genesis files the cut would have written: chain 22's shape, the collected commitments. +cat > "$T/mkgen.py" <<'PY' +import json, sys +out, tsv, ts, variant = sys.argv[1:5] +pins = ["4dc1f18bdd6d9104ba1622710cf2c146f71759676dbdcb70dad036152242e8b4", "6392e5e9b7348f3729ba2031e0e3ec9788d6ed62a14521526482c92615f55106", + "c31636aa281004163e6fe0debd81854f11d7bca6c8062a91b1501c7e593e5bf0", "ae3d0dc276061fd2a87b2e91b239f6dcccb746b49dedeecfdb2aab977c95c370"] +rows = [l.split() for l in open(tsv) if l.strip() and not l.startswith("#")] +g = {"chain_id": 22, "timestamp_ms": int(ts), "validators": [{"public_key": "pk-" + n} for n, _ in rows], + "bridge": {"burn_sequence": 14}, "tokens": {"tokens": [{"backings": [{"locked": "100001999"}, {"locked": "0"}]}]}, + "max_proof_bytes": 6291456, "max_block_bytes": 19922944, + "perps": {"engine_program_log_height": 14, "state_proof_keys": [{"tier": 18, "public_log_height": 7 + i, "sha256": p} for i, p in enumerate(pins)]}, + "hc_auth_multisig": "41db73c507ab1f24ebe76d9c425b42a08ed9a02b4b1284d410502729ce6b9cd1", + "beacon": {"commit_delay_blocks": 256, "commitments": [{"validator": "addr-" + n, "commitment": c} for n, c in rows]}} +if variant == "chain21": g["chain_id"] = 21 +if variant == "badhc": g["hc_auth_multisig"] = "9705f8e2f1bc8561848b29cb8efd57c37e3e39788fa58a5f03304d8019668247" +if variant == "nopvk": del g["perps"]["state_proof_keys"] +if variant == "nobeacon": del g["beacon"] +if variant == "otherbeacon": g["beacon"]["commitments"][4]["commitment"] = "22" * 32 +json.dump(g, open(out, "w"), indent=1) +PY +G=$T/genesis-chain22.json; GB=$T/genesis-chain22-B.json +python3 "$T/mkgen.py" "$G" "$BEACON_TSV" 1763600000000 ok; python3 "$T/mkgen.py" "$GB" "$BEACON_TSV" 1763600099000 ok +for v in chain21 badhc nopvk nobeacon otherbeacon; do python3 "$T/mkgen.py" "$T/g-$v.json" "$BEACON_TSV" 1763600000000 $v; done +H22=$(hash_of "$G"); H22B=$(hash_of "$GB"); P=${H22:0:8}; PB=${H22B:0:8} +# ── push, chain 21 still running ── +t "push without LOCAL_NODE" 1 "LOCAL_NODE is unset" env "${NEWENV[@]}" "$F" push "$G" +t "push with chain 21's v0.8.6 as LOCAL_NODE" 1 "nothing done" env "${NEWENV[@]}" LOCAL_NODE="$BIN/old/rand-node" "$F" push "$G" +t "push of a chain-21 genesis" 1 "has chain_id 21, not 22" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$T/g-chain21.json" +t "push of a genesis with chain 21's msauth guest" 1 "not constraint set 11's" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$T/g-badhc.json" +t "push of a genesis without the pinned perps keys" 1 "CON-601" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$T/g-nopvk.json" +t "push of a genesis without a beacon" 1 "has no beacon section" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$T/g-nobeacon.json" +t "push of a genesis cut from other commitments than the hosts gave" 1 "(beacon-collect)" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$T/g-otherbeacon.json" +t "push with a LOCAL_NODE whose init says constraint set 9" 1 "the binary was not built from this tree" env "${NEWENV[@]}" "${LN[@]}" REHEARSE_SET=9 "$F" push "$G" +t "push with no cut record" 1 "no cut record" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$G" +record "" +t "push with the record's second-hash empty" 1 "no 'second-hash:' line" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$G" +record "$H22B" +t "push with the second operator's hash differing" 1 "differ" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$G" +chk "…nothing copied" '! ls $H/*/root/stage-c22/genesis-chain22.json >/dev/null 2>&1' +record "$H22" +t "push when one host's staged binary writes constraint set 9" 1 "not constraint set 11" env "${NEWENV[@]}" "${LN[@]}" REHEARSE_SET=9 REHEARSE_SET_HOST=192.0.2.7 "$F" push "$G" +t "push of the chain-22 genesis while chain 21 runs" 0 "PUSHED — chain 21 still runs" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$G" +has "…the constraint set: LOCAL_NODE 11 = this tree = the vendored zkVM" "LOCAL_NODE 11 = this tree = the vendored zkVM; chain 22's row is (22, \"$H22\", 11)," +has "…and the row is not committed yet (check-limits / verify will refuse)" "does not hold (22, \"$H22\", 11), yet" +chk "25 hosts hold the genesis, byte-identical; all 25 still active" '[ "$(shasum -a 256 $H/*/root/stage-c22/genesis-chain22.json | cut -d" " -f1 | sort -u)" = "$(sha $G)" ] && [ "$(ls $H/*/root/stage-c22/genesis-chain22.json | wc -l | tr -d " ")" = 25 ] && [ "$(active)" = 25 ]' +t "start with no marker" 1 "no switched-

/ rollback-* marker" "$F" start +t "switch while the fleet still runs" 1 "some hosts are not ready" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "$F" switch "$G" +chk "…installed nothing, no marker" '[ "$(units)" = "$U0" ] && [ "$(installed rand-node)" = "25:$OLD_SHA " ] && [ "$(nmark)" = 0 ]' +# ── the halt guard ── +N0=$(sshn) +t "stop without CONFIRM_HALT_CHAIN21" 1 "REFUSED — this halts chain 21 on all 25 hosts" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "$F" stop "$G" +has "…and names the phrase, bound to this genesis" "CONFIRM_HALT_CHAIN21=halt-chain-21-for-$P" +t "stop with CONFIRM_HALT_CHAIN21=yes" 1 "got 'yes'" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" CONFIRM_HALT_CHAIN21=yes "$F" stop "$G" +t "stop with the phrase of ANOTHER genesis (a re-cut's)" 1 "REFUSED" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" CONFIRM_HALT_CHAIN21="halt-chain-21-for-$PB" "$F" stop "$G" +chk "…the refused stops contacted no host and stopped nothing" '[ "$(sshn)" = "$N0" ] && [ "$(active)" = 25 ]' +GO=(CONFIRM_HALT_CHAIN21="halt-chain-21-for-$P") +# ── the gated stop ── +mv "$H/192.0.2.14/root/stage-c22/genesis-chain22.json" "$T/g.aside" +t "stop when one host lacks the pushed genesis" 1 "NOTHING STOPPED" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "${GO[@]}" "$F" stop "$G" +has "…that host says so" "the chain-22 genesis is not pushed here" +mv "$T/g.aside" "$H/192.0.2.14/root/stage-c22/genesis-chain22.json" +mv "$(bdir 192.0.2.9)/beacon.seed.json" "$T/seed.aside" +t "stop when one validator has no beacon seed" 1 "NOTHING STOPPED" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "${GO[@]}" "$F" stop "$G" +has "…that host's beacon check refused" "beacon seed NOT verified" +mv "$T/seed.aside" "$(bdir 192.0.2.9)/beacon.seed.json" +cp "$(bdir 192.0.2.10)/beacon.seed.json" "$T/s10.keep"; cp "$(bdir 192.0.2.11)/beacon.seed.json" "$(bdir 192.0.2.10)/beacon.seed.json" +t "stop when one validator's seed is another host's" 1 "a seed file copied from another host" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "${GO[@]}" "$F" stop "$G" +cp "$T/s10.keep" "$(bdir 192.0.2.10)/beacon.seed.json"; chmod 600 "$(bdir 192.0.2.10)/beacon.seed.json" +echo active > "$H/192.0.2.104/state/rand-guardian" +t "stop while one guardian's rand-guardian runs" 1 "rand-guardian still active" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "${GO[@]}" "$F" stop "$G" +echo inactive > "$H/192.0.2.104/state/rand-guardian" +cp "$TESTBLOB" "$H/192.0.2.13/usr/local/bin/msauth-vk-production.bin" +t "stop while one host's installed msauth blob is not chain 21's" 1 "switch would refuse to back it up" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "${GO[@]}" "$F" stop "$G" +cp "$OLDBLOB" "$H/192.0.2.13/usr/local/bin/msauth-vk-production.bin" +cp "$KEYS/$PVK8" "$H/192.0.2.16/root/stage-c22/$PVK7" +t "stop when one host's staged perps key changed since stage" 1 "NOTHING STOPPED" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "${GO[@]}" "$F" stop "$G" +cp "$KEYS/$PVK7" "$H/192.0.2.16/root/stage-c22/$PVK7" +chk "…all 25 still active after every refused stop" '[ "$(active)" = 25 ]' +t "stop (the phrase given; gated: all 25 ready)" 0 "STOPPED" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "${GO[@]}" "$F" stop "$G" +has "…every validator's seed opened its genesis commitment before the halt" "beacon ok" +chk "all 25 inactive" '[ "$(active)" = 0 ]' +# ── switch ── +record "" +t "switch with the record's second-hash blanked after push" 1 "no 'second-hash:' line" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "$F" switch "$G" +record "$H22" +t "switch without v0.8.6's pins" 1 "OLD_SHA / OLD_SHA_WALLET" env "${NEWENV[@]}" "${LN[@]}" "$F" switch "$G" +cp "$KEYS/$PVK8" "$H/192.0.2.5/root/stage-c22/$PVK7" +cp "$BIN/badrel/rand-node" "$H/192.0.2.12/usr/local/bin/rand-node" +t "switch with host 5's staged perps key replaced and host 12 not on v0.8.6" 1 "rollback → start → wait21" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "$F" switch "$G" +has "…host 5: its staged key is not the pinned file (before anything)" "staged $PVK7 is not the pinned v0.8.7 file" +has "…host 12 made no backup" "no backup made" +chk "…host 5 installed nothing: still v0.8.6, chain 21's blob, no perps key, unit on chain 21; no marker" '[ "$(sha $H/192.0.2.5/usr/local/bin/rand-node)" = "$OLD_SHA" ] && [ "$(sha $H/192.0.2.5/usr/local/bin/msauth-vk-production.bin)" = "$OLD_VK" ] && ! ls $H/192.0.2.5/usr/local/bin/perp-vk-* >/dev/null 2>&1 && grep -q -- "-ca76c276 " $H/192.0.2.5/etc/systemd/system/rand-node.service && [ "$(nmark)" = 0 ]' +chk "…host 12's unit still names chain 21, no .pre-c22 there" '[ ! -e $H/192.0.2.12/root/rand-node.pre-c22 ] && grep -q -- "-ca76c276 " $H/192.0.2.12/etc/systemd/system/rand-node.service' +cp "$KEYS/$PVK7" "$H/192.0.2.5/root/stage-c22/$PVK7"; cp "$BIN/old/rand-node" "$H/192.0.2.12/usr/local/bin/rand-node" +t "switch with the staged binary pinning another perps key (every host re-checks before anything)" 1 "perps keys NOT verified" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" REHEARSE_PVK_PIN="$(sha "$TESTBLOB")" "$F" switch "$G" +chk "…host 5 still installed nothing" '[ "$(sha $H/192.0.2.5/usr/local/bin/rand-node)" = "$OLD_SHA" ] && [ "$(nmark)" = 0 ]' +echo active > "$H/192.0.2.102/state/rand-guardian" +t "switch with a rand-guardian still active" 1 "rand-guardian still active" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "$F" switch "$G" +t "start after a failed switch" 1 "refusing" "$F" start +echo inactive > "$H/192.0.2.102/state/rand-guardian" +t "switch (re-run over the hosts already switched)" 0 "SWITCHED" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "$F" switch "$G" +has "…each host's beacon seed checked in its new datadir" "beacon ok" +chk "one marker, switched-$P" '[ "$(nmark)" = 1 ] && [ -f $T/markers/switched-$P ]' +chk "every host runs v0.8.7 + the set-11 blob + the four perps keys beside the binary" '[ "$(installed rand-node)" = "25:$NEW_SHA " ] && [ "$(installed rand)" = "25:$NEW_SHA_WALLET " ] && [ "$(installed msauth-vk-production.bin)" = "25:$VK_SHA " ] && for f in $KEYS/perp-vk-production-*.bin; do [ "$(installed $(basename $f))" = "25:$(sha $f) " ] || exit 1; done' +chk "…and kept v0.8.6's rand-node, rand AND chain 21's msauth blob as *.pre-c22 (the rollback)" '[ "$(sha $H/192.0.2.*/root/rand-node.pre-c22 | sort -u)" = "$OLD_SHA" ] && [ "$(sha $H/192.0.2.*/root/rand.pre-c22 | sort -u)" = "$OLD_SHA_WALLET" ] && [ "$(for h in $H/192.0.2.{1..19} $H/192.0.2.10{1..6}; do sha $h/root/msauth-vk-production.bin.pre-c22; done | sort -u)" = "$OLD_VK" ]' +chk "19 node units name -$P, none -ca76c276; each new datadir initialised with constraint-set 11" '[ "$(grep -l -- "-$P " $H/192.0.2.{1..19}/etc/systemd/system/rand-node.service | wc -l | tr -d " ")" = 19 ] && ! grep -q -- "-ca76c276 " $H/192.0.2.{1..19}/etc/systemd/system/rand-node.service && [ "$(cat $H/192.0.2.{1..19}/root/data-*-$P/constraint-set $H/192.0.2.10{1..6}/var/lib/randnode/data-22/constraint-set | sort -u)" = 11 ]' +chk "the archives e and obs1 got their chain-22 datadir on the volume" '[ -L $H/$(ip_of e)/root/data-rand-node-e-$P ] && [ -d $H/$(ip_of e)/mnt/volume/root/data-rand-node-e-$P/db ] && [ -d $H/$(ip_of obs1)/mnt/volume/root/data-obs1-$P/db ]' +chk "each validator's new datadir holds ITS OWN staging seed (0600), byte-identical, nowhere else" 'for i in $(seq 1 19) 101 102 103 104 105 106; do case $i in 10?) d=$H/192.0.2.$i/var/lib/randnode/data-22;; *) d=$(ls -d $H/192.0.2.$i/root/data-*-$P);; esac; cmp -s $(bdir 192.0.2.$i)/beacon.seed.json $d/beacon.seed.json && [ "$(stat -f %Lp $d/beacon.seed.json)" = 600 ] || exit 1; done' +chk "chain-21 datadirs untouched; node unit backups kept" '[ "$(ls -d $H/192.0.2.{1..19}/root/data-*-ca76c276/db | wc -l | tr -d " ")" = 19 ] && [ "$(ls $H/192.0.2.{1..19}/root/rand-node.service.ca76c276.bak | wc -l | tr -d " ")" = 19 ] && [ -d $H/192.0.2.106/var/lib/randnode/data-21/db ]' +chk "6 guardian hosts: chain22.conf on data-22, chain21.conf moved aside, stamp = the genesis" '[ "$(grep -l -- "/data-22 " $H/192.0.2.10{1..6}/etc/systemd/system/rand-node.service.d/chain22.conf | wc -l | tr -d " ")" = 6 ] && ! ls $H/192.0.2.10{1..6}/etc/systemd/system/rand-node.service.d/chain21.conf >/dev/null 2>&1 && [ "$(ls $H/192.0.2.10{1..6}/root/chain21.conf.c21 | wc -l | tr -d " ")" = 6 ] && [ "$(cat $H/192.0.2.10{1..6}/var/lib/randnode/data-22.genesis | sort -u)" = "$H22" ]' +# ── start: the first group (23 ≥ quorum 17), the gates ── +cp "$KEYS/$PVK8" "$H/192.0.2.15/usr/local/bin/$PVK7" +t "start with one host's installed perps key not the pinned one" 1 "nothing started" "$F" start +has "…that host's key check says so" "perps key NOT verified" +cp "$KEYS/$PVK7" "$H/192.0.2.15/usr/local/bin/$PVK7" +D6=$(ls -d "$H"/192.0.2.6/root/data-*-"$P"); cp "$D6/beacon.seed.json" "$T/d6.keep"; cp "$(bdir 192.0.2.7)/beacon.seed.json" "$D6/beacon.seed.json" +t "start with one validator's datadir seed swapped for another host's" 1 "nothing started" "$F" start +has "…its beacon check names the copy" "a seed file copied from another host" +cp "$T/d6.keep" "$D6/beacon.seed.json"; chmod 600 "$D6/beacon.seed.json" +echo 9 > "$D6/constraint-set" +t "start with one datadir's constraint-set marker at 9" 1 "not constraint set 11" "$F" start +echo 11 > "$D6/constraint-set" +chk "…no node started anywhere, the marker kept" '[ "$(active)" = 0 ] && [ -f $T/markers/switched-$P ]' +touch "$H/$(ip_of lon1)/state/refuse-start" +t "start with one first-group host that does not come up" 1 "NOT STARTED on: lon1" "$F" start +has "…22 of 23 active: still above the quorum" "22 of 23 active (≥ quorum 17: chain 22 can commit)" +chk "…22 active, the marker KEPT" '[ "$(active)" = 22 ] && [ -f $T/markers/switched-$P ]' +rm "$H/$(ip_of lon1)/state/refuse-start" +t "start again once that host is fixed" 0 "START-DONE" "$F" start +chk "the first group (23) active, e and obs1 NOT yet; marker first-$P" '[ "$(active)" = 23 ] && [ "$(cat $H/$(ip_of e)/state/rand-node)" = inactive ] && [ "$(cat $H/$(ip_of obs1)/state/rand-node)" = inactive ] && [ -f $T/markers/first-$P ] && [ ! -e $T/markers/switched-$P ]' +t "a second start" 1 "refusing" "$F" start +t "start-archives before check-limits" 1 "run check-limits against c first" "$F" start-archives +t "check-limits the cut script refuses" 1 "the archives are not started" env "${LN[@]}" STUB_LIMITS_RC=1 "$F" check-limits "$G" +t "check-limits" 0 "CHECK-LIMITS OK" env "${LN[@]}" "$F" check-limits "$G" +chk "…the cut script's check-limits ran against the first node with NODE=LOCAL_NODE" 'tail -1 $T/stub.log | grep -q "^STUB check-limits $CHECK_LIMITS_RPC $G .*NODE=$BIN/new/rand-node"' +t "start-archives" 0 "START-ARCHIVES-DONE" "$F" start-archives +chk "all 25 active; no marker left" '[ "$(active)" = 25 ] && [ "$(nmark)" = 0 ]' +t "wait" 0 "CUTOVER DONE: all 25 on $H22" env "${LN[@]}" "$F" wait "$G" +has "…every validator's seed against its running node (--rpc)" "beacon seed opens the commitment in force" +mv "$H/192.0.2.17/usr/local/bin/$PVK10" "$T/pvk.aside" +t "wait with one node's perps key gone (perps_keys absent)" 1 "24/25 on chain 22" env "${LN[@]}" WAIT_TRIES=2 "$F" wait "$G" +has "…and names it" "not yet: mkc1" +mv "$T/pvk.aside" "$H/192.0.2.17/usr/local/bin/$PVK10" +t "bridge-start without BRIDGE_ON_CHAIN22=yes" 1 "BRIDGE_ON_CHAIN22=yes is required" env RELAYER_HANDLED=yes "$F" bridge-start +t "bridge-start" 0 "BRIDGE-START DONE" env BRIDGE_ON_CHAIN22=yes RELAYER_HOST=192.0.2.150 RELAYER_SSH_KEY="$T/key/rel" "$F" bridge-start +chk "…six guardians and the relayer active" '[ "$(cat $H/192.0.2.10{1..6}/state/rand-guardian | sort -u)" = active ] && [ "$(cat $H/192.0.2.150/state/rand-relayer)" = active ]' +t "verify without the CHAIN_CONSTRAINT_SETS row" 1 "does not hold chain 22's row" env "${LN[@]}" "$F" verify "$G" +sed "s/^ (21, \"ca76c2766ed6090ac6ba5dad86b724dd5d2b80c0edfaed287cd8b129e663ca9a\", 9),\$/&\\ + (22, \"$H22\", 11),/" "$T/cs/no-row.rs" > "$T/cs/row.rs" +t "verify with the row committed" 0 "VERIFIED" env "${LN[@]}" CONSTRAINT_SET_RS="$T/cs/row.rs" "$F" verify "$G" +t "status" 0 "archive2: dropped, not contacted" "$F" status +t "quiet: nothing burned or minted on chain 22" 0 "QUIET" "$F" quiet "$G" +t "quiet when a deposit was minted" 1 "NOT the genesis values" env REHEARSE_MINTED=5 "$F" quiet "$G" +# ── a re-cut over a switched fleet, without a rollback, is refused ── +t "halt with an active host that serves chain 21's genesis" 1 "SERVES CHAIN 21" env REHEARSE_SERVE_OLD=192.0.2.9 "$F" halt +t "halt the chain-22 fleet" 0 "HALTED" "$F" halt +record "$H22B" +t "push of a re-cut while chain 22 is down" 0 "PUSHED" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$GB" +UB=$(units) +t "switch to a re-cut over the switched fleet (no rollback)" 1 "needs rollback first" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "$F" switch "$GB" +chk "…no unit changed, no marker" '[ "$(units)" = "$UB" ] && [ "$(nmark)" = 0 ]' +# ── rollback ── +t "rollback while a guardian runs" 1 "rand-guardian still active" env "${OLDENV[@]}" "$F" rollback +t "bridge-stop before the rollback" 0 "BRIDGE-STOP DONE" env RELAYER_HANDLED=yes "$F" bridge-stop +t "rollback" 0 "ROLLED BACK" env "${OLDENV[@]}" "$F" rollback +chk "every unit / drop-in is byte for byte chain 21's again; v0.8.6 binaries and CHAIN 21's msauth blob restored" '[ "$(units)" = "$U0" ] && [ "$(installed rand-node)" = "25:$OLD_SHA " ] && [ "$(installed rand)" = "25:$OLD_SHA_WALLET " ] && [ "$(installed msauth-vk-production.bin)" = "25:$OLD_VK " ]' +t "start after rollback (all 25, C and D first, no chain-22 gate)" 0 "all 25 active on chain 21" "$F" start +hasnt "…the chain-22 start gate was skipped" "the start gate" +t "wait21: chain 21 on all 25, C's height moves" 0 "CHAIN 21 COMMITTING AGAIN" "$F" wait21 +t "rollback while running" 1 "STILL RUNNING" env "${OLDENV[@]}" "$F" rollback +t "preflight after the rollback flags the leftovers (*.pre-c22, data-22)" 1 "not every host is ready" env "${OLDENV[@]}" "${NEWENV[@]}" "$F" preflight +# ── the re-cut, in order: push (chain 21 running), stop, switch, start, … — the seeds are reused ── +record "$H22B" +t "push of the re-cut while chain 21 runs" 0 "PUSHED" env "${NEWENV[@]}" "${LN[@]}" "$F" push "$GB" +t "stop for the re-cut with the FIRST cut's phrase" 1 "REFUSED" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "${GO[@]}" "$F" stop "$GB" +t "stop for the re-cut (its own phrase)" 0 "STOPPED" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" CONFIRM_HALT_CHAIN21="halt-chain-21-for-$PB" "$F" stop "$GB" +t "switch to the re-cut (guardians still hold the first cut's data-22)" 0 "SWITCHED" env "${NEWENV[@]}" "${OLDENV[@]}" "${LN[@]}" "$F" switch "$GB" +chk "each guardian host moved the first cut's data-22 aside and initialised a fresh one, with the same seed" '[ "$(ls -d $H/192.0.2.10{1..6}/var/lib/randnode/data-22.stale-$P/db | wc -l | tr -d " ")" = 6 ] && [ "$(cat $H/192.0.2.10{1..6}/var/lib/randnode/data-22.genesis | sort -u)" = "$H22B" ] && for g in 1 2 3 4 5 6; do cmp -s $H/192.0.2.10$g/var/lib/randnode/beacon-c22/beacon.seed.json $H/192.0.2.10$g/var/lib/randnode/data-22/beacon.seed.json || exit 1; done' +chk "…v0.8.7 installed again over the restored v0.8.6, keeping the one set of backups" '[ "$(installed rand-node)" = "25:$NEW_SHA " ] && [ "$(sha $H/192.0.2.*/root/rand-node.pre-c22 | sort -u)" = "$OLD_SHA" ] && [ "$(for h in $H/192.0.2.{1..19}; do sha $h/root/msauth-vk-production.bin.pre-c22; done | sort -u)" = "$OLD_VK" ]' +t "start on the re-cut" 0 "START-DONE" "$F" start +t "check-limits on the re-cut" 0 "CHECK-LIMITS OK" env "${LN[@]}" "$F" check-limits "$GB" +t "start-archives on the re-cut" 0 "START-ARCHIVES-DONE" "$F" start-archives +t "wait on the re-cut" 0 "CUTOVER DONE: all 25 on $H22B" env "${LN[@]}" "$F" wait "$GB" +# ── the waiver ── +record "" +t "push with SECOND_REBUILD_WAIVED=1 records the waiver" 0 "PUSHED" env "${NEWENV[@]}" "${LN[@]}" SECOND_REBUILD_WAIVED=1 "$F" push "$GB" +chk "…once" '[ "$(grep -c "^second-rebuild-waived: .*genesis $H22B" $CUT_RECORD)" = 1 ]' +# ── archive2, the record ── +chk "archive2 was NEVER contacted: the fake ssh/scp were not once asked for its address ($(sshn) ssh/scp calls in all)" '[ "$(sshn)" -gt 1000 ] && ! grep -qx "$ARCHIVE2_IP" $T/ssh.log && [ ! -e $H/$ARCHIVE2_IP ]' +chk "the roll record names every phase that changed something, and no address" 'for p in stage beacon-init beacon-collect bridge-stop push stop switch start check-limits start-archives wait bridge-start verify halt rollback wait21; do grep -q "^$p: " $ROLL_RECORD || { echo "no $p: line"; exit 1; }; done && ! grep -q "192\.0\.2\." $ROLL_RECORD && grep -q "^skipped: archive2" $ROLL_RECORD && grep -q "CHAIN 22 LIVE" $ROLL_RECORD && grep -q "CONFIRM_HALT_CHAIN21 given" $ROLL_RECORD' +echo "fleet rehearsal (chain 22): $pass passed, $fail failed" +[ $fail = 0 ]