diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5cce5d3943..8117e74e17 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -102,7 +102,7 @@ App 15368 is GitHub Actions, the app the current checks are pinned to. | main.yml | `run-ci` | ubuntu-latest, Python 3.12 | ubuntu-latest, Python 3.12 | ubuntu-latest and macos-latest, Python 3.10 to 3.12 | | main.yml | `unsupported-python-guard` | no | no | Python 3.13 and 3.14 | | packages.yml | `types`, `agent` | Python 3.12 | Python 3.12 | Python 3.10 and 3.12 | -| packages.yml | `flow-lint`, `flow-critical` | yes | yes | yes | +| packages.yml | `flow-lint`, `flow-critical`, `flow-onprem-offline` | yes | yes | yes | | ci.yml | `smoke`, `ci` | yes | yes | no | | packages.yml | `flow-unit` | 3 shards packed by measured time, fast lane (without `nightly_only.txt`) | no | 2 shards | | packages.yml | `flow-native-census` | no | no | Windows and macOS | diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 30656bfec9..d59cd2105d 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -630,6 +630,46 @@ jobs: sys.exit(1 if bad else 0) PY + # The on-prem install path for real: build two signed offline releases of + # openadapt-flow[privacy] from uv.lock on this connected runner, then, as + # root in a network namespace with no route out, install the first from its + # wheelhouse, refuse a tampered second, update to the signed second, roll + # back, and run the air-gap attestation (egress probe, audit chain). pip has + # no index and the machine has no network, so a wheel missing from the + # archive fails the install. deploy/on-prem/bin/test-offline-install.sh. + # First, bin/test-update.sh: the update state machine's refusal, migration + # and recovery paths with a fake venv build, which nothing else runs. + flow-onprem-offline: + name: flow on-prem offline install, update and rollback (egress blocked) + needs: scope + if: github.event.pull_request.draft != true && (needs.scope.outputs.flow == 'true') + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: packages/flow + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + - name: Signed update and rollback state machine (hermetic, fake build) + run: bash deploy/on-prem/bin/test-update.sh + - name: Build two signed offline releases from the lock (connected) + run: bash deploy/on-prem/bin/test-offline-install.sh stage "$RUNNER_TEMP/offline" + - name: Install, update and roll back with egress blocked + # env -u SUDO_UID keeps the command root inside the namespace: + # install.sh owns the release tree as root. + run: >- + sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" + deploy/on-prem/bin/run-egress-blocked.sh + bash deploy/on-prem/bin/test-offline-install.sh verify "$RUNNER_TEMP/offline" + # The agent-framework adapters (openadapt_flow.integrations.agents): each # framework's own agent loop with a scripted model, writing through a real # HTTP app, from the lock with every framework extra (CrewAI pinned beside @@ -1425,6 +1465,7 @@ jobs: - flow-lint - flow-unit - flow-critical + - flow-onprem-offline - flow-agent-frameworks - flow-native-census - flow-linux-atspi @@ -1478,6 +1519,7 @@ jobs: "flow-lint": "flow", "flow-unit": "flow", "flow-critical": "flow", + "flow-onprem-offline": "flow", "flow-agent-frameworks": "flow", "flow-native-census": "flow", "flow-linux-atspi": "flow", diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index a9a36396a9..c68dff4b87 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -21,6 +21,8 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow). | `bin/verify-airgap.sh` | Asserts no outbound path: config/env/deployment URL scan, optional active egress probe, optional audit-chain walk. | REAL (best-effort; firewall is primary) | | `bin/run-egress-blocked.sh` | Runs a command in a loopback-only Linux network namespace, refusing to start unless no other interface or route exists. CI runs the golden free path through it (`tests/e2e/test_egress_blocked_e2e.py`): VERIFIED, zero model calls, and no outbound attempt in strict no-outbound mode. | REAL (Linux, root) | | `bin/test-update.sh` | Hermetic offline proof of signed update, refusal paths, legacy migration, data preservation, rollback/recovery, locks, and audit-chain serialization. | REAL (43 baseline assertions; 45 when local GPG key generation is available) | +| `bin/build-release.sh` | Builds a release archive on a connected host from `uv.lock`: the exact pinned wheels, hash-checked, plus the `openadapt-flow` wheel, a `.sha256`, and an openssl `.sig` when given the key. | REAL | +| `bin/test-offline-install.sh` | Builds two signed releases, then as root with egress blocked installs the real engine from the first, refuses a tampered second, updates to the signed second, rolls back, and runs the air-gap attestation. CI runs it with `bin/test-update.sh` (job `flow-onprem-offline`). | REAL (Linux, root) | | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) | | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) | | `docker-compose.yml` + `Dockerfile` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. | REAL topology; STUB image build (needs offline `./wheels`) | diff --git a/packages/flow/deploy/on-prem/UPDATE.md b/packages/flow/deploy/on-prem/UPDATE.md index 0e594e8395..9346e2c8a8 100644 --- a/packages/flow/deploy/on-prem/UPDATE.md +++ b/packages/flow/deploy/on-prem/UPDATE.md @@ -42,7 +42,23 @@ all survive across any number of updates. ## Build a signed release bundle (off-site, on a connected host) The bundle is a tarball containing a `VERSION` file and an offline pip -wheelhouse, plus a required detached signature and an optional checksum: +wheelhouse, plus a required detached signature and an optional checksum. + +`bin/build-release.sh` builds it from a source checkout: it downloads exactly +the wheels `packages/flow/uv.lock` pins, checks each against the lock's hashes, +adds the `openadapt-flow` wheel, and writes the archive, its `.sha256`, and an +openssl `.sig` when given the signing key. Run it on a host with the same OS, +CPU architecture and Python minor version as the clinic box: + +```bash +deploy/on-prem/bin/build-release.sh --out dist/ --sign-key vendor_priv.pem +``` + +CI builds two releases this way and installs, updates and rolls back the real +engine from them with the network cut off (`bin/test-offline-install.sh`). + +To assemble one by hand instead (this resolves the newest matching +dependencies rather than the locked ones): ```bash # 1. Assemble the payload diff --git a/packages/flow/deploy/on-prem/bin/audit-log.sh b/packages/flow/deploy/on-prem/bin/audit-log.sh index 091add53c3..a963a59a01 100755 --- a/packages/flow/deploy/on-prem/bin/audit-log.sh +++ b/packages/flow/deploy/on-prem/bin/audit-log.sh @@ -68,21 +68,50 @@ chmod 0700 "$(dirname "$AUDIT_LOG")" 2>/dev/null || true # Serialize the read-last-hash + append transaction. O_APPEND prevents byte # interleaving, but without this lock two writers can both chain to the same # parent and produce a structurally broken audit log. +# +# Ownership is the exclusive creation of the pid file, done by the shell itself +# with noclobber (open(2) with O_CREAT|O_EXCL). The exit status of mkdir(1) is +# not trusted for exclusion: the uutils mkdir that Ubuntu 26.04 ships reports +# success to both callers when two processes race to create one directory. lock_dir="${AUDIT_LOG}.lock" +lock_file="$lock_dir/pid" +break_file="${AUDIT_LOG}.lock-breaker" lock_acquired=0 + +# Create $1 holding this process's pid; fails if $1 already exists. +create_exclusive() { + ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null +} + +# Remove a lock whose owner has exited. Between reading the owner and removing +# the lock, that owner can release it and another writer can take it, so the +# removal happens only under a second exclusive lock that serializes breakers, +# and only if the lock still names the same dead owner. Without this, a breaker +# could delete a live writer's lock and let two writers chain to the same parent. +break_stale_lock() { + local stale="$1" current="" + create_exclusive "$break_file" || return 0 + current="$(cat "$lock_file" 2>/dev/null || true)" + if [[ "$current" == "$stale" ]] && ! kill -0 "$stale" 2>/dev/null; then + rm -f "$lock_file" 2>/dev/null || true + fi + rm -f "$break_file" 2>/dev/null || true +} + for ((attempt = 0; attempt < 200; attempt++)); do - if mkdir "$lock_dir" 2>/dev/null; then - printf '%s\n' "$$" > "$lock_dir/pid" - lock_acquired=1 - break + if [[ ! -L "$lock_dir" ]]; then + mkdir -p "$lock_dir" 2>/dev/null || true + if [[ -d "$lock_dir" && ! -L "$lock_dir" ]] && create_exclusive "$lock_file"; then + lock_acquired=1 + break + fi fi owner="" - if [[ -d "$lock_dir" && ! -L "$lock_dir" && -r "$lock_dir/pid" ]]; then - owner="$(cat "$lock_dir/pid" 2>/dev/null || true)" + if [[ -d "$lock_dir" && ! -L "$lock_dir" && -f "$lock_file" && ! -L "$lock_file" ]]; then + owner="$(cat "$lock_file" 2>/dev/null || true)" fi if [[ -n "$owner" && "$owner" =~ ^[0-9]+$ ]] && ! kill -0 "$owner" 2>/dev/null; then - rm -f "$lock_dir/pid" 2>/dev/null || true - rmdir "$lock_dir" 2>/dev/null || true + break_stale_lock "$owner" continue fi sleep 0.05 @@ -92,7 +121,7 @@ if [[ "$lock_acquired" -ne 1 ]]; then exit 1 fi release_audit_lock() { - rm -f "$lock_dir/pid" 2>/dev/null || true + rm -f "$lock_file" 2>/dev/null || true rmdir "$lock_dir" 2>/dev/null || true } trap release_audit_lock EXIT diff --git a/packages/flow/deploy/on-prem/bin/build-release.sh b/packages/flow/deploy/on-prem/bin/build-release.sh new file mode 100755 index 0000000000..5b62d7878f --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/build-release.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# build-release.sh — build an offline release archive from the lock file. +# +# Run on a CONNECTED build host with the same OS, CPU architecture and Python +# minor version as the air-gapped target. Produces, in --out: +# +# release-.tar.gz VERSION, wheels/, requirements.lock.txt +# release-.tar.gz.sha256 checksum (corruption check only) +# release-.tar.gz.sig openssl signature, when --sign-key is given +# +# Every third-party wheel is the exact version pinned in packages/flow/uv.lock +# and is checked against the lock's hashes while downloading (pip +# --require-hashes), so the archive holds what CI tested and nothing newer. +# The openadapt-flow wheel is built from this checkout unless --flow-wheel +# names one. Stage the archive with `install.sh --update` (see UPDATE.md), or +# extract it and pass release/wheels to `install.sh --wheelhouse` for a first +# install. +# +# Usage: +# build-release.sh --out DIR [--flow-wheel FILE] [--sign-key PRIVATE.pem] +# [--python PYTHON] +# +# Needs uv and the target Python (default python3). Exit 0 on success. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FLOW_DIR="$(cd "$HERE/../../.." && pwd)" +OUT="" +FLOW_WHEEL="" +SIGN_KEY="" +PYTHON="python3" + +while [[ $# -gt 0 ]]; do + case "$1" in + --out) OUT="${2:-}"; shift 2 ;; + --flow-wheel) FLOW_WHEEL="${2:-}"; shift 2 ;; + --sign-key) SIGN_KEY="${2:-}"; shift 2 ;; + --python) PYTHON="${2:-}"; shift 2 ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "build-release.sh: unknown arg $1" >&2; exit 2 ;; + esac +done +[[ -n "$OUT" ]] || { echo "build-release.sh: --out DIR is required" >&2; exit 2; } +command -v uv >/dev/null || { echo "build-release.sh: uv is required" >&2; exit 2; } +[[ -f "$FLOW_DIR/uv.lock" ]] || { echo "build-release.sh: no uv.lock in $FLOW_DIR" >&2; exit 2; } +if [[ -n "$SIGN_KEY" && ! -f "$SIGN_KEY" ]]; then + echo "build-release.sh: signing key $SIGN_KEY not found" >&2; exit 2 +fi + +# shellcheck source=bin/lib-release.sh +source "$HERE/lib-release.sh" + +mkdir -p "$OUT" +OUT="$(cd "$OUT" && pwd)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/oa-build-release.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/release/wheels" + +if [[ -z "$FLOW_WHEEL" ]]; then + echo "== building the openadapt-flow wheel from $FLOW_DIR ==" + (cd "$FLOW_DIR" && uv build --wheel --out-dir "$WORK/flow" >/dev/null) + FLOW_WHEEL="$(find "$WORK/flow" -maxdepth 1 -name 'openadapt_flow-*.whl' -print)" +fi +[[ -f "$FLOW_WHEEL" ]] || { echo "build-release.sh: flow wheel not found" >&2; exit 2; } +cp "$FLOW_WHEEL" "$WORK/release/wheels/" +VERSION="$(rel_wheelhouse_version "$WORK/release/wheels")" +rel_validate_version "$VERSION" + +echo "== exporting the locked dependencies of openadapt-flow[privacy] $VERSION ==" +(cd "$FLOW_DIR" && uv export --frozen --no-dev --extra privacy --no-emit-project \ + --format requirements-txt --no-header -o "$WORK/release/requirements.lock.txt" >/dev/null) + +echo "== downloading the locked wheels for $("$PYTHON" -c 'import platform, sys; print(sys.platform, platform.machine(), "py%d.%d" % sys.version_info[:2])') ==" +"$PYTHON" -m pip download --quiet --disable-pip-version-check \ + --only-binary=:all: --no-deps --require-hashes \ + -r "$WORK/release/requirements.lock.txt" -d "$WORK/release/wheels" + +printf '%s\n' "$VERSION" > "$WORK/release/VERSION" +ARCHIVE="$OUT/release-$VERSION.tar.gz" +COPYFILE_DISABLE=1 tar czf "$ARCHIVE" -C "$WORK" release +sha256_of "$ARCHIVE" > "$ARCHIVE.sha256" +if [[ -n "$SIGN_KEY" ]]; then + openssl dgst -sha256 -sign "$SIGN_KEY" -out "$ARCHIVE.sig" "$ARCHIVE" +fi + +count="$(find "$WORK/release/wheels" -name '*.whl' | wc -l | tr -d ' ')" +echo "built $ARCHIVE ($count wheels)" +[[ -n "$SIGN_KEY" ]] && echo "signed $ARCHIVE.sig" || echo "unsigned: sign it before staging (UPDATE.md)" diff --git a/packages/flow/deploy/on-prem/bin/lib-release.sh b/packages/flow/deploy/on-prem/bin/lib-release.sh index 7676dc4900..295dd5c38b 100755 --- a/packages/flow/deploy/on-prem/bin/lib-release.sh +++ b/packages/flow/deploy/on-prem/bin/lib-release.sh @@ -344,32 +344,45 @@ rel_validate_release_dir() { && -x "$release_dir/venv/bin/openadapt-flow" ]] } +# Create $1 holding this process's pid; fails if $1 already exists. The shell +# opens it with noclobber (O_CREAT|O_EXCL), so exclusion does not depend on +# mkdir(1): the uutils mkdir that Ubuntu 26.04 ships reports success to both +# callers when two processes race to create one directory. +_rel_create_exclusive() { + ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null +} + _rel_lock_acquire() { - local root="$1" owner="" + local root="$1" owner="" current="" local lock="$root/releases/.update.lock" + local breaker="$root/releases/.update.lock-breaker" mkdir -p "$root/releases" || return 1 - if mkdir "$lock" 2>/dev/null; then - printf '%s\n' "$$" > "$lock/pid" || { - rmdir "$lock" 2>/dev/null || true - return 1 - } + if [[ -L "$lock" ]]; then + _rel_err "release lock is a symlink; refusing: $lock" + return 1 + fi + mkdir -p "$lock" 2>/dev/null || true + if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then REL_LOCK_DIR="$lock" return 0 fi - if [[ -d "$lock" && ! -L "$lock" && -r "$lock/pid" ]]; then + if [[ -d "$lock" && ! -L "$lock" && -f "$lock/pid" && ! -L "$lock/pid" ]]; then owner="$(cat "$lock/pid" 2>/dev/null || true)" fi if [[ "$owner" =~ ^[0-9]+$ ]] && kill -0 "$owner" 2>/dev/null; then _rel_err "another update or rollback is active (pid $owner)" return 1 fi - if [[ -d "$lock" && ! -L "$lock" ]]; then - rm -f "$lock/pid" 2>/dev/null || true - if rmdir "$lock" 2>/dev/null && mkdir "$lock" 2>/dev/null; then - printf '%s\n' "$$" > "$lock/pid" || { - rmdir "$lock" 2>/dev/null || true - return 1 - } + # The owner has exited (or left an unreadable pid). Remove its lock only + # under the breaker lock and only if it still names that owner, so a + # concurrent breaker cannot remove a lock another process has just taken. + if [[ -n "$owner" ]] && _rel_create_exclusive "$breaker"; then + current="$(cat "$lock/pid" 2>/dev/null || true)" + if [[ "$current" == "$owner" ]]; then + rm -f "$lock/pid" 2>/dev/null || true + fi + rm -f "$breaker" 2>/dev/null || true + if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then REL_LOCK_DIR="$lock" return 0 fi diff --git a/packages/flow/deploy/on-prem/bin/test-offline-install.sh b/packages/flow/deploy/on-prem/bin/test-offline-install.sh new file mode 100755 index 0000000000..c3e6d391cd --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/test-offline-install.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# +# test-offline-install.sh — install, upgrade and roll back the real engine on a +# machine with no network. +# +# Unlike test-update.sh, which fakes the venv build, this installs the real +# openadapt-flow[privacy] wheels pinned by uv.lock, so it proves the offline +# release archive is complete: pip gets no index and the machine has no route +# out, so any missing wheel fails the install. +# +# stage DIR on a connected host: build two signed releases (this checkout +# as V1, the same source as V1.post1 as V2) and a throwaway +# signing key, with bin/build-release.sh. +# verify DIR as root, inside bin/run-egress-blocked.sh: first install from +# V1's wheelhouse, refuse a tampered V2, update to the signed V2, +# roll back to V1, then run the air-gap attestation with its +# egress probe and audit-chain walk. Data must survive each step. +# +# CI runs both (packages.yml, flow on-prem offline install). By hand on Linux: +# bin/test-offline-install.sh stage /tmp/oa-offline +# sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" \ +# bin/run-egress-blocked.sh bin/test-offline-install.sh verify /tmp/oa-offline +# +# Exit 0 when every check passes. + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ONPREM="$(cd "$HERE/.." && pwd)" +FLOW_DIR="$(cd "$ONPREM/../.." && pwd)" +CMD="${1:-}" +DIR="${2:-}" +if [[ -z "$CMD" || -z "$DIR" || ( "$CMD" != stage && "$CMD" != verify ) ]]; then + grep '^#' "$0" | sed 's/^# \{0,1\}//' + exit 2 +fi + +stage() { + set -e + mkdir -p "$DIR/keys" "$DIR/v1" "$DIR/v2" + DIR="$(cd "$DIR" && pwd)" + openssl genrsa -out "$DIR/keys/vendor.pem" 2048 2>/dev/null + openssl rsa -in "$DIR/keys/vendor.pem" -pubout -out "$DIR/keys/vendor.pub" 2>/dev/null + bash "$HERE/build-release.sh" --out "$DIR/v1" --sign-key "$DIR/keys/vendor.pem" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + + # V2: the same source under a new version, built the same way. + local v1 src + v1="$(find "$DIR/v1" -maxdepth 1 -name 'release-*.tar.gz' -print)" + v1="$(basename "$v1" .tar.gz)" + v1="${v1#release-}" + src="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-src.XXXXXX")" + for item in pyproject.toml README.md LICENSE THIRD_PARTY_NOTICES.md \ + public-artifacts.json openadapt_flow schemas; do + cp -R "$FLOW_DIR/$item" "$src/" + done + find "$src" -name __pycache__ -type d -prune -exec rm -rf {} + + sed -i.bak "s/^version = \"$v1\"$/version = \"$v1.post1\"/" "$src/pyproject.toml" + sed -i.bak "s/^__version__ = \"$v1\"$/__version__ = \"$v1.post1\"/" "$src/openadapt_flow/__init__.py" + rm -f "$src/pyproject.toml.bak" "$src/openadapt_flow/__init__.py.bak" + grep -q "^version = \"$v1.post1\"$" "$src/pyproject.toml" + (cd "$src" && uv build --wheel --out-dir "$src/dist" >/dev/null) + bash "$HERE/build-release.sh" --out "$DIR/v2" --sign-key "$DIR/keys/vendor.pem" \ + --flow-wheel "$src/dist/openadapt_flow-$v1.post1-py3-none-any.whl" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + rm -rf "$src" + # The private key never goes near the install. + rm -f "$DIR/keys/vendor.pem" + printf '%s\n' "$v1" > "$DIR/V1" + printf '%s\n' "$v1.post1" > "$DIR/V2" + echo "staged $v1 and $v1.post1 in $DIR" +} + +PASS=0 +FAIL=0 +ok() { printf ' PASS %s\n' "$1"; PASS=$((PASS + 1)); } +bad() { printf ' FAIL %s\n' "$1"; FAIL=$((FAIL + 1)); } +check() { local name="$1"; shift; if "$@"; then ok "$name"; else bad "$name"; fi; } +check_eq() { if [[ "$2" == "$3" ]]; then ok "$1"; else bad "$1 (expected '$2', got '$3')"; fi; } + +verify() { + DIR="$(cd "$DIR" && pwd)" + if [[ "$(id -u)" -ne 0 ]]; then + echo "test-offline-install.sh verify: run as root (install.sh needs it)" >&2; exit 2 + fi + if [[ "${OPENADAPT_EGRESS_BLOCKED:-}" != netns ]]; then + echo "test-offline-install.sh verify: run inside bin/run-egress-blocked.sh" >&2; exit 2 + fi + local v1 v2 py work root config rc + v1="$(cat "$DIR/V1")"; v2="$(cat "$DIR/V2")" + py="${OPENADAPT_ONPREM_PY:-python3}" + work="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-verify.XXXXXX")" + root="$work/srv" + config="$work/onprem.yaml" + cat > "$config" <&1) + } + + echo "== offline install, upgrade and rollback ($v1 -> $v2 -> $v1), no network ==" + check "the machine has no route out" "$py" -c ' +import socket, sys +try: + socket.create_connection(("1.1.1.1", 443), timeout=3) +except OSError: + sys.exit(0) +sys.exit(1)' + + echo "-- first install from the $v1 wheelhouse --" + mkdir -p "$work/media" + tar xzf "$DIR/v1/release-$v1.tar.gz" -C "$work/media" + bash "$ONPREM/install.sh" --config "$config" --wheelhouse "$work/media/release/wheels" \ + > "$work/install.log" 2>&1 + rc=$? + check_eq "install: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/install.log" + check_eq "install: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "install: engine and privacy extra import, version $v1" "$v1" "$(engine_version)" + check_eq "install: engine CLI reports $v1" "openadapt-flow $v1" \ + "$(cd / && "$root/current/venv/bin/openadapt-flow" --version 2>&1)" + + mkdir -p "$root/bundles" "$root/jobs/inbox" + printf 'bundle\n' > "$root/bundles/keepme.txt" + printf 'job\n' > "$root/jobs/inbox/queued-0001.job" + local bundle_sha job_sha + bundle_sha="$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + job_sha="$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + cp "$DIR/keys/vendor.pub" "$root/keys/vendor.pub" + + local arc="$DIR/v2/release-$v2.tar.gz" + echo "-- a tampered $v2 archive is refused --" + cp "$arc" "$work/media/tampered.tar.gz" + printf 'x' >> "$work/media/tampered.tar.gz" + bash "$ONPREM/install.sh" --update --config "$config" --release "$work/media/tampered.tar.gz" \ + --signature "$arc.sig" > "$work/tampered.log" 2>&1 + rc=$? + check "tampered: update refused" test "$rc" -ne 0 + check_eq "tampered: current still $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check "tampered: nothing built for $v2" test ! -e "$root/releases/$v2" + + echo "-- signed update to $v2 --" + bash "$ONPREM/install.sh" --update --config "$config" --release "$arc" \ + --checksum "$arc.sha256" --signature "$arc.sig" > "$work/update.log" 2>&1 + rc=$? + check_eq "update: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/update.log" + check_eq "update: current -> $v2" "$root/releases/$v2" "$(readlink "$root/current")" + check_eq "update: previous -> $v1" "$root/releases/$v1" "$(readlink "$root/previous")" + check_eq "update: engine version $v2" "$v2" "$(engine_version)" + check_eq "update: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + check_eq "update: queued job unchanged" "$job_sha" "$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + + echo "-- rollback to $v1 --" + bash "$ONPREM/install.sh" --rollback --config "$config" > "$work/rollback.log" 2>&1 + rc=$? + check_eq "rollback: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/rollback.log" + check_eq "rollback: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "rollback: engine version $v1" "$v1" "$(engine_version)" + check_eq "rollback: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + + echo "-- air-gap attestation --" + OPENADAPT_FLOW_SCRUB=on bash "$HERE/verify-airgap.sh" --config "$config" --probe --audit \ + > "$work/airgap.log" 2>&1 + rc=$? + check_eq "attestation: exit 0" 0 "$rc" + check "attestation: egress probe could not reach the internet" \ + grep -q "outbound egress is blocked" "$work/airgap.log" + check "attestation: audit chain intact" grep -q "audit chain intact" "$work/airgap.log" + [[ "$rc" -eq 0 ]] || cat "$work/airgap.log" + + rm -rf "$work" + echo + echo "== result: $PASS passed, $FAIL failed ==" + [[ "$FAIL" -eq 0 ]] +} + +"$CMD" diff --git a/packages/flow/deploy/on-prem/bin/test-update.sh b/packages/flow/deploy/on-prem/bin/test-update.sh index c2778b4602..cb16fdecba 100755 --- a/packages/flow/deploy/on-prem/bin/test-update.sh +++ b/packages/flow/deploy/on-prem/bin/test-update.sh @@ -244,6 +244,28 @@ assert "concurrent update lock is refused" test "$?" -ne 0 rm -f "$ROOT/releases/.update.lock/pid" rmdir "$ROOT/releases/.update.lock" +# Ubuntu 26.04 ships uutils mkdir, which reports success to both callers when +# two processes race to create one directory. Model that with a mkdir shim that +# never reports EEXIST: the locks must still exclude, so they cannot rest on +# the exit status of mkdir(1). +EEXIST_SHIM="$WORK/eexist-shim" +mkdir -p "$EEXIST_SHIM" +REAL_MKDIR="$(command -v mkdir)" +cat > "$EEXIST_SHIM/mkdir" </dev/null || true +exit 0 +SHIM_EOF +chmod +x "$EEXIST_SHIM/mkdir" +mkdir "$ROOT/releases/.update.lock" +printf '%s\n' "$$" > "$ROOT/releases/.update.lock/pid" +PATH="$EEXIST_SHIM:$PATH" apply_release "$ROOT" "$ARC6" >/dev/null 2>&1 +assert "update lock holds when mkdir(1) never reports EEXIST" test "$?" -ne 0 +assert_eq "a refused update leaves the live owner's lock" "$$" \ + "$(cat "$ROOT/releases/.update.lock/pid" 2>/dev/null || echo none)" +rm -f "$ROOT/releases/.update.lock/pid" +rmdir "$ROOT/releases/.update.lock" + chmod -x "$ROOT/releases/2.0.0/venv/bin/openadapt-flow" rel_do_rollback "$ROOT" "" >/dev/null 2>&1 assert "incomplete rollback target is refused" test "$?" -ne 0 @@ -290,7 +312,64 @@ if grep -q 'audit chain intact' <<< "$audit_output"; then pass "concurrent audit writers preserve one hash chain" else fail "concurrent audit writers preserve one hash chain" + printf '%s\n' "$audit_output" | sed 's/^/ /' +fi + +SHIM_LOG="$WORK/eexist-audit/audit.log" +for index in 1 2 3 4 5 6 7 8 9 10 11 12; do + OPENADAPT_ONPREM_AUDIT_LOG="$SHIM_LOG" PATH="$EEXIST_SHIM:$PATH" \ + bash "$OPENADAPT_ONPREM_AUDIT_BIN" started "shim-$index" --note test >/dev/null & +done +wait +audit_output="$(OPENADAPT_ONPREM_AUDIT_LOG="$SHIM_LOG" \ + bash "$HERE/verify-airgap.sh" --config "$WORK/missing.yaml" --audit 2>&1 || true)" +if grep -q 'audit chain intact across 12 record' <<< "$audit_output"; then + pass "audit writers stay serialized when mkdir(1) never reports EEXIST" +else + fail "audit writers stay serialized when mkdir(1) never reports EEXIST" + printf '%s\n' "$audit_output" | sed 's/^/ /' +fi + +# A writer that finds a lock owned by an exited process may break it. Force the +# race where, between that writer reading the dead owner and breaking the +# lock, the owner releases and a live writer takes the lock: a shimmed cat +# returns the dead owner once, then hands the lock to a live process. The +# breaker must leave the live writer's lock alone. +RACE="$WORK/stale-race" +mkdir -p "$RACE/shim" +RACE_LOG="$RACE/audit.log" +bash -c 'exit 0' & +dead_pid=$! +wait "$dead_pid" +sleep 30 & +live_pid=$! +mkdir "$RACE_LOG.lock" +printf '%s\n' "$dead_pid" > "$RACE_LOG.lock/pid" +REAL_CAT="$(command -v cat)" +cat > "$RACE/shim/cat" < "$RACE/handed-over" + printf '%s\n' "$live_pid" > "$RACE_LOG.lock/pid" + exit 0 fi +exec "$REAL_CAT" "\$@" +SHIM_EOF +chmod +x "$RACE/shim/cat" +OPENADAPT_ONPREM_AUDIT_LOG="$RACE_LOG" PATH="$RACE/shim:$PATH" \ + bash "$OPENADAPT_ONPREM_AUDIT_BIN" started stale-race --note test >/dev/null 2>&1 & +race_writer=$! +sleep 2 +race_owner="$(cat "$RACE_LOG.lock/pid" 2>/dev/null || echo none)" +assert_eq "stale-lock breaker leaves a live writer's lock alone" "$live_pid" "$race_owner" +rm -f "$RACE_LOG.lock/pid" +rmdir "$RACE_LOG.lock" 2>/dev/null || true +kill "$live_pid" 2>/dev/null || true +wait "$live_pid" 2>/dev/null || true +wait "$race_writer" +assert_eq "writer appends once the live lock is released" 0 "$?" +assert_eq "writer appends exactly one record" 1 "$(wc -l < "$RACE_LOG" | tr -d ' ')" if ! grep -q 'identity evidence' "$OPENADAPT_ONPREM_AUDIT_LOG"; then pass "release audit remains PHI-free" else