From 919b60abe0efde0b0df0166ced4ba0a0b4ff49be Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 18:11:14 -0500 Subject: [PATCH 1/4] on-prem: install, update and roll back the real engine offline in CI bin/build-release.sh builds a release archive from uv.lock (pinned wheels, hash-checked) plus the flow wheel, a checksum and an openssl signature. bin/test-offline-install.sh builds two signed releases, then as root inside run-egress-blocked.sh installs the first from its wheelhouse, refuses a tampered second, updates to the signed second, rolls back and runs the air-gap attestation. The new flow-onprem-offline job runs it, and runs bin/test-update.sh, which no job ran before. --- .github/workflows/README.md | 2 +- .github/workflows/packages.yml | 42 ++++ packages/flow/deploy/on-prem/README.md | 2 + packages/flow/deploy/on-prem/UPDATE.md | 18 +- .../flow/deploy/on-prem/bin/build-release.sh | 90 ++++++++ .../on-prem/bin/test-offline-install.sh | 198 ++++++++++++++++++ 6 files changed, 350 insertions(+), 2 deletions(-) create mode 100755 packages/flow/deploy/on-prem/bin/build-release.sh create mode 100755 packages/flow/deploy/on-prem/bin/test-offline-install.sh diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5cce5d3943..8117e74e17 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -102,7 +102,7 @@ App 15368 is GitHub Actions, the app the current checks are pinned to. | main.yml | `run-ci` | ubuntu-latest, Python 3.12 | ubuntu-latest, Python 3.12 | ubuntu-latest and macos-latest, Python 3.10 to 3.12 | | main.yml | `unsupported-python-guard` | no | no | Python 3.13 and 3.14 | | packages.yml | `types`, `agent` | Python 3.12 | Python 3.12 | Python 3.10 and 3.12 | -| packages.yml | `flow-lint`, `flow-critical` | yes | yes | yes | +| packages.yml | `flow-lint`, `flow-critical`, `flow-onprem-offline` | yes | yes | yes | | ci.yml | `smoke`, `ci` | yes | yes | no | | packages.yml | `flow-unit` | 3 shards packed by measured time, fast lane (without `nightly_only.txt`) | no | 2 shards | | packages.yml | `flow-native-census` | no | no | Windows and macOS | diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 30656bfec9..d59cd2105d 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -630,6 +630,46 @@ jobs: sys.exit(1 if bad else 0) PY + # The on-prem install path for real: build two signed offline releases of + # openadapt-flow[privacy] from uv.lock on this connected runner, then, as + # root in a network namespace with no route out, install the first from its + # wheelhouse, refuse a tampered second, update to the signed second, roll + # back, and run the air-gap attestation (egress probe, audit chain). pip has + # no index and the machine has no network, so a wheel missing from the + # archive fails the install. deploy/on-prem/bin/test-offline-install.sh. + # First, bin/test-update.sh: the update state machine's refusal, migration + # and recovery paths with a fake venv build, which nothing else runs. + flow-onprem-offline: + name: flow on-prem offline install, update and rollback (egress blocked) + needs: scope + if: github.event.pull_request.draft != true && (needs.scope.outputs.flow == 'true') + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: packages/flow + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + - name: Signed update and rollback state machine (hermetic, fake build) + run: bash deploy/on-prem/bin/test-update.sh + - name: Build two signed offline releases from the lock (connected) + run: bash deploy/on-prem/bin/test-offline-install.sh stage "$RUNNER_TEMP/offline" + - name: Install, update and roll back with egress blocked + # env -u SUDO_UID keeps the command root inside the namespace: + # install.sh owns the release tree as root. + run: >- + sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" + deploy/on-prem/bin/run-egress-blocked.sh + bash deploy/on-prem/bin/test-offline-install.sh verify "$RUNNER_TEMP/offline" + # The agent-framework adapters (openadapt_flow.integrations.agents): each # framework's own agent loop with a scripted model, writing through a real # HTTP app, from the lock with every framework extra (CrewAI pinned beside @@ -1425,6 +1465,7 @@ jobs: - flow-lint - flow-unit - flow-critical + - flow-onprem-offline - flow-agent-frameworks - flow-native-census - flow-linux-atspi @@ -1478,6 +1519,7 @@ jobs: "flow-lint": "flow", "flow-unit": "flow", "flow-critical": "flow", + "flow-onprem-offline": "flow", "flow-agent-frameworks": "flow", "flow-native-census": "flow", "flow-linux-atspi": "flow", diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index a9a36396a9..c68dff4b87 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -21,6 +21,8 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow). | `bin/verify-airgap.sh` | Asserts no outbound path: config/env/deployment URL scan, optional active egress probe, optional audit-chain walk. | REAL (best-effort; firewall is primary) | | `bin/run-egress-blocked.sh` | Runs a command in a loopback-only Linux network namespace, refusing to start unless no other interface or route exists. CI runs the golden free path through it (`tests/e2e/test_egress_blocked_e2e.py`): VERIFIED, zero model calls, and no outbound attempt in strict no-outbound mode. | REAL (Linux, root) | | `bin/test-update.sh` | Hermetic offline proof of signed update, refusal paths, legacy migration, data preservation, rollback/recovery, locks, and audit-chain serialization. | REAL (43 baseline assertions; 45 when local GPG key generation is available) | +| `bin/build-release.sh` | Builds a release archive on a connected host from `uv.lock`: the exact pinned wheels, hash-checked, plus the `openadapt-flow` wheel, a `.sha256`, and an openssl `.sig` when given the key. | REAL | +| `bin/test-offline-install.sh` | Builds two signed releases, then as root with egress blocked installs the real engine from the first, refuses a tampered second, updates to the signed second, rolls back, and runs the air-gap attestation. CI runs it with `bin/test-update.sh` (job `flow-onprem-offline`). | REAL (Linux, root) | | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) | | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) | | `docker-compose.yml` + `Dockerfile` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. | REAL topology; STUB image build (needs offline `./wheels`) | diff --git a/packages/flow/deploy/on-prem/UPDATE.md b/packages/flow/deploy/on-prem/UPDATE.md index 0e594e8395..9346e2c8a8 100644 --- a/packages/flow/deploy/on-prem/UPDATE.md +++ b/packages/flow/deploy/on-prem/UPDATE.md @@ -42,7 +42,23 @@ all survive across any number of updates. ## Build a signed release bundle (off-site, on a connected host) The bundle is a tarball containing a `VERSION` file and an offline pip -wheelhouse, plus a required detached signature and an optional checksum: +wheelhouse, plus a required detached signature and an optional checksum. + +`bin/build-release.sh` builds it from a source checkout: it downloads exactly +the wheels `packages/flow/uv.lock` pins, checks each against the lock's hashes, +adds the `openadapt-flow` wheel, and writes the archive, its `.sha256`, and an +openssl `.sig` when given the signing key. Run it on a host with the same OS, +CPU architecture and Python minor version as the clinic box: + +```bash +deploy/on-prem/bin/build-release.sh --out dist/ --sign-key vendor_priv.pem +``` + +CI builds two releases this way and installs, updates and rolls back the real +engine from them with the network cut off (`bin/test-offline-install.sh`). + +To assemble one by hand instead (this resolves the newest matching +dependencies rather than the locked ones): ```bash # 1. Assemble the payload diff --git a/packages/flow/deploy/on-prem/bin/build-release.sh b/packages/flow/deploy/on-prem/bin/build-release.sh new file mode 100755 index 0000000000..5b62d7878f --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/build-release.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# build-release.sh — build an offline release archive from the lock file. +# +# Run on a CONNECTED build host with the same OS, CPU architecture and Python +# minor version as the air-gapped target. Produces, in --out: +# +# release-.tar.gz VERSION, wheels/, requirements.lock.txt +# release-.tar.gz.sha256 checksum (corruption check only) +# release-.tar.gz.sig openssl signature, when --sign-key is given +# +# Every third-party wheel is the exact version pinned in packages/flow/uv.lock +# and is checked against the lock's hashes while downloading (pip +# --require-hashes), so the archive holds what CI tested and nothing newer. +# The openadapt-flow wheel is built from this checkout unless --flow-wheel +# names one. Stage the archive with `install.sh --update` (see UPDATE.md), or +# extract it and pass release/wheels to `install.sh --wheelhouse` for a first +# install. +# +# Usage: +# build-release.sh --out DIR [--flow-wheel FILE] [--sign-key PRIVATE.pem] +# [--python PYTHON] +# +# Needs uv and the target Python (default python3). Exit 0 on success. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FLOW_DIR="$(cd "$HERE/../../.." && pwd)" +OUT="" +FLOW_WHEEL="" +SIGN_KEY="" +PYTHON="python3" + +while [[ $# -gt 0 ]]; do + case "$1" in + --out) OUT="${2:-}"; shift 2 ;; + --flow-wheel) FLOW_WHEEL="${2:-}"; shift 2 ;; + --sign-key) SIGN_KEY="${2:-}"; shift 2 ;; + --python) PYTHON="${2:-}"; shift 2 ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "build-release.sh: unknown arg $1" >&2; exit 2 ;; + esac +done +[[ -n "$OUT" ]] || { echo "build-release.sh: --out DIR is required" >&2; exit 2; } +command -v uv >/dev/null || { echo "build-release.sh: uv is required" >&2; exit 2; } +[[ -f "$FLOW_DIR/uv.lock" ]] || { echo "build-release.sh: no uv.lock in $FLOW_DIR" >&2; exit 2; } +if [[ -n "$SIGN_KEY" && ! -f "$SIGN_KEY" ]]; then + echo "build-release.sh: signing key $SIGN_KEY not found" >&2; exit 2 +fi + +# shellcheck source=bin/lib-release.sh +source "$HERE/lib-release.sh" + +mkdir -p "$OUT" +OUT="$(cd "$OUT" && pwd)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/oa-build-release.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/release/wheels" + +if [[ -z "$FLOW_WHEEL" ]]; then + echo "== building the openadapt-flow wheel from $FLOW_DIR ==" + (cd "$FLOW_DIR" && uv build --wheel --out-dir "$WORK/flow" >/dev/null) + FLOW_WHEEL="$(find "$WORK/flow" -maxdepth 1 -name 'openadapt_flow-*.whl' -print)" +fi +[[ -f "$FLOW_WHEEL" ]] || { echo "build-release.sh: flow wheel not found" >&2; exit 2; } +cp "$FLOW_WHEEL" "$WORK/release/wheels/" +VERSION="$(rel_wheelhouse_version "$WORK/release/wheels")" +rel_validate_version "$VERSION" + +echo "== exporting the locked dependencies of openadapt-flow[privacy] $VERSION ==" +(cd "$FLOW_DIR" && uv export --frozen --no-dev --extra privacy --no-emit-project \ + --format requirements-txt --no-header -o "$WORK/release/requirements.lock.txt" >/dev/null) + +echo "== downloading the locked wheels for $("$PYTHON" -c 'import platform, sys; print(sys.platform, platform.machine(), "py%d.%d" % sys.version_info[:2])') ==" +"$PYTHON" -m pip download --quiet --disable-pip-version-check \ + --only-binary=:all: --no-deps --require-hashes \ + -r "$WORK/release/requirements.lock.txt" -d "$WORK/release/wheels" + +printf '%s\n' "$VERSION" > "$WORK/release/VERSION" +ARCHIVE="$OUT/release-$VERSION.tar.gz" +COPYFILE_DISABLE=1 tar czf "$ARCHIVE" -C "$WORK" release +sha256_of "$ARCHIVE" > "$ARCHIVE.sha256" +if [[ -n "$SIGN_KEY" ]]; then + openssl dgst -sha256 -sign "$SIGN_KEY" -out "$ARCHIVE.sig" "$ARCHIVE" +fi + +count="$(find "$WORK/release/wheels" -name '*.whl' | wc -l | tr -d ' ')" +echo "built $ARCHIVE ($count wheels)" +[[ -n "$SIGN_KEY" ]] && echo "signed $ARCHIVE.sig" || echo "unsigned: sign it before staging (UPDATE.md)" diff --git a/packages/flow/deploy/on-prem/bin/test-offline-install.sh b/packages/flow/deploy/on-prem/bin/test-offline-install.sh new file mode 100755 index 0000000000..e910abf488 --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/test-offline-install.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# +# test-offline-install.sh — install, upgrade and roll back the real engine on a +# machine with no network. +# +# Unlike test-update.sh, which fakes the venv build, this installs the real +# openadapt-flow[privacy] wheels pinned by uv.lock, so it proves the offline +# release archive is complete: pip gets no index and the machine has no route +# out, so any missing wheel fails the install. +# +# stage DIR on a connected host: build two signed releases (this checkout +# as V1, the same source as V1.post1 as V2) and a throwaway +# signing key, with bin/build-release.sh. +# verify DIR as root, inside bin/run-egress-blocked.sh: first install from +# V1's wheelhouse, refuse a tampered V2, update to the signed V2, +# roll back to V1, then run the air-gap attestation with its +# egress probe and audit-chain walk. Data must survive each step. +# +# CI runs both (packages.yml, flow on-prem offline install). By hand on Linux: +# bin/test-offline-install.sh stage /tmp/oa-offline +# sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" \ +# bin/run-egress-blocked.sh bin/test-offline-install.sh verify /tmp/oa-offline +# +# Exit 0 when every check passes. + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ONPREM="$(cd "$HERE/.." && pwd)" +FLOW_DIR="$(cd "$ONPREM/../.." && pwd)" +CMD="${1:-}" +DIR="${2:-}" +if [[ -z "$CMD" || -z "$DIR" || ( "$CMD" != stage && "$CMD" != verify ) ]]; then + grep '^#' "$0" | sed 's/^# \{0,1\}//' + exit 2 +fi + +stage() { + set -e + mkdir -p "$DIR/keys" "$DIR/v1" "$DIR/v2" + DIR="$(cd "$DIR" && pwd)" + openssl genrsa -out "$DIR/keys/vendor.pem" 2048 2>/dev/null + openssl rsa -in "$DIR/keys/vendor.pem" -pubout -out "$DIR/keys/vendor.pub" 2>/dev/null + bash "$HERE/build-release.sh" --out "$DIR/v1" --sign-key "$DIR/keys/vendor.pem" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + + # V2: the same source under a new version, built the same way. + local v1 src + v1="$(find "$DIR/v1" -maxdepth 1 -name 'release-*.tar.gz' -print)" + v1="$(basename "$v1" .tar.gz)" + v1="${v1#release-}" + src="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-src.XXXXXX")" + for item in pyproject.toml README.md LICENSE THIRD_PARTY_NOTICES.md \ + public-artifacts.json openadapt_flow schemas; do + cp -R "$FLOW_DIR/$item" "$src/" + done + find "$src" -name __pycache__ -type d -prune -exec rm -rf {} + + sed -i.bak "s/^version = \"$v1\"$/version = \"$v1.post1\"/" "$src/pyproject.toml" + sed -i.bak "s/^__version__ = \"$v1\"$/__version__ = \"$v1.post1\"/" "$src/openadapt_flow/__init__.py" + rm -f "$src/pyproject.toml.bak" "$src/openadapt_flow/__init__.py.bak" + grep -q "^version = \"$v1.post1\"$" "$src/pyproject.toml" + (cd "$src" && uv build --wheel --out-dir "$src/dist" >/dev/null) + bash "$HERE/build-release.sh" --out "$DIR/v2" --sign-key "$DIR/keys/vendor.pem" \ + --flow-wheel "$src/dist/openadapt_flow-$v1.post1-py3-none-any.whl" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + rm -rf "$src" + # The private key never goes near the install. + rm -f "$DIR/keys/vendor.pem" + printf '%s\n' "$v1" > "$DIR/V1" + printf '%s\n' "$v1.post1" > "$DIR/V2" + echo "staged $v1 and $v1.post1 in $DIR" +} + +PASS=0 +FAIL=0 +ok() { printf ' PASS %s\n' "$1"; PASS=$((PASS + 1)); } +bad() { printf ' FAIL %s\n' "$1"; FAIL=$((FAIL + 1)); } +check() { local name="$1"; shift; if "$@"; then ok "$name"; else bad "$name"; fi; } +check_eq() { if [[ "$2" == "$3" ]]; then ok "$1"; else bad "$1 (expected '$2', got '$3')"; fi; } + +verify() { + DIR="$(cd "$DIR" && pwd)" + if [[ "$(id -u)" -ne 0 ]]; then + echo "test-offline-install.sh verify: run as root (install.sh needs it)" >&2; exit 2 + fi + if [[ "${OPENADAPT_EGRESS_BLOCKED:-}" != netns ]]; then + echo "test-offline-install.sh verify: run inside bin/run-egress-blocked.sh" >&2; exit 2 + fi + local v1 v2 py work root config rc + v1="$(cat "$DIR/V1")"; v2="$(cat "$DIR/V2")" + py="${OPENADAPT_ONPREM_PY:-python3}" + work="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-verify.XXXXXX")" + root="$work/srv" + config="$work/onprem.yaml" + cat > "$config" <&1 + } + + echo "== offline install, upgrade and rollback ($v1 -> $v2 -> $v1), no network ==" + check "the machine has no route out" "$py" -c ' +import socket, sys +try: + socket.create_connection(("1.1.1.1", 443), timeout=3) +except OSError: + sys.exit(0) +sys.exit(1)' + + echo "-- first install from the $v1 wheelhouse --" + mkdir -p "$work/media" + tar xzf "$DIR/v1/release-$v1.tar.gz" -C "$work/media" + bash "$ONPREM/install.sh" --config "$config" --wheelhouse "$work/media/release/wheels" \ + > "$work/install.log" 2>&1 + rc=$? + check_eq "install: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/install.log" + check_eq "install: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "install: engine and privacy extra import, version $v1" "$v1" "$(engine_version)" + check "install: engine CLI runs" "$root/current/venv/bin/openadapt-flow" --help + + mkdir -p "$root/bundles" "$root/jobs/inbox" + printf 'bundle\n' > "$root/bundles/keepme.txt" + printf 'job\n' > "$root/jobs/inbox/queued-0001.job" + local bundle_sha job_sha + bundle_sha="$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + job_sha="$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + cp "$DIR/keys/vendor.pub" "$root/keys/vendor.pub" + + local arc="$DIR/v2/release-$v2.tar.gz" + echo "-- a tampered $v2 archive is refused --" + cp "$arc" "$work/media/tampered.tar.gz" + printf 'x' >> "$work/media/tampered.tar.gz" + bash "$ONPREM/install.sh" --update --config "$config" --release "$work/media/tampered.tar.gz" \ + --signature "$arc.sig" > "$work/tampered.log" 2>&1 + rc=$? + check "tampered: update refused" test "$rc" -ne 0 + check_eq "tampered: current still $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check "tampered: nothing built for $v2" test ! -e "$root/releases/$v2" + + echo "-- signed update to $v2 --" + bash "$ONPREM/install.sh" --update --config "$config" --release "$arc" \ + --checksum "$arc.sha256" --signature "$arc.sig" > "$work/update.log" 2>&1 + rc=$? + check_eq "update: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/update.log" + check_eq "update: current -> $v2" "$root/releases/$v2" "$(readlink "$root/current")" + check_eq "update: previous -> $v1" "$root/releases/$v1" "$(readlink "$root/previous")" + check_eq "update: engine version $v2" "$v2" "$(engine_version)" + check_eq "update: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + check_eq "update: queued job unchanged" "$job_sha" "$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + + echo "-- rollback to $v1 --" + bash "$ONPREM/install.sh" --rollback --config "$config" > "$work/rollback.log" 2>&1 + rc=$? + check_eq "rollback: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/rollback.log" + check_eq "rollback: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "rollback: engine version $v1" "$v1" "$(engine_version)" + check_eq "rollback: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + + echo "-- air-gap attestation --" + OPENADAPT_FLOW_SCRUB=on bash "$HERE/verify-airgap.sh" --config "$config" --probe --audit \ + > "$work/airgap.log" 2>&1 + rc=$? + check_eq "attestation: exit 0" 0 "$rc" + check "attestation: egress probe could not reach the internet" \ + grep -q "outbound egress is blocked" "$work/airgap.log" + check "attestation: audit chain intact" grep -q "audit chain intact" "$work/airgap.log" + [[ "$rc" -eq 0 ]] || cat "$work/airgap.log" + + rm -rf "$work" + echo + echo "== result: $PASS passed, $FAIL failed ==" + [[ "$FAIL" -eq 0 ]] +} + +"$CMD" From f5eb3c39c35e70ce0de4b98e52023629300f939c Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 18:17:42 -0500 Subject: [PATCH 2/4] on-prem: check the engine version from the release venv, not the checkout The version check ran python -c from packages/flow, so it imported the source checkout and reported 2.6.0 after the update to 2.6.0.post1. Run it isolated from /, require the module to live in the venv, and compare the module with the installed distribution and the CLI's --version. --- .../deploy/on-prem/bin/test-offline-install.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/flow/deploy/on-prem/bin/test-offline-install.sh b/packages/flow/deploy/on-prem/bin/test-offline-install.sh index e910abf488..c3e6d391cd 100755 --- a/packages/flow/deploy/on-prem/bin/test-offline-install.sh +++ b/packages/flow/deploy/on-prem/bin/test-offline-install.sh @@ -113,9 +113,20 @@ updates: YAML export OPENADAPT_ONPREM_PY="$py" export OPENADAPT_ONPREM_AUDIT_LOG="$root/audit/audit.log" + # -I and cd /: import only what the release venv installed, never a source + # checkout in the working directory. Prints the installed distribution's + # version, and fails unless the module is the one inside the venv. engine_version() { - "$root/current/venv/bin/python" -c \ - 'import openadapt_flow, presidio_analyzer, openadapt_privacy; print(openadapt_flow.__version__)' 2>&1 + (cd / && "$root/current/venv/bin/python" -I -c ' +import importlib.metadata, os, sys +import openadapt_flow, openadapt_privacy, presidio_analyzer +here = os.path.realpath(openadapt_flow.__file__) +if not here.startswith(os.path.realpath(sys.prefix) + os.sep): + sys.exit("openadapt_flow imported from outside the release venv: " + here) +version = importlib.metadata.version("openadapt-flow") +if openadapt_flow.__version__ != version: + sys.exit("module says %s, distribution says %s" % (openadapt_flow.__version__, version)) +print(version)' 2>&1) } echo "== offline install, upgrade and rollback ($v1 -> $v2 -> $v1), no network ==" @@ -137,7 +148,8 @@ sys.exit(1)' [[ "$rc" -eq 0 ]] || tail -n 40 "$work/install.log" check_eq "install: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" check_eq "install: engine and privacy extra import, version $v1" "$v1" "$(engine_version)" - check "install: engine CLI runs" "$root/current/venv/bin/openadapt-flow" --help + check_eq "install: engine CLI reports $v1" "openadapt-flow $v1" \ + "$(cd / && "$root/current/venv/bin/openadapt-flow" --version 2>&1)" mkdir -p "$root/bundles" "$root/jobs/inbox" printf 'bundle\n' > "$root/bundles/keepme.txt" From efbb57c2abebf39fc5626fbb5c45d4de28020c60 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 18:32:14 -0500 Subject: [PATCH 3/4] on-prem audit: do not break a lock that a live writer just took A writer that found the audit lock owned by an exited process removed it without re-checking. If the owner released between the read and the removal and another writer took the lock, the breaker deleted the live lock and two writers could chain to the same parent. Break stale locks only under a second lock and only if the lock still names the same dead owner. Add a deterministic test that forces the handover, and print the verifier output when the concurrent-writer check fails. --- packages/flow/deploy/on-prem/bin/audit-log.sh | 21 +++++++++- .../flow/deploy/on-prem/bin/test-update.sh | 42 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/packages/flow/deploy/on-prem/bin/audit-log.sh b/packages/flow/deploy/on-prem/bin/audit-log.sh index 091add53c3..9b51ce4e4a 100755 --- a/packages/flow/deploy/on-prem/bin/audit-log.sh +++ b/packages/flow/deploy/on-prem/bin/audit-log.sh @@ -69,7 +69,25 @@ chmod 0700 "$(dirname "$AUDIT_LOG")" 2>/dev/null || true # interleaving, but without this lock two writers can both chain to the same # parent and produce a structurally broken audit log. lock_dir="${AUDIT_LOG}.lock" +break_dir="${AUDIT_LOG}.lock-break" lock_acquired=0 + +# Remove a lock whose owner has exited. Between reading the owner and removing +# the lock, that owner can release it and another writer can take it, so the +# removal happens only under a second lock that serializes breakers, and only +# if the lock still names the same dead owner. Without this, a breaker could +# delete a live writer's lock and let two writers chain to the same parent. +break_stale_lock() { + local stale="$1" current="" + mkdir "$break_dir" 2>/dev/null || return 0 + current="$(cat "$lock_dir/pid" 2>/dev/null || true)" + if [[ "$current" == "$stale" ]] && ! kill -0 "$stale" 2>/dev/null; then + rm -f "$lock_dir/pid" 2>/dev/null || true + rmdir "$lock_dir" 2>/dev/null || true + fi + rmdir "$break_dir" 2>/dev/null || true +} + for ((attempt = 0; attempt < 200; attempt++)); do if mkdir "$lock_dir" 2>/dev/null; then printf '%s\n' "$$" > "$lock_dir/pid" @@ -81,8 +99,7 @@ for ((attempt = 0; attempt < 200; attempt++)); do owner="$(cat "$lock_dir/pid" 2>/dev/null || true)" fi if [[ -n "$owner" && "$owner" =~ ^[0-9]+$ ]] && ! kill -0 "$owner" 2>/dev/null; then - rm -f "$lock_dir/pid" 2>/dev/null || true - rmdir "$lock_dir" 2>/dev/null || true + break_stale_lock "$owner" continue fi sleep 0.05 diff --git a/packages/flow/deploy/on-prem/bin/test-update.sh b/packages/flow/deploy/on-prem/bin/test-update.sh index c2778b4602..26be744811 100755 --- a/packages/flow/deploy/on-prem/bin/test-update.sh +++ b/packages/flow/deploy/on-prem/bin/test-update.sh @@ -290,7 +290,49 @@ if grep -q 'audit chain intact' <<< "$audit_output"; then pass "concurrent audit writers preserve one hash chain" else fail "concurrent audit writers preserve one hash chain" + printf '%s\n' "$audit_output" | sed 's/^/ /' fi + +# A writer that finds a lock owned by an exited process may break it. Force the +# race where, between that writer reading the dead owner and breaking the +# lock, the owner releases and a live writer takes the lock: a shimmed cat +# returns the dead owner once, then hands the lock to a live process. The +# breaker must leave the live writer's lock alone. +RACE="$WORK/stale-race" +mkdir -p "$RACE/shim" +RACE_LOG="$RACE/audit.log" +bash -c 'exit 0' & +dead_pid=$! +wait "$dead_pid" +sleep 30 & +live_pid=$! +mkdir "$RACE_LOG.lock" +printf '%s\n' "$dead_pid" > "$RACE_LOG.lock/pid" +REAL_CAT="$(command -v cat)" +cat > "$RACE/shim/cat" < "$RACE/handed-over" + printf '%s\n' "$live_pid" > "$RACE_LOG.lock/pid" + exit 0 +fi +exec "$REAL_CAT" "\$@" +SHIM_EOF +chmod +x "$RACE/shim/cat" +OPENADAPT_ONPREM_AUDIT_LOG="$RACE_LOG" PATH="$RACE/shim:$PATH" \ + bash "$OPENADAPT_ONPREM_AUDIT_BIN" started stale-race --note test >/dev/null 2>&1 & +race_writer=$! +sleep 2 +race_owner="$(cat "$RACE_LOG.lock/pid" 2>/dev/null || echo none)" +assert_eq "stale-lock breaker leaves a live writer's lock alone" "$live_pid" "$race_owner" +rm -f "$RACE_LOG.lock/pid" +rmdir "$RACE_LOG.lock" 2>/dev/null || true +kill "$live_pid" 2>/dev/null || true +wait "$live_pid" 2>/dev/null || true +wait "$race_writer" +assert_eq "writer appends once the live lock is released" 0 "$?" +assert_eq "writer appends exactly one record" 1 "$(wc -l < "$RACE_LOG" | tr -d ' ')" if ! grep -q 'identity evidence' "$OPENADAPT_ONPREM_AUDIT_LOG"; then pass "release audit remains PHI-free" else From 7770839e32569ff40e1774672b661609a3813b6b Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 19:15:34 -0500 Subject: [PATCH 4/4] on-prem locks: take ownership with an exclusive file create, not mkdir(1) Ubuntu 26.04 ships uutils mkdir, which reports success to both callers when two processes race to create one directory. The audit-log writer lock and the release update lock both treated a zero exit from mkdir as ownership, so on that image concurrent audit writers chained to the same parent (the "concurrent audit writers preserve one hash chain" CI failure) and a second update could proceed beside a live one. Ownership is now the shell's own noclobber create of the pid file inside the lock directory (O_CREAT|O_EXCL), and the stale-lock breaker lock is an exclusive file too. Tests model the uutils behavior with a mkdir shim that never reports EEXIST; on the previous code they fail. --- packages/flow/deploy/on-prem/bin/audit-log.sh | 44 ++++++++++++------- .../flow/deploy/on-prem/bin/lib-release.sh | 41 +++++++++++------ .../flow/deploy/on-prem/bin/test-update.sh | 37 ++++++++++++++++ 3 files changed, 92 insertions(+), 30 deletions(-) diff --git a/packages/flow/deploy/on-prem/bin/audit-log.sh b/packages/flow/deploy/on-prem/bin/audit-log.sh index 9b51ce4e4a..a963a59a01 100755 --- a/packages/flow/deploy/on-prem/bin/audit-log.sh +++ b/packages/flow/deploy/on-prem/bin/audit-log.sh @@ -68,35 +68,47 @@ chmod 0700 "$(dirname "$AUDIT_LOG")" 2>/dev/null || true # Serialize the read-last-hash + append transaction. O_APPEND prevents byte # interleaving, but without this lock two writers can both chain to the same # parent and produce a structurally broken audit log. +# +# Ownership is the exclusive creation of the pid file, done by the shell itself +# with noclobber (open(2) with O_CREAT|O_EXCL). The exit status of mkdir(1) is +# not trusted for exclusion: the uutils mkdir that Ubuntu 26.04 ships reports +# success to both callers when two processes race to create one directory. lock_dir="${AUDIT_LOG}.lock" -break_dir="${AUDIT_LOG}.lock-break" +lock_file="$lock_dir/pid" +break_file="${AUDIT_LOG}.lock-breaker" lock_acquired=0 +# Create $1 holding this process's pid; fails if $1 already exists. +create_exclusive() { + ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null +} + # Remove a lock whose owner has exited. Between reading the owner and removing # the lock, that owner can release it and another writer can take it, so the -# removal happens only under a second lock that serializes breakers, and only -# if the lock still names the same dead owner. Without this, a breaker could -# delete a live writer's lock and let two writers chain to the same parent. +# removal happens only under a second exclusive lock that serializes breakers, +# and only if the lock still names the same dead owner. Without this, a breaker +# could delete a live writer's lock and let two writers chain to the same parent. break_stale_lock() { local stale="$1" current="" - mkdir "$break_dir" 2>/dev/null || return 0 - current="$(cat "$lock_dir/pid" 2>/dev/null || true)" + create_exclusive "$break_file" || return 0 + current="$(cat "$lock_file" 2>/dev/null || true)" if [[ "$current" == "$stale" ]] && ! kill -0 "$stale" 2>/dev/null; then - rm -f "$lock_dir/pid" 2>/dev/null || true - rmdir "$lock_dir" 2>/dev/null || true + rm -f "$lock_file" 2>/dev/null || true fi - rmdir "$break_dir" 2>/dev/null || true + rm -f "$break_file" 2>/dev/null || true } for ((attempt = 0; attempt < 200; attempt++)); do - if mkdir "$lock_dir" 2>/dev/null; then - printf '%s\n' "$$" > "$lock_dir/pid" - lock_acquired=1 - break + if [[ ! -L "$lock_dir" ]]; then + mkdir -p "$lock_dir" 2>/dev/null || true + if [[ -d "$lock_dir" && ! -L "$lock_dir" ]] && create_exclusive "$lock_file"; then + lock_acquired=1 + break + fi fi owner="" - if [[ -d "$lock_dir" && ! -L "$lock_dir" && -r "$lock_dir/pid" ]]; then - owner="$(cat "$lock_dir/pid" 2>/dev/null || true)" + if [[ -d "$lock_dir" && ! -L "$lock_dir" && -f "$lock_file" && ! -L "$lock_file" ]]; then + owner="$(cat "$lock_file" 2>/dev/null || true)" fi if [[ -n "$owner" && "$owner" =~ ^[0-9]+$ ]] && ! kill -0 "$owner" 2>/dev/null; then break_stale_lock "$owner" @@ -109,7 +121,7 @@ if [[ "$lock_acquired" -ne 1 ]]; then exit 1 fi release_audit_lock() { - rm -f "$lock_dir/pid" 2>/dev/null || true + rm -f "$lock_file" 2>/dev/null || true rmdir "$lock_dir" 2>/dev/null || true } trap release_audit_lock EXIT diff --git a/packages/flow/deploy/on-prem/bin/lib-release.sh b/packages/flow/deploy/on-prem/bin/lib-release.sh index 7676dc4900..295dd5c38b 100755 --- a/packages/flow/deploy/on-prem/bin/lib-release.sh +++ b/packages/flow/deploy/on-prem/bin/lib-release.sh @@ -344,32 +344,45 @@ rel_validate_release_dir() { && -x "$release_dir/venv/bin/openadapt-flow" ]] } +# Create $1 holding this process's pid; fails if $1 already exists. The shell +# opens it with noclobber (O_CREAT|O_EXCL), so exclusion does not depend on +# mkdir(1): the uutils mkdir that Ubuntu 26.04 ships reports success to both +# callers when two processes race to create one directory. +_rel_create_exclusive() { + ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null +} + _rel_lock_acquire() { - local root="$1" owner="" + local root="$1" owner="" current="" local lock="$root/releases/.update.lock" + local breaker="$root/releases/.update.lock-breaker" mkdir -p "$root/releases" || return 1 - if mkdir "$lock" 2>/dev/null; then - printf '%s\n' "$$" > "$lock/pid" || { - rmdir "$lock" 2>/dev/null || true - return 1 - } + if [[ -L "$lock" ]]; then + _rel_err "release lock is a symlink; refusing: $lock" + return 1 + fi + mkdir -p "$lock" 2>/dev/null || true + if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then REL_LOCK_DIR="$lock" return 0 fi - if [[ -d "$lock" && ! -L "$lock" && -r "$lock/pid" ]]; then + if [[ -d "$lock" && ! -L "$lock" && -f "$lock/pid" && ! -L "$lock/pid" ]]; then owner="$(cat "$lock/pid" 2>/dev/null || true)" fi if [[ "$owner" =~ ^[0-9]+$ ]] && kill -0 "$owner" 2>/dev/null; then _rel_err "another update or rollback is active (pid $owner)" return 1 fi - if [[ -d "$lock" && ! -L "$lock" ]]; then - rm -f "$lock/pid" 2>/dev/null || true - if rmdir "$lock" 2>/dev/null && mkdir "$lock" 2>/dev/null; then - printf '%s\n' "$$" > "$lock/pid" || { - rmdir "$lock" 2>/dev/null || true - return 1 - } + # The owner has exited (or left an unreadable pid). Remove its lock only + # under the breaker lock and only if it still names that owner, so a + # concurrent breaker cannot remove a lock another process has just taken. + if [[ -n "$owner" ]] && _rel_create_exclusive "$breaker"; then + current="$(cat "$lock/pid" 2>/dev/null || true)" + if [[ "$current" == "$owner" ]]; then + rm -f "$lock/pid" 2>/dev/null || true + fi + rm -f "$breaker" 2>/dev/null || true + if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then REL_LOCK_DIR="$lock" return 0 fi diff --git a/packages/flow/deploy/on-prem/bin/test-update.sh b/packages/flow/deploy/on-prem/bin/test-update.sh index 26be744811..cb16fdecba 100755 --- a/packages/flow/deploy/on-prem/bin/test-update.sh +++ b/packages/flow/deploy/on-prem/bin/test-update.sh @@ -244,6 +244,28 @@ assert "concurrent update lock is refused" test "$?" -ne 0 rm -f "$ROOT/releases/.update.lock/pid" rmdir "$ROOT/releases/.update.lock" +# Ubuntu 26.04 ships uutils mkdir, which reports success to both callers when +# two processes race to create one directory. Model that with a mkdir shim that +# never reports EEXIST: the locks must still exclude, so they cannot rest on +# the exit status of mkdir(1). +EEXIST_SHIM="$WORK/eexist-shim" +mkdir -p "$EEXIST_SHIM" +REAL_MKDIR="$(command -v mkdir)" +cat > "$EEXIST_SHIM/mkdir" </dev/null || true +exit 0 +SHIM_EOF +chmod +x "$EEXIST_SHIM/mkdir" +mkdir "$ROOT/releases/.update.lock" +printf '%s\n' "$$" > "$ROOT/releases/.update.lock/pid" +PATH="$EEXIST_SHIM:$PATH" apply_release "$ROOT" "$ARC6" >/dev/null 2>&1 +assert "update lock holds when mkdir(1) never reports EEXIST" test "$?" -ne 0 +assert_eq "a refused update leaves the live owner's lock" "$$" \ + "$(cat "$ROOT/releases/.update.lock/pid" 2>/dev/null || echo none)" +rm -f "$ROOT/releases/.update.lock/pid" +rmdir "$ROOT/releases/.update.lock" + chmod -x "$ROOT/releases/2.0.0/venv/bin/openadapt-flow" rel_do_rollback "$ROOT" "" >/dev/null 2>&1 assert "incomplete rollback target is refused" test "$?" -ne 0 @@ -293,6 +315,21 @@ else printf '%s\n' "$audit_output" | sed 's/^/ /' fi +SHIM_LOG="$WORK/eexist-audit/audit.log" +for index in 1 2 3 4 5 6 7 8 9 10 11 12; do + OPENADAPT_ONPREM_AUDIT_LOG="$SHIM_LOG" PATH="$EEXIST_SHIM:$PATH" \ + bash "$OPENADAPT_ONPREM_AUDIT_BIN" started "shim-$index" --note test >/dev/null & +done +wait +audit_output="$(OPENADAPT_ONPREM_AUDIT_LOG="$SHIM_LOG" \ + bash "$HERE/verify-airgap.sh" --config "$WORK/missing.yaml" --audit 2>&1 || true)" +if grep -q 'audit chain intact across 12 record' <<< "$audit_output"; then + pass "audit writers stay serialized when mkdir(1) never reports EEXIST" +else + fail "audit writers stay serialized when mkdir(1) never reports EEXIST" + printf '%s\n' "$audit_output" | sed 's/^/ /' +fi + # A writer that finds a lock owned by an exited process may break it. Force the # race where, between that writer reading the dead owner and breaking the # lock, the owner releases and a live writer takes the lock: a shimmed cat