abrichr · Nov 5, 2026 · Nov 4, 2026 · Nov 4, 2026 · Nov 4, 2026 · Nov 5, 2026
@@ -102,7 +102,7 @@ App 15368 is GitHub Actions, the app the current checks are pinned to.
 | main.yml | `run-ci` | ubuntu-latest, Python 3.12 | ubuntu-latest, Python 3.12 | ubuntu-latest and macos-latest, Python 3.10 to 3.12 |
 | main.yml | `unsupported-python-guard` | no | no | Python 3.13 and 3.14 |
 | packages.yml | `types`, `agent` | Python 3.12 | Python 3.12 | Python 3.10 and 3.12 |
-| packages.yml | `flow-lint`, `flow-critical` | yes | yes | yes |
+| packages.yml | `flow-lint`, `flow-critical`, `flow-onprem-offline` | yes | yes | yes |
 | ci.yml | `smoke`, `ci` | yes | yes | no |
 | packages.yml | `flow-unit` | 3 shards packed by measured time, fast lane (without `nightly_only.txt`) | no | 2 shards |
 | packages.yml | `flow-native-census` | no | no | Windows and macOS |

@@ -630,6 +630,46 @@ jobs:
           sys.exit(1 if bad else 0)
           PY
 
+  # The on-prem install path for real: build two signed offline releases of
+  # openadapt-flow[privacy] from uv.lock on this connected runner, then, as
+  # root in a network namespace with no route out, install the first from its
+  # wheelhouse, refuse a tampered second, update to the signed second, roll
+  # back, and run the air-gap attestation (egress probe, audit chain). pip has
+  # no index and the machine has no network, so a wheel missing from the
+  # archive fails the install. deploy/on-prem/bin/test-offline-install.sh.
+  # First, bin/test-update.sh: the update state machine's refusal, migration
+  # and recovery paths with a fake venv build, which nothing else runs.
+  flow-onprem-offline:
+    name: flow on-prem offline install, update and rollback (egress blocked)
+    needs: scope
+    if: github.event.pull_request.draft != true && (needs.scope.outputs.flow == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    defaults:
+      run:
+        working-directory: packages/flow
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+        with:
+          enable-cache: false
+      - name: Signed update and rollback state machine (hermetic, fake build)
+        run: bash deploy/on-prem/bin/test-update.sh
+      - name: Build two signed offline releases from the lock (connected)
+        run: bash deploy/on-prem/bin/test-offline-install.sh stage "$RUNNER_TEMP/offline"
+      - name: Install, update and roll back with egress blocked
+        # env -u SUDO_UID keeps the command root inside the namespace:
+        # install.sh owns the release tree as root.
+        run: >-
+          sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH"
+          deploy/on-prem/bin/run-egress-blocked.sh
+          bash deploy/on-prem/bin/test-offline-install.sh verify "$RUNNER_TEMP/offline"
+
   # The agent-framework adapters (openadapt_flow.integrations.agents): each
   # framework's own agent loop with a scripted model, writing through a real
   # HTTP app, from the lock with every framework extra (CrewAI pinned beside
@@ -1425,6 +1465,7 @@ jobs:
       - flow-lint
       - flow-unit
       - flow-critical
+      - flow-onprem-offline
       - flow-agent-frameworks
       - flow-native-census
       - flow-linux-atspi
@@ -1478,6 +1519,7 @@ jobs:
               "flow-lint": "flow",
               "flow-unit": "flow",
               "flow-critical": "flow",
+              "flow-onprem-offline": "flow",
               "flow-agent-frameworks": "flow",
               "flow-native-census": "flow",
               "flow-linux-atspi": "flow",

@@ -21,6 +21,8 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow).
 | `bin/verify-airgap.sh` | Asserts no outbound path: config/env/deployment URL scan, optional active egress probe, optional audit-chain walk. | REAL (best-effort; firewall is primary) |
 | `bin/run-egress-blocked.sh` | Runs a command in a loopback-only Linux network namespace, refusing to start unless no other interface or route exists. CI runs the golden free path through it (`tests/e2e/test_egress_blocked_e2e.py`): VERIFIED, zero model calls, and no outbound attempt in strict no-outbound mode. | REAL (Linux, root) |
 | `bin/test-update.sh` | Hermetic offline proof of signed update, refusal paths, legacy migration, data preservation, rollback/recovery, locks, and audit-chain serialization. | REAL (43 baseline assertions; 45 when local GPG key generation is available) |
+| `bin/build-release.sh` | Builds a release archive on a connected host from `uv.lock`: the exact pinned wheels, hash-checked, plus the `openadapt-flow` wheel, a `.sha256`, and an openssl `.sig` when given the key. | REAL |
+| `bin/test-offline-install.sh` | Builds two signed releases, then as root with egress blocked installs the real engine from the first, refuses a tampered second, updates to the signed second, rolls back, and runs the air-gap attestation. CI runs it with `bin/test-update.sh` (job `flow-onprem-offline`). | REAL (Linux, root) |
 | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) |
 | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) |
 | `docker-compose.yml` + `Dockerfile` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. | REAL topology; STUB image build (needs offline `./wheels`) |

@@ -42,7 +42,23 @@ all survive across any number of updates.
 ## Build a signed release bundle (off-site, on a connected host)
 
 The bundle is a tarball containing a `VERSION` file and an offline pip
-wheelhouse, plus a required detached signature and an optional checksum:
+wheelhouse, plus a required detached signature and an optional checksum.
+
+`bin/build-release.sh` builds it from a source checkout: it downloads exactly
+the wheels `packages/flow/uv.lock` pins, checks each against the lock's hashes,
+adds the `openadapt-flow` wheel, and writes the archive, its `.sha256`, and an
+openssl `.sig` when given the signing key. Run it on a host with the same OS,
+CPU architecture and Python minor version as the clinic box:
+
+```bash
+deploy/on-prem/bin/build-release.sh --out dist/ --sign-key vendor_priv.pem
+```
+
+CI builds two releases this way and installs, updates and rolls back the real
+engine from them with the network cut off (`bin/test-offline-install.sh`).
+
+To assemble one by hand instead (this resolves the newest matching
+dependencies rather than the locked ones):
 
 ```bash
 # 1. Assemble the payload

@@ -68,21 +68,50 @@ chmod 0700 "$(dirname "$AUDIT_LOG")" 2>/dev/null || true
 # Serialize the read-last-hash + append transaction. O_APPEND prevents byte
 # interleaving, but without this lock two writers can both chain to the same
 # parent and produce a structurally broken audit log.
+#
+# Ownership is the exclusive creation of the pid file, done by the shell itself
+# with noclobber (open(2) with O_CREAT|O_EXCL). The exit status of mkdir(1) is
+# not trusted for exclusion: the uutils mkdir that Ubuntu 26.04 ships reports
+# success to both callers when two processes race to create one directory.
 lock_dir="${AUDIT_LOG}.lock"
+lock_file="$lock_dir/pid"
+break_file="${AUDIT_LOG}.lock-breaker"
 lock_acquired=0
+
+# Create $1 holding this process's pid; fails if $1 already exists.
+create_exclusive() {
+  ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null
+}
+
+# Remove a lock whose owner has exited. Between reading the owner and removing
+# the lock, that owner can release it and another writer can take it, so the
+# removal happens only under a second exclusive lock that serializes breakers,
+# and only if the lock still names the same dead owner. Without this, a breaker
+# could delete a live writer's lock and let two writers chain to the same parent.
+break_stale_lock() {
+  local stale="$1" current=""
+  create_exclusive "$break_file" || return 0
+  current="$(cat "$lock_file" 2>/dev/null || true)"
+  if [[ "$current" == "$stale" ]] && ! kill -0 "$stale" 2>/dev/null; then
+    rm -f "$lock_file" 2>/dev/null || true
+  fi
+  rm -f "$break_file" 2>/dev/null || true
+}
+
 for ((attempt = 0; attempt < 200; attempt++)); do
-  if mkdir "$lock_dir" 2>/dev/null; then
-    printf '%s\n' "$$" > "$lock_dir/pid"
-    lock_acquired=1
-    break
+  if [[ ! -L "$lock_dir" ]]; then
+    mkdir -p "$lock_dir" 2>/dev/null || true
+    if [[ -d "$lock_dir" && ! -L "$lock_dir" ]] && create_exclusive "$lock_file"; then
+      lock_acquired=1
+      break
+    fi
   fi
   owner=""
-  if [[ -d "$lock_dir" && ! -L "$lock_dir" && -r "$lock_dir/pid" ]]; then
-    owner="$(cat "$lock_dir/pid" 2>/dev/null || true)"
+  if [[ -d "$lock_dir" && ! -L "$lock_dir" && -f "$lock_file" && ! -L "$lock_file" ]]; then
+    owner="$(cat "$lock_file" 2>/dev/null || true)"
   fi
   if [[ -n "$owner" && "$owner" =~ ^[0-9]+$ ]] && ! kill -0 "$owner" 2>/dev/null; then
-    rm -f "$lock_dir/pid" 2>/dev/null || true
-    rmdir "$lock_dir" 2>/dev/null || true
+    break_stale_lock "$owner"
     continue
   fi
   sleep 0.05
@@ -92,7 +121,7 @@ if [[ "$lock_acquired" -ne 1 ]]; then
   exit 1
 fi
 release_audit_lock() {
-  rm -f "$lock_dir/pid" 2>/dev/null || true
+  rm -f "$lock_file" 2>/dev/null || true
   rmdir "$lock_dir" 2>/dev/null || true
 }
 trap release_audit_lock EXIT

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+#
+# build-release.sh — build an offline release archive from the lock file.
+#
+# Run on a CONNECTED build host with the same OS, CPU architecture and Python
+# minor version as the air-gapped target. Produces, in --out:
+#
+#   release-<version>.tar.gz          VERSION, wheels/, requirements.lock.txt
+#   release-<version>.tar.gz.sha256   checksum (corruption check only)
+#   release-<version>.tar.gz.sig      openssl signature, when --sign-key is given
+#
+# Every third-party wheel is the exact version pinned in packages/flow/uv.lock
+# and is checked against the lock's hashes while downloading (pip
+# --require-hashes), so the archive holds what CI tested and nothing newer.
+# The openadapt-flow wheel is built from this checkout unless --flow-wheel
+# names one. Stage the archive with `install.sh --update` (see UPDATE.md), or
+# extract it and pass release/wheels to `install.sh --wheelhouse` for a first
+# install.
+#
+# Usage:
+#   build-release.sh --out DIR [--flow-wheel FILE] [--sign-key PRIVATE.pem]
+#                    [--python PYTHON]
+#
+# Needs uv and the target Python (default python3). Exit 0 on success.
+
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+FLOW_DIR="$(cd "$HERE/../../.." && pwd)"
+OUT=""
+FLOW_WHEEL=""
+SIGN_KEY=""
+PYTHON="python3"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --out)        OUT="${2:-}"; shift 2 ;;
+    --flow-wheel) FLOW_WHEEL="${2:-}"; shift 2 ;;
+    --sign-key)   SIGN_KEY="${2:-}"; shift 2 ;;
+    --python)     PYTHON="${2:-}"; shift 2 ;;
+    -h|--help)    grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "build-release.sh: unknown arg $1" >&2; exit 2 ;;
+  esac
+done
+[[ -n "$OUT" ]] || { echo "build-release.sh: --out DIR is required" >&2; exit 2; }
+command -v uv >/dev/null || { echo "build-release.sh: uv is required" >&2; exit 2; }
+[[ -f "$FLOW_DIR/uv.lock" ]] || { echo "build-release.sh: no uv.lock in $FLOW_DIR" >&2; exit 2; }
+if [[ -n "$SIGN_KEY" && ! -f "$SIGN_KEY" ]]; then
+  echo "build-release.sh: signing key $SIGN_KEY not found" >&2; exit 2
+fi
+
+# shellcheck source=bin/lib-release.sh
+source "$HERE/lib-release.sh"
+
+mkdir -p "$OUT"
+OUT="$(cd "$OUT" && pwd)"
+WORK="$(mktemp -d "${TMPDIR:-/tmp}/oa-build-release.XXXXXX")"
+trap 'rm -rf "$WORK"' EXIT
+mkdir -p "$WORK/release/wheels"
+
+if [[ -z "$FLOW_WHEEL" ]]; then
+  echo "== building the openadapt-flow wheel from $FLOW_DIR =="
+  (cd "$FLOW_DIR" && uv build --wheel --out-dir "$WORK/flow" >/dev/null)
+  FLOW_WHEEL="$(find "$WORK/flow" -maxdepth 1 -name 'openadapt_flow-*.whl' -print)"
+fi
+[[ -f "$FLOW_WHEEL" ]] || { echo "build-release.sh: flow wheel not found" >&2; exit 2; }
+cp "$FLOW_WHEEL" "$WORK/release/wheels/"
+VERSION="$(rel_wheelhouse_version "$WORK/release/wheels")"
+rel_validate_version "$VERSION"
+
+echo "== exporting the locked dependencies of openadapt-flow[privacy] $VERSION =="
+(cd "$FLOW_DIR" && uv export --frozen --no-dev --extra privacy --no-emit-project \
+  --format requirements-txt --no-header -o "$WORK/release/requirements.lock.txt" >/dev/null)
+
+echo "== downloading the locked wheels for $("$PYTHON" -c 'import platform, sys; print(sys.platform, platform.machine(), "py%d.%d" % sys.version_info[:2])') =="
+"$PYTHON" -m pip download --quiet --disable-pip-version-check \
+  --only-binary=:all: --no-deps --require-hashes \
+  -r "$WORK/release/requirements.lock.txt" -d "$WORK/release/wheels"
+
+printf '%s\n' "$VERSION" > "$WORK/release/VERSION"
+ARCHIVE="$OUT/release-$VERSION.tar.gz"
+COPYFILE_DISABLE=1 tar czf "$ARCHIVE" -C "$WORK" release
+sha256_of "$ARCHIVE" > "$ARCHIVE.sha256"
+if [[ -n "$SIGN_KEY" ]]; then
+  openssl dgst -sha256 -sign "$SIGN_KEY" -out "$ARCHIVE.sig" "$ARCHIVE"
+fi
+
+count="$(find "$WORK/release/wheels" -name '*.whl' | wc -l | tr -d ' ')"
+echo "built $ARCHIVE ($count wheels)"
+[[ -n "$SIGN_KEY" ]] && echo "signed $ARCHIVE.sig" || echo "unsigned: sign it before staging (UPDATE.md)"
@@ -344,32 +344,45 @@ rel_validate_release_dir() {
     && -x "$release_dir/venv/bin/openadapt-flow" ]]
 }
 
+# Create $1 holding this process's pid; fails if $1 already exists. The shell
+# opens it with noclobber (O_CREAT|O_EXCL), so exclusion does not depend on
+# mkdir(1): the uutils mkdir that Ubuntu 26.04 ships reports success to both
+# callers when two processes race to create one directory.
+_rel_create_exclusive() {
+  ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null
+}
+
 _rel_lock_acquire() {
-  local root="$1" owner=""
+  local root="$1" owner="" current=""
   local lock="$root/releases/.update.lock"
+  local breaker="$root/releases/.update.lock-breaker"
   mkdir -p "$root/releases" || return 1
-  if mkdir "$lock" 2>/dev/null; then
-    printf '%s\n' "$$" > "$lock/pid" || {
-      rmdir "$lock" 2>/dev/null || true
-      return 1
-    }
+  if [[ -L "$lock" ]]; then
+    _rel_err "release lock is a symlink; refusing: $lock"
+    return 1
+  fi
+  mkdir -p "$lock" 2>/dev/null || true
+  if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then
     REL_LOCK_DIR="$lock"
     return 0
   fi
-  if [[ -d "$lock" && ! -L "$lock" && -r "$lock/pid" ]]; then
+  if [[ -d "$lock" && ! -L "$lock" && -f "$lock/pid" && ! -L "$lock/pid" ]]; then
     owner="$(cat "$lock/pid" 2>/dev/null || true)"
   fi
   if [[ "$owner" =~ ^[0-9]+$ ]] && kill -0 "$owner" 2>/dev/null; then
     _rel_err "another update or rollback is active (pid $owner)"
     return 1
   fi
-  if [[ -d "$lock" && ! -L "$lock" ]]; then
-    rm -f "$lock/pid" 2>/dev/null || true
-    if rmdir "$lock" 2>/dev/null && mkdir "$lock" 2>/dev/null; then
-      printf '%s\n' "$$" > "$lock/pid" || {
-        rmdir "$lock" 2>/dev/null || true
-        return 1
-      }
+  # The owner has exited (or left an unreadable pid). Remove its lock only
+  # under the breaker lock and only if it still names that owner, so a
+  # concurrent breaker cannot remove a lock another process has just taken.
+  if [[ -n "$owner" ]] && _rel_create_exclusive "$breaker"; then
+    current="$(cat "$lock/pid" 2>/dev/null || true)"
+    if [[ "$current" == "$owner" ]]; then
+      rm -f "$lock/pid" 2>/dev/null || true
+    fi
+    rm -f "$breaker" 2>/dev/null || true
+    if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then
       REL_LOCK_DIR="$lock"
       return 0
     fi