diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5cce5d3943..8117e74e17 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -102,7 +102,7 @@ App 15368 is GitHub Actions, the app the current checks are pinned to. | main.yml | `run-ci` | ubuntu-latest, Python 3.12 | ubuntu-latest, Python 3.12 | ubuntu-latest and macos-latest, Python 3.10 to 3.12 | | main.yml | `unsupported-python-guard` | no | no | Python 3.13 and 3.14 | | packages.yml | `types`, `agent` | Python 3.12 | Python 3.12 | Python 3.10 and 3.12 | -| packages.yml | `flow-lint`, `flow-critical` | yes | yes | yes | +| packages.yml | `flow-lint`, `flow-critical`, `flow-onprem-offline` | yes | yes | yes | | ci.yml | `smoke`, `ci` | yes | yes | no | | packages.yml | `flow-unit` | 3 shards packed by measured time, fast lane (without `nightly_only.txt`) | no | 2 shards | | packages.yml | `flow-native-census` | no | no | Windows and macOS | diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 48e9ba0828..ee994aa3b4 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -604,13 +604,28 @@ jobs: tests/test_admitted_composition_runtime.py \ tests/test_agent_guard.py \ --basetemp=runs/ci-critical --junitxml=runs/critical-junit.xml + - name: Golden free path with egress blocked (loopback-only network namespace) + # Direct IP sockets stay in a loopback-only namespace. The tutorial + # must still reach VERIFIED with no Python-hook refusal recorded. + # A native-child control measures the hook observation boundary. + run: | + sudo -n env "PATH=$PATH" "HOME=$HOME" \ + deploy/on-prem/bin/run-egress-blocked.sh \ + python -m pytest -v tests/e2e/test_egress_blocked_e2e.py \ + --basetemp=runs/ci-onprem-egress --junitxml=runs/onprem-egress-junit.xml + - name: Network namespace setup and owned Unix broker boundary + env: + OPENADAPT_TEST_NETNS: "1" + run: | + python -m pytest -v tests/e2e/test_egress_namespace_e2e.py \ + --basetemp=runs/ci-onprem-boundary --junitxml=runs/onprem-boundary-junit.xml - name: Fail if any critical test was skipped or none ran run: | python - <<'PY' import sys import xml.etree.ElementTree as ET bad = False - for name in ("critical-e2e", "critical"): + for name in ("critical-e2e", "critical", "onprem-egress", "onprem-boundary"): root = ET.parse(f"runs/{name}-junit.xml").getroot() suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite")) tests = sum(int(s.get("tests", 0)) for s in suites) @@ -621,6 +636,46 @@ jobs: sys.exit(1 if bad else 0) PY + # The on-prem install path for real: build two signed offline releases of + # openadapt-flow[privacy] from uv.lock on this connected runner, then, as + # root in a network namespace with no route out, install the first from its + # wheelhouse, refuse a tampered second, update to the signed second, roll + # back, and run the air-gap attestation (egress probe, audit chain). pip has + # no index and the machine has no network, so a wheel missing from the + # archive fails the install. deploy/on-prem/bin/test-offline-install.sh. + # First, bin/test-update.sh: the update state machine's refusal, migration + # and recovery paths with a fake venv build, which nothing else runs. + flow-onprem-offline: + name: flow on-prem offline install, update and rollback (egress blocked) + needs: scope + if: github.event.pull_request.draft != true && (needs.scope.outputs.flow == 'true') + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: packages/flow + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + - name: Signed update and rollback state machine (hermetic, fake build) + run: bash deploy/on-prem/bin/test-update.sh + - name: Build two signed offline releases from the lock (connected) + run: bash deploy/on-prem/bin/test-offline-install.sh stage "$RUNNER_TEMP/offline" + - name: Install, update and roll back with egress blocked + # env -u SUDO_UID keeps the command root inside the namespace: + # install.sh owns the release tree as root. + run: >- + sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" + deploy/on-prem/bin/run-egress-blocked.sh + bash deploy/on-prem/bin/test-offline-install.sh verify "$RUNNER_TEMP/offline" + # The agent-framework adapters (openadapt_flow.integrations.agents): each # framework's own agent loop with a scripted model, writing through a real # HTTP app, from the lock with every framework extra (CrewAI pinned beside @@ -1416,6 +1471,7 @@ jobs: - flow-lint - flow-unit - flow-critical + - flow-onprem-offline - flow-agent-frameworks - flow-native-census - flow-linux-atspi @@ -1469,6 +1525,7 @@ jobs: "flow-lint": "flow", "flow-unit": "flow", "flow-critical": "flow", + "flow-onprem-offline": "flow", "flow-agent-frameworks": "flow", "flow-native-census": "flow", "flow-linux-atspi": "flow", diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index ac88003aa4..efd8483042 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -19,7 +19,10 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow). | `bin/run-queue.sh` | Local, directory-based run queue. Claims jobs, runs `openadapt-flow run`, records outcomes, files done/failed. Fail-closed on `SCRUB!=on`. | REAL (thin wrapper over the shipped CLI) | | `bin/audit-log.sh` | Append-only, hash-chained, PHI-free audit writer (run + release-lifecycle events). | REAL | | `bin/verify-airgap.sh` | Asserts no outbound path: config/env/deployment URL scan, optional active egress probe, optional audit-chain walk. | REAL (best-effort; firewall is primary) | +| `bin/run-egress-blocked.sh` | Runs a command in a loopback-only Linux network namespace, refusing to start unless no other interface or route exists. CI runs the golden free path through it (`tests/e2e/test_egress_blocked_e2e.py`): VERIFIED, zero model calls, and no refusal recorded by the strict Python audit hook. | REAL (Linux, root) | | `bin/test-update.sh` | Hermetic offline proof of signed update, refusal paths, legacy migration, data preservation, rollback/recovery, locks, and audit-chain serialization. | REAL (43 baseline assertions; 45 when local GPG key generation is available) | +| `bin/build-release.sh` | Builds a release archive on a connected host from `uv.lock`: the exact pinned wheels, hash-checked, plus the `openadapt-flow` wheel, a `.sha256`, and an openssl `.sig` when given the key. | REAL | +| `bin/test-offline-install.sh` | Builds two signed releases, then as root with egress blocked installs the real engine from the first, refuses a tampered second, updates to the signed second, rolls back, and runs the air-gap attestation. CI runs it with `bin/test-update.sh` (job `flow-onprem-offline`). | REAL (Linux, root) | | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) | | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) | | `docker-compose.yml` + `Dockerfile` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. | REAL topology; STUB image build (needs offline `./wheels`) | @@ -47,6 +50,29 @@ PHI-free record to `audit/audit.log`, and files the job under `jobs/done` (exit OPENADAPT_FLOW_SCRUB=on ./bin/verify-airgap.sh --config onprem.yaml --probe --audit ``` +## Golden run with direct IP networking restricted + +On Linux with the engine installed, run the free path in a new network +namespace containing only loopback: + +```bash +sudo -E env "PATH=$PATH" "HOME=$HOME" deploy/on-prem/bin/run-egress-blocked.sh \ + python -m pytest -v tests/e2e/test_egress_blocked_e2e.py +``` + +The wrapper starts only after all interface and route inspections succeed +and confirm exactly loopback with no routes. The tests check +direct IP refusal independently, then require the tutorial to reach VERIFIED +with zero model calls and a receipt. CI runs this on every flow change. + +The restriction covers direct IP sockets in the command and its children. +Filesystem access stays shared: pathname Unix sockets can reach a process +outside the namespace that forwards traffic. Isolate such brokers separately +when enforcing a deployment boundary. An empty strict-mode refusal log means +the Python hook recorded no refusals; it does not observe native sockets or +children that never install the hook. Native-child and owned Unix-relay +controls exercise these boundaries in CI. + ## Update & rollback Patch the box offline; revert instantly if a release misbehaves. Full runbook in diff --git a/packages/flow/deploy/on-prem/UPDATE.md b/packages/flow/deploy/on-prem/UPDATE.md index 0e594e8395..9346e2c8a8 100644 --- a/packages/flow/deploy/on-prem/UPDATE.md +++ b/packages/flow/deploy/on-prem/UPDATE.md @@ -42,7 +42,23 @@ all survive across any number of updates. ## Build a signed release bundle (off-site, on a connected host) The bundle is a tarball containing a `VERSION` file and an offline pip -wheelhouse, plus a required detached signature and an optional checksum: +wheelhouse, plus a required detached signature and an optional checksum. + +`bin/build-release.sh` builds it from a source checkout: it downloads exactly +the wheels `packages/flow/uv.lock` pins, checks each against the lock's hashes, +adds the `openadapt-flow` wheel, and writes the archive, its `.sha256`, and an +openssl `.sig` when given the signing key. Run it on a host with the same OS, +CPU architecture and Python minor version as the clinic box: + +```bash +deploy/on-prem/bin/build-release.sh --out dist/ --sign-key vendor_priv.pem +``` + +CI builds two releases this way and installs, updates and rolls back the real +engine from them with the network cut off (`bin/test-offline-install.sh`). + +To assemble one by hand instead (this resolves the newest matching +dependencies rather than the locked ones): ```bash # 1. Assemble the payload diff --git a/packages/flow/deploy/on-prem/bin/build-release.sh b/packages/flow/deploy/on-prem/bin/build-release.sh new file mode 100755 index 0000000000..5b62d7878f --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/build-release.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# build-release.sh — build an offline release archive from the lock file. +# +# Run on a CONNECTED build host with the same OS, CPU architecture and Python +# minor version as the air-gapped target. Produces, in --out: +# +# release-.tar.gz VERSION, wheels/, requirements.lock.txt +# release-.tar.gz.sha256 checksum (corruption check only) +# release-.tar.gz.sig openssl signature, when --sign-key is given +# +# Every third-party wheel is the exact version pinned in packages/flow/uv.lock +# and is checked against the lock's hashes while downloading (pip +# --require-hashes), so the archive holds what CI tested and nothing newer. +# The openadapt-flow wheel is built from this checkout unless --flow-wheel +# names one. Stage the archive with `install.sh --update` (see UPDATE.md), or +# extract it and pass release/wheels to `install.sh --wheelhouse` for a first +# install. +# +# Usage: +# build-release.sh --out DIR [--flow-wheel FILE] [--sign-key PRIVATE.pem] +# [--python PYTHON] +# +# Needs uv and the target Python (default python3). Exit 0 on success. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FLOW_DIR="$(cd "$HERE/../../.." && pwd)" +OUT="" +FLOW_WHEEL="" +SIGN_KEY="" +PYTHON="python3" + +while [[ $# -gt 0 ]]; do + case "$1" in + --out) OUT="${2:-}"; shift 2 ;; + --flow-wheel) FLOW_WHEEL="${2:-}"; shift 2 ;; + --sign-key) SIGN_KEY="${2:-}"; shift 2 ;; + --python) PYTHON="${2:-}"; shift 2 ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "build-release.sh: unknown arg $1" >&2; exit 2 ;; + esac +done +[[ -n "$OUT" ]] || { echo "build-release.sh: --out DIR is required" >&2; exit 2; } +command -v uv >/dev/null || { echo "build-release.sh: uv is required" >&2; exit 2; } +[[ -f "$FLOW_DIR/uv.lock" ]] || { echo "build-release.sh: no uv.lock in $FLOW_DIR" >&2; exit 2; } +if [[ -n "$SIGN_KEY" && ! -f "$SIGN_KEY" ]]; then + echo "build-release.sh: signing key $SIGN_KEY not found" >&2; exit 2 +fi + +# shellcheck source=bin/lib-release.sh +source "$HERE/lib-release.sh" + +mkdir -p "$OUT" +OUT="$(cd "$OUT" && pwd)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/oa-build-release.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/release/wheels" + +if [[ -z "$FLOW_WHEEL" ]]; then + echo "== building the openadapt-flow wheel from $FLOW_DIR ==" + (cd "$FLOW_DIR" && uv build --wheel --out-dir "$WORK/flow" >/dev/null) + FLOW_WHEEL="$(find "$WORK/flow" -maxdepth 1 -name 'openadapt_flow-*.whl' -print)" +fi +[[ -f "$FLOW_WHEEL" ]] || { echo "build-release.sh: flow wheel not found" >&2; exit 2; } +cp "$FLOW_WHEEL" "$WORK/release/wheels/" +VERSION="$(rel_wheelhouse_version "$WORK/release/wheels")" +rel_validate_version "$VERSION" + +echo "== exporting the locked dependencies of openadapt-flow[privacy] $VERSION ==" +(cd "$FLOW_DIR" && uv export --frozen --no-dev --extra privacy --no-emit-project \ + --format requirements-txt --no-header -o "$WORK/release/requirements.lock.txt" >/dev/null) + +echo "== downloading the locked wheels for $("$PYTHON" -c 'import platform, sys; print(sys.platform, platform.machine(), "py%d.%d" % sys.version_info[:2])') ==" +"$PYTHON" -m pip download --quiet --disable-pip-version-check \ + --only-binary=:all: --no-deps --require-hashes \ + -r "$WORK/release/requirements.lock.txt" -d "$WORK/release/wheels" + +printf '%s\n' "$VERSION" > "$WORK/release/VERSION" +ARCHIVE="$OUT/release-$VERSION.tar.gz" +COPYFILE_DISABLE=1 tar czf "$ARCHIVE" -C "$WORK" release +sha256_of "$ARCHIVE" > "$ARCHIVE.sha256" +if [[ -n "$SIGN_KEY" ]]; then + openssl dgst -sha256 -sign "$SIGN_KEY" -out "$ARCHIVE.sig" "$ARCHIVE" +fi + +count="$(find "$WORK/release/wheels" -name '*.whl' | wc -l | tr -d ' ')" +echo "built $ARCHIVE ($count wheels)" +[[ -n "$SIGN_KEY" ]] && echo "signed $ARCHIVE.sig" || echo "unsigned: sign it before staging (UPDATE.md)" diff --git a/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh b/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh new file mode 100755 index 0000000000..b80e14a3dc --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# run-egress-blocked.sh — run a command with no route off this machine. +# +# Linux only. Starts the command in a new network namespace containing only +# loopback. Direct IP connections from the command and its children cannot +# reach networks outside that namespace. Services started inside it can use +# loopback to communicate with each other. +# +# Filesystem access is unchanged: pathname Unix sockets can reach processes +# outside the namespace, including brokers that forward network traffic. This +# is a direct-IP restriction, not a filesystem or untrusted-code sandbox. +# +# Before running the command, all interface and route inspections must succeed +# and confirm only loopback and no routes. The command runs as the invoking +# sudo user, with OPENADAPT_EGRESS_BLOCKED=netns as a test-location hint. Tests +# independently check the restriction rather than trusting that variable. +# +# Usage (as root). Pass every needed variable on the env command line; do not +# rely on sudo -E (GitHub-hosted runners ignore it): +# sudo -n env "PATH=$PATH" "HOME=$HOME" KEY=VAL \ +# deploy/on-prem/bin/run-egress-blocked.sh CMD [ARGS...] +# +# Exit code: the command's, or 2 when the namespace cannot be set up. + +set -euo pipefail + +if [[ $# -eq 0 || "$1" == "-h" || "$1" == "--help" ]]; then + grep '^#' "$0" | sed 's/^# \{0,1\}//' + [[ $# -eq 0 ]] && exit 2 || exit 0 +fi +if [[ "$(uname -s)" != "Linux" ]]; then + echo "run-egress-blocked.sh: needs Linux network namespaces" >&2 + exit 2 +fi +if [[ "$(id -u)" -ne 0 ]]; then + echo "run-egress-blocked.sh: run as root (sudo); the command runs as the invoking user" >&2 + exit 2 +fi +for tool in unshare ip setpriv; do + command -v "$tool" >/dev/null || { echo "run-egress-blocked.sh: missing $tool" >&2; exit 2; } +done + +RUN_UID="${SUDO_UID:-0}" +RUN_GID="${SUDO_GID:-0}" +export RUN_UID RUN_GID + +# The inner script expands its own variables, inside the namespace. +# shellcheck disable=SC2016 +exec unshare --net -- bash -euo pipefail -c ' + ip link set lo up + if ! links="$(ip -o link show)"; then + echo "run-egress-blocked.sh: cannot inspect interfaces" >&2 + exit 2 + fi + interfaces="$(awk -F": " "{print \$2}" <<< "$links")" + if [[ "$interfaces" != lo ]]; then + echo "run-egress-blocked.sh: expected only loopback, found: $interfaces" >&2 + exit 2 + fi + if ! routes4="$(ip route show)"; then + echo "run-egress-blocked.sh: cannot inspect IPv4 routes" >&2 + exit 2 + fi + if ! routes6="$(ip -6 route show)"; then + echo "run-egress-blocked.sh: cannot inspect IPv6 routes" >&2 + exit 2 + fi + if [[ -n "$routes4" || -n "$routes6" ]]; then + echo "run-egress-blocked.sh: unexpected routes: $routes4 $routes6" >&2 + exit 2 + fi + export OPENADAPT_EGRESS_BLOCKED=netns + if [[ "$RUN_UID" == 0 ]]; then + exec "$@" + fi + exec setpriv --reuid="$RUN_UID" --regid="$RUN_GID" --init-groups -- "$@" +' run-egress-blocked "$@" diff --git a/packages/flow/deploy/on-prem/bin/test-offline-install.sh b/packages/flow/deploy/on-prem/bin/test-offline-install.sh new file mode 100755 index 0000000000..c3e6d391cd --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/test-offline-install.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# +# test-offline-install.sh — install, upgrade and roll back the real engine on a +# machine with no network. +# +# Unlike test-update.sh, which fakes the venv build, this installs the real +# openadapt-flow[privacy] wheels pinned by uv.lock, so it proves the offline +# release archive is complete: pip gets no index and the machine has no route +# out, so any missing wheel fails the install. +# +# stage DIR on a connected host: build two signed releases (this checkout +# as V1, the same source as V1.post1 as V2) and a throwaway +# signing key, with bin/build-release.sh. +# verify DIR as root, inside bin/run-egress-blocked.sh: first install from +# V1's wheelhouse, refuse a tampered V2, update to the signed V2, +# roll back to V1, then run the air-gap attestation with its +# egress probe and audit-chain walk. Data must survive each step. +# +# CI runs both (packages.yml, flow on-prem offline install). By hand on Linux: +# bin/test-offline-install.sh stage /tmp/oa-offline +# sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" \ +# bin/run-egress-blocked.sh bin/test-offline-install.sh verify /tmp/oa-offline +# +# Exit 0 when every check passes. + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ONPREM="$(cd "$HERE/.." && pwd)" +FLOW_DIR="$(cd "$ONPREM/../.." && pwd)" +CMD="${1:-}" +DIR="${2:-}" +if [[ -z "$CMD" || -z "$DIR" || ( "$CMD" != stage && "$CMD" != verify ) ]]; then + grep '^#' "$0" | sed 's/^# \{0,1\}//' + exit 2 +fi + +stage() { + set -e + mkdir -p "$DIR/keys" "$DIR/v1" "$DIR/v2" + DIR="$(cd "$DIR" && pwd)" + openssl genrsa -out "$DIR/keys/vendor.pem" 2048 2>/dev/null + openssl rsa -in "$DIR/keys/vendor.pem" -pubout -out "$DIR/keys/vendor.pub" 2>/dev/null + bash "$HERE/build-release.sh" --out "$DIR/v1" --sign-key "$DIR/keys/vendor.pem" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + + # V2: the same source under a new version, built the same way. + local v1 src + v1="$(find "$DIR/v1" -maxdepth 1 -name 'release-*.tar.gz' -print)" + v1="$(basename "$v1" .tar.gz)" + v1="${v1#release-}" + src="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-src.XXXXXX")" + for item in pyproject.toml README.md LICENSE THIRD_PARTY_NOTICES.md \ + public-artifacts.json openadapt_flow schemas; do + cp -R "$FLOW_DIR/$item" "$src/" + done + find "$src" -name __pycache__ -type d -prune -exec rm -rf {} + + sed -i.bak "s/^version = \"$v1\"$/version = \"$v1.post1\"/" "$src/pyproject.toml" + sed -i.bak "s/^__version__ = \"$v1\"$/__version__ = \"$v1.post1\"/" "$src/openadapt_flow/__init__.py" + rm -f "$src/pyproject.toml.bak" "$src/openadapt_flow/__init__.py.bak" + grep -q "^version = \"$v1.post1\"$" "$src/pyproject.toml" + (cd "$src" && uv build --wheel --out-dir "$src/dist" >/dev/null) + bash "$HERE/build-release.sh" --out "$DIR/v2" --sign-key "$DIR/keys/vendor.pem" \ + --flow-wheel "$src/dist/openadapt_flow-$v1.post1-py3-none-any.whl" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + rm -rf "$src" + # The private key never goes near the install. + rm -f "$DIR/keys/vendor.pem" + printf '%s\n' "$v1" > "$DIR/V1" + printf '%s\n' "$v1.post1" > "$DIR/V2" + echo "staged $v1 and $v1.post1 in $DIR" +} + +PASS=0 +FAIL=0 +ok() { printf ' PASS %s\n' "$1"; PASS=$((PASS + 1)); } +bad() { printf ' FAIL %s\n' "$1"; FAIL=$((FAIL + 1)); } +check() { local name="$1"; shift; if "$@"; then ok "$name"; else bad "$name"; fi; } +check_eq() { if [[ "$2" == "$3" ]]; then ok "$1"; else bad "$1 (expected '$2', got '$3')"; fi; } + +verify() { + DIR="$(cd "$DIR" && pwd)" + if [[ "$(id -u)" -ne 0 ]]; then + echo "test-offline-install.sh verify: run as root (install.sh needs it)" >&2; exit 2 + fi + if [[ "${OPENADAPT_EGRESS_BLOCKED:-}" != netns ]]; then + echo "test-offline-install.sh verify: run inside bin/run-egress-blocked.sh" >&2; exit 2 + fi + local v1 v2 py work root config rc + v1="$(cat "$DIR/V1")"; v2="$(cat "$DIR/V2")" + py="${OPENADAPT_ONPREM_PY:-python3}" + work="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-verify.XXXXXX")" + root="$work/srv" + config="$work/onprem.yaml" + cat > "$config" <&1) + } + + echo "== offline install, upgrade and rollback ($v1 -> $v2 -> $v1), no network ==" + check "the machine has no route out" "$py" -c ' +import socket, sys +try: + socket.create_connection(("1.1.1.1", 443), timeout=3) +except OSError: + sys.exit(0) +sys.exit(1)' + + echo "-- first install from the $v1 wheelhouse --" + mkdir -p "$work/media" + tar xzf "$DIR/v1/release-$v1.tar.gz" -C "$work/media" + bash "$ONPREM/install.sh" --config "$config" --wheelhouse "$work/media/release/wheels" \ + > "$work/install.log" 2>&1 + rc=$? + check_eq "install: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/install.log" + check_eq "install: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "install: engine and privacy extra import, version $v1" "$v1" "$(engine_version)" + check_eq "install: engine CLI reports $v1" "openadapt-flow $v1" \ + "$(cd / && "$root/current/venv/bin/openadapt-flow" --version 2>&1)" + + mkdir -p "$root/bundles" "$root/jobs/inbox" + printf 'bundle\n' > "$root/bundles/keepme.txt" + printf 'job\n' > "$root/jobs/inbox/queued-0001.job" + local bundle_sha job_sha + bundle_sha="$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + job_sha="$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + cp "$DIR/keys/vendor.pub" "$root/keys/vendor.pub" + + local arc="$DIR/v2/release-$v2.tar.gz" + echo "-- a tampered $v2 archive is refused --" + cp "$arc" "$work/media/tampered.tar.gz" + printf 'x' >> "$work/media/tampered.tar.gz" + bash "$ONPREM/install.sh" --update --config "$config" --release "$work/media/tampered.tar.gz" \ + --signature "$arc.sig" > "$work/tampered.log" 2>&1 + rc=$? + check "tampered: update refused" test "$rc" -ne 0 + check_eq "tampered: current still $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check "tampered: nothing built for $v2" test ! -e "$root/releases/$v2" + + echo "-- signed update to $v2 --" + bash "$ONPREM/install.sh" --update --config "$config" --release "$arc" \ + --checksum "$arc.sha256" --signature "$arc.sig" > "$work/update.log" 2>&1 + rc=$? + check_eq "update: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/update.log" + check_eq "update: current -> $v2" "$root/releases/$v2" "$(readlink "$root/current")" + check_eq "update: previous -> $v1" "$root/releases/$v1" "$(readlink "$root/previous")" + check_eq "update: engine version $v2" "$v2" "$(engine_version)" + check_eq "update: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + check_eq "update: queued job unchanged" "$job_sha" "$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + + echo "-- rollback to $v1 --" + bash "$ONPREM/install.sh" --rollback --config "$config" > "$work/rollback.log" 2>&1 + rc=$? + check_eq "rollback: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/rollback.log" + check_eq "rollback: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "rollback: engine version $v1" "$v1" "$(engine_version)" + check_eq "rollback: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + + echo "-- air-gap attestation --" + OPENADAPT_FLOW_SCRUB=on bash "$HERE/verify-airgap.sh" --config "$config" --probe --audit \ + > "$work/airgap.log" 2>&1 + rc=$? + check_eq "attestation: exit 0" 0 "$rc" + check "attestation: egress probe could not reach the internet" \ + grep -q "outbound egress is blocked" "$work/airgap.log" + check "attestation: audit chain intact" grep -q "audit chain intact" "$work/airgap.log" + [[ "$rc" -eq 0 ]] || cat "$work/airgap.log" + + rm -rf "$work" + echo + echo "== result: $PASS passed, $FAIL failed ==" + [[ "$FAIL" -eq 0 ]] +} + +"$CMD" diff --git a/packages/flow/tests/e2e/test_egress_blocked_e2e.py b/packages/flow/tests/e2e/test_egress_blocked_e2e.py new file mode 100644 index 0000000000..fb6c468e55 --- /dev/null +++ b/packages/flow/tests/e2e/test_egress_blocked_e2e.py @@ -0,0 +1,160 @@ +"""Golden free path inside a loopback-only Linux network namespace. + +Run through ``deploy/on-prem/bin/run-egress-blocked.sh``. The first test +checks direct IP connections independently of the location environment +variable. The real tutorial still records, compiles, certifies, runs and +verifies a write through independent read-back. + +Strict no-outbound mode adds a Python audit hook. Its empty refusal log means +no refusal was recorded by that hook. Native sockets, children without the +hook and allowed pathname Unix brokers are outside that observation. The +native-child positive control demonstrates this limit with a real attempt +that the network namespace refuses. +""" + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +FLOW = Path(__file__).resolve().parents[2] + +pytestmark = pytest.mark.skipif( + os.environ.get("OPENADAPT_EGRESS_BLOCKED") != "netns", + reason="runs inside deploy/on-prem/bin/run-egress-blocked.sh (Linux, root)", +) + +#: Public anycast hosts every connected machine reaches. +CANARIES = (("1.1.1.1", 443), ("8.8.8.8", 53), ("2606:4700:4700::1111", 443)) + + +def _strict_env(log: Path) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if not k.startswith("OPENADAPT_NO_")} + env.update( + OPENADAPT_NO_OUTBOUND="1", + # Children inherit the direct-IP restriction; their sockets are not audited. + OPENADAPT_NO_OUTBOUND_CHILDREN="sandbox", + OPENADAPT_NO_OUTBOUND_LOG=str(log), + PYTHONPATH=os.pathsep.join( + [str(FLOW)] + [p for p in [env.get("PYTHONPATH", "")] if p] + ), + ) + return env + + +def test_nothing_off_the_machine_is_reachable() -> None: + for host, port in CANARIES: + family = socket.AF_INET6 if ":" in host else socket.AF_INET + with pytest.raises(OSError): + with socket.socket(family, socket.SOCK_STREAM) as sock: + sock.settimeout(5) + sock.connect((host, port)) + # Loopback still works: the local system of record lives there. + with socket.socket() as server: + server.bind(("127.0.0.1", 0)) + server.listen(1) + socket.create_connection(server.getsockname(), timeout=5).close() + + +def test_strict_mode_logs_an_attempt_to_leave(tmp_path: Path) -> None: + log = tmp_path / "outbound-blocked.jsonl" + probe = textwrap.dedent( + """\ + import socket + import openadapt_flow + from openadapt_flow import no_outbound + try: + socket.create_connection(("1.1.1.1", 443), timeout=1) + except no_outbound.OutboundBlocked: + print("blocked") + """ + ) + proc = subprocess.run( + [sys.executable, "-c", probe], + env=_strict_env(log), + capture_output=True, + text=True, + timeout=120, + ) + assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "blocked" + assert json.loads(log.read_text().splitlines()[0])["host"] == "1.1.1.1" + + +def test_the_free_path_verifies_without_reaching_another_machine( + tmp_path: Path, +) -> None: + log = tmp_path / "outbound-blocked.jsonl" + out = tmp_path / "tutorial" + proc = subprocess.run( + [sys.executable, "-m", "openadapt_flow", "tutorial", "--out", str(out)], + env=_strict_env(log), + capture_output=True, + text=True, + timeout=900, + ) + assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:] + report = json.loads((out / "run" / "report.json").read_text()) + assert report["execution_outcome"] == "VERIFIED" + envelope = report["outcome_envelope"] + assert envelope["model_calls"] == 0 + assert envelope["external_network_calls"] == "none" + assert envelope["passed_contracts"]["effect"] >= 1 + assert (out / "run" / "receipt.json").is_file() + # No refusal was recorded by the Python hook; native attempts are not logged. + assert not log.exists() or log.read_text() == "" + + +def test_native_child_attempt_is_blocked_without_a_python_refusal( + tmp_path: Path, +) -> None: + log = tmp_path / "native-outbound-blocked.jsonl" + probe = textwrap.dedent( + """\ + import json + import subprocess + import openadapt_flow + # A location environment variable alone must not authorize this probe. + links = subprocess.check_output(["ip", "-o", "link", "show"], text=True) + routes4 = subprocess.check_output(["ip", "route", "show"], text=True) + routes6 = subprocess.check_output(["ip", "-6", "route", "show"], text=True) + assert [line.split(": ", 2)[1] for line in links.splitlines()] == ["lo"] + assert routes4 == routes6 == "" + proc = subprocess.run( + ["/bin/bash", "-c", "printf native-marker >/dev/tcp/192.0.2.123/443"], + capture_output=True, text=True, timeout=5, + ) + print(json.dumps({"returncode": proc.returncode, "stderr": proc.stderr, + "links": links, "routes4": routes4, "routes6": routes6})) + """ + ) + env = _strict_env(log) + env["LC_ALL"] = "C" + proc = subprocess.run( + [sys.executable, "-c", probe], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + (tmp_path / "native-child-result.json").write_text( + json.dumps( + { + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + ) + ) + assert proc.returncode == 0, proc.stderr + child = json.loads(proc.stdout) + assert child["returncode"] != 0 + assert "Network is unreachable" in child["stderr"] + assert not log.exists() or log.read_text() == "" diff --git a/packages/flow/tests/e2e/test_egress_namespace_e2e.py b/packages/flow/tests/e2e/test_egress_namespace_e2e.py new file mode 100644 index 0000000000..33833eba54 --- /dev/null +++ b/packages/flow/tests/e2e/test_egress_namespace_e2e.py @@ -0,0 +1,289 @@ +"""Real namespace setup and an owned pathname Unix broker boundary. + +Opt in with OPENADAPT_TEST_NETNS=1 on Linux with passwordless sudo. These +tests run outside the wrapper and create only fresh synthetic listeners. +""" + +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +from pathlib import Path + +import pytest + +FLOW = Path(__file__).resolve().parents[2] +WRAPPER = FLOW / "deploy/on-prem/bin/run-egress-blocked.sh" +pytestmark = pytest.mark.skipif( + os.environ.get("OPENADAPT_TEST_NETNS") != "1", + reason="opt in to real Linux namespace creation with OPENADAPT_TEST_NETNS=1", +) + + +def _invoke( + argv: list[str], *, env: dict[str, str] | None = None +) -> subprocess.CompletedProcess[str]: + return subprocess.run(argv, env=env, capture_output=True, text=True, timeout=30) + + +def _namespace(*argv: str, forward: dict[str, str] | None = None) -> list[str]: + """Build a sudo + env + wrapper argv that does not rely on sudo -E. + + GitHub-hosted runners ignore sudo -E ("preserving the entire environment + is not supported"). Every variable the child needs must appear as an + explicit KEY=VAL on the env command line. + """ + assignments = [f"PATH={os.environ['PATH']}"] + if forward: + for key, value in forward.items(): + if key == "PATH": + continue + assignments.append(f"{key}={value}") + return [ + "sudo", + "-n", + "env", + *assignments, + str(WRAPPER), + *argv, + ] + + +@pytest.mark.parametrize("root", [False, True]) +def test_real_namespace_and_invoking_identity(root: bool, tmp_path: Path) -> None: + code = ( + "import json,os,subprocess; print(json.dumps(dict(" + "uid=os.getuid(),gid=os.getgid(),netns=os.readlink('/proc/self/ns/net')," + "links=subprocess.check_output(['ip','-o','link','show'],text=True)," + "routes4=subprocess.check_output(['ip','route','show'],text=True)," + "routes6=subprocess.check_output(['ip','-6','route','show'],text=True))))" + ) + argv = _namespace(sys.executable, "-c", code) + if root: + # env OPTIONS must precede NAME=VALUE. Index 4 was correct when argv + # was ``sudo -n -E env PATH=...``; after dropping -E, insert after env. + env_i = argv.index("env") + argv[env_i + 1 : env_i + 1] = ["-u", "SUDO_UID", "-u", "SUDO_GID"] + proc = _invoke(argv) + (tmp_path / "namespace-identity.json").write_text( + json.dumps( + { + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + ) + ) + assert proc.returncode == 0, proc.stderr + result = json.loads(proc.stdout) + assert result["uid"] == (0 if root else os.getuid()) + assert result["gid"] == (0 if root else os.getgid()) + assert result["netns"] != os.readlink("/proc/self/ns/net") + assert [line.split(": ", 2)[1] for line in result["links"].splitlines()] == ["lo"] + assert result["routes4"] == "" + assert result["routes6"] == "" + + +@pytest.mark.parametrize( + ("inspection", "output", "status"), + [ + ("-o link show", "", 69), + ("route show", "", 69), + ("-6 route show", "", 69), + ("-o link show", "", 0), + ("-o link show", "2: synthetic0: mtu 1500", 0), + ("route show", "default via 192.0.2.1 dev synthetic0", 0), + ("-6 route show", "default via 2001:db8::1 dev synthetic0", 0), + ], +) +def test_unreadable_or_unexpected_inspection_prevents_command( + tmp_path: Path, inspection: str, output: str, status: int +) -> None: + tools = tmp_path / "tools" + tools.mkdir() + for name in ("uname", "id", "bash", "unshare", "setpriv", "awk", "grep"): + real = shutil.which(name) + assert real is not None, name + (tools / name).symlink_to(real) + real_ip = shutil.which("ip") + assert real_ip is not None + fake_ip = tools / "ip" + fake_ip.write_text( + "#!/bin/bash\n" + f'if [[ "$*" == {inspection!r} ]]; then\n' + f" printf '%s' {output!r}\n exit {status}\nfi\n" + f'exec {real_ip!r} "$@"\n' + ) + fake_ip.chmod(0o755) + marker = tmp_path / "command-ran" + proc = _invoke( + [ + "sudo", + "-n", + "env", + f"PATH={tools}", + str(WRAPPER), + "/usr/bin/touch", + str(marker), + ] + ) + assert proc.returncode == 2, (proc.stdout, proc.stderr) + assert not marker.exists() + + +@pytest.mark.parametrize("mode", ["nonroot", "missing-unshare", "failed-unshare"]) +def test_setup_failure_prevents_command(tmp_path: Path, mode: str) -> None: + marker = tmp_path / "command-ran" + argv = [str(WRAPPER), "/usr/bin/touch", str(marker)] + expected = 2 + if mode == "nonroot": + assert os.getuid() != 0, "this outer test must run as an unprivileged user" + else: + tools = tmp_path / "tools" + tools.mkdir() + for name in ("uname", "id", "bash", "ip", "setpriv"): + real = shutil.which(name) + assert real is not None, name + (tools / name).symlink_to(real) + if mode == "failed-unshare": + unshare = tools / "unshare" + unshare.write_text("#!/bin/bash\nexit 73\n") + unshare.chmod(0o755) + expected = 73 + argv = ["sudo", "-n", "env", f"PATH={tools}", *argv] + proc = _invoke(argv) + assert proc.returncode == expected, (proc.stdout, proc.stderr) + assert not marker.exists() + + +def test_owned_pathname_unix_relay_remains_outside_the_ip_restriction( + tmp_path: Path, +) -> None: + marker = b"synthetic-boundary-marker" + ack = b"synthetic-boundary-ack" + receipts: list[bytes] = [] + errors: list[str] = [] + observations: dict[str, object] = {} + # AF_UNIX paths have a short OS limit; pytest's evidence path may be long. + with tempfile.TemporaryDirectory(prefix="oa-egress-") as directory: + unix_path = str(Path(directory) / "relay.sock") + with socket.socket() as tcp, socket.socket(socket.AF_UNIX) as unix: + tcp.bind(("127.0.0.1", 0)) + tcp.listen(3) + tcp.settimeout(15) + address = tcp.getsockname() + unix.bind(unix_path) + unix.listen(2) + unix.settimeout(15) + + def sink() -> None: + try: + for _ in range(3): + conn, _ = tcp.accept() + with conn: + conn.settimeout(5) + receipts.append(conn.recv(128)) + conn.sendall(ack) + except Exception as exc: + errors.append(repr(exc)) + + def relay() -> None: + try: + for _ in range(2): + conn, _ = unix.accept() + with ( + conn, + socket.create_connection(address, timeout=5) as upstream, + ): + conn.settimeout(5) + upstream.sendall(conn.recv(128)) + conn.sendall(upstream.recv(128)) + except Exception as exc: + errors.append(repr(exc)) + + workers = [threading.Thread(target=sink), threading.Thread(target=relay)] + for worker in workers: + worker.start() + try: + # First prove the independent host-side listener actually works. + with socket.create_connection(address, timeout=5) as direct: + direct.sendall(b"direct-control") + assert direct.recv(128) == ack + code = ( + "import json,os,socket; import openadapt_flow; " + "from openadapt_flow import no_outbound; " + "assert no_outbound.installed(); " + f"s=socket.socket(socket.AF_UNIX); s.settimeout(5); s.connect({unix_path!r}); " + f"s.sendall({marker!r}); ack=s.recv(128).decode(); s.close(); " + "print(json.dumps(dict(ack=ack,netns=os.readlink('/proc/self/ns/net'))))" + ) + log = tmp_path / "relay-refusals.jsonl" + env = { + k: v + for k, v in os.environ.items() + if not k.startswith("OPENADAPT_NO_") + } + env.update( + OPENADAPT_NO_OUTBOUND="1", + OPENADAPT_NO_OUTBOUND_CHILDREN="sandbox", + OPENADAPT_NO_OUTBOUND_LOG=str(log), + PYTHONPATH=str(FLOW), + ) + baseline = _invoke([sys.executable, "-c", code], env=env) + observations["baseline"] = { + "returncode": baseline.returncode, + "stdout": baseline.stdout, + "stderr": baseline.stderr, + } + assert baseline.returncode == 0, baseline.stderr + assert json.loads(baseline.stdout)["ack"] == ack.decode() + assert receipts == [b"direct-control", marker] + # The same outside TCP listener is unreachable by direct IP. + direct_code = ( + "import socket,errno; s=socket.socket(); s.settimeout(2); " + f"\ntry: s.connect({address!r}); raise AssertionError('host TCP reachable')" + "\nexcept OSError as exc: assert exc.errno in (errno.ENETUNREACH,errno.ECONNREFUSED), exc" + ) + direct = _invoke( + _namespace(sys.executable, "-c", direct_code, forward=env), env=env + ) + observations["direct"] = { + "returncode": direct.returncode, + "stdout": direct.stdout, + "stderr": direct.stderr, + } + assert direct.returncode == 0, direct.stderr + isolated = _invoke( + _namespace(sys.executable, "-c", code, forward=env), env=env + ) + observations["isolated"] = { + "returncode": isolated.returncode, + "stdout": isolated.stdout, + "stderr": isolated.stderr, + } + assert isolated.returncode == 0, isolated.stderr + result = json.loads(isolated.stdout) + assert result["netns"] != os.readlink("/proc/self/ns/net") + assert result["ack"] == ack.decode() + finally: + for worker in workers: + worker.join(20) + (tmp_path / "independent-relay-receipt.json").write_text( + json.dumps( + { + "received": [x.decode() for x in receipts], + "errors": errors, + "observations": observations, + } + ) + ) + assert not any(worker.is_alive() for worker in workers) + assert not errors, errors + assert receipts == [b"direct-control", marker, marker] + assert not log.exists() or log.read_text() == "" diff --git a/packages/flow/tests/test_ci_workflow_contract.py b/packages/flow/tests/test_ci_workflow_contract.py index a2dcb61133..62a0f07460 100644 --- a/packages/flow/tests/test_ci_workflow_contract.py +++ b/packages/flow/tests/test_ci_workflow_contract.py @@ -531,3 +531,17 @@ def test_clean_machine_lifecycle_declares_utf8_on_every_os() -> None: assert 'PYTHONUTF8: "1"' in lifecycle_header assert 'PYTHONIOENCODING: "utf-8"' in lifecycle_header + + +def test_egress_boundary_controls_are_required_without_skips() -> None: + workflow = ROOT_PACKAGES.read_text(encoding="utf-8") + start = workflow.index("\n flow-critical:") + end = workflow.index("\n flow-agent-frameworks:", start) + critical = workflow[start:end] + assert "tests/e2e/test_egress_blocked_e2e.py" in critical + assert "tests/e2e/test_egress_namespace_e2e.py" in critical + assert 'OPENADAPT_TEST_NETNS: "1"' in critical + assert "--junitxml=runs/onprem-boundary-junit.xml" in critical + gate = critical[critical.index("- name: Fail if any critical test") :] + assert '"onprem-egress", "onprem-boundary"' in gate + assert "if tests == 0 or skipped:" in gate diff --git a/packages/flow/tests/test_egress_namespace_argv.py b/packages/flow/tests/test_egress_namespace_argv.py new file mode 100644 index 0000000000..3b5e46f6ea --- /dev/null +++ b/packages/flow/tests/test_egress_namespace_argv.py @@ -0,0 +1,54 @@ +"""_namespace must pass child env on the env argv; sudo -E is not available in CI.""" + +from __future__ import annotations + +import os + +from tests.e2e import test_egress_namespace_e2e as ns + + +def test_namespace_forwards_env_without_sudo_dash_e() -> None: + forward = { + "OPENADAPT_NO_OUTBOUND": "1", + "OPENADAPT_NO_OUTBOUND_CHILDREN": "sandbox", + "PYTHONPATH": "/tmp/flow-src", + "PATH": "/should-not-override", + } + argv = ns._namespace("python", "-c", "pass", forward=forward) + assert argv[0] == "sudo" + assert argv[1] == "-n" + assert argv[2] == "env" + assert "-E" not in argv + assert f"PATH={os.environ['PATH']}" in argv + assert "PATH=/should-not-override" not in argv + assert "OPENADAPT_NO_OUTBOUND=1" in argv + assert "OPENADAPT_NO_OUTBOUND_CHILDREN=sandbox" in argv + assert "PYTHONPATH=/tmp/flow-src" in argv + assert str(ns.WRAPPER) in argv + assert argv[-3:] == ["python", "-c", "pass"] + + +def test_namespace_without_forward_only_sets_path() -> None: + argv = ns._namespace("/bin/true") + assert "-E" not in argv + assert argv[2] == "env" + assert any(a.startswith("PATH=") for a in argv) + assert not any(a.startswith("OPENADAPT_") for a in argv) + + +def test_root_unsets_sit_before_assignments() -> None: + """GNU env treats anything after NAME=VALUE as the command; -u must precede.""" + argv = ns._namespace("/bin/true") + env_i = argv.index("env") + argv[env_i + 1 : env_i + 1] = ["-u", "SUDO_UID", "-u", "SUDO_GID"] + assert argv[env_i : env_i + 5] == [ + "env", + "-u", + "SUDO_UID", + "-u", + "SUDO_GID", + ] + path_i = next(i for i, a in enumerate(argv) if a.startswith("PATH=")) + assert path_i > env_i + 4 + # Regression: the old argv[4:4] insert put -u after PATH= and broke CI. + assert argv[4] != "-u"