From 9b97567003ab009902711a34be452b08dbd0652a Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 17:48:16 -0500 Subject: [PATCH 1/9] on-prem: prove the golden free path verifies with egress blocked by the kernel --- .github/workflows/packages.yml | 13 +- packages/flow/deploy/on-prem/README.md | 14 +++ .../deploy/on-prem/bin/run-egress-blocked.sh | 63 ++++++++++ .../flow/tests/e2e/test_egress_blocked_e2e.py | 118 ++++++++++++++++++ 4 files changed, 207 insertions(+), 1 deletion(-) create mode 100755 packages/flow/deploy/on-prem/bin/run-egress-blocked.sh create mode 100644 packages/flow/tests/e2e/test_egress_blocked_e2e.py diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index c8e4aa0e12..30656bfec9 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -602,13 +602,24 @@ jobs: tests/test_admitted_composition_runtime.py \ tests/test_agent_guard.py \ --basetemp=runs/ci-critical --junitxml=runs/critical-junit.xml + - name: Golden free path with egress blocked (loopback-only network namespace) + # The on-prem proof: the free path still reaches VERIFIED when the + # kernel refuses every packet bound off the machine, and strict + # no-outbound mode logs no attempt to leave. The script refuses to run + # unless the namespace holds only loopback; the first test then shows + # public hosts are unreachable, so it cannot pass with a network. + run: | + sudo -E env "PATH=$PATH" "HOME=$HOME" \ + deploy/on-prem/bin/run-egress-blocked.sh \ + python -m pytest -v tests/e2e/test_egress_blocked_e2e.py \ + --basetemp=runs/ci-onprem-egress --junitxml=runs/onprem-egress-junit.xml - name: Fail if any critical test was skipped or none ran run: | python - <<'PY' import sys import xml.etree.ElementTree as ET bad = False - for name in ("critical-e2e", "critical"): + for name in ("critical-e2e", "critical", "onprem-egress"): root = ET.parse(f"runs/{name}-junit.xml").getroot() suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite")) tests = sum(int(s.get("tests", 0)) for s in suites) diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index ac88003aa4..a9a36396a9 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -19,6 +19,7 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow). | `bin/run-queue.sh` | Local, directory-based run queue. Claims jobs, runs `openadapt-flow run`, records outcomes, files done/failed. Fail-closed on `SCRUB!=on`. | REAL (thin wrapper over the shipped CLI) | | `bin/audit-log.sh` | Append-only, hash-chained, PHI-free audit writer (run + release-lifecycle events). | REAL | | `bin/verify-airgap.sh` | Asserts no outbound path: config/env/deployment URL scan, optional active egress probe, optional audit-chain walk. | REAL (best-effort; firewall is primary) | +| `bin/run-egress-blocked.sh` | Runs a command in a loopback-only Linux network namespace, refusing to start unless no other interface or route exists. CI runs the golden free path through it (`tests/e2e/test_egress_blocked_e2e.py`): VERIFIED, zero model calls, and no outbound attempt in strict no-outbound mode. | REAL (Linux, root) | | `bin/test-update.sh` | Hermetic offline proof of signed update, refusal paths, legacy migration, data preservation, rollback/recovery, locks, and audit-chain serialization. | REAL (43 baseline assertions; 45 when local GPG key generation is available) | | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) | | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) | @@ -47,6 +48,19 @@ PHI-free record to `audit/audit.log`, and files the job under `jobs/done` (exit OPENADAPT_FLOW_SCRUB=on ./bin/verify-airgap.sh --config onprem.yaml --probe --audit ``` +## Proof that a run needs no network + +On any Linux host with the engine installed, run the free path with every +packet bound off the machine refused by the kernel: + +```bash +sudo -E env "PATH=$PATH" "HOME=$HOME" deploy/on-prem/bin/run-egress-blocked.sh \ + python -m pytest -v tests/e2e/test_egress_blocked_e2e.py +``` + +The first test shows public hosts are unreachable inside the namespace, so the +proof cannot pass on a connected machine. CI runs this on every flow change. + ## Update & rollback Patch the box offline; revert instantly if a release misbehaves. Full runbook in diff --git a/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh b/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh new file mode 100755 index 0000000000..4c9286097a --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# run-egress-blocked.sh — run a command with no route off this machine. +# +# Linux only. Starts the command in a new network namespace that holds only +# the loopback interface, so the kernel refuses every connection and DNS query +# that would leave the machine, from Python, from native code, and from child +# processes alike (a browser included). Loopback still works, so a local +# system of record on 127.0.0.1 stays reachable. +# +# Before running the command it checks that the namespace really has no +# interface besides loopback and no route, and refuses to run otherwise. The +# command runs as the user who invoked sudo, with OPENADAPT_EGRESS_BLOCKED=netns +# in its environment. That variable only tells a test where it is running; the +# test still has to show for itself that nothing leaves the machine. +# +# Usage (as root, normally through sudo -E so PATH and HOME are kept): +# sudo -E env "PATH=$PATH" deploy/on-prem/bin/run-egress-blocked.sh CMD [ARGS...] +# +# Exit code: the command's, or 2 when the namespace cannot be set up. + +set -euo pipefail + +if [[ $# -eq 0 || "$1" == "-h" || "$1" == "--help" ]]; then + grep '^#' "$0" | sed 's/^# \{0,1\}//' + [[ $# -eq 0 ]] && exit 2 || exit 0 +fi +if [[ "$(uname -s)" != "Linux" ]]; then + echo "run-egress-blocked.sh: needs Linux network namespaces" >&2 + exit 2 +fi +if [[ "$(id -u)" -ne 0 ]]; then + echo "run-egress-blocked.sh: run as root (sudo); the command runs as the invoking user" >&2 + exit 2 +fi +for tool in unshare ip setpriv; do + command -v "$tool" >/dev/null || { echo "run-egress-blocked.sh: missing $tool" >&2; exit 2; } +done + +RUN_UID="${SUDO_UID:-0}" +RUN_GID="${SUDO_GID:-0}" +export RUN_UID RUN_GID + +# The inner script expands its own variables, inside the namespace. +# shellcheck disable=SC2016 +exec unshare --net -- bash -euo pipefail -c ' + ip link set lo up + others="$(ip -o link show | awk -F": " "{print \$2}" | grep -v "^lo$" || true)" + if [[ -n "$others" ]]; then + echo "run-egress-blocked.sh: interfaces besides loopback: $others" >&2 + exit 2 + fi + routes="$(ip route show; ip -6 route show | grep -v "^::1 " | grep -v "^fe80::" || true)" + if [[ -n "$routes" ]]; then + echo "run-egress-blocked.sh: unexpected routes: $routes" >&2 + exit 2 + fi + export OPENADAPT_EGRESS_BLOCKED=netns + if [[ "$RUN_UID" == 0 ]]; then + exec "$@" + fi + exec setpriv --reuid="$RUN_UID" --regid="$RUN_GID" --init-groups -- "$@" +' run-egress-blocked "$@" diff --git a/packages/flow/tests/e2e/test_egress_blocked_e2e.py b/packages/flow/tests/e2e/test_egress_blocked_e2e.py new file mode 100644 index 0000000000..b60d9ea923 --- /dev/null +++ b/packages/flow/tests/e2e/test_egress_blocked_e2e.py @@ -0,0 +1,118 @@ +"""The golden free path with egress blocked: VERIFIED with no route off the machine. + +The on-prem promise is that a run needs nothing outside the building. These +tests prove it on the real free path (record, compile, certify, run, verify +the write by an independent read) inside +``deploy/on-prem/bin/run-egress-blocked.sh``, a Linux network namespace that +holds only loopback. The kernel refuses every packet bound off the machine, +from Python, native code and the browser alike. + +They run only inside that namespace (``OPENADAPT_EGRESS_BLOCKED=netns``), and +the first test does not trust the variable: it shows that public hosts, +public DNS resolvers included, are unreachable, so on a connected machine it +fails instead of passing. + +The run also goes through strict no-outbound mode (:mod:`openadapt_flow.no_outbound`) +with its refusal log on. An empty log means the engine did not even try to +reach another machine; the second test shows that the log does fill when +something tries, so the empty log is not vacuous. +""" + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +FLOW = Path(__file__).resolve().parents[2] + +pytestmark = pytest.mark.skipif( + os.environ.get("OPENADAPT_EGRESS_BLOCKED") != "netns", + reason="runs inside deploy/on-prem/bin/run-egress-blocked.sh (Linux, root)", +) + +#: Public anycast hosts every connected machine reaches. +CANARIES = (("1.1.1.1", 443), ("8.8.8.8", 53), ("2606:4700:4700::1111", 443)) + + +def _strict_env(log: Path) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if not k.startswith("OPENADAPT_NO_")} + env.update( + OPENADAPT_NO_OUTBOUND="1", + # The namespace contains every child, the browser included. + OPENADAPT_NO_OUTBOUND_CHILDREN="sandbox", + OPENADAPT_NO_OUTBOUND_LOG=str(log), + PYTHONPATH=os.pathsep.join( + [str(FLOW)] + [p for p in [env.get("PYTHONPATH", "")] if p] + ), + ) + return env + + +def test_nothing_off_the_machine_is_reachable() -> None: + for host, port in CANARIES: + family = socket.AF_INET6 if ":" in host else socket.AF_INET + with pytest.raises(OSError): + with socket.socket(family, socket.SOCK_STREAM) as sock: + sock.settimeout(5) + sock.connect((host, port)) + # Loopback still works: the local system of record lives there. + with socket.socket() as server: + server.bind(("127.0.0.1", 0)) + server.listen(1) + socket.create_connection(server.getsockname(), timeout=5).close() + + +def test_strict_mode_logs_an_attempt_to_leave(tmp_path: Path) -> None: + log = tmp_path / "outbound-blocked.jsonl" + probe = textwrap.dedent( + """\ + import socket + import openadapt_flow + from openadapt_flow import no_outbound + try: + socket.create_connection(("1.1.1.1", 443), timeout=1) + except no_outbound.OutboundBlocked: + print("blocked") + """ + ) + proc = subprocess.run( + [sys.executable, "-c", probe], + env=_strict_env(log), + capture_output=True, + text=True, + timeout=120, + ) + assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "blocked" + assert json.loads(log.read_text().splitlines()[0])["host"] == "1.1.1.1" + + +def test_the_free_path_verifies_without_reaching_another_machine( + tmp_path: Path, +) -> None: + log = tmp_path / "outbound-blocked.jsonl" + out = tmp_path / "tutorial" + proc = subprocess.run( + [sys.executable, "-m", "openadapt_flow", "tutorial", "--out", str(out)], + env=_strict_env(log), + capture_output=True, + text=True, + timeout=900, + ) + assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:] + report = json.loads((out / "run" / "report.json").read_text()) + assert report["execution_outcome"] == "VERIFIED" + envelope = report["outcome_envelope"] + assert envelope["model_calls"] == 0 + assert envelope["external_network_calls"] == "none" + assert envelope["passed_contracts"]["effect"] >= 1 + assert (out / "run" / "receipt.json").is_file() + # Not one refused attempt: the engine never tried to leave. + assert not log.exists() or log.read_text() == "" From ca3557eac28f10e067e9f966102e114c1bb38e3a Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 21:10:08 -0500 Subject: [PATCH 2/9] fix: fail closed on network namespace inspections Signed-off-by: abrichr --- .github/workflows/packages.yml | 16 +- packages/flow/deploy/on-prem/README.md | 22 +- .../deploy/on-prem/bin/run-egress-blocked.sh | 45 ++-- .../flow/tests/e2e/test_egress_blocked_e2e.py | 61 +++-- .../tests/e2e/test_egress_namespace_e2e.py | 218 ++++++++++++++++++ .../flow/tests/test_ci_workflow_contract.py | 15 ++ 6 files changed, 329 insertions(+), 48 deletions(-) create mode 100644 packages/flow/tests/e2e/test_egress_namespace_e2e.py diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 30656bfec9..84487c25ee 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -603,23 +603,27 @@ jobs: tests/test_agent_guard.py \ --basetemp=runs/ci-critical --junitxml=runs/critical-junit.xml - name: Golden free path with egress blocked (loopback-only network namespace) - # The on-prem proof: the free path still reaches VERIFIED when the - # kernel refuses every packet bound off the machine, and strict - # no-outbound mode logs no attempt to leave. The script refuses to run - # unless the namespace holds only loopback; the first test then shows - # public hosts are unreachable, so it cannot pass with a network. + # Direct IP sockets stay in a loopback-only namespace. The tutorial + # must still reach VERIFIED with no Python-hook refusal recorded. + # A native-child control measures the hook observation boundary. run: | sudo -E env "PATH=$PATH" "HOME=$HOME" \ deploy/on-prem/bin/run-egress-blocked.sh \ python -m pytest -v tests/e2e/test_egress_blocked_e2e.py \ --basetemp=runs/ci-onprem-egress --junitxml=runs/onprem-egress-junit.xml + - name: Network namespace setup and owned Unix broker boundary + env: + OPENADAPT_TEST_NETNS: "1" + run: | + python -m pytest -v tests/e2e/test_egress_namespace_e2e.py \ + --basetemp=runs/ci-onprem-boundary --junitxml=runs/onprem-boundary-junit.xml - name: Fail if any critical test was skipped or none ran run: | python - <<'PY' import sys import xml.etree.ElementTree as ET bad = False - for name in ("critical-e2e", "critical", "onprem-egress"): + for name in ("critical-e2e", "critical", "onprem-egress", "onprem-boundary"): root = ET.parse(f"runs/{name}-junit.xml").getroot() suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite")) tests = sum(int(s.get("tests", 0)) for s in suites) diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index a9a36396a9..8b394f9dbd 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -19,7 +19,7 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow). | `bin/run-queue.sh` | Local, directory-based run queue. Claims jobs, runs `openadapt-flow run`, records outcomes, files done/failed. Fail-closed on `SCRUB!=on`. | REAL (thin wrapper over the shipped CLI) | | `bin/audit-log.sh` | Append-only, hash-chained, PHI-free audit writer (run + release-lifecycle events). | REAL | | `bin/verify-airgap.sh` | Asserts no outbound path: config/env/deployment URL scan, optional active egress probe, optional audit-chain walk. | REAL (best-effort; firewall is primary) | -| `bin/run-egress-blocked.sh` | Runs a command in a loopback-only Linux network namespace, refusing to start unless no other interface or route exists. CI runs the golden free path through it (`tests/e2e/test_egress_blocked_e2e.py`): VERIFIED, zero model calls, and no outbound attempt in strict no-outbound mode. | REAL (Linux, root) | +| `bin/run-egress-blocked.sh` | Runs a command in a loopback-only Linux network namespace, refusing to start unless no other interface or route exists. CI runs the golden free path through it (`tests/e2e/test_egress_blocked_e2e.py`): VERIFIED, zero model calls, and no refusal recorded by the strict Python audit hook. | REAL (Linux, root) | | `bin/test-update.sh` | Hermetic offline proof of signed update, refusal paths, legacy migration, data preservation, rollback/recovery, locks, and audit-chain serialization. | REAL (43 baseline assertions; 45 when local GPG key generation is available) | | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) | | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) | @@ -48,18 +48,28 @@ PHI-free record to `audit/audit.log`, and files the job under `jobs/done` (exit OPENADAPT_FLOW_SCRUB=on ./bin/verify-airgap.sh --config onprem.yaml --probe --audit ``` -## Proof that a run needs no network +## Golden run with direct IP networking restricted -On any Linux host with the engine installed, run the free path with every -packet bound off the machine refused by the kernel: +On Linux with the engine installed, run the free path in a new network +namespace containing only loopback: ```bash sudo -E env "PATH=$PATH" "HOME=$HOME" deploy/on-prem/bin/run-egress-blocked.sh \ python -m pytest -v tests/e2e/test_egress_blocked_e2e.py ``` -The first test shows public hosts are unreachable inside the namespace, so the -proof cannot pass on a connected machine. CI runs this on every flow change. +The wrapper refuses to start if any interface or route inspection fails, +or if it finds anything besides loopback and no routes. The tests check +direct IP refusal independently, then require the tutorial to reach VERIFIED +with zero model calls and a receipt. CI runs this on every flow change. + +The restriction covers direct IP sockets in the command and its children. +Filesystem access stays shared: pathname Unix sockets can reach a process +outside the namespace that forwards traffic. Isolate such brokers separately +when enforcing a deployment boundary. An empty strict-mode refusal log means +the Python hook recorded no refusals; it does not observe native sockets or +children that never install the hook. Native-child and owned Unix-relay +controls exercise these boundaries in CI. ## Update & rollback diff --git a/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh b/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh index 4c9286097a..8dd95f45bf 100755 --- a/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh +++ b/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh @@ -2,17 +2,19 @@ # # run-egress-blocked.sh — run a command with no route off this machine. # -# Linux only. Starts the command in a new network namespace that holds only -# the loopback interface, so the kernel refuses every connection and DNS query -# that would leave the machine, from Python, from native code, and from child -# processes alike (a browser included). Loopback still works, so a local -# system of record on 127.0.0.1 stays reachable. +# Linux only. Starts the command in a new network namespace containing only +# loopback. Direct IP connections from the command and its children cannot +# reach networks outside that namespace. Services started inside it can use +# loopback to communicate with each other. # -# Before running the command it checks that the namespace really has no -# interface besides loopback and no route, and refuses to run otherwise. The -# command runs as the user who invoked sudo, with OPENADAPT_EGRESS_BLOCKED=netns -# in its environment. That variable only tells a test where it is running; the -# test still has to show for itself that nothing leaves the machine. +# Filesystem access is unchanged: pathname Unix sockets can reach processes +# outside the namespace, including brokers that forward network traffic. This +# is a direct-IP restriction, not a filesystem or untrusted-code sandbox. +# +# Before running the command, all interface and route inspections must succeed +# and confirm only loopback and no routes. The command runs as the invoking +# sudo user, with OPENADAPT_EGRESS_BLOCKED=netns as a test-location hint. Tests +# independently check the restriction rather than trusting that variable. # # Usage (as root, normally through sudo -E so PATH and HOME are kept): # sudo -E env "PATH=$PATH" deploy/on-prem/bin/run-egress-blocked.sh CMD [ARGS...] @@ -45,14 +47,25 @@ export RUN_UID RUN_GID # shellcheck disable=SC2016 exec unshare --net -- bash -euo pipefail -c ' ip link set lo up - others="$(ip -o link show | awk -F": " "{print \$2}" | grep -v "^lo$" || true)" - if [[ -n "$others" ]]; then - echo "run-egress-blocked.sh: interfaces besides loopback: $others" >&2 + if ! links="$(ip -o link show)"; then + echo "run-egress-blocked.sh: cannot inspect interfaces" >&2 + exit 2 + fi + interfaces="$(awk -F": " "{print \$2}" <<< "$links")" + if [[ "$interfaces" != lo ]]; then + echo "run-egress-blocked.sh: expected only loopback, found: $interfaces" >&2 + exit 2 + fi + if ! routes4="$(ip route show)"; then + echo "run-egress-blocked.sh: cannot inspect IPv4 routes" >&2 + exit 2 + fi + if ! routes6="$(ip -6 route show)"; then + echo "run-egress-blocked.sh: cannot inspect IPv6 routes" >&2 exit 2 fi - routes="$(ip route show; ip -6 route show | grep -v "^::1 " | grep -v "^fe80::" || true)" - if [[ -n "$routes" ]]; then - echo "run-egress-blocked.sh: unexpected routes: $routes" >&2 + if [[ -n "$routes4" || -n "$routes6" ]]; then + echo "run-egress-blocked.sh: unexpected routes: $routes4 $routes6" >&2 exit 2 fi export OPENADAPT_EGRESS_BLOCKED=netns diff --git a/packages/flow/tests/e2e/test_egress_blocked_e2e.py b/packages/flow/tests/e2e/test_egress_blocked_e2e.py index b60d9ea923..1363b60e1f 100644 --- a/packages/flow/tests/e2e/test_egress_blocked_e2e.py +++ b/packages/flow/tests/e2e/test_egress_blocked_e2e.py @@ -1,21 +1,15 @@ -"""The golden free path with egress blocked: VERIFIED with no route off the machine. - -The on-prem promise is that a run needs nothing outside the building. These -tests prove it on the real free path (record, compile, certify, run, verify -the write by an independent read) inside -``deploy/on-prem/bin/run-egress-blocked.sh``, a Linux network namespace that -holds only loopback. The kernel refuses every packet bound off the machine, -from Python, native code and the browser alike. - -They run only inside that namespace (``OPENADAPT_EGRESS_BLOCKED=netns``), and -the first test does not trust the variable: it shows that public hosts, -public DNS resolvers included, are unreachable, so on a connected machine it -fails instead of passing. - -The run also goes through strict no-outbound mode (:mod:`openadapt_flow.no_outbound`) -with its refusal log on. An empty log means the engine did not even try to -reach another machine; the second test shows that the log does fill when -something tries, so the empty log is not vacuous. +"""Golden free path inside a loopback-only Linux network namespace. + +Run through ``deploy/on-prem/bin/run-egress-blocked.sh``. The first test +checks direct IP connections independently of the location environment +variable. The real tutorial still records, compiles, certifies, runs and +verifies a write through independent read-back. + +Strict no-outbound mode adds a Python audit hook. Its empty refusal log means +no refusal was recorded by that hook. Native sockets, children without the +hook and allowed pathname Unix brokers are outside that observation. The +native-child positive control demonstrates this limit with a real attempt +that the network namespace refuses. """ from __future__ import annotations @@ -45,7 +39,7 @@ def _strict_env(log: Path) -> dict[str, str]: env = {k: v for k, v in os.environ.items() if not k.startswith("OPENADAPT_NO_")} env.update( OPENADAPT_NO_OUTBOUND="1", - # The namespace contains every child, the browser included. + # Children inherit the direct-IP restriction; their sockets are not audited. OPENADAPT_NO_OUTBOUND_CHILDREN="sandbox", OPENADAPT_NO_OUTBOUND_LOG=str(log), PYTHONPATH=os.pathsep.join( @@ -114,5 +108,32 @@ def test_the_free_path_verifies_without_reaching_another_machine( assert envelope["external_network_calls"] == "none" assert envelope["passed_contracts"]["effect"] >= 1 assert (out / "run" / "receipt.json").is_file() - # Not one refused attempt: the engine never tried to leave. + # No refusal was recorded by the Python hook; native attempts are not logged. + assert not log.exists() or log.read_text() == "" + + +def test_native_child_attempt_is_blocked_without_a_python_refusal(tmp_path: Path) -> None: + log = tmp_path / "native-outbound-blocked.jsonl" + probe = textwrap.dedent( + """\ + import json + import subprocess + import openadapt_flow + proc = subprocess.run( + ["/bin/bash", "-c", "printf native-marker >/dev/tcp/192.0.2.123/443"], + capture_output=True, text=True, timeout=5, + ) + print(json.dumps({"returncode": proc.returncode, "stderr": proc.stderr})) + """ + ) + env = _strict_env(log) + env["LC_ALL"] = "C" + proc = subprocess.run( + [sys.executable, "-c", probe], env=env, + capture_output=True, text=True, timeout=30, + ) + assert proc.returncode == 0, proc.stderr + child = json.loads(proc.stdout) + assert child["returncode"] != 0 + assert "Network is unreachable" in child["stderr"] assert not log.exists() or log.read_text() == "" diff --git a/packages/flow/tests/e2e/test_egress_namespace_e2e.py b/packages/flow/tests/e2e/test_egress_namespace_e2e.py new file mode 100644 index 0000000000..293c3f25cf --- /dev/null +++ b/packages/flow/tests/e2e/test_egress_namespace_e2e.py @@ -0,0 +1,218 @@ +"""Real namespace setup and an owned pathname Unix broker boundary. + +Opt in with OPENADAPT_TEST_NETNS=1 on Linux with passwordless sudo. These +tests run outside the wrapper and create only fresh synthetic listeners. +""" + +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +from pathlib import Path + +import pytest + +FLOW = Path(__file__).resolve().parents[2] +WRAPPER = FLOW / "deploy/on-prem/bin/run-egress-blocked.sh" +pytestmark = pytest.mark.skipif( + os.environ.get("OPENADAPT_TEST_NETNS") != "1", + reason="opt in to real Linux namespace creation with OPENADAPT_TEST_NETNS=1", +) + + +def _invoke( + argv: list[str], *, env: dict[str, str] | None = None +) -> subprocess.CompletedProcess[str]: + return subprocess.run(argv, env=env, capture_output=True, text=True, timeout=30) + + +def _namespace(*argv: str) -> list[str]: + return ["sudo", "-n", "-E", "env", f"PATH={os.environ['PATH']}", str(WRAPPER), *argv] + + +@pytest.mark.parametrize("root", [False, True]) +def test_real_namespace_and_invoking_identity(root: bool) -> None: + code = ( + "import json,os,subprocess; print(json.dumps(dict(" + "uid=os.getuid(),gid=os.getgid(),netns=os.readlink('/proc/self/ns/net')," + "links=subprocess.check_output(['ip','-o','link','show'],text=True)," + "routes4=subprocess.check_output(['ip','route','show'],text=True)," + "routes6=subprocess.check_output(['ip','-6','route','show'],text=True))))" + ) + argv = _namespace(sys.executable, "-c", code) + if root: + argv[4:4] = ["-u", "SUDO_UID", "-u", "SUDO_GID"] + proc = _invoke(argv) + assert proc.returncode == 0, proc.stderr + result = json.loads(proc.stdout) + assert result["uid"] == (0 if root else os.getuid()) + assert result["gid"] == (0 if root else os.getgid()) + assert result["netns"] != os.readlink("/proc/self/ns/net") + assert [line.split(": ", 2)[1] for line in result["links"].splitlines()] == ["lo"] + assert result["routes4"] == "" + assert result["routes6"] == "" + + +@pytest.mark.parametrize( + ("inspection", "output", "status"), + [ + ("-o link show", "", 69), + ("route show", "", 69), + ("-6 route show", "", 69), + ("-o link show", "", 0), + ("-o link show", "2: synthetic0: mtu 1500", 0), + ("route show", "default via 192.0.2.1 dev synthetic0", 0), + ("-6 route show", "default via 2001:db8::1 dev synthetic0", 0), + ], +) +def test_unreadable_or_unexpected_inspection_prevents_command( + tmp_path: Path, inspection: str, output: str, status: int +) -> None: + tools = tmp_path / "tools" + tools.mkdir() + for name in ("uname", "id", "bash", "unshare", "setpriv", "awk", "grep"): + real = shutil.which(name) + assert real is not None, name + (tools / name).symlink_to(real) + real_ip = shutil.which("ip") + assert real_ip is not None + fake_ip = tools / "ip" + fake_ip.write_text( + "#!/bin/bash\n" + f"if [[ \"$*\" == {inspection!r} ]]; then\n" + f" printf '%s' {output!r}\n exit {status}\nfi\n" + f"exec {real_ip!r} \"$@\"\n" + ) + fake_ip.chmod(0o755) + marker = tmp_path / "command-ran" + proc = _invoke( + ["sudo", "-n", "env", f"PATH={tools}", str(WRAPPER), + "/usr/bin/touch", str(marker)] + ) + assert proc.returncode == 2, (proc.stdout, proc.stderr) + assert not marker.exists() + + +@pytest.mark.parametrize("mode", ["nonroot", "missing-unshare", "failed-unshare"]) +def test_setup_failure_prevents_command(tmp_path: Path, mode: str) -> None: + marker = tmp_path / "command-ran" + argv = [str(WRAPPER), "/usr/bin/touch", str(marker)] + expected = 2 + if mode == "nonroot": + assert os.getuid() != 0, "this outer test must run as an unprivileged user" + else: + tools = tmp_path / "tools" + tools.mkdir() + for name in ("uname", "id", "bash", "ip", "setpriv"): + real = shutil.which(name) + assert real is not None, name + (tools / name).symlink_to(real) + if mode == "failed-unshare": + unshare = tools / "unshare" + unshare.write_text("#!/bin/bash\nexit 73\n") + unshare.chmod(0o755) + expected = 73 + argv = ["sudo", "-n", "env", f"PATH={tools}", *argv] + proc = _invoke(argv) + assert proc.returncode == expected, (proc.stdout, proc.stderr) + assert not marker.exists() + + +def test_owned_pathname_unix_relay_remains_outside_the_ip_restriction( + tmp_path: Path, +) -> None: + marker = b"synthetic-boundary-marker" + ack = b"synthetic-boundary-ack" + receipts: list[bytes] = [] + errors: list[str] = [] + # AF_UNIX paths have a short OS limit; pytest's evidence path may be long. + with tempfile.TemporaryDirectory(prefix="oa-egress-") as directory: + unix_path = str(Path(directory) / "relay.sock") + with socket.socket() as tcp, socket.socket(socket.AF_UNIX) as unix: + tcp.bind(("127.0.0.1", 0)) + tcp.listen(3) + tcp.settimeout(15) + address = tcp.getsockname() + unix.bind(unix_path) + unix.listen(2) + unix.settimeout(15) + + def sink() -> None: + try: + for _ in range(3): + conn, _ = tcp.accept() + with conn: + conn.settimeout(5) + receipts.append(conn.recv(128)) + conn.sendall(ack) + except Exception as exc: + errors.append(repr(exc)) + + def relay() -> None: + try: + for _ in range(2): + conn, _ = unix.accept() + with conn, socket.create_connection(address, timeout=5) as upstream: + conn.settimeout(5) + upstream.sendall(conn.recv(128)) + conn.sendall(upstream.recv(128)) + except Exception as exc: + errors.append(repr(exc)) + + workers = [threading.Thread(target=sink), threading.Thread(target=relay)] + for worker in workers: + worker.start() + try: + # First prove the independent host-side listener actually works. + with socket.create_connection(address, timeout=5) as direct: + direct.sendall(b"direct-control") + assert direct.recv(128) == ack + code = ( + "import json,os,socket; import openadapt_flow; " + "from openadapt_flow import no_outbound; " + "assert no_outbound.installed(); " + f"s=socket.socket(socket.AF_UNIX); s.settimeout(5); s.connect({unix_path!r}); " + f"s.sendall({marker!r}); ack=s.recv(128).decode(); s.close(); " + "print(json.dumps(dict(ack=ack,netns=os.readlink('/proc/self/ns/net'))))" + ) + log = tmp_path / "relay-refusals.jsonl" + env = {k: v for k, v in os.environ.items() if not k.startswith("OPENADAPT_NO_")} + env.update( + OPENADAPT_NO_OUTBOUND="1", + OPENADAPT_NO_OUTBOUND_CHILDREN="sandbox", + OPENADAPT_NO_OUTBOUND_LOG=str(log), + PYTHONPATH=str(FLOW), + ) + baseline = _invoke([sys.executable, "-c", code], env=env) + assert baseline.returncode == 0, baseline.stderr + assert json.loads(baseline.stdout)["ack"] == ack.decode() + assert receipts == [b"direct-control", marker] + # The same outside TCP listener is unreachable by direct IP. + direct_code = ( + "import socket,errno; s=socket.socket(); s.settimeout(2); " + f"\ntry: s.connect({address!r}); raise AssertionError('host TCP reachable')" + "\nexcept OSError as exc: assert exc.errno in (errno.ENETUNREACH,errno.ECONNREFUSED), exc" + ) + direct = _invoke(_namespace(sys.executable, "-c", direct_code), env=env) + assert direct.returncode == 0, direct.stderr + isolated = _invoke(_namespace(sys.executable, "-c", code), env=env) + assert isolated.returncode == 0, isolated.stderr + result = json.loads(isolated.stdout) + assert result["netns"] != os.readlink("/proc/self/ns/net") + assert result["ack"] == ack.decode() + finally: + for worker in workers: + worker.join(20) + (tmp_path / "independent-relay-receipt.json").write_text( + json.dumps({"received": [x.decode() for x in receipts], "errors": errors}) + ) + assert not any(worker.is_alive() for worker in workers) + assert not errors, errors + assert receipts == [b"direct-control", marker, marker] + assert not log.exists() or log.read_text() == "" diff --git a/packages/flow/tests/test_ci_workflow_contract.py b/packages/flow/tests/test_ci_workflow_contract.py index a2dcb61133..01afae740e 100644 --- a/packages/flow/tests/test_ci_workflow_contract.py +++ b/packages/flow/tests/test_ci_workflow_contract.py @@ -531,3 +531,18 @@ def test_clean_machine_lifecycle_declares_utf8_on_every_os() -> None: assert 'PYTHONUTF8: "1"' in lifecycle_header assert 'PYTHONIOENCODING: "utf-8"' in lifecycle_header + + + +def test_egress_boundary_controls_are_required_without_skips() -> None: + workflow = ROOT_PACKAGES.read_text(encoding="utf-8") + start = workflow.index("\n flow-critical:") + end = workflow.index("\n flow-agent-frameworks:", start) + critical = workflow[start:end] + assert "tests/e2e/test_egress_blocked_e2e.py" in critical + assert "tests/e2e/test_egress_namespace_e2e.py" in critical + assert 'OPENADAPT_TEST_NETNS: "1"' in critical + assert "--junitxml=runs/onprem-boundary-junit.xml" in critical + gate = critical[critical.index("- name: Fail if any critical test") :] + assert '"onprem-egress", "onprem-boundary"' in gate + assert "if tests == 0 or skipped:" in gate From 7ab677f34b61c4956a918f6dbab1929082d83bcb Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 21:12:03 -0500 Subject: [PATCH 3/9] style: format namespace boundary controls Signed-off-by: abrichr --- packages/flow/deploy/on-prem/README.md | 4 +- .../flow/tests/e2e/test_egress_blocked_e2e.py | 11 +++-- .../tests/e2e/test_egress_namespace_e2e.py | 40 +++++++++++++++---- .../flow/tests/test_ci_workflow_contract.py | 1 - 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index 8b394f9dbd..ba784895f3 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -58,8 +58,8 @@ sudo -E env "PATH=$PATH" "HOME=$HOME" deploy/on-prem/bin/run-egress-blocked.sh \ python -m pytest -v tests/e2e/test_egress_blocked_e2e.py ``` -The wrapper refuses to start if any interface or route inspection fails, -or if it finds anything besides loopback and no routes. The tests check +The wrapper starts only after all interface and route inspections succeed +and confirm exactly loopback with no routes. The tests check direct IP refusal independently, then require the tutorial to reach VERIFIED with zero model calls and a receipt. CI runs this on every flow change. diff --git a/packages/flow/tests/e2e/test_egress_blocked_e2e.py b/packages/flow/tests/e2e/test_egress_blocked_e2e.py index 1363b60e1f..07fea6fad4 100644 --- a/packages/flow/tests/e2e/test_egress_blocked_e2e.py +++ b/packages/flow/tests/e2e/test_egress_blocked_e2e.py @@ -112,7 +112,9 @@ def test_the_free_path_verifies_without_reaching_another_machine( assert not log.exists() or log.read_text() == "" -def test_native_child_attempt_is_blocked_without_a_python_refusal(tmp_path: Path) -> None: +def test_native_child_attempt_is_blocked_without_a_python_refusal( + tmp_path: Path, +) -> None: log = tmp_path / "native-outbound-blocked.jsonl" probe = textwrap.dedent( """\ @@ -129,8 +131,11 @@ def test_native_child_attempt_is_blocked_without_a_python_refusal(tmp_path: Path env = _strict_env(log) env["LC_ALL"] = "C" proc = subprocess.run( - [sys.executable, "-c", probe], env=env, - capture_output=True, text=True, timeout=30, + [sys.executable, "-c", probe], + env=env, + capture_output=True, + text=True, + timeout=30, ) assert proc.returncode == 0, proc.stderr child = json.loads(proc.stdout) diff --git a/packages/flow/tests/e2e/test_egress_namespace_e2e.py b/packages/flow/tests/e2e/test_egress_namespace_e2e.py index 293c3f25cf..7c6444dd87 100644 --- a/packages/flow/tests/e2e/test_egress_namespace_e2e.py +++ b/packages/flow/tests/e2e/test_egress_namespace_e2e.py @@ -33,7 +33,15 @@ def _invoke( def _namespace(*argv: str) -> list[str]: - return ["sudo", "-n", "-E", "env", f"PATH={os.environ['PATH']}", str(WRAPPER), *argv] + return [ + "sudo", + "-n", + "-E", + "env", + f"PATH={os.environ['PATH']}", + str(WRAPPER), + *argv, + ] @pytest.mark.parametrize("root", [False, True]) @@ -85,15 +93,22 @@ def test_unreadable_or_unexpected_inspection_prevents_command( fake_ip = tools / "ip" fake_ip.write_text( "#!/bin/bash\n" - f"if [[ \"$*\" == {inspection!r} ]]; then\n" + f'if [[ "$*" == {inspection!r} ]]; then\n' f" printf '%s' {output!r}\n exit {status}\nfi\n" - f"exec {real_ip!r} \"$@\"\n" + f'exec {real_ip!r} "$@"\n' ) fake_ip.chmod(0o755) marker = tmp_path / "command-ran" proc = _invoke( - ["sudo", "-n", "env", f"PATH={tools}", str(WRAPPER), - "/usr/bin/touch", str(marker)] + [ + "sudo", + "-n", + "env", + f"PATH={tools}", + str(WRAPPER), + "/usr/bin/touch", + str(marker), + ] ) assert proc.returncode == 2, (proc.stdout, proc.stderr) assert not marker.exists() @@ -158,7 +173,10 @@ def relay() -> None: try: for _ in range(2): conn, _ = unix.accept() - with conn, socket.create_connection(address, timeout=5) as upstream: + with ( + conn, + socket.create_connection(address, timeout=5) as upstream, + ): conn.settimeout(5) upstream.sendall(conn.recv(128)) conn.sendall(upstream.recv(128)) @@ -182,7 +200,11 @@ def relay() -> None: "print(json.dumps(dict(ack=ack,netns=os.readlink('/proc/self/ns/net'))))" ) log = tmp_path / "relay-refusals.jsonl" - env = {k: v for k, v in os.environ.items() if not k.startswith("OPENADAPT_NO_")} + env = { + k: v + for k, v in os.environ.items() + if not k.startswith("OPENADAPT_NO_") + } env.update( OPENADAPT_NO_OUTBOUND="1", OPENADAPT_NO_OUTBOUND_CHILDREN="sandbox", @@ -210,7 +232,9 @@ def relay() -> None: for worker in workers: worker.join(20) (tmp_path / "independent-relay-receipt.json").write_text( - json.dumps({"received": [x.decode() for x in receipts], "errors": errors}) + json.dumps( + {"received": [x.decode() for x in receipts], "errors": errors} + ) ) assert not any(worker.is_alive() for worker in workers) assert not errors, errors diff --git a/packages/flow/tests/test_ci_workflow_contract.py b/packages/flow/tests/test_ci_workflow_contract.py index 01afae740e..62a0f07460 100644 --- a/packages/flow/tests/test_ci_workflow_contract.py +++ b/packages/flow/tests/test_ci_workflow_contract.py @@ -533,7 +533,6 @@ def test_clean_machine_lifecycle_declares_utf8_on_every_os() -> None: assert 'PYTHONIOENCODING: "utf-8"' in lifecycle_header - def test_egress_boundary_controls_are_required_without_skips() -> None: workflow = ROOT_PACKAGES.read_text(encoding="utf-8") start = workflow.index("\n flow-critical:") From 9a71fa419085d1a55efe413f45174342aaf9e050 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 21:19:29 -0500 Subject: [PATCH 4/9] test: qualify native probe isolation and retain raw results Signed-off-by: abrichr --- .../flow/tests/e2e/test_egress_blocked_e2e.py | 14 ++++++++- .../tests/e2e/test_egress_namespace_e2e.py | 29 +++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/packages/flow/tests/e2e/test_egress_blocked_e2e.py b/packages/flow/tests/e2e/test_egress_blocked_e2e.py index 07fea6fad4..6b874f0f64 100644 --- a/packages/flow/tests/e2e/test_egress_blocked_e2e.py +++ b/packages/flow/tests/e2e/test_egress_blocked_e2e.py @@ -121,11 +121,18 @@ def test_native_child_attempt_is_blocked_without_a_python_refusal( import json import subprocess import openadapt_flow + # A location environment variable alone must not authorize this probe. + links = subprocess.check_output(["ip", "-o", "link", "show"], text=True) + routes4 = subprocess.check_output(["ip", "route", "show"], text=True) + routes6 = subprocess.check_output(["ip", "-6", "route", "show"], text=True) + assert [line.split(": ", 2)[1] for line in links.splitlines()] == ["lo"] + assert routes4 == routes6 == "" proc = subprocess.run( ["/bin/bash", "-c", "printf native-marker >/dev/tcp/192.0.2.123/443"], capture_output=True, text=True, timeout=5, ) - print(json.dumps({"returncode": proc.returncode, "stderr": proc.stderr})) + print(json.dumps({"returncode": proc.returncode, "stderr": proc.stderr, + "links": links, "routes4": routes4, "routes6": routes6})) """ ) env = _strict_env(log) @@ -137,6 +144,11 @@ def test_native_child_attempt_is_blocked_without_a_python_refusal( text=True, timeout=30, ) + (tmp_path / "native-child-result.json").write_text( + json.dumps( + {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + ) + ) assert proc.returncode == 0, proc.stderr child = json.loads(proc.stdout) assert child["returncode"] != 0 diff --git a/packages/flow/tests/e2e/test_egress_namespace_e2e.py b/packages/flow/tests/e2e/test_egress_namespace_e2e.py index 7c6444dd87..644e06da41 100644 --- a/packages/flow/tests/e2e/test_egress_namespace_e2e.py +++ b/packages/flow/tests/e2e/test_egress_namespace_e2e.py @@ -45,7 +45,7 @@ def _namespace(*argv: str) -> list[str]: @pytest.mark.parametrize("root", [False, True]) -def test_real_namespace_and_invoking_identity(root: bool) -> None: +def test_real_namespace_and_invoking_identity(root: bool, tmp_path: Path) -> None: code = ( "import json,os,subprocess; print(json.dumps(dict(" "uid=os.getuid(),gid=os.getgid(),netns=os.readlink('/proc/self/ns/net')," @@ -57,6 +57,11 @@ def test_real_namespace_and_invoking_identity(root: bool) -> None: if root: argv[4:4] = ["-u", "SUDO_UID", "-u", "SUDO_GID"] proc = _invoke(argv) + (tmp_path / "namespace-identity.json").write_text( + json.dumps( + {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + ) + ) assert proc.returncode == 0, proc.stderr result = json.loads(proc.stdout) assert result["uid"] == (0 if root else os.getuid()) @@ -146,6 +151,7 @@ def test_owned_pathname_unix_relay_remains_outside_the_ip_restriction( ack = b"synthetic-boundary-ack" receipts: list[bytes] = [] errors: list[str] = [] + observations: dict[str, object] = {} # AF_UNIX paths have a short OS limit; pytest's evidence path may be long. with tempfile.TemporaryDirectory(prefix="oa-egress-") as directory: unix_path = str(Path(directory) / "relay.sock") @@ -212,6 +218,11 @@ def relay() -> None: PYTHONPATH=str(FLOW), ) baseline = _invoke([sys.executable, "-c", code], env=env) + observations["baseline"] = { + "returncode": baseline.returncode, + "stdout": baseline.stdout, + "stderr": baseline.stderr, + } assert baseline.returncode == 0, baseline.stderr assert json.loads(baseline.stdout)["ack"] == ack.decode() assert receipts == [b"direct-control", marker] @@ -222,8 +233,18 @@ def relay() -> None: "\nexcept OSError as exc: assert exc.errno in (errno.ENETUNREACH,errno.ECONNREFUSED), exc" ) direct = _invoke(_namespace(sys.executable, "-c", direct_code), env=env) + observations["direct"] = { + "returncode": direct.returncode, + "stdout": direct.stdout, + "stderr": direct.stderr, + } assert direct.returncode == 0, direct.stderr isolated = _invoke(_namespace(sys.executable, "-c", code), env=env) + observations["isolated"] = { + "returncode": isolated.returncode, + "stdout": isolated.stdout, + "stderr": isolated.stderr, + } assert isolated.returncode == 0, isolated.stderr result = json.loads(isolated.stdout) assert result["netns"] != os.readlink("/proc/self/ns/net") @@ -233,7 +254,11 @@ def relay() -> None: worker.join(20) (tmp_path / "independent-relay-receipt.json").write_text( json.dumps( - {"received": [x.decode() for x in receipts], "errors": errors} + { + "received": [x.decode() for x in receipts], + "errors": errors, + "observations": observations, + } ) ) assert not any(worker.is_alive() for worker in workers) From 381f52b148f20f4c91a10602cd3b9968fd7f87e9 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 21:29:54 -0500 Subject: [PATCH 5/9] style: format retained namespace probe results Signed-off-by: abrichr --- packages/flow/tests/e2e/test_egress_blocked_e2e.py | 6 +++++- packages/flow/tests/e2e/test_egress_namespace_e2e.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/flow/tests/e2e/test_egress_blocked_e2e.py b/packages/flow/tests/e2e/test_egress_blocked_e2e.py index 6b874f0f64..fb6c468e55 100644 --- a/packages/flow/tests/e2e/test_egress_blocked_e2e.py +++ b/packages/flow/tests/e2e/test_egress_blocked_e2e.py @@ -146,7 +146,11 @@ def test_native_child_attempt_is_blocked_without_a_python_refusal( ) (tmp_path / "native-child-result.json").write_text( json.dumps( - {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + { + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + } ) ) assert proc.returncode == 0, proc.stderr diff --git a/packages/flow/tests/e2e/test_egress_namespace_e2e.py b/packages/flow/tests/e2e/test_egress_namespace_e2e.py index 644e06da41..e8eaadbd0a 100644 --- a/packages/flow/tests/e2e/test_egress_namespace_e2e.py +++ b/packages/flow/tests/e2e/test_egress_namespace_e2e.py @@ -59,7 +59,11 @@ def test_real_namespace_and_invoking_identity(root: bool, tmp_path: Path) -> Non proc = _invoke(argv) (tmp_path / "namespace-identity.json").write_text( json.dumps( - {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + { + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + } ) ) assert proc.returncode == 0, proc.stderr From 2d85ae25967041dfe3219dbb01243671cbbb33e8 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 20:45:34 -0800 Subject: [PATCH 6/9] on-prem: install, update and roll back the real engine offline in CI (#1918) * on-prem: install, update and roll back the real engine offline in CI bin/build-release.sh builds a release archive from uv.lock (pinned wheels, hash-checked) plus the flow wheel, a checksum and an openssl signature. bin/test-offline-install.sh builds two signed releases, then as root inside run-egress-blocked.sh installs the first from its wheelhouse, refuses a tampered second, updates to the signed second, rolls back and runs the air-gap attestation. The new flow-onprem-offline job runs it, and runs bin/test-update.sh, which no job ran before. * on-prem: check the engine version from the release venv, not the checkout The version check ran python -c from packages/flow, so it imported the source checkout and reported 2.6.0 after the update to 2.6.0.post1. Run it isolated from /, require the module to live in the venv, and compare the module with the installed distribution and the CLI's --version. * on-prem audit: do not break a lock that a live writer just took A writer that found the audit lock owned by an exited process removed it without re-checking. If the owner released between the read and the removal and another writer took the lock, the breaker deleted the live lock and two writers could chain to the same parent. Break stale locks only under a second lock and only if the lock still names the same dead owner. Add a deterministic test that forces the handover, and print the verifier output when the concurrent-writer check fails. * on-prem locks: take ownership with an exclusive file create, not mkdir(1) Ubuntu 26.04 ships uutils mkdir, which reports success to both callers when two processes race to create one directory. The audit-log writer lock and the release update lock both treated a zero exit from mkdir as ownership, so on that image concurrent audit writers chained to the same parent (the "concurrent audit writers preserve one hash chain" CI failure) and a second update could proceed beside a live one. Ownership is now the shell's own noclobber create of the pid file inside the lock directory (O_CREAT|O_EXCL), and the stale-lock breaker lock is an exclusive file too. Tests model the uutils behavior with a mkdir shim that never reports EEXIST; on the previous code they fail. --- .github/workflows/README.md | 2 +- .github/workflows/packages.yml | 42 ++++ packages/flow/deploy/on-prem/README.md | 2 + packages/flow/deploy/on-prem/UPDATE.md | 18 +- packages/flow/deploy/on-prem/bin/audit-log.sh | 47 +++- .../flow/deploy/on-prem/bin/build-release.sh | 90 ++++++++ .../flow/deploy/on-prem/bin/lib-release.sh | 41 ++-- .../on-prem/bin/test-offline-install.sh | 210 ++++++++++++++++++ .../flow/deploy/on-prem/bin/test-update.sh | 79 +++++++ 9 files changed, 506 insertions(+), 25 deletions(-) create mode 100755 packages/flow/deploy/on-prem/bin/build-release.sh create mode 100755 packages/flow/deploy/on-prem/bin/test-offline-install.sh diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5cce5d3943..8117e74e17 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -102,7 +102,7 @@ App 15368 is GitHub Actions, the app the current checks are pinned to. | main.yml | `run-ci` | ubuntu-latest, Python 3.12 | ubuntu-latest, Python 3.12 | ubuntu-latest and macos-latest, Python 3.10 to 3.12 | | main.yml | `unsupported-python-guard` | no | no | Python 3.13 and 3.14 | | packages.yml | `types`, `agent` | Python 3.12 | Python 3.12 | Python 3.10 and 3.12 | -| packages.yml | `flow-lint`, `flow-critical` | yes | yes | yes | +| packages.yml | `flow-lint`, `flow-critical`, `flow-onprem-offline` | yes | yes | yes | | ci.yml | `smoke`, `ci` | yes | yes | no | | packages.yml | `flow-unit` | 3 shards packed by measured time, fast lane (without `nightly_only.txt`) | no | 2 shards | | packages.yml | `flow-native-census` | no | no | Windows and macOS | diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 84487c25ee..cde3990d65 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -634,6 +634,46 @@ jobs: sys.exit(1 if bad else 0) PY + # The on-prem install path for real: build two signed offline releases of + # openadapt-flow[privacy] from uv.lock on this connected runner, then, as + # root in a network namespace with no route out, install the first from its + # wheelhouse, refuse a tampered second, update to the signed second, roll + # back, and run the air-gap attestation (egress probe, audit chain). pip has + # no index and the machine has no network, so a wheel missing from the + # archive fails the install. deploy/on-prem/bin/test-offline-install.sh. + # First, bin/test-update.sh: the update state machine's refusal, migration + # and recovery paths with a fake venv build, which nothing else runs. + flow-onprem-offline: + name: flow on-prem offline install, update and rollback (egress blocked) + needs: scope + if: github.event.pull_request.draft != true && (needs.scope.outputs.flow == 'true') + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: packages/flow + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + - name: Signed update and rollback state machine (hermetic, fake build) + run: bash deploy/on-prem/bin/test-update.sh + - name: Build two signed offline releases from the lock (connected) + run: bash deploy/on-prem/bin/test-offline-install.sh stage "$RUNNER_TEMP/offline" + - name: Install, update and roll back with egress blocked + # env -u SUDO_UID keeps the command root inside the namespace: + # install.sh owns the release tree as root. + run: >- + sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" + deploy/on-prem/bin/run-egress-blocked.sh + bash deploy/on-prem/bin/test-offline-install.sh verify "$RUNNER_TEMP/offline" + # The agent-framework adapters (openadapt_flow.integrations.agents): each # framework's own agent loop with a scripted model, writing through a real # HTTP app, from the lock with every framework extra (CrewAI pinned beside @@ -1429,6 +1469,7 @@ jobs: - flow-lint - flow-unit - flow-critical + - flow-onprem-offline - flow-agent-frameworks - flow-native-census - flow-linux-atspi @@ -1482,6 +1523,7 @@ jobs: "flow-lint": "flow", "flow-unit": "flow", "flow-critical": "flow", + "flow-onprem-offline": "flow", "flow-agent-frameworks": "flow", "flow-native-census": "flow", "flow-linux-atspi": "flow", diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index ba784895f3..efd8483042 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -21,6 +21,8 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow). | `bin/verify-airgap.sh` | Asserts no outbound path: config/env/deployment URL scan, optional active egress probe, optional audit-chain walk. | REAL (best-effort; firewall is primary) | | `bin/run-egress-blocked.sh` | Runs a command in a loopback-only Linux network namespace, refusing to start unless no other interface or route exists. CI runs the golden free path through it (`tests/e2e/test_egress_blocked_e2e.py`): VERIFIED, zero model calls, and no refusal recorded by the strict Python audit hook. | REAL (Linux, root) | | `bin/test-update.sh` | Hermetic offline proof of signed update, refusal paths, legacy migration, data preservation, rollback/recovery, locks, and audit-chain serialization. | REAL (43 baseline assertions; 45 when local GPG key generation is available) | +| `bin/build-release.sh` | Builds a release archive on a connected host from `uv.lock`: the exact pinned wheels, hash-checked, plus the `openadapt-flow` wheel, a `.sha256`, and an openssl `.sig` when given the key. | REAL | +| `bin/test-offline-install.sh` | Builds two signed releases, then as root with egress blocked installs the real engine from the first, refuses a tampered second, updates to the signed second, rolls back, and runs the air-gap attestation. CI runs it with `bin/test-update.sh` (job `flow-onprem-offline`). | REAL (Linux, root) | | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) | | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) | | `docker-compose.yml` + `Dockerfile` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. | REAL topology; STUB image build (needs offline `./wheels`) | diff --git a/packages/flow/deploy/on-prem/UPDATE.md b/packages/flow/deploy/on-prem/UPDATE.md index 0e594e8395..9346e2c8a8 100644 --- a/packages/flow/deploy/on-prem/UPDATE.md +++ b/packages/flow/deploy/on-prem/UPDATE.md @@ -42,7 +42,23 @@ all survive across any number of updates. ## Build a signed release bundle (off-site, on a connected host) The bundle is a tarball containing a `VERSION` file and an offline pip -wheelhouse, plus a required detached signature and an optional checksum: +wheelhouse, plus a required detached signature and an optional checksum. + +`bin/build-release.sh` builds it from a source checkout: it downloads exactly +the wheels `packages/flow/uv.lock` pins, checks each against the lock's hashes, +adds the `openadapt-flow` wheel, and writes the archive, its `.sha256`, and an +openssl `.sig` when given the signing key. Run it on a host with the same OS, +CPU architecture and Python minor version as the clinic box: + +```bash +deploy/on-prem/bin/build-release.sh --out dist/ --sign-key vendor_priv.pem +``` + +CI builds two releases this way and installs, updates and rolls back the real +engine from them with the network cut off (`bin/test-offline-install.sh`). + +To assemble one by hand instead (this resolves the newest matching +dependencies rather than the locked ones): ```bash # 1. Assemble the payload diff --git a/packages/flow/deploy/on-prem/bin/audit-log.sh b/packages/flow/deploy/on-prem/bin/audit-log.sh index 091add53c3..a963a59a01 100755 --- a/packages/flow/deploy/on-prem/bin/audit-log.sh +++ b/packages/flow/deploy/on-prem/bin/audit-log.sh @@ -68,21 +68,50 @@ chmod 0700 "$(dirname "$AUDIT_LOG")" 2>/dev/null || true # Serialize the read-last-hash + append transaction. O_APPEND prevents byte # interleaving, but without this lock two writers can both chain to the same # parent and produce a structurally broken audit log. +# +# Ownership is the exclusive creation of the pid file, done by the shell itself +# with noclobber (open(2) with O_CREAT|O_EXCL). The exit status of mkdir(1) is +# not trusted for exclusion: the uutils mkdir that Ubuntu 26.04 ships reports +# success to both callers when two processes race to create one directory. lock_dir="${AUDIT_LOG}.lock" +lock_file="$lock_dir/pid" +break_file="${AUDIT_LOG}.lock-breaker" lock_acquired=0 + +# Create $1 holding this process's pid; fails if $1 already exists. +create_exclusive() { + ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null +} + +# Remove a lock whose owner has exited. Between reading the owner and removing +# the lock, that owner can release it and another writer can take it, so the +# removal happens only under a second exclusive lock that serializes breakers, +# and only if the lock still names the same dead owner. Without this, a breaker +# could delete a live writer's lock and let two writers chain to the same parent. +break_stale_lock() { + local stale="$1" current="" + create_exclusive "$break_file" || return 0 + current="$(cat "$lock_file" 2>/dev/null || true)" + if [[ "$current" == "$stale" ]] && ! kill -0 "$stale" 2>/dev/null; then + rm -f "$lock_file" 2>/dev/null || true + fi + rm -f "$break_file" 2>/dev/null || true +} + for ((attempt = 0; attempt < 200; attempt++)); do - if mkdir "$lock_dir" 2>/dev/null; then - printf '%s\n' "$$" > "$lock_dir/pid" - lock_acquired=1 - break + if [[ ! -L "$lock_dir" ]]; then + mkdir -p "$lock_dir" 2>/dev/null || true + if [[ -d "$lock_dir" && ! -L "$lock_dir" ]] && create_exclusive "$lock_file"; then + lock_acquired=1 + break + fi fi owner="" - if [[ -d "$lock_dir" && ! -L "$lock_dir" && -r "$lock_dir/pid" ]]; then - owner="$(cat "$lock_dir/pid" 2>/dev/null || true)" + if [[ -d "$lock_dir" && ! -L "$lock_dir" && -f "$lock_file" && ! -L "$lock_file" ]]; then + owner="$(cat "$lock_file" 2>/dev/null || true)" fi if [[ -n "$owner" && "$owner" =~ ^[0-9]+$ ]] && ! kill -0 "$owner" 2>/dev/null; then - rm -f "$lock_dir/pid" 2>/dev/null || true - rmdir "$lock_dir" 2>/dev/null || true + break_stale_lock "$owner" continue fi sleep 0.05 @@ -92,7 +121,7 @@ if [[ "$lock_acquired" -ne 1 ]]; then exit 1 fi release_audit_lock() { - rm -f "$lock_dir/pid" 2>/dev/null || true + rm -f "$lock_file" 2>/dev/null || true rmdir "$lock_dir" 2>/dev/null || true } trap release_audit_lock EXIT diff --git a/packages/flow/deploy/on-prem/bin/build-release.sh b/packages/flow/deploy/on-prem/bin/build-release.sh new file mode 100755 index 0000000000..5b62d7878f --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/build-release.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# build-release.sh — build an offline release archive from the lock file. +# +# Run on a CONNECTED build host with the same OS, CPU architecture and Python +# minor version as the air-gapped target. Produces, in --out: +# +# release-.tar.gz VERSION, wheels/, requirements.lock.txt +# release-.tar.gz.sha256 checksum (corruption check only) +# release-.tar.gz.sig openssl signature, when --sign-key is given +# +# Every third-party wheel is the exact version pinned in packages/flow/uv.lock +# and is checked against the lock's hashes while downloading (pip +# --require-hashes), so the archive holds what CI tested and nothing newer. +# The openadapt-flow wheel is built from this checkout unless --flow-wheel +# names one. Stage the archive with `install.sh --update` (see UPDATE.md), or +# extract it and pass release/wheels to `install.sh --wheelhouse` for a first +# install. +# +# Usage: +# build-release.sh --out DIR [--flow-wheel FILE] [--sign-key PRIVATE.pem] +# [--python PYTHON] +# +# Needs uv and the target Python (default python3). Exit 0 on success. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FLOW_DIR="$(cd "$HERE/../../.." && pwd)" +OUT="" +FLOW_WHEEL="" +SIGN_KEY="" +PYTHON="python3" + +while [[ $# -gt 0 ]]; do + case "$1" in + --out) OUT="${2:-}"; shift 2 ;; + --flow-wheel) FLOW_WHEEL="${2:-}"; shift 2 ;; + --sign-key) SIGN_KEY="${2:-}"; shift 2 ;; + --python) PYTHON="${2:-}"; shift 2 ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "build-release.sh: unknown arg $1" >&2; exit 2 ;; + esac +done +[[ -n "$OUT" ]] || { echo "build-release.sh: --out DIR is required" >&2; exit 2; } +command -v uv >/dev/null || { echo "build-release.sh: uv is required" >&2; exit 2; } +[[ -f "$FLOW_DIR/uv.lock" ]] || { echo "build-release.sh: no uv.lock in $FLOW_DIR" >&2; exit 2; } +if [[ -n "$SIGN_KEY" && ! -f "$SIGN_KEY" ]]; then + echo "build-release.sh: signing key $SIGN_KEY not found" >&2; exit 2 +fi + +# shellcheck source=bin/lib-release.sh +source "$HERE/lib-release.sh" + +mkdir -p "$OUT" +OUT="$(cd "$OUT" && pwd)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/oa-build-release.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/release/wheels" + +if [[ -z "$FLOW_WHEEL" ]]; then + echo "== building the openadapt-flow wheel from $FLOW_DIR ==" + (cd "$FLOW_DIR" && uv build --wheel --out-dir "$WORK/flow" >/dev/null) + FLOW_WHEEL="$(find "$WORK/flow" -maxdepth 1 -name 'openadapt_flow-*.whl' -print)" +fi +[[ -f "$FLOW_WHEEL" ]] || { echo "build-release.sh: flow wheel not found" >&2; exit 2; } +cp "$FLOW_WHEEL" "$WORK/release/wheels/" +VERSION="$(rel_wheelhouse_version "$WORK/release/wheels")" +rel_validate_version "$VERSION" + +echo "== exporting the locked dependencies of openadapt-flow[privacy] $VERSION ==" +(cd "$FLOW_DIR" && uv export --frozen --no-dev --extra privacy --no-emit-project \ + --format requirements-txt --no-header -o "$WORK/release/requirements.lock.txt" >/dev/null) + +echo "== downloading the locked wheels for $("$PYTHON" -c 'import platform, sys; print(sys.platform, platform.machine(), "py%d.%d" % sys.version_info[:2])') ==" +"$PYTHON" -m pip download --quiet --disable-pip-version-check \ + --only-binary=:all: --no-deps --require-hashes \ + -r "$WORK/release/requirements.lock.txt" -d "$WORK/release/wheels" + +printf '%s\n' "$VERSION" > "$WORK/release/VERSION" +ARCHIVE="$OUT/release-$VERSION.tar.gz" +COPYFILE_DISABLE=1 tar czf "$ARCHIVE" -C "$WORK" release +sha256_of "$ARCHIVE" > "$ARCHIVE.sha256" +if [[ -n "$SIGN_KEY" ]]; then + openssl dgst -sha256 -sign "$SIGN_KEY" -out "$ARCHIVE.sig" "$ARCHIVE" +fi + +count="$(find "$WORK/release/wheels" -name '*.whl' | wc -l | tr -d ' ')" +echo "built $ARCHIVE ($count wheels)" +[[ -n "$SIGN_KEY" ]] && echo "signed $ARCHIVE.sig" || echo "unsigned: sign it before staging (UPDATE.md)" diff --git a/packages/flow/deploy/on-prem/bin/lib-release.sh b/packages/flow/deploy/on-prem/bin/lib-release.sh index 7676dc4900..295dd5c38b 100755 --- a/packages/flow/deploy/on-prem/bin/lib-release.sh +++ b/packages/flow/deploy/on-prem/bin/lib-release.sh @@ -344,32 +344,45 @@ rel_validate_release_dir() { && -x "$release_dir/venv/bin/openadapt-flow" ]] } +# Create $1 holding this process's pid; fails if $1 already exists. The shell +# opens it with noclobber (O_CREAT|O_EXCL), so exclusion does not depend on +# mkdir(1): the uutils mkdir that Ubuntu 26.04 ships reports success to both +# callers when two processes race to create one directory. +_rel_create_exclusive() { + ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null +} + _rel_lock_acquire() { - local root="$1" owner="" + local root="$1" owner="" current="" local lock="$root/releases/.update.lock" + local breaker="$root/releases/.update.lock-breaker" mkdir -p "$root/releases" || return 1 - if mkdir "$lock" 2>/dev/null; then - printf '%s\n' "$$" > "$lock/pid" || { - rmdir "$lock" 2>/dev/null || true - return 1 - } + if [[ -L "$lock" ]]; then + _rel_err "release lock is a symlink; refusing: $lock" + return 1 + fi + mkdir -p "$lock" 2>/dev/null || true + if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then REL_LOCK_DIR="$lock" return 0 fi - if [[ -d "$lock" && ! -L "$lock" && -r "$lock/pid" ]]; then + if [[ -d "$lock" && ! -L "$lock" && -f "$lock/pid" && ! -L "$lock/pid" ]]; then owner="$(cat "$lock/pid" 2>/dev/null || true)" fi if [[ "$owner" =~ ^[0-9]+$ ]] && kill -0 "$owner" 2>/dev/null; then _rel_err "another update or rollback is active (pid $owner)" return 1 fi - if [[ -d "$lock" && ! -L "$lock" ]]; then - rm -f "$lock/pid" 2>/dev/null || true - if rmdir "$lock" 2>/dev/null && mkdir "$lock" 2>/dev/null; then - printf '%s\n' "$$" > "$lock/pid" || { - rmdir "$lock" 2>/dev/null || true - return 1 - } + # The owner has exited (or left an unreadable pid). Remove its lock only + # under the breaker lock and only if it still names that owner, so a + # concurrent breaker cannot remove a lock another process has just taken. + if [[ -n "$owner" ]] && _rel_create_exclusive "$breaker"; then + current="$(cat "$lock/pid" 2>/dev/null || true)" + if [[ "$current" == "$owner" ]]; then + rm -f "$lock/pid" 2>/dev/null || true + fi + rm -f "$breaker" 2>/dev/null || true + if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then REL_LOCK_DIR="$lock" return 0 fi diff --git a/packages/flow/deploy/on-prem/bin/test-offline-install.sh b/packages/flow/deploy/on-prem/bin/test-offline-install.sh new file mode 100755 index 0000000000..c3e6d391cd --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/test-offline-install.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# +# test-offline-install.sh — install, upgrade and roll back the real engine on a +# machine with no network. +# +# Unlike test-update.sh, which fakes the venv build, this installs the real +# openadapt-flow[privacy] wheels pinned by uv.lock, so it proves the offline +# release archive is complete: pip gets no index and the machine has no route +# out, so any missing wheel fails the install. +# +# stage DIR on a connected host: build two signed releases (this checkout +# as V1, the same source as V1.post1 as V2) and a throwaway +# signing key, with bin/build-release.sh. +# verify DIR as root, inside bin/run-egress-blocked.sh: first install from +# V1's wheelhouse, refuse a tampered V2, update to the signed V2, +# roll back to V1, then run the air-gap attestation with its +# egress probe and audit-chain walk. Data must survive each step. +# +# CI runs both (packages.yml, flow on-prem offline install). By hand on Linux: +# bin/test-offline-install.sh stage /tmp/oa-offline +# sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" \ +# bin/run-egress-blocked.sh bin/test-offline-install.sh verify /tmp/oa-offline +# +# Exit 0 when every check passes. + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ONPREM="$(cd "$HERE/.." && pwd)" +FLOW_DIR="$(cd "$ONPREM/../.." && pwd)" +CMD="${1:-}" +DIR="${2:-}" +if [[ -z "$CMD" || -z "$DIR" || ( "$CMD" != stage && "$CMD" != verify ) ]]; then + grep '^#' "$0" | sed 's/^# \{0,1\}//' + exit 2 +fi + +stage() { + set -e + mkdir -p "$DIR/keys" "$DIR/v1" "$DIR/v2" + DIR="$(cd "$DIR" && pwd)" + openssl genrsa -out "$DIR/keys/vendor.pem" 2048 2>/dev/null + openssl rsa -in "$DIR/keys/vendor.pem" -pubout -out "$DIR/keys/vendor.pub" 2>/dev/null + bash "$HERE/build-release.sh" --out "$DIR/v1" --sign-key "$DIR/keys/vendor.pem" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + + # V2: the same source under a new version, built the same way. + local v1 src + v1="$(find "$DIR/v1" -maxdepth 1 -name 'release-*.tar.gz' -print)" + v1="$(basename "$v1" .tar.gz)" + v1="${v1#release-}" + src="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-src.XXXXXX")" + for item in pyproject.toml README.md LICENSE THIRD_PARTY_NOTICES.md \ + public-artifacts.json openadapt_flow schemas; do + cp -R "$FLOW_DIR/$item" "$src/" + done + find "$src" -name __pycache__ -type d -prune -exec rm -rf {} + + sed -i.bak "s/^version = \"$v1\"$/version = \"$v1.post1\"/" "$src/pyproject.toml" + sed -i.bak "s/^__version__ = \"$v1\"$/__version__ = \"$v1.post1\"/" "$src/openadapt_flow/__init__.py" + rm -f "$src/pyproject.toml.bak" "$src/openadapt_flow/__init__.py.bak" + grep -q "^version = \"$v1.post1\"$" "$src/pyproject.toml" + (cd "$src" && uv build --wheel --out-dir "$src/dist" >/dev/null) + bash "$HERE/build-release.sh" --out "$DIR/v2" --sign-key "$DIR/keys/vendor.pem" \ + --flow-wheel "$src/dist/openadapt_flow-$v1.post1-py3-none-any.whl" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + rm -rf "$src" + # The private key never goes near the install. + rm -f "$DIR/keys/vendor.pem" + printf '%s\n' "$v1" > "$DIR/V1" + printf '%s\n' "$v1.post1" > "$DIR/V2" + echo "staged $v1 and $v1.post1 in $DIR" +} + +PASS=0 +FAIL=0 +ok() { printf ' PASS %s\n' "$1"; PASS=$((PASS + 1)); } +bad() { printf ' FAIL %s\n' "$1"; FAIL=$((FAIL + 1)); } +check() { local name="$1"; shift; if "$@"; then ok "$name"; else bad "$name"; fi; } +check_eq() { if [[ "$2" == "$3" ]]; then ok "$1"; else bad "$1 (expected '$2', got '$3')"; fi; } + +verify() { + DIR="$(cd "$DIR" && pwd)" + if [[ "$(id -u)" -ne 0 ]]; then + echo "test-offline-install.sh verify: run as root (install.sh needs it)" >&2; exit 2 + fi + if [[ "${OPENADAPT_EGRESS_BLOCKED:-}" != netns ]]; then + echo "test-offline-install.sh verify: run inside bin/run-egress-blocked.sh" >&2; exit 2 + fi + local v1 v2 py work root config rc + v1="$(cat "$DIR/V1")"; v2="$(cat "$DIR/V2")" + py="${OPENADAPT_ONPREM_PY:-python3}" + work="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-verify.XXXXXX")" + root="$work/srv" + config="$work/onprem.yaml" + cat > "$config" <&1) + } + + echo "== offline install, upgrade and rollback ($v1 -> $v2 -> $v1), no network ==" + check "the machine has no route out" "$py" -c ' +import socket, sys +try: + socket.create_connection(("1.1.1.1", 443), timeout=3) +except OSError: + sys.exit(0) +sys.exit(1)' + + echo "-- first install from the $v1 wheelhouse --" + mkdir -p "$work/media" + tar xzf "$DIR/v1/release-$v1.tar.gz" -C "$work/media" + bash "$ONPREM/install.sh" --config "$config" --wheelhouse "$work/media/release/wheels" \ + > "$work/install.log" 2>&1 + rc=$? + check_eq "install: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/install.log" + check_eq "install: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "install: engine and privacy extra import, version $v1" "$v1" "$(engine_version)" + check_eq "install: engine CLI reports $v1" "openadapt-flow $v1" \ + "$(cd / && "$root/current/venv/bin/openadapt-flow" --version 2>&1)" + + mkdir -p "$root/bundles" "$root/jobs/inbox" + printf 'bundle\n' > "$root/bundles/keepme.txt" + printf 'job\n' > "$root/jobs/inbox/queued-0001.job" + local bundle_sha job_sha + bundle_sha="$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + job_sha="$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + cp "$DIR/keys/vendor.pub" "$root/keys/vendor.pub" + + local arc="$DIR/v2/release-$v2.tar.gz" + echo "-- a tampered $v2 archive is refused --" + cp "$arc" "$work/media/tampered.tar.gz" + printf 'x' >> "$work/media/tampered.tar.gz" + bash "$ONPREM/install.sh" --update --config "$config" --release "$work/media/tampered.tar.gz" \ + --signature "$arc.sig" > "$work/tampered.log" 2>&1 + rc=$? + check "tampered: update refused" test "$rc" -ne 0 + check_eq "tampered: current still $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check "tampered: nothing built for $v2" test ! -e "$root/releases/$v2" + + echo "-- signed update to $v2 --" + bash "$ONPREM/install.sh" --update --config "$config" --release "$arc" \ + --checksum "$arc.sha256" --signature "$arc.sig" > "$work/update.log" 2>&1 + rc=$? + check_eq "update: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/update.log" + check_eq "update: current -> $v2" "$root/releases/$v2" "$(readlink "$root/current")" + check_eq "update: previous -> $v1" "$root/releases/$v1" "$(readlink "$root/previous")" + check_eq "update: engine version $v2" "$v2" "$(engine_version)" + check_eq "update: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + check_eq "update: queued job unchanged" "$job_sha" "$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + + echo "-- rollback to $v1 --" + bash "$ONPREM/install.sh" --rollback --config "$config" > "$work/rollback.log" 2>&1 + rc=$? + check_eq "rollback: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/rollback.log" + check_eq "rollback: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "rollback: engine version $v1" "$v1" "$(engine_version)" + check_eq "rollback: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + + echo "-- air-gap attestation --" + OPENADAPT_FLOW_SCRUB=on bash "$HERE/verify-airgap.sh" --config "$config" --probe --audit \ + > "$work/airgap.log" 2>&1 + rc=$? + check_eq "attestation: exit 0" 0 "$rc" + check "attestation: egress probe could not reach the internet" \ + grep -q "outbound egress is blocked" "$work/airgap.log" + check "attestation: audit chain intact" grep -q "audit chain intact" "$work/airgap.log" + [[ "$rc" -eq 0 ]] || cat "$work/airgap.log" + + rm -rf "$work" + echo + echo "== result: $PASS passed, $FAIL failed ==" + [[ "$FAIL" -eq 0 ]] +} + +"$CMD" diff --git a/packages/flow/deploy/on-prem/bin/test-update.sh b/packages/flow/deploy/on-prem/bin/test-update.sh index c2778b4602..cb16fdecba 100755 --- a/packages/flow/deploy/on-prem/bin/test-update.sh +++ b/packages/flow/deploy/on-prem/bin/test-update.sh @@ -244,6 +244,28 @@ assert "concurrent update lock is refused" test "$?" -ne 0 rm -f "$ROOT/releases/.update.lock/pid" rmdir "$ROOT/releases/.update.lock" +# Ubuntu 26.04 ships uutils mkdir, which reports success to both callers when +# two processes race to create one directory. Model that with a mkdir shim that +# never reports EEXIST: the locks must still exclude, so they cannot rest on +# the exit status of mkdir(1). +EEXIST_SHIM="$WORK/eexist-shim" +mkdir -p "$EEXIST_SHIM" +REAL_MKDIR="$(command -v mkdir)" +cat > "$EEXIST_SHIM/mkdir" </dev/null || true +exit 0 +SHIM_EOF +chmod +x "$EEXIST_SHIM/mkdir" +mkdir "$ROOT/releases/.update.lock" +printf '%s\n' "$$" > "$ROOT/releases/.update.lock/pid" +PATH="$EEXIST_SHIM:$PATH" apply_release "$ROOT" "$ARC6" >/dev/null 2>&1 +assert "update lock holds when mkdir(1) never reports EEXIST" test "$?" -ne 0 +assert_eq "a refused update leaves the live owner's lock" "$$" \ + "$(cat "$ROOT/releases/.update.lock/pid" 2>/dev/null || echo none)" +rm -f "$ROOT/releases/.update.lock/pid" +rmdir "$ROOT/releases/.update.lock" + chmod -x "$ROOT/releases/2.0.0/venv/bin/openadapt-flow" rel_do_rollback "$ROOT" "" >/dev/null 2>&1 assert "incomplete rollback target is refused" test "$?" -ne 0 @@ -290,7 +312,64 @@ if grep -q 'audit chain intact' <<< "$audit_output"; then pass "concurrent audit writers preserve one hash chain" else fail "concurrent audit writers preserve one hash chain" + printf '%s\n' "$audit_output" | sed 's/^/ /' +fi + +SHIM_LOG="$WORK/eexist-audit/audit.log" +for index in 1 2 3 4 5 6 7 8 9 10 11 12; do + OPENADAPT_ONPREM_AUDIT_LOG="$SHIM_LOG" PATH="$EEXIST_SHIM:$PATH" \ + bash "$OPENADAPT_ONPREM_AUDIT_BIN" started "shim-$index" --note test >/dev/null & +done +wait +audit_output="$(OPENADAPT_ONPREM_AUDIT_LOG="$SHIM_LOG" \ + bash "$HERE/verify-airgap.sh" --config "$WORK/missing.yaml" --audit 2>&1 || true)" +if grep -q 'audit chain intact across 12 record' <<< "$audit_output"; then + pass "audit writers stay serialized when mkdir(1) never reports EEXIST" +else + fail "audit writers stay serialized when mkdir(1) never reports EEXIST" + printf '%s\n' "$audit_output" | sed 's/^/ /' +fi + +# A writer that finds a lock owned by an exited process may break it. Force the +# race where, between that writer reading the dead owner and breaking the +# lock, the owner releases and a live writer takes the lock: a shimmed cat +# returns the dead owner once, then hands the lock to a live process. The +# breaker must leave the live writer's lock alone. +RACE="$WORK/stale-race" +mkdir -p "$RACE/shim" +RACE_LOG="$RACE/audit.log" +bash -c 'exit 0' & +dead_pid=$! +wait "$dead_pid" +sleep 30 & +live_pid=$! +mkdir "$RACE_LOG.lock" +printf '%s\n' "$dead_pid" > "$RACE_LOG.lock/pid" +REAL_CAT="$(command -v cat)" +cat > "$RACE/shim/cat" < "$RACE/handed-over" + printf '%s\n' "$live_pid" > "$RACE_LOG.lock/pid" + exit 0 fi +exec "$REAL_CAT" "\$@" +SHIM_EOF +chmod +x "$RACE/shim/cat" +OPENADAPT_ONPREM_AUDIT_LOG="$RACE_LOG" PATH="$RACE/shim:$PATH" \ + bash "$OPENADAPT_ONPREM_AUDIT_BIN" started stale-race --note test >/dev/null 2>&1 & +race_writer=$! +sleep 2 +race_owner="$(cat "$RACE_LOG.lock/pid" 2>/dev/null || echo none)" +assert_eq "stale-lock breaker leaves a live writer's lock alone" "$live_pid" "$race_owner" +rm -f "$RACE_LOG.lock/pid" +rmdir "$RACE_LOG.lock" 2>/dev/null || true +kill "$live_pid" 2>/dev/null || true +wait "$live_pid" 2>/dev/null || true +wait "$race_writer" +assert_eq "writer appends once the live lock is released" 0 "$?" +assert_eq "writer appends exactly one record" 1 "$(wc -l < "$RACE_LOG" | tr -d ' ')" if ! grep -q 'identity evidence' "$OPENADAPT_ONPREM_AUDIT_LOG"; then pass "release audit remains PHI-free" else From 1ef78c0ccac378e0239d10f773fd064b08df156f Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 5 Nov 2026 04:22:33 -0500 Subject: [PATCH 7/9] fix(on-prem): forward egress-test env without sudo -E GitHub-hosted runners ignore sudo -E. Put OPENADAPT_* and PYTHONPATH on the env argv so the netns child still installs the no-outbound hook, and stop asking sudo to preserve the whole environment. --- .github/workflows/packages.yml | 2 +- .../deploy/on-prem/bin/run-egress-blocked.sh | 6 ++-- .../tests/e2e/test_egress_namespace_e2e.py | 25 ++++++++++--- .../flow/tests/test_egress_namespace_argv.py | 36 +++++++++++++++++++ 4 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 packages/flow/tests/test_egress_namespace_argv.py diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index cde3990d65..5ba0718087 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -607,7 +607,7 @@ jobs: # must still reach VERIFIED with no Python-hook refusal recorded. # A native-child control measures the hook observation boundary. run: | - sudo -E env "PATH=$PATH" "HOME=$HOME" \ + sudo -n env "PATH=$PATH" "HOME=$HOME" \ deploy/on-prem/bin/run-egress-blocked.sh \ python -m pytest -v tests/e2e/test_egress_blocked_e2e.py \ --basetemp=runs/ci-onprem-egress --junitxml=runs/onprem-egress-junit.xml diff --git a/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh b/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh index 8dd95f45bf..b80e14a3dc 100755 --- a/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh +++ b/packages/flow/deploy/on-prem/bin/run-egress-blocked.sh @@ -16,8 +16,10 @@ # sudo user, with OPENADAPT_EGRESS_BLOCKED=netns as a test-location hint. Tests # independently check the restriction rather than trusting that variable. # -# Usage (as root, normally through sudo -E so PATH and HOME are kept): -# sudo -E env "PATH=$PATH" deploy/on-prem/bin/run-egress-blocked.sh CMD [ARGS...] +# Usage (as root). Pass every needed variable on the env command line; do not +# rely on sudo -E (GitHub-hosted runners ignore it): +# sudo -n env "PATH=$PATH" "HOME=$HOME" KEY=VAL \ +# deploy/on-prem/bin/run-egress-blocked.sh CMD [ARGS...] # # Exit code: the command's, or 2 when the namespace cannot be set up. diff --git a/packages/flow/tests/e2e/test_egress_namespace_e2e.py b/packages/flow/tests/e2e/test_egress_namespace_e2e.py index e8eaadbd0a..6f91162bb1 100644 --- a/packages/flow/tests/e2e/test_egress_namespace_e2e.py +++ b/packages/flow/tests/e2e/test_egress_namespace_e2e.py @@ -32,13 +32,24 @@ def _invoke( return subprocess.run(argv, env=env, capture_output=True, text=True, timeout=30) -def _namespace(*argv: str) -> list[str]: +def _namespace(*argv: str, forward: dict[str, str] | None = None) -> list[str]: + """Build a sudo + env + wrapper argv that does not rely on sudo -E. + + GitHub-hosted runners ignore sudo -E ("preserving the entire environment + is not supported"). Every variable the child needs must appear as an + explicit KEY=VAL on the env command line. + """ + assignments = [f"PATH={os.environ['PATH']}"] + if forward: + for key, value in forward.items(): + if key == "PATH": + continue + assignments.append(f"{key}={value}") return [ "sudo", "-n", - "-E", "env", - f"PATH={os.environ['PATH']}", + *assignments, str(WRAPPER), *argv, ] @@ -236,14 +247,18 @@ def relay() -> None: f"\ntry: s.connect({address!r}); raise AssertionError('host TCP reachable')" "\nexcept OSError as exc: assert exc.errno in (errno.ENETUNREACH,errno.ECONNREFUSED), exc" ) - direct = _invoke(_namespace(sys.executable, "-c", direct_code), env=env) + direct = _invoke( + _namespace(sys.executable, "-c", direct_code, forward=env), env=env + ) observations["direct"] = { "returncode": direct.returncode, "stdout": direct.stdout, "stderr": direct.stderr, } assert direct.returncode == 0, direct.stderr - isolated = _invoke(_namespace(sys.executable, "-c", code), env=env) + isolated = _invoke( + _namespace(sys.executable, "-c", code, forward=env), env=env + ) observations["isolated"] = { "returncode": isolated.returncode, "stdout": isolated.stdout, diff --git a/packages/flow/tests/test_egress_namespace_argv.py b/packages/flow/tests/test_egress_namespace_argv.py new file mode 100644 index 0000000000..9bdb47ce85 --- /dev/null +++ b/packages/flow/tests/test_egress_namespace_argv.py @@ -0,0 +1,36 @@ +"""_namespace must pass child env on the env argv; sudo -E is not available in CI.""" + +from __future__ import annotations + +import os + +from tests.e2e import test_egress_namespace_e2e as ns + + +def test_namespace_forwards_env_without_sudo_dash_e() -> None: + forward = { + "OPENADAPT_NO_OUTBOUND": "1", + "OPENADAPT_NO_OUTBOUND_CHILDREN": "sandbox", + "PYTHONPATH": "/tmp/flow-src", + "PATH": "/should-not-override", + } + argv = ns._namespace("python", "-c", "pass", forward=forward) + assert argv[0] == "sudo" + assert argv[1] == "-n" + assert argv[2] == "env" + assert "-E" not in argv + assert f"PATH={os.environ['PATH']}" in argv + assert "PATH=/should-not-override" not in argv + assert "OPENADAPT_NO_OUTBOUND=1" in argv + assert "OPENADAPT_NO_OUTBOUND_CHILDREN=sandbox" in argv + assert "PYTHONPATH=/tmp/flow-src" in argv + assert str(ns.WRAPPER) in argv + assert argv[-3:] == ["python", "-c", "pass"] + + +def test_namespace_without_forward_only_sets_path() -> None: + argv = ns._namespace("/bin/true") + assert "-E" not in argv + assert argv[2] == "env" + assert any(a.startswith("PATH=") for a in argv) + assert not any(a.startswith("OPENADAPT_") for a in argv) From d59b6e26c2a90ac1127512160ab7682dd891f93d Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 5 Nov 2026 04:35:51 -0500 Subject: [PATCH 8/9] fix(on-prem): put env -u before NAME=VALUE in root netns probe Dropping sudo -E shifted PATH= to argv[3], so the root case's argv[4:4] -u insert landed after the assignment. GNU env then treated -u as the command (env: '-u': No such file or directory) and failed critical CI on test_real_namespace_and_invoking_identity[True]. --- .../tests/e2e/test_egress_namespace_e2e.py | 5 ++++- .../flow/tests/test_egress_namespace_argv.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/flow/tests/e2e/test_egress_namespace_e2e.py b/packages/flow/tests/e2e/test_egress_namespace_e2e.py index 6f91162bb1..0e6a7dfca1 100644 --- a/packages/flow/tests/e2e/test_egress_namespace_e2e.py +++ b/packages/flow/tests/e2e/test_egress_namespace_e2e.py @@ -66,7 +66,10 @@ def test_real_namespace_and_invoking_identity(root: bool, tmp_path: Path) -> Non ) argv = _namespace(sys.executable, "-c", code) if root: - argv[4:4] = ["-u", "SUDO_UID", "-u", "SUDO_GID"] + # env OPTIONS must precede NAME=VALUE. Index 4 was correct when argv + # was sudo -n -E env PATH=...; after dropping -E, insert after env. + env_i = argv.index("env") + argv[env_i + 1 : env_i + 1] = ["-u", "SUDO_UID", "-u", "SUDO_GID"] proc = _invoke(argv) (tmp_path / "namespace-identity.json").write_text( json.dumps( diff --git a/packages/flow/tests/test_egress_namespace_argv.py b/packages/flow/tests/test_egress_namespace_argv.py index 9bdb47ce85..3b5e46f6ea 100644 --- a/packages/flow/tests/test_egress_namespace_argv.py +++ b/packages/flow/tests/test_egress_namespace_argv.py @@ -34,3 +34,21 @@ def test_namespace_without_forward_only_sets_path() -> None: assert argv[2] == "env" assert any(a.startswith("PATH=") for a in argv) assert not any(a.startswith("OPENADAPT_") for a in argv) + + +def test_root_unsets_sit_before_assignments() -> None: + """GNU env treats anything after NAME=VALUE as the command; -u must precede.""" + argv = ns._namespace("/bin/true") + env_i = argv.index("env") + argv[env_i + 1 : env_i + 1] = ["-u", "SUDO_UID", "-u", "SUDO_GID"] + assert argv[env_i : env_i + 5] == [ + "env", + "-u", + "SUDO_UID", + "-u", + "SUDO_GID", + ] + path_i = next(i for i, a in enumerate(argv) if a.startswith("PATH=")) + assert path_i > env_i + 4 + # Regression: the old argv[4:4] insert put -u after PATH= and broke CI. + assert argv[4] != "-u" From 6d7c1f0fed324e6ff876632367b829f3ca8203bd Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 5 Nov 2026 04:41:14 -0500 Subject: [PATCH 9/9] test(on-prem): clarify root netns env -u argv comment No behavior change; retriggers PR CI for d59b6e26c. --- packages/flow/tests/e2e/test_egress_namespace_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/flow/tests/e2e/test_egress_namespace_e2e.py b/packages/flow/tests/e2e/test_egress_namespace_e2e.py index 0e6a7dfca1..33833eba54 100644 --- a/packages/flow/tests/e2e/test_egress_namespace_e2e.py +++ b/packages/flow/tests/e2e/test_egress_namespace_e2e.py @@ -67,7 +67,7 @@ def test_real_namespace_and_invoking_identity(root: bool, tmp_path: Path) -> Non argv = _namespace(sys.executable, "-c", code) if root: # env OPTIONS must precede NAME=VALUE. Index 4 was correct when argv - # was sudo -n -E env PATH=...; after dropping -E, insert after env. + # was ``sudo -n -E env PATH=...``; after dropping -E, insert after env. env_i = argv.index("env") argv[env_i + 1 : env_i + 1] = ["-u", "SUDO_UID", "-u", "SUDO_GID"] proc = _invoke(argv)