From 919b60abe0efde0b0df0166ced4ba0a0b4ff49be Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 18:11:14 -0500 Subject: [PATCH 01/21] on-prem: install, update and roll back the real engine offline in CI bin/build-release.sh builds a release archive from uv.lock (pinned wheels, hash-checked) plus the flow wheel, a checksum and an openssl signature. bin/test-offline-install.sh builds two signed releases, then as root inside run-egress-blocked.sh installs the first from its wheelhouse, refuses a tampered second, updates to the signed second, rolls back and runs the air-gap attestation. The new flow-onprem-offline job runs it, and runs bin/test-update.sh, which no job ran before. --- .github/workflows/README.md | 2 +- .github/workflows/packages.yml | 42 ++++ packages/flow/deploy/on-prem/README.md | 2 + packages/flow/deploy/on-prem/UPDATE.md | 18 +- .../flow/deploy/on-prem/bin/build-release.sh | 90 ++++++++ .../on-prem/bin/test-offline-install.sh | 198 ++++++++++++++++++ 6 files changed, 350 insertions(+), 2 deletions(-) create mode 100755 packages/flow/deploy/on-prem/bin/build-release.sh create mode 100755 packages/flow/deploy/on-prem/bin/test-offline-install.sh diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5cce5d3943..8117e74e17 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -102,7 +102,7 @@ App 15368 is GitHub Actions, the app the current checks are pinned to. | main.yml | `run-ci` | ubuntu-latest, Python 3.12 | ubuntu-latest, Python 3.12 | ubuntu-latest and macos-latest, Python 3.10 to 3.12 | | main.yml | `unsupported-python-guard` | no | no | Python 3.13 and 3.14 | | packages.yml | `types`, `agent` | Python 3.12 | Python 3.12 | Python 3.10 and 3.12 | -| packages.yml | `flow-lint`, `flow-critical` | yes | yes | yes | +| packages.yml | `flow-lint`, `flow-critical`, `flow-onprem-offline` | yes | yes | yes | | ci.yml | `smoke`, `ci` | yes | yes | no | | packages.yml | `flow-unit` | 3 shards packed by measured time, fast lane (without `nightly_only.txt`) | no | 2 shards | | packages.yml | `flow-native-census` | no | no | Windows and macOS | diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 30656bfec9..d59cd2105d 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -630,6 +630,46 @@ jobs: sys.exit(1 if bad else 0) PY + # The on-prem install path for real: build two signed offline releases of + # openadapt-flow[privacy] from uv.lock on this connected runner, then, as + # root in a network namespace with no route out, install the first from its + # wheelhouse, refuse a tampered second, update to the signed second, roll + # back, and run the air-gap attestation (egress probe, audit chain). pip has + # no index and the machine has no network, so a wheel missing from the + # archive fails the install. deploy/on-prem/bin/test-offline-install.sh. + # First, bin/test-update.sh: the update state machine's refusal, migration + # and recovery paths with a fake venv build, which nothing else runs. + flow-onprem-offline: + name: flow on-prem offline install, update and rollback (egress blocked) + needs: scope + if: github.event.pull_request.draft != true && (needs.scope.outputs.flow == 'true') + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: packages/flow + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + - name: Signed update and rollback state machine (hermetic, fake build) + run: bash deploy/on-prem/bin/test-update.sh + - name: Build two signed offline releases from the lock (connected) + run: bash deploy/on-prem/bin/test-offline-install.sh stage "$RUNNER_TEMP/offline" + - name: Install, update and roll back with egress blocked + # env -u SUDO_UID keeps the command root inside the namespace: + # install.sh owns the release tree as root. + run: >- + sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" + deploy/on-prem/bin/run-egress-blocked.sh + bash deploy/on-prem/bin/test-offline-install.sh verify "$RUNNER_TEMP/offline" + # The agent-framework adapters (openadapt_flow.integrations.agents): each # framework's own agent loop with a scripted model, writing through a real # HTTP app, from the lock with every framework extra (CrewAI pinned beside @@ -1425,6 +1465,7 @@ jobs: - flow-lint - flow-unit - flow-critical + - flow-onprem-offline - flow-agent-frameworks - flow-native-census - flow-linux-atspi @@ -1478,6 +1519,7 @@ jobs: "flow-lint": "flow", "flow-unit": "flow", "flow-critical": "flow", + "flow-onprem-offline": "flow", "flow-agent-frameworks": "flow", "flow-native-census": "flow", "flow-linux-atspi": "flow", diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index a9a36396a9..c68dff4b87 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -21,6 +21,8 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow). | `bin/verify-airgap.sh` | Asserts no outbound path: config/env/deployment URL scan, optional active egress probe, optional audit-chain walk. | REAL (best-effort; firewall is primary) | | `bin/run-egress-blocked.sh` | Runs a command in a loopback-only Linux network namespace, refusing to start unless no other interface or route exists. CI runs the golden free path through it (`tests/e2e/test_egress_blocked_e2e.py`): VERIFIED, zero model calls, and no outbound attempt in strict no-outbound mode. | REAL (Linux, root) | | `bin/test-update.sh` | Hermetic offline proof of signed update, refusal paths, legacy migration, data preservation, rollback/recovery, locks, and audit-chain serialization. | REAL (43 baseline assertions; 45 when local GPG key generation is available) | +| `bin/build-release.sh` | Builds a release archive on a connected host from `uv.lock`: the exact pinned wheels, hash-checked, plus the `openadapt-flow` wheel, a `.sha256`, and an openssl `.sig` when given the key. | REAL | +| `bin/test-offline-install.sh` | Builds two signed releases, then as root with egress blocked installs the real engine from the first, refuses a tampered second, updates to the signed second, rolls back, and runs the air-gap attestation. CI runs it with `bin/test-update.sh` (job `flow-onprem-offline`). | REAL (Linux, root) | | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) | | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) | | `docker-compose.yml` + `Dockerfile` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. | REAL topology; STUB image build (needs offline `./wheels`) | diff --git a/packages/flow/deploy/on-prem/UPDATE.md b/packages/flow/deploy/on-prem/UPDATE.md index 0e594e8395..9346e2c8a8 100644 --- a/packages/flow/deploy/on-prem/UPDATE.md +++ b/packages/flow/deploy/on-prem/UPDATE.md @@ -42,7 +42,23 @@ all survive across any number of updates. ## Build a signed release bundle (off-site, on a connected host) The bundle is a tarball containing a `VERSION` file and an offline pip -wheelhouse, plus a required detached signature and an optional checksum: +wheelhouse, plus a required detached signature and an optional checksum. + +`bin/build-release.sh` builds it from a source checkout: it downloads exactly +the wheels `packages/flow/uv.lock` pins, checks each against the lock's hashes, +adds the `openadapt-flow` wheel, and writes the archive, its `.sha256`, and an +openssl `.sig` when given the signing key. Run it on a host with the same OS, +CPU architecture and Python minor version as the clinic box: + +```bash +deploy/on-prem/bin/build-release.sh --out dist/ --sign-key vendor_priv.pem +``` + +CI builds two releases this way and installs, updates and rolls back the real +engine from them with the network cut off (`bin/test-offline-install.sh`). + +To assemble one by hand instead (this resolves the newest matching +dependencies rather than the locked ones): ```bash # 1. Assemble the payload diff --git a/packages/flow/deploy/on-prem/bin/build-release.sh b/packages/flow/deploy/on-prem/bin/build-release.sh new file mode 100755 index 0000000000..5b62d7878f --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/build-release.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# build-release.sh — build an offline release archive from the lock file. +# +# Run on a CONNECTED build host with the same OS, CPU architecture and Python +# minor version as the air-gapped target. Produces, in --out: +# +# release-.tar.gz VERSION, wheels/, requirements.lock.txt +# release-.tar.gz.sha256 checksum (corruption check only) +# release-.tar.gz.sig openssl signature, when --sign-key is given +# +# Every third-party wheel is the exact version pinned in packages/flow/uv.lock +# and is checked against the lock's hashes while downloading (pip +# --require-hashes), so the archive holds what CI tested and nothing newer. +# The openadapt-flow wheel is built from this checkout unless --flow-wheel +# names one. Stage the archive with `install.sh --update` (see UPDATE.md), or +# extract it and pass release/wheels to `install.sh --wheelhouse` for a first +# install. +# +# Usage: +# build-release.sh --out DIR [--flow-wheel FILE] [--sign-key PRIVATE.pem] +# [--python PYTHON] +# +# Needs uv and the target Python (default python3). Exit 0 on success. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FLOW_DIR="$(cd "$HERE/../../.." && pwd)" +OUT="" +FLOW_WHEEL="" +SIGN_KEY="" +PYTHON="python3" + +while [[ $# -gt 0 ]]; do + case "$1" in + --out) OUT="${2:-}"; shift 2 ;; + --flow-wheel) FLOW_WHEEL="${2:-}"; shift 2 ;; + --sign-key) SIGN_KEY="${2:-}"; shift 2 ;; + --python) PYTHON="${2:-}"; shift 2 ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "build-release.sh: unknown arg $1" >&2; exit 2 ;; + esac +done +[[ -n "$OUT" ]] || { echo "build-release.sh: --out DIR is required" >&2; exit 2; } +command -v uv >/dev/null || { echo "build-release.sh: uv is required" >&2; exit 2; } +[[ -f "$FLOW_DIR/uv.lock" ]] || { echo "build-release.sh: no uv.lock in $FLOW_DIR" >&2; exit 2; } +if [[ -n "$SIGN_KEY" && ! -f "$SIGN_KEY" ]]; then + echo "build-release.sh: signing key $SIGN_KEY not found" >&2; exit 2 +fi + +# shellcheck source=bin/lib-release.sh +source "$HERE/lib-release.sh" + +mkdir -p "$OUT" +OUT="$(cd "$OUT" && pwd)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/oa-build-release.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/release/wheels" + +if [[ -z "$FLOW_WHEEL" ]]; then + echo "== building the openadapt-flow wheel from $FLOW_DIR ==" + (cd "$FLOW_DIR" && uv build --wheel --out-dir "$WORK/flow" >/dev/null) + FLOW_WHEEL="$(find "$WORK/flow" -maxdepth 1 -name 'openadapt_flow-*.whl' -print)" +fi +[[ -f "$FLOW_WHEEL" ]] || { echo "build-release.sh: flow wheel not found" >&2; exit 2; } +cp "$FLOW_WHEEL" "$WORK/release/wheels/" +VERSION="$(rel_wheelhouse_version "$WORK/release/wheels")" +rel_validate_version "$VERSION" + +echo "== exporting the locked dependencies of openadapt-flow[privacy] $VERSION ==" +(cd "$FLOW_DIR" && uv export --frozen --no-dev --extra privacy --no-emit-project \ + --format requirements-txt --no-header -o "$WORK/release/requirements.lock.txt" >/dev/null) + +echo "== downloading the locked wheels for $("$PYTHON" -c 'import platform, sys; print(sys.platform, platform.machine(), "py%d.%d" % sys.version_info[:2])') ==" +"$PYTHON" -m pip download --quiet --disable-pip-version-check \ + --only-binary=:all: --no-deps --require-hashes \ + -r "$WORK/release/requirements.lock.txt" -d "$WORK/release/wheels" + +printf '%s\n' "$VERSION" > "$WORK/release/VERSION" +ARCHIVE="$OUT/release-$VERSION.tar.gz" +COPYFILE_DISABLE=1 tar czf "$ARCHIVE" -C "$WORK" release +sha256_of "$ARCHIVE" > "$ARCHIVE.sha256" +if [[ -n "$SIGN_KEY" ]]; then + openssl dgst -sha256 -sign "$SIGN_KEY" -out "$ARCHIVE.sig" "$ARCHIVE" +fi + +count="$(find "$WORK/release/wheels" -name '*.whl' | wc -l | tr -d ' ')" +echo "built $ARCHIVE ($count wheels)" +[[ -n "$SIGN_KEY" ]] && echo "signed $ARCHIVE.sig" || echo "unsigned: sign it before staging (UPDATE.md)" diff --git a/packages/flow/deploy/on-prem/bin/test-offline-install.sh b/packages/flow/deploy/on-prem/bin/test-offline-install.sh new file mode 100755 index 0000000000..e910abf488 --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/test-offline-install.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# +# test-offline-install.sh — install, upgrade and roll back the real engine on a +# machine with no network. +# +# Unlike test-update.sh, which fakes the venv build, this installs the real +# openadapt-flow[privacy] wheels pinned by uv.lock, so it proves the offline +# release archive is complete: pip gets no index and the machine has no route +# out, so any missing wheel fails the install. +# +# stage DIR on a connected host: build two signed releases (this checkout +# as V1, the same source as V1.post1 as V2) and a throwaway +# signing key, with bin/build-release.sh. +# verify DIR as root, inside bin/run-egress-blocked.sh: first install from +# V1's wheelhouse, refuse a tampered V2, update to the signed V2, +# roll back to V1, then run the air-gap attestation with its +# egress probe and audit-chain walk. Data must survive each step. +# +# CI runs both (packages.yml, flow on-prem offline install). By hand on Linux: +# bin/test-offline-install.sh stage /tmp/oa-offline +# sudo env -u SUDO_UID -u SUDO_GID "PATH=$PATH" \ +# bin/run-egress-blocked.sh bin/test-offline-install.sh verify /tmp/oa-offline +# +# Exit 0 when every check passes. + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ONPREM="$(cd "$HERE/.." && pwd)" +FLOW_DIR="$(cd "$ONPREM/../.." && pwd)" +CMD="${1:-}" +DIR="${2:-}" +if [[ -z "$CMD" || -z "$DIR" || ( "$CMD" != stage && "$CMD" != verify ) ]]; then + grep '^#' "$0" | sed 's/^# \{0,1\}//' + exit 2 +fi + +stage() { + set -e + mkdir -p "$DIR/keys" "$DIR/v1" "$DIR/v2" + DIR="$(cd "$DIR" && pwd)" + openssl genrsa -out "$DIR/keys/vendor.pem" 2048 2>/dev/null + openssl rsa -in "$DIR/keys/vendor.pem" -pubout -out "$DIR/keys/vendor.pub" 2>/dev/null + bash "$HERE/build-release.sh" --out "$DIR/v1" --sign-key "$DIR/keys/vendor.pem" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + + # V2: the same source under a new version, built the same way. + local v1 src + v1="$(find "$DIR/v1" -maxdepth 1 -name 'release-*.tar.gz' -print)" + v1="$(basename "$v1" .tar.gz)" + v1="${v1#release-}" + src="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-src.XXXXXX")" + for item in pyproject.toml README.md LICENSE THIRD_PARTY_NOTICES.md \ + public-artifacts.json openadapt_flow schemas; do + cp -R "$FLOW_DIR/$item" "$src/" + done + find "$src" -name __pycache__ -type d -prune -exec rm -rf {} + + sed -i.bak "s/^version = \"$v1\"$/version = \"$v1.post1\"/" "$src/pyproject.toml" + sed -i.bak "s/^__version__ = \"$v1\"$/__version__ = \"$v1.post1\"/" "$src/openadapt_flow/__init__.py" + rm -f "$src/pyproject.toml.bak" "$src/openadapt_flow/__init__.py.bak" + grep -q "^version = \"$v1.post1\"$" "$src/pyproject.toml" + (cd "$src" && uv build --wheel --out-dir "$src/dist" >/dev/null) + bash "$HERE/build-release.sh" --out "$DIR/v2" --sign-key "$DIR/keys/vendor.pem" \ + --flow-wheel "$src/dist/openadapt_flow-$v1.post1-py3-none-any.whl" \ + ${OPENADAPT_ONPREM_PY:+--python "$OPENADAPT_ONPREM_PY"} + rm -rf "$src" + # The private key never goes near the install. + rm -f "$DIR/keys/vendor.pem" + printf '%s\n' "$v1" > "$DIR/V1" + printf '%s\n' "$v1.post1" > "$DIR/V2" + echo "staged $v1 and $v1.post1 in $DIR" +} + +PASS=0 +FAIL=0 +ok() { printf ' PASS %s\n' "$1"; PASS=$((PASS + 1)); } +bad() { printf ' FAIL %s\n' "$1"; FAIL=$((FAIL + 1)); } +check() { local name="$1"; shift; if "$@"; then ok "$name"; else bad "$name"; fi; } +check_eq() { if [[ "$2" == "$3" ]]; then ok "$1"; else bad "$1 (expected '$2', got '$3')"; fi; } + +verify() { + DIR="$(cd "$DIR" && pwd)" + if [[ "$(id -u)" -ne 0 ]]; then + echo "test-offline-install.sh verify: run as root (install.sh needs it)" >&2; exit 2 + fi + if [[ "${OPENADAPT_EGRESS_BLOCKED:-}" != netns ]]; then + echo "test-offline-install.sh verify: run inside bin/run-egress-blocked.sh" >&2; exit 2 + fi + local v1 v2 py work root config rc + v1="$(cat "$DIR/V1")"; v2="$(cat "$DIR/V2")" + py="${OPENADAPT_ONPREM_PY:-python3}" + work="$(mktemp -d "${TMPDIR:-/tmp}/oa-offline-verify.XXXXXX")" + root="$work/srv" + config="$work/onprem.yaml" + cat > "$config" <&1 + } + + echo "== offline install, upgrade and rollback ($v1 -> $v2 -> $v1), no network ==" + check "the machine has no route out" "$py" -c ' +import socket, sys +try: + socket.create_connection(("1.1.1.1", 443), timeout=3) +except OSError: + sys.exit(0) +sys.exit(1)' + + echo "-- first install from the $v1 wheelhouse --" + mkdir -p "$work/media" + tar xzf "$DIR/v1/release-$v1.tar.gz" -C "$work/media" + bash "$ONPREM/install.sh" --config "$config" --wheelhouse "$work/media/release/wheels" \ + > "$work/install.log" 2>&1 + rc=$? + check_eq "install: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/install.log" + check_eq "install: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "install: engine and privacy extra import, version $v1" "$v1" "$(engine_version)" + check "install: engine CLI runs" "$root/current/venv/bin/openadapt-flow" --help + + mkdir -p "$root/bundles" "$root/jobs/inbox" + printf 'bundle\n' > "$root/bundles/keepme.txt" + printf 'job\n' > "$root/jobs/inbox/queued-0001.job" + local bundle_sha job_sha + bundle_sha="$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + job_sha="$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + cp "$DIR/keys/vendor.pub" "$root/keys/vendor.pub" + + local arc="$DIR/v2/release-$v2.tar.gz" + echo "-- a tampered $v2 archive is refused --" + cp "$arc" "$work/media/tampered.tar.gz" + printf 'x' >> "$work/media/tampered.tar.gz" + bash "$ONPREM/install.sh" --update --config "$config" --release "$work/media/tampered.tar.gz" \ + --signature "$arc.sig" > "$work/tampered.log" 2>&1 + rc=$? + check "tampered: update refused" test "$rc" -ne 0 + check_eq "tampered: current still $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check "tampered: nothing built for $v2" test ! -e "$root/releases/$v2" + + echo "-- signed update to $v2 --" + bash "$ONPREM/install.sh" --update --config "$config" --release "$arc" \ + --checksum "$arc.sha256" --signature "$arc.sig" > "$work/update.log" 2>&1 + rc=$? + check_eq "update: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/update.log" + check_eq "update: current -> $v2" "$root/releases/$v2" "$(readlink "$root/current")" + check_eq "update: previous -> $v1" "$root/releases/$v1" "$(readlink "$root/previous")" + check_eq "update: engine version $v2" "$v2" "$(engine_version)" + check_eq "update: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + check_eq "update: queued job unchanged" "$job_sha" "$(shasum -a 256 "$root/jobs/inbox/queued-0001.job" | awk '{print $1}')" + + echo "-- rollback to $v1 --" + bash "$ONPREM/install.sh" --rollback --config "$config" > "$work/rollback.log" 2>&1 + rc=$? + check_eq "rollback: exit 0" 0 "$rc" + [[ "$rc" -eq 0 ]] || tail -n 40 "$work/rollback.log" + check_eq "rollback: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" + check_eq "rollback: engine version $v1" "$v1" "$(engine_version)" + check_eq "rollback: bundle unchanged" "$bundle_sha" "$(shasum -a 256 "$root/bundles/keepme.txt" | awk '{print $1}')" + + echo "-- air-gap attestation --" + OPENADAPT_FLOW_SCRUB=on bash "$HERE/verify-airgap.sh" --config "$config" --probe --audit \ + > "$work/airgap.log" 2>&1 + rc=$? + check_eq "attestation: exit 0" 0 "$rc" + check "attestation: egress probe could not reach the internet" \ + grep -q "outbound egress is blocked" "$work/airgap.log" + check "attestation: audit chain intact" grep -q "audit chain intact" "$work/airgap.log" + [[ "$rc" -eq 0 ]] || cat "$work/airgap.log" + + rm -rf "$work" + echo + echo "== result: $PASS passed, $FAIL failed ==" + [[ "$FAIL" -eq 0 ]] +} + +"$CMD" From f5eb3c39c35e70ce0de4b98e52023629300f939c Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 18:17:42 -0500 Subject: [PATCH 02/21] on-prem: check the engine version from the release venv, not the checkout The version check ran python -c from packages/flow, so it imported the source checkout and reported 2.6.0 after the update to 2.6.0.post1. Run it isolated from /, require the module to live in the venv, and compare the module with the installed distribution and the CLI's --version. --- .../deploy/on-prem/bin/test-offline-install.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/flow/deploy/on-prem/bin/test-offline-install.sh b/packages/flow/deploy/on-prem/bin/test-offline-install.sh index e910abf488..c3e6d391cd 100755 --- a/packages/flow/deploy/on-prem/bin/test-offline-install.sh +++ b/packages/flow/deploy/on-prem/bin/test-offline-install.sh @@ -113,9 +113,20 @@ updates: YAML export OPENADAPT_ONPREM_PY="$py" export OPENADAPT_ONPREM_AUDIT_LOG="$root/audit/audit.log" + # -I and cd /: import only what the release venv installed, never a source + # checkout in the working directory. Prints the installed distribution's + # version, and fails unless the module is the one inside the venv. engine_version() { - "$root/current/venv/bin/python" -c \ - 'import openadapt_flow, presidio_analyzer, openadapt_privacy; print(openadapt_flow.__version__)' 2>&1 + (cd / && "$root/current/venv/bin/python" -I -c ' +import importlib.metadata, os, sys +import openadapt_flow, openadapt_privacy, presidio_analyzer +here = os.path.realpath(openadapt_flow.__file__) +if not here.startswith(os.path.realpath(sys.prefix) + os.sep): + sys.exit("openadapt_flow imported from outside the release venv: " + here) +version = importlib.metadata.version("openadapt-flow") +if openadapt_flow.__version__ != version: + sys.exit("module says %s, distribution says %s" % (openadapt_flow.__version__, version)) +print(version)' 2>&1) } echo "== offline install, upgrade and rollback ($v1 -> $v2 -> $v1), no network ==" @@ -137,7 +148,8 @@ sys.exit(1)' [[ "$rc" -eq 0 ]] || tail -n 40 "$work/install.log" check_eq "install: current -> $v1" "$root/releases/$v1" "$(readlink "$root/current")" check_eq "install: engine and privacy extra import, version $v1" "$v1" "$(engine_version)" - check "install: engine CLI runs" "$root/current/venv/bin/openadapt-flow" --help + check_eq "install: engine CLI reports $v1" "openadapt-flow $v1" \ + "$(cd / && "$root/current/venv/bin/openadapt-flow" --version 2>&1)" mkdir -p "$root/bundles" "$root/jobs/inbox" printf 'bundle\n' > "$root/bundles/keepme.txt" From efbb57c2abebf39fc5626fbb5c45d4de28020c60 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 18:32:14 -0500 Subject: [PATCH 03/21] on-prem audit: do not break a lock that a live writer just took A writer that found the audit lock owned by an exited process removed it without re-checking. If the owner released between the read and the removal and another writer took the lock, the breaker deleted the live lock and two writers could chain to the same parent. Break stale locks only under a second lock and only if the lock still names the same dead owner. Add a deterministic test that forces the handover, and print the verifier output when the concurrent-writer check fails. --- packages/flow/deploy/on-prem/bin/audit-log.sh | 21 +++++++++- .../flow/deploy/on-prem/bin/test-update.sh | 42 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/packages/flow/deploy/on-prem/bin/audit-log.sh b/packages/flow/deploy/on-prem/bin/audit-log.sh index 091add53c3..9b51ce4e4a 100755 --- a/packages/flow/deploy/on-prem/bin/audit-log.sh +++ b/packages/flow/deploy/on-prem/bin/audit-log.sh @@ -69,7 +69,25 @@ chmod 0700 "$(dirname "$AUDIT_LOG")" 2>/dev/null || true # interleaving, but without this lock two writers can both chain to the same # parent and produce a structurally broken audit log. lock_dir="${AUDIT_LOG}.lock" +break_dir="${AUDIT_LOG}.lock-break" lock_acquired=0 + +# Remove a lock whose owner has exited. Between reading the owner and removing +# the lock, that owner can release it and another writer can take it, so the +# removal happens only under a second lock that serializes breakers, and only +# if the lock still names the same dead owner. Without this, a breaker could +# delete a live writer's lock and let two writers chain to the same parent. +break_stale_lock() { + local stale="$1" current="" + mkdir "$break_dir" 2>/dev/null || return 0 + current="$(cat "$lock_dir/pid" 2>/dev/null || true)" + if [[ "$current" == "$stale" ]] && ! kill -0 "$stale" 2>/dev/null; then + rm -f "$lock_dir/pid" 2>/dev/null || true + rmdir "$lock_dir" 2>/dev/null || true + fi + rmdir "$break_dir" 2>/dev/null || true +} + for ((attempt = 0; attempt < 200; attempt++)); do if mkdir "$lock_dir" 2>/dev/null; then printf '%s\n' "$$" > "$lock_dir/pid" @@ -81,8 +99,7 @@ for ((attempt = 0; attempt < 200; attempt++)); do owner="$(cat "$lock_dir/pid" 2>/dev/null || true)" fi if [[ -n "$owner" && "$owner" =~ ^[0-9]+$ ]] && ! kill -0 "$owner" 2>/dev/null; then - rm -f "$lock_dir/pid" 2>/dev/null || true - rmdir "$lock_dir" 2>/dev/null || true + break_stale_lock "$owner" continue fi sleep 0.05 diff --git a/packages/flow/deploy/on-prem/bin/test-update.sh b/packages/flow/deploy/on-prem/bin/test-update.sh index c2778b4602..26be744811 100755 --- a/packages/flow/deploy/on-prem/bin/test-update.sh +++ b/packages/flow/deploy/on-prem/bin/test-update.sh @@ -290,7 +290,49 @@ if grep -q 'audit chain intact' <<< "$audit_output"; then pass "concurrent audit writers preserve one hash chain" else fail "concurrent audit writers preserve one hash chain" + printf '%s\n' "$audit_output" | sed 's/^/ /' fi + +# A writer that finds a lock owned by an exited process may break it. Force the +# race where, between that writer reading the dead owner and breaking the +# lock, the owner releases and a live writer takes the lock: a shimmed cat +# returns the dead owner once, then hands the lock to a live process. The +# breaker must leave the live writer's lock alone. +RACE="$WORK/stale-race" +mkdir -p "$RACE/shim" +RACE_LOG="$RACE/audit.log" +bash -c 'exit 0' & +dead_pid=$! +wait "$dead_pid" +sleep 30 & +live_pid=$! +mkdir "$RACE_LOG.lock" +printf '%s\n' "$dead_pid" > "$RACE_LOG.lock/pid" +REAL_CAT="$(command -v cat)" +cat > "$RACE/shim/cat" < "$RACE/handed-over" + printf '%s\n' "$live_pid" > "$RACE_LOG.lock/pid" + exit 0 +fi +exec "$REAL_CAT" "\$@" +SHIM_EOF +chmod +x "$RACE/shim/cat" +OPENADAPT_ONPREM_AUDIT_LOG="$RACE_LOG" PATH="$RACE/shim:$PATH" \ + bash "$OPENADAPT_ONPREM_AUDIT_BIN" started stale-race --note test >/dev/null 2>&1 & +race_writer=$! +sleep 2 +race_owner="$(cat "$RACE_LOG.lock/pid" 2>/dev/null || echo none)" +assert_eq "stale-lock breaker leaves a live writer's lock alone" "$live_pid" "$race_owner" +rm -f "$RACE_LOG.lock/pid" +rmdir "$RACE_LOG.lock" 2>/dev/null || true +kill "$live_pid" 2>/dev/null || true +wait "$live_pid" 2>/dev/null || true +wait "$race_writer" +assert_eq "writer appends once the live lock is released" 0 "$?" +assert_eq "writer appends exactly one record" 1 "$(wc -l < "$RACE_LOG" | tr -d ' ')" if ! grep -q 'identity evidence' "$OPENADAPT_ONPREM_AUDIT_LOG"; then pass "release audit remains PHI-free" else From 7f2de3449d43c59ff768b7858694b7f38b14768c Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 18:55:47 -0500 Subject: [PATCH 04/21] on-prem: build the runner image from a signed release with no network, and test it in CI The Dockerfile installed from an unpinned ./wheels directory that nothing produced, and its command (bin/run-queue.sh relative to the storage volume) could not resolve. build-image.sh now verifies a signed release archive against the pinned vendor key, extracts it, and builds with --network none from the release's lock-pinned wheels. docker-compose.yml runs a named, never-pulled image with an absolute command. test-container.sh builds it in CI, refuses a tampered archive, shows a missing wheel fails the build, and runs the runner on the compose network with no route out. --- .github/workflows/packages.yml | 6 +- packages/flow/deploy/on-prem/Dockerfile | 53 +++-- packages/flow/deploy/on-prem/README.md | 2 +- .../flow/deploy/on-prem/bin/build-image.sh | 82 +++++++ .../flow/deploy/on-prem/bin/test-container.sh | 209 ++++++++++++++++++ .../flow/deploy/on-prem/docker-compose.yml | 31 +-- packages/flow/docs/ON_PREM.md | 6 +- 7 files changed, 348 insertions(+), 41 deletions(-) create mode 100755 packages/flow/deploy/on-prem/bin/build-image.sh create mode 100755 packages/flow/deploy/on-prem/bin/test-container.sh diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index d59cd2105d..346a428817 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -638,7 +638,9 @@ jobs: # no index and the machine has no network, so a wheel missing from the # archive fails the install. deploy/on-prem/bin/test-offline-install.sh. # First, bin/test-update.sh: the update state machine's refusal, migration - # and recovery paths with a fake venv build, which nothing else runs. + # and recovery paths with a fake venv build, which nothing else runs. Then + # bin/test-container.sh: the runner image built from the first release with + # no network, run on docker-compose.yml's internal network. flow-onprem-offline: name: flow on-prem offline install, update and rollback (egress blocked) needs: scope @@ -662,6 +664,8 @@ jobs: run: bash deploy/on-prem/bin/test-update.sh - name: Build two signed offline releases from the lock (connected) run: bash deploy/on-prem/bin/test-offline-install.sh stage "$RUNNER_TEMP/offline" + - name: Container image from the signed release (no-network build, internal network) + run: bash deploy/on-prem/bin/test-container.sh "$RUNNER_TEMP/offline" - name: Install, update and roll back with egress blocked # env -u SUDO_UID keeps the command root inside the namespace: # install.sh owns the release tree as root. diff --git a/packages/flow/deploy/on-prem/Dockerfile b/packages/flow/deploy/on-prem/Dockerfile index 1e176cf75a..b71c2bdffa 100644 --- a/packages/flow/deploy/on-prem/Dockerfile +++ b/packages/flow/deploy/on-prem/Dockerfile @@ -1,37 +1,46 @@ -# openadapt-flow on-prem runner image — built OFFLINE from a local wheelhouse. +# openadapt-flow on-prem runner image, built with no network from a signed +# release archive (bin/build-release.sh). # -# Air-gapped hosts do not reach PyPI. Build this on a connected staging host (or -# with a pre-populated ./wheels) and copy the resulting image tar into the -# clinic via removable media (`docker save` / `docker load`). +# Do not run `docker build` on this file directly. bin/build-image.sh checks the +# archive's signature against the pinned vendor key, extracts it, lays out the +# build context below and builds with `--network none`, so pip can only use +# the release's own wheels (the ones pinned by uv.lock): # -# Prepare the wheelhouse off-site: -# pip download 'openadapt-flow[privacy]' -d deploy/on-prem/wheels -# Then build (context = repo root, per docker-compose.yml): -# docker build -f deploy/on-prem/Dockerfile -t openadapt-flow-onprem:local . +# release/ the extracted release: VERSION, wheels/, requirements.lock.txt +# on-prem/ this directory (scripts, units, this Dockerfile) # -# [STUB] the ./wheels wheelhouse is NOT committed (it is large and version- -# specific). Without it the build fails LOUDLY rather than reaching the network. +# The base image must already be in the local image store (`docker load` it +# on an air-gapped host). Its default is pinned by digest; the release must be +# built for the same Python minor version and CPU architecture (see +# build-release.sh). Move the result in with `docker save` / `docker load`. -FROM python:3.11-slim +ARG BASE_IMAGE=python:3.12-slim@sha256:b18b261e1b4974b4f321451ad5251ded85abb22c2b8b262da527dda1d031ab30 +FROM ${BASE_IMAGE} -# Never let pip reach the internet from inside the image build. ENV PIP_NO_INDEX=1 \ PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PYTHONDONTWRITEBYTECODE=1 \ OPENADAPT_FLOW_SCRUB=on -WORKDIR /opt/openadapt-flow +# The same install as install.sh (lib-release.sh rel_build_release): only the +# release's wheels, the exact version the archive declares, then a dependency +# consistency check. +COPY release /tmp/release +RUN set -eu; \ + version="$(cat /tmp/release/VERSION)"; \ + python -m venv /opt/openadapt-flow/venv; \ + /opt/openadapt-flow/venv/bin/pip install --no-index --only-binary=:all: \ + --find-links /tmp/release/wheels "openadapt-flow[privacy]==${version}"; \ + /opt/openadapt-flow/venv/bin/pip check; \ + printf '%s\n' "$version" > /opt/openadapt-flow/VERSION; \ + rm -rf /tmp/release -# Copy the offline wheelhouse (staged off-site) and the deploy scripts. -COPY deploy/on-prem/wheels /tmp/wheels -COPY deploy/on-prem /opt/openadapt-flow/deploy/on-prem +COPY on-prem /opt/openadapt-flow/deploy/on-prem -# Install the engine + privacy extra from local wheels only. -RUN pip install --no-index --find-links /tmp/wheels 'openadapt-flow[privacy]' \ - && rm -rf /tmp/wheels +ENV PATH="/opt/openadapt-flow/venv/bin:/opt/openadapt-flow/deploy/on-prem/bin:${PATH}" # Storage root is a bind-mounted, host-encrypted volume at runtime. WORKDIR /srv/openadapt -ENV PATH="/opt/openadapt-flow/deploy/on-prem:${PATH}" -# Default: process the local job queue. Overridden by docker-compose command. -CMD ["deploy/on-prem/bin/run-queue.sh", "watch"] +# Absolute path: the working directory is the storage volume, not the scripts. +CMD ["/opt/openadapt-flow/deploy/on-prem/bin/run-queue.sh", "watch"] diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index c68dff4b87..a2f1f81afa 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -25,7 +25,7 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow). | `bin/test-offline-install.sh` | Builds two signed releases, then as root with egress blocked installs the real engine from the first, refuses a tampered second, updates to the signed second, rolls back, and runs the air-gap attestation. CI runs it with `bin/test-update.sh` (job `flow-onprem-offline`). | REAL (Linux, root) | | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) | | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) | -| `docker-compose.yml` + `Dockerfile` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. | REAL topology; STUB image build (needs offline `./wheels`) | +| `docker-compose.yml` + `Dockerfile` + `bin/build-image.sh` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. `bin/build-image.sh` verifies a signed release and builds the image from its wheels with `--network none`. `bin/test-container.sh` (CI, job `flow-onprem-offline`) builds it, refuses a tampered archive, checks a missing wheel fails the build, and runs the runner on the compose network with no route out. | REAL (Linux, Docker) | | `COMPLIANCE.md` | Regulatory posture, data-flow diagram, what's encrypted, boundaries. | REAL (doc; non-legal) | ## 60-second install (systemd host) diff --git a/packages/flow/deploy/on-prem/bin/build-image.sh b/packages/flow/deploy/on-prem/bin/build-image.sh new file mode 100755 index 0000000000..f75b2d04ef --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/build-image.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# build-image.sh — build the on-prem runner image from a signed release +# archive, with no network. +# +# Checks the archive against the pinned vendor key (the same check as +# `install.sh --update`), extracts it with the bounded extractor, and runs +# `docker build --network none` on ../Dockerfile, so the image holds exactly +# the release's wheels. A missing wheel fails the build instead of reaching +# PyPI. The base image must already be loaded locally. +# +# Usage: +# build-image.sh --archive release-V.tar.gz --signature release-V.tar.gz.sig +# --pubkey vendor.pub [--sig-tool openssl|gpg|minisign|signify] +# [--checksum FILE] [--tag NAME:TAG] [--base IMAGE] +# [--save image.tar] +# +# The default tag is openadapt-flow-onprem:. Run the image +# with docker-compose.yml (OPENADAPT_FLOW_IMAGE=). Exit 0 on success. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ONPREM="$(cd "$HERE/.." && pwd)" +ARCHIVE="" +SIGNATURE="" +PUBKEY="" +SIG_TOOL="" +CHECKSUM="" +TAG="" +BASE="" +SAVE="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --archive) ARCHIVE="${2:-}"; shift 2 ;; + --signature) SIGNATURE="${2:-}"; shift 2 ;; + --pubkey) PUBKEY="${2:-}"; shift 2 ;; + --sig-tool) SIG_TOOL="${2:-}"; shift 2 ;; + --checksum) CHECKSUM="${2:-}"; shift 2 ;; + --tag) TAG="${2:-}"; shift 2 ;; + --base) BASE="${2:-}"; shift 2 ;; + --save) SAVE="${2:-}"; shift 2 ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "build-image.sh: unknown arg $1" >&2; exit 2 ;; + esac +done +[[ -n "$ARCHIVE" ]] || { echo "build-image.sh: --archive is required" >&2; exit 2; } +command -v docker >/dev/null || { echo "build-image.sh: docker is required" >&2; exit 2; } + +# shellcheck source=bin/lib-release.sh +source "$HERE/lib-release.sh" + +rel_verify_integrity "$ARCHIVE" "$CHECKSUM" "$SIGNATURE" "$PUBKEY" "$SIG_TOOL" || { + echo "build-image.sh: refusing to build from an unverified archive" >&2; exit 1 +} + +CTX="$(mktemp -d "${TMPDIR:-/tmp}/oa-build-image.XXXXXX")" +trap 'rm -rf "$CTX"' EXIT +SRC="$(rel_extract "$ARCHIVE" "$CTX/extract")" || { + echo "build-image.sh: release archive could not be extracted" >&2; exit 1 +} +VERSION="$(rel_declared_version "$SRC" "$SRC/wheels")" || exit 1 +mv "$SRC" "$CTX/release" +rm -rf "$CTX/extract" + +# The deploy scripts, without anything a site may have left beside them. +mkdir -p "$CTX/on-prem" +tar -C "$ONPREM" --exclude=./wheels --exclude=./storage --exclude=./release \ + --exclude='*.pem' --exclude='*.env' -cf - . | tar -C "$CTX/on-prem" -xf - + +TAG="${TAG:-openadapt-flow-onprem:$VERSION}" +args=(build --network none --pull=false -f "$CTX/on-prem/Dockerfile" -t "$TAG") +[[ -n "$BASE" ]] && args+=(--build-arg "BASE_IMAGE=$BASE") +echo "== building $TAG from release $VERSION with no network ==" +docker "${args[@]}" "$CTX" + +if [[ -n "$SAVE" ]]; then + docker save -o "$SAVE" "$TAG" + echo "saved $TAG to $SAVE" +fi +echo "built $TAG" diff --git a/packages/flow/deploy/on-prem/bin/test-container.sh b/packages/flow/deploy/on-prem/bin/test-container.sh new file mode 100755 index 0000000000..2abf6090bc --- /dev/null +++ b/packages/flow/deploy/on-prem/bin/test-container.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# +# test-container.sh — build the on-prem runner image from a signed release +# with no network, and run it on the compose file's internal network. +# +# test-container.sh DIR +# +# DIR is what `bin/test-offline-install.sh stage DIR` writes (a signed V1 +# release and the vendor public key). Needs Linux, Docker with compose, and a +# connected host only to pull the pinned base image. Checks: +# - build-image.sh refuses a tampered archive and builds no image; +# - the image build has no network: the same release with one dependency +# wheel removed fails to build; +# - the signed release builds, and inside the image (no network) the engine +# and privacy extra import from the image's venv at the release version; +# - the default command resolves, and run-queue.sh files a job and writes +# the audit log on a bind-mounted storage root; it refuses to start with +# scrubbing off; +# - docker-compose.yml needs the image named, never pulls, publishes no +# ports, and its network is internal: a TCP connect to the internet that +# works from the same image on Docker's default network fails from the +# compose runner. +# +# CI runs it in packages.yml (flow on-prem offline install). Exit 0 when every +# check passes. + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ONPREM="$(cd "$HERE/.." && pwd)" +DIR="${1:-}" +if [[ -z "$DIR" || ! -f "$DIR/V1" ]]; then + grep '^#' "$0" | sed 's/^# \{0,1\}//' + exit 2 +fi +DIR="$(cd "$DIR" && pwd)" +command -v docker >/dev/null || { echo "test-container.sh: docker is required" >&2; exit 2; } + +PASS=0 +FAIL=0 +ok() { printf ' PASS %s\n' "$1"; PASS=$((PASS + 1)); } +bad() { printf ' FAIL %s\n' "$1"; FAIL=$((FAIL + 1)); } +check_eq() { if [[ "$2" == "$3" ]]; then ok "$1"; else bad "$1 (expected '$2', got '$3')"; fi; } + +v1="$(cat "$DIR/V1")" +archive="$DIR/v1/release-$v1.tar.gz" +sig="$archive.sig" +pub="$DIR/keys/vendor.pub" +base="$(awk -F= '/^ARG BASE_IMAGE=/ {print $2; exit}' "$ONPREM/Dockerfile")" +tag="openadapt-flow-onprem-test:$v1" +project="oa-onprem-test-$$" +compose=(docker compose -p "$project" -f "$ONPREM/docker-compose.yml") +work="$(mktemp -d "${TMPDIR:-/tmp}/oa-test-container.XXXXXX")" +store="$work/storage" +mkdir -p "$store" + +cleanup() { + OPENADAPT_FLOW_IMAGE="$tag" OPENADAPT_ONPREM_STORAGE="$store" \ + "${compose[@]}" down --remove-orphans >/dev/null 2>&1 || true + # The container writes the storage root as root; empty it from inside. + docker run --rm --network none -v "$store:/s" "$tag" \ + sh -c 'rm -rf /s/* /s/.[!.]*' >/dev/null 2>&1 || true + docker image rm -f "$tag" "$tag-tampered" "$tag-missing" >/dev/null 2>&1 || true + rm -rf "$work" +} +trap cleanup EXIT + +echo "== pulling the pinned base image (connected; the build itself has no network) ==" +docker pull -q "$base" >/dev/null || { echo "test-container.sh: cannot pull $base" >&2; exit 1; } + +echo "== build-image.sh refuses a tampered archive ==" +cp "$archive" "$work/tampered.tar.gz" +printf 'x' >> "$work/tampered.tar.gz" +if bash "$HERE/build-image.sh" --archive "$work/tampered.tar.gz" --signature "$sig" \ + --pubkey "$pub" --sig-tool openssl --tag "$tag-tampered" >"$work/tampered.log" 2>&1; then + bad "tampered archive refused" +else + ok "tampered archive refused" +fi +if docker image inspect "$tag-tampered" >/dev/null 2>&1; then + bad "no image built from the tampered archive" +else + ok "no image built from the tampered archive" +fi + +echo "== the image build has no network: a release missing one wheel fails ==" +mkdir -p "$work/missing/on-prem" +tar xzf "$archive" -C "$work/missing" +victim="$(find "$work/missing/release/wheels" -name 'presidio_analyzer-*.whl' -print | head -n 1)" +if [[ -z "$victim" ]]; then + bad "release holds the presidio_analyzer wheel to remove" +else + rm -f "$victim" + tar -C "$ONPREM" -cf - . | tar -C "$work/missing/on-prem" -xf - + if docker build --network none --pull=false -q -f "$work/missing/on-prem/Dockerfile" \ + -t "$tag-missing" "$work/missing" >"$work/missing.log" 2>&1; then + bad "build without $(basename "$victim") fails (it reached a package index?)" + else + if grep -q "presidio" "$work/missing.log"; then + ok "build without $(basename "$victim") fails on that package" + else + bad "build without $(basename "$victim") failed for another reason" + tail -n 20 "$work/missing.log" + fi + fi +fi + +echo "== build the signed release ==" +if bash "$HERE/build-image.sh" --archive "$archive" --signature "$sig" --pubkey "$pub" \ + --sig-tool openssl --tag "$tag" >"$work/build.log" 2>&1; then + ok "signed release $v1 builds with no network" +else + bad "signed release $v1 builds with no network" + tail -n 40 "$work/build.log" + echo "== result: $PASS passed, $FAIL failed ==" + exit 1 +fi + +run() { docker run --rm --network none "$@"; } + +check_eq "image: engine CLI reports $v1" "openadapt-flow $v1" \ + "$(run "$tag" openadapt-flow --version 2>&1)" +check_eq "image: engine and privacy extra import from the image venv, version $v1" "$v1" \ + "$(run -w / "$tag" python -I -c ' +import importlib.metadata, os, sys +import openadapt_flow, openadapt_privacy, presidio_analyzer +here = os.path.realpath(openadapt_flow.__file__) +if not here.startswith("/opt/openadapt-flow/venv/"): + sys.exit("openadapt_flow imported from outside the image venv: " + here) +if openadapt_flow.__version__ != importlib.metadata.version("openadapt-flow"): + sys.exit("module and distribution versions differ") +print(openadapt_flow.__version__) +' 2>&1)" +check_eq "image: VERSION file" "$v1" "$(run "$tag" cat /opt/openadapt-flow/VERSION 2>&1)" +check_eq "image: default command is the absolute run-queue path" \ + '["/opt/openadapt-flow/deploy/on-prem/bin/run-queue.sh","watch"]' \ + "$(docker image inspect -f '{{json .Config.Cmd}}' "$tag")" + +echo "== run-queue.sh in the image, on a bind-mounted storage root ==" +mkdir -p "$store/jobs/inbox" +printf 'params=none\n' > "$store/jobs/inbox/no-bundle.job" +qenv=(-e OPENADAPT_ONPREM_ROOT=/srv/openadapt + -e OPENADAPT_ONPREM_AUDIT_LOG=/srv/openadapt/audit/audit.log + -v "$store:/srv/openadapt") +if run "${qenv[@]}" "$tag" /opt/openadapt-flow/deploy/on-prem/bin/run-queue.sh once \ + >"$work/queue.log" 2>&1; then + ok "run-queue.sh once exits 0" +else + bad "run-queue.sh once exits 0"; cat "$work/queue.log" +fi +if [[ -f "$store/jobs/failed/no-bundle.job" ]]; then + ok "malformed job filed under failed/" +else + bad "malformed job filed under failed/" +fi +if grep -q "no-bundle" "$store/audit/audit.log" 2>/dev/null; then + ok "audit log records the job" +else + bad "audit log records the job" +fi +if run "${qenv[@]}" -e OPENADAPT_FLOW_SCRUB=off "$tag" \ + /opt/openadapt-flow/deploy/on-prem/bin/run-queue.sh once >/dev/null 2>&1; then + bad "run-queue.sh refuses to start with scrubbing off" +else + ok "run-queue.sh refuses to start with scrubbing off" +fi + +echo "== docker-compose.yml ==" +if OPENADAPT_ONPREM_STORAGE="$store" "${compose[@]}" config >/dev/null 2>&1; then + bad "compose refuses to run without OPENADAPT_FLOW_IMAGE" +else + ok "compose refuses to run without OPENADAPT_FLOW_IMAGE" +fi +export OPENADAPT_FLOW_IMAGE="$tag" OPENADAPT_ONPREM_STORAGE="$store" +cfg="$("${compose[@]}" config --format json 2>"$work/config.err")" +check_eq "compose: runner never pulls, publishes no ports, network internal" \ + "never|none|True" \ + "$(printf '%s' "$cfg" | python3 -c ' +import json, sys +c = json.load(sys.stdin) +r = c["services"]["runner"] +nets = [c["networks"][n].get("internal", False) for n in r["networks"]] +print("%s|%s|%s" % (r.get("pull_policy"), r.get("ports") or "none", all(nets) and bool(nets))) +' 2>&1)" + +probe='import socket; socket.create_connection(("1.1.1.1", 443), timeout=5).close()' +if docker run --rm "$tag" python -c "$probe" >/dev/null 2>&1; then + ok "control: the probe connects from this image on Docker's default network" + if "${compose[@]}" run --rm -T --no-deps runner python -c "$probe" >"$work/probe.log" 2>&1; then + bad "compose runner cannot connect to the internet" + else + ok "compose runner cannot connect to the internet" + fi +else + bad "control: the probe connects from this image on Docker's default network (cannot tell a blocked network from no network)" +fi + +"${compose[@]}" up -d runner >"$work/up.log" 2>&1 || { bad "compose up runner"; cat "$work/up.log"; } +sleep 5 +state="$(docker inspect -f '{{.State.Status}}' "$("${compose[@]}" ps -q runner)" 2>/dev/null)" +check_eq "compose: runner keeps watching the queue" "running" "$state" +if "${compose[@]}" logs runner 2>&1 | grep -q "watching /srv/openadapt/jobs/inbox"; then + ok "compose: runner logs that it is watching the inbox" +else + bad "compose: runner logs that it is watching the inbox"; "${compose[@]}" logs runner +fi + +echo "== result: $PASS passed, $FAIL failed ==" +[[ "$FAIL" -eq 0 ]] diff --git a/packages/flow/deploy/on-prem/docker-compose.yml b/packages/flow/deploy/on-prem/docker-compose.yml index 850f158b9f..9da3be2e5b 100644 --- a/packages/flow/deploy/on-prem/docker-compose.yml +++ b/packages/flow/deploy/on-prem/docker-compose.yml @@ -8,28 +8,31 @@ # * the `clinic-lan` network is `internal: true` -> Docker installs no default # gateway, so containers on it CANNOT reach the internet (only each other). # * no ports are published to the host by the runner (nothing inbound either). -# * the image is built from a LOCAL wheelhouse (see Dockerfile) — no PyPI. +# * the image is built with no network from a signed release archive +# (bin/build-image.sh, see Dockerfile) and never pulled: pull_policy never. # # What is REAL vs STUB: # [REAL] the compose topology, internal network, volumes, fail-closed env. # [REAL] `runner` runs the shipped `openadapt-flow run` via bin/run-queue.sh. -# [STUB] the images must be BUILT OFFLINE first (Dockerfile needs a staged -# ./wheels wheelhouse); this file does not fetch anything. +# [REAL] the image: build it first with bin/build-image.sh (or `docker load` +# one built elsewhere) and name it in OPENADAPT_FLOW_IMAGE. CI builds +# it with no network and runs it on this file's internal network +# (bin/test-container.sh). This file does not build or fetch anything. # [STUB] `vlm` profile needs a real GPU box + a locally-staged model; the # appliance is opt-in and LAN-only even here. # -# Validate without building: docker compose -f docker-compose.yml config +# Every command needs the image name, e.g. +# export OPENADAPT_FLOW_IMAGE=openadapt-flow-onprem: +# Validate: docker compose -f docker-compose.yml config # Bring up the runner: docker compose up -d runner # With the VLM appliance: docker compose --profile vlm up -d services: runner: - build: - # Repo root is the build context so the engine source is available to the - # offline wheelhouse install. See ./Dockerfile. - context: ../.. - dockerfile: deploy/on-prem/Dockerfile - image: openadapt-flow-onprem:local + # Built by bin/build-image.sh from a signed release. No default: running + # an unnamed or stale version is worse than refusing to start. + image: ${OPENADAPT_FLOW_IMAGE:?set OPENADAPT_FLOW_IMAGE to the image bin/build-image.sh built} + pull_policy: never restart: unless-stopped # Fail-closed PHI scrubbing + local storage/audit wiring. NO cloud keys. environment: @@ -45,7 +48,7 @@ services: - type: bind source: ${OPENADAPT_ONPREM_STORAGE:-./storage} target: /srv/openadapt - command: ["bin/run-queue.sh", "watch"] + command: ["/opt/openadapt-flow/deploy/on-prem/bin/run-queue.sh", "watch"] networks: - clinic-lan # No `ports:` — nothing is published to the host or beyond. @@ -55,10 +58,8 @@ services: # docs/deployment/ON_PREM_VLM.md. Enabled only with `--profile vlm`. vlm-appliance: profiles: ["vlm"] - build: - context: ../.. - dockerfile: deploy/on-prem/Dockerfile - image: openadapt-flow-onprem:local + image: ${OPENADAPT_FLOW_IMAGE:?set OPENADAPT_FLOW_IMAGE to the image bin/build-image.sh built} + pull_policy: never restart: unless-stopped environment: VLM_BACKEND: "vllm" diff --git a/packages/flow/docs/ON_PREM.md b/packages/flow/docs/ON_PREM.md index fc909f21d5..cb4d2db6f6 100644 --- a/packages/flow/docs/ON_PREM.md +++ b/packages/flow/docs/ON_PREM.md @@ -189,8 +189,10 @@ sudo systemctl enable --now openadapt-flow-runner.path OPENADAPT_FLOW_SCRUB=on ./bin/verify-airgap.sh --config onprem.yaml --probe ``` -Containers instead of systemd: `docker compose -f docker-compose.yml config` -(validate), then `docker compose up -d runner`. See +Containers instead of systemd: build the image from a signed release with +`bin/build-image.sh` (no network; `docker save` / `docker load` it across), +set `OPENADAPT_FLOW_IMAGE` to its tag, then `docker compose -f +docker-compose.yml config` (validate) and `docker compose up -d runner`. See [`deploy/on-prem/README.md`](../deploy/on-prem/README.md) and [`deploy/on-prem/UPDATE.md`](../deploy/on-prem/UPDATE.md) for the runnable package and update/rollback runbook, and From 7770839e32569ff40e1774672b661609a3813b6b Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 19:15:34 -0500 Subject: [PATCH 05/21] on-prem locks: take ownership with an exclusive file create, not mkdir(1) Ubuntu 26.04 ships uutils mkdir, which reports success to both callers when two processes race to create one directory. The audit-log writer lock and the release update lock both treated a zero exit from mkdir as ownership, so on that image concurrent audit writers chained to the same parent (the "concurrent audit writers preserve one hash chain" CI failure) and a second update could proceed beside a live one. Ownership is now the shell's own noclobber create of the pid file inside the lock directory (O_CREAT|O_EXCL), and the stale-lock breaker lock is an exclusive file too. Tests model the uutils behavior with a mkdir shim that never reports EEXIST; on the previous code they fail. --- packages/flow/deploy/on-prem/bin/audit-log.sh | 44 ++++++++++++------- .../flow/deploy/on-prem/bin/lib-release.sh | 41 +++++++++++------ .../flow/deploy/on-prem/bin/test-update.sh | 37 ++++++++++++++++ 3 files changed, 92 insertions(+), 30 deletions(-) diff --git a/packages/flow/deploy/on-prem/bin/audit-log.sh b/packages/flow/deploy/on-prem/bin/audit-log.sh index 9b51ce4e4a..a963a59a01 100755 --- a/packages/flow/deploy/on-prem/bin/audit-log.sh +++ b/packages/flow/deploy/on-prem/bin/audit-log.sh @@ -68,35 +68,47 @@ chmod 0700 "$(dirname "$AUDIT_LOG")" 2>/dev/null || true # Serialize the read-last-hash + append transaction. O_APPEND prevents byte # interleaving, but without this lock two writers can both chain to the same # parent and produce a structurally broken audit log. +# +# Ownership is the exclusive creation of the pid file, done by the shell itself +# with noclobber (open(2) with O_CREAT|O_EXCL). The exit status of mkdir(1) is +# not trusted for exclusion: the uutils mkdir that Ubuntu 26.04 ships reports +# success to both callers when two processes race to create one directory. lock_dir="${AUDIT_LOG}.lock" -break_dir="${AUDIT_LOG}.lock-break" +lock_file="$lock_dir/pid" +break_file="${AUDIT_LOG}.lock-breaker" lock_acquired=0 +# Create $1 holding this process's pid; fails if $1 already exists. +create_exclusive() { + ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null +} + # Remove a lock whose owner has exited. Between reading the owner and removing # the lock, that owner can release it and another writer can take it, so the -# removal happens only under a second lock that serializes breakers, and only -# if the lock still names the same dead owner. Without this, a breaker could -# delete a live writer's lock and let two writers chain to the same parent. +# removal happens only under a second exclusive lock that serializes breakers, +# and only if the lock still names the same dead owner. Without this, a breaker +# could delete a live writer's lock and let two writers chain to the same parent. break_stale_lock() { local stale="$1" current="" - mkdir "$break_dir" 2>/dev/null || return 0 - current="$(cat "$lock_dir/pid" 2>/dev/null || true)" + create_exclusive "$break_file" || return 0 + current="$(cat "$lock_file" 2>/dev/null || true)" if [[ "$current" == "$stale" ]] && ! kill -0 "$stale" 2>/dev/null; then - rm -f "$lock_dir/pid" 2>/dev/null || true - rmdir "$lock_dir" 2>/dev/null || true + rm -f "$lock_file" 2>/dev/null || true fi - rmdir "$break_dir" 2>/dev/null || true + rm -f "$break_file" 2>/dev/null || true } for ((attempt = 0; attempt < 200; attempt++)); do - if mkdir "$lock_dir" 2>/dev/null; then - printf '%s\n' "$$" > "$lock_dir/pid" - lock_acquired=1 - break + if [[ ! -L "$lock_dir" ]]; then + mkdir -p "$lock_dir" 2>/dev/null || true + if [[ -d "$lock_dir" && ! -L "$lock_dir" ]] && create_exclusive "$lock_file"; then + lock_acquired=1 + break + fi fi owner="" - if [[ -d "$lock_dir" && ! -L "$lock_dir" && -r "$lock_dir/pid" ]]; then - owner="$(cat "$lock_dir/pid" 2>/dev/null || true)" + if [[ -d "$lock_dir" && ! -L "$lock_dir" && -f "$lock_file" && ! -L "$lock_file" ]]; then + owner="$(cat "$lock_file" 2>/dev/null || true)" fi if [[ -n "$owner" && "$owner" =~ ^[0-9]+$ ]] && ! kill -0 "$owner" 2>/dev/null; then break_stale_lock "$owner" @@ -109,7 +121,7 @@ if [[ "$lock_acquired" -ne 1 ]]; then exit 1 fi release_audit_lock() { - rm -f "$lock_dir/pid" 2>/dev/null || true + rm -f "$lock_file" 2>/dev/null || true rmdir "$lock_dir" 2>/dev/null || true } trap release_audit_lock EXIT diff --git a/packages/flow/deploy/on-prem/bin/lib-release.sh b/packages/flow/deploy/on-prem/bin/lib-release.sh index 7676dc4900..295dd5c38b 100755 --- a/packages/flow/deploy/on-prem/bin/lib-release.sh +++ b/packages/flow/deploy/on-prem/bin/lib-release.sh @@ -344,32 +344,45 @@ rel_validate_release_dir() { && -x "$release_dir/venv/bin/openadapt-flow" ]] } +# Create $1 holding this process's pid; fails if $1 already exists. The shell +# opens it with noclobber (O_CREAT|O_EXCL), so exclusion does not depend on +# mkdir(1): the uutils mkdir that Ubuntu 26.04 ships reports success to both +# callers when two processes race to create one directory. +_rel_create_exclusive() { + ( set -C; printf '%s\n' "$$" > "$1" ) 2>/dev/null +} + _rel_lock_acquire() { - local root="$1" owner="" + local root="$1" owner="" current="" local lock="$root/releases/.update.lock" + local breaker="$root/releases/.update.lock-breaker" mkdir -p "$root/releases" || return 1 - if mkdir "$lock" 2>/dev/null; then - printf '%s\n' "$$" > "$lock/pid" || { - rmdir "$lock" 2>/dev/null || true - return 1 - } + if [[ -L "$lock" ]]; then + _rel_err "release lock is a symlink; refusing: $lock" + return 1 + fi + mkdir -p "$lock" 2>/dev/null || true + if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then REL_LOCK_DIR="$lock" return 0 fi - if [[ -d "$lock" && ! -L "$lock" && -r "$lock/pid" ]]; then + if [[ -d "$lock" && ! -L "$lock" && -f "$lock/pid" && ! -L "$lock/pid" ]]; then owner="$(cat "$lock/pid" 2>/dev/null || true)" fi if [[ "$owner" =~ ^[0-9]+$ ]] && kill -0 "$owner" 2>/dev/null; then _rel_err "another update or rollback is active (pid $owner)" return 1 fi - if [[ -d "$lock" && ! -L "$lock" ]]; then - rm -f "$lock/pid" 2>/dev/null || true - if rmdir "$lock" 2>/dev/null && mkdir "$lock" 2>/dev/null; then - printf '%s\n' "$$" > "$lock/pid" || { - rmdir "$lock" 2>/dev/null || true - return 1 - } + # The owner has exited (or left an unreadable pid). Remove its lock only + # under the breaker lock and only if it still names that owner, so a + # concurrent breaker cannot remove a lock another process has just taken. + if [[ -n "$owner" ]] && _rel_create_exclusive "$breaker"; then + current="$(cat "$lock/pid" 2>/dev/null || true)" + if [[ "$current" == "$owner" ]]; then + rm -f "$lock/pid" 2>/dev/null || true + fi + rm -f "$breaker" 2>/dev/null || true + if [[ -d "$lock" && ! -L "$lock" ]] && _rel_create_exclusive "$lock/pid"; then REL_LOCK_DIR="$lock" return 0 fi diff --git a/packages/flow/deploy/on-prem/bin/test-update.sh b/packages/flow/deploy/on-prem/bin/test-update.sh index 26be744811..cb16fdecba 100755 --- a/packages/flow/deploy/on-prem/bin/test-update.sh +++ b/packages/flow/deploy/on-prem/bin/test-update.sh @@ -244,6 +244,28 @@ assert "concurrent update lock is refused" test "$?" -ne 0 rm -f "$ROOT/releases/.update.lock/pid" rmdir "$ROOT/releases/.update.lock" +# Ubuntu 26.04 ships uutils mkdir, which reports success to both callers when +# two processes race to create one directory. Model that with a mkdir shim that +# never reports EEXIST: the locks must still exclude, so they cannot rest on +# the exit status of mkdir(1). +EEXIST_SHIM="$WORK/eexist-shim" +mkdir -p "$EEXIST_SHIM" +REAL_MKDIR="$(command -v mkdir)" +cat > "$EEXIST_SHIM/mkdir" </dev/null || true +exit 0 +SHIM_EOF +chmod +x "$EEXIST_SHIM/mkdir" +mkdir "$ROOT/releases/.update.lock" +printf '%s\n' "$$" > "$ROOT/releases/.update.lock/pid" +PATH="$EEXIST_SHIM:$PATH" apply_release "$ROOT" "$ARC6" >/dev/null 2>&1 +assert "update lock holds when mkdir(1) never reports EEXIST" test "$?" -ne 0 +assert_eq "a refused update leaves the live owner's lock" "$$" \ + "$(cat "$ROOT/releases/.update.lock/pid" 2>/dev/null || echo none)" +rm -f "$ROOT/releases/.update.lock/pid" +rmdir "$ROOT/releases/.update.lock" + chmod -x "$ROOT/releases/2.0.0/venv/bin/openadapt-flow" rel_do_rollback "$ROOT" "" >/dev/null 2>&1 assert "incomplete rollback target is refused" test "$?" -ne 0 @@ -293,6 +315,21 @@ else printf '%s\n' "$audit_output" | sed 's/^/ /' fi +SHIM_LOG="$WORK/eexist-audit/audit.log" +for index in 1 2 3 4 5 6 7 8 9 10 11 12; do + OPENADAPT_ONPREM_AUDIT_LOG="$SHIM_LOG" PATH="$EEXIST_SHIM:$PATH" \ + bash "$OPENADAPT_ONPREM_AUDIT_BIN" started "shim-$index" --note test >/dev/null & +done +wait +audit_output="$(OPENADAPT_ONPREM_AUDIT_LOG="$SHIM_LOG" \ + bash "$HERE/verify-airgap.sh" --config "$WORK/missing.yaml" --audit 2>&1 || true)" +if grep -q 'audit chain intact across 12 record' <<< "$audit_output"; then + pass "audit writers stay serialized when mkdir(1) never reports EEXIST" +else + fail "audit writers stay serialized when mkdir(1) never reports EEXIST" + printf '%s\n' "$audit_output" | sed 's/^/ /' +fi + # A writer that finds a lock owned by an exited process may break it. Force the # race where, between that writer reading the dead owner and breaking the # lock, the owner releases and a live writer takes the lock: a shimmed cat From d6db42e0a02fdb8dd08015c2d560ba26d2cdf873 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 19:24:05 -0500 Subject: [PATCH 06/21] on-prem container test: keep pip's error in the missing-wheel log; read the root-owned audit log from inside the image --- packages/flow/deploy/on-prem/bin/test-container.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/flow/deploy/on-prem/bin/test-container.sh b/packages/flow/deploy/on-prem/bin/test-container.sh index 2abf6090bc..cc4a07c4a3 100755 --- a/packages/flow/deploy/on-prem/bin/test-container.sh +++ b/packages/flow/deploy/on-prem/bin/test-container.sh @@ -92,7 +92,9 @@ if [[ -z "$victim" ]]; then else rm -f "$victim" tar -C "$ONPREM" -cf - . | tar -C "$work/missing/on-prem" -xf - - if docker build --network none --pull=false -q -f "$work/missing/on-prem/Dockerfile" \ + # --progress=plain keeps pip's own error in the log; -q would drop it. + if docker build --network none --pull=false --progress=plain \ + -f "$work/missing/on-prem/Dockerfile" \ -t "$tag-missing" "$work/missing" >"$work/missing.log" 2>&1; then bad "build without $(basename "$victim") fails (it reached a package index?)" else @@ -153,7 +155,10 @@ if [[ -f "$store/jobs/failed/no-bundle.job" ]]; then else bad "malformed job filed under failed/" fi -if grep -q "no-bundle" "$store/audit/audit.log" 2>/dev/null; then +# The runner is root in the image and audit-log.sh makes the audit directory +# 0700, so read the log the way the deployment does: from inside the image. +if run -v "$store:/srv/openadapt" "$tag" cat /srv/openadapt/audit/audit.log 2>/dev/null \ + | grep -q '"job":"no-bundle"'; then ok "audit log records the job" else bad "audit log records the job" From acf95a41dee4eaee50788dfe7c2f413ef9d8b4ae Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 19:07:32 -0800 Subject: [PATCH 07/21] fix(flow): name shared rows by task values and keep changed columns in anchor scoring (#1852) Part of #1675. --- packages/flow/docs/GENERATE_CONTRACTS.md | 9 +- .../openadapt_flow/generate/demo_analysis.py | 30 +-- .../generate/verify_contract.py | 55 +++--- ...est_generate_verify_contract_attacks_r2.py | 46 +++++ ...st_generate_verify_contract_false_stops.py | 182 ++++++++++++++++++ 5 files changed, 279 insertions(+), 43 deletions(-) create mode 100644 packages/flow/tests/test_generate_verify_contract_false_stops.py diff --git a/packages/flow/docs/GENERATE_CONTRACTS.md b/packages/flow/docs/GENERATE_CONTRACTS.md index 34f5a331d2..719f38a1af 100644 --- a/packages/flow/docs/GENERATE_CONTRACTS.md +++ b/packages/flow/docs/GENERATE_CONTRACTS.md @@ -292,9 +292,12 @@ is a log from its name. **Changes the app makes to existing rows.** A column of an existing row that the demonstrations changed with no task value is excused only on that row: -the target itself, a row with the same key in every demonstration (a counter -row), or the row the task's values name through a key of the main record -(the account a contact belongs to). It is never excused by name on other +the target itself, the row the task's values name through a key of the main +record (the account a contact belongs to), or a row with the same key in +every demonstration (a counter row). A row the task's values name is named +by them even when every demonstration happened to touch the same one (both +demonstrations were in one project), so a correct write in another project +is not stopped. It is never excused by name on other rows or tables. Time stamps set during the write, counters that went up and columns set to the acting user are proposed; anything else is listed and excused only if the person adds it (`columns_add`). A changed row the diff --git a/packages/flow/openadapt_flow/generate/demo_analysis.py b/packages/flow/openadapt_flow/generate/demo_analysis.py index f09d7e26bb..cbf69ba1f1 100644 --- a/packages/flow/openadapt_flow/generate/demo_analysis.py +++ b/packages/flow/openadapt_flow/generate/demo_analysis.py @@ -551,22 +551,28 @@ def pick_anchor( v = demo.values.get(p) if v is None or isinstance(v, (list, dict)): continue - if mode == "update": - hit = locate(schema, demo, ch, v, before=True, max_hops=1) or locate( - schema, demo, ch, v, changed_only=True, max_hops=1 - ) - else: - hit = locate(schema, demo, ch, v, max_hops=1) + # Only the row's own columns, and keys named after the table they # point to: a row the write added beside the main record (an # issue's journal) reaches the record's values only through a # link back, and a key equal to some row's id by chance proves - # nothing. - hit = [ - loc - for loc in hit - if all(h.kind == "fk" and _hop_cost(h) <= 1 for h in loc.hops) - ] + # nothing. Filtered before an update falls back to its changed + # columns: a key equal to some row's id by chance (a project id + # that is also an issue id) must not hide the value the write + # put in the row itself. + def kept(locs: list[Location]) -> list[Location]: + return [ + loc + for loc in locs + if all(h.kind == "fk" and _hop_cost(h) <= 1 for h in loc.hops) + ] + + if mode == "update": + hit = kept( + locate(schema, demo, ch, v, before=True, max_hops=1) + ) or kept(locate(schema, demo, ch, v, changed_only=True, max_hops=1)) + else: + hit = kept(locate(schema, demo, ch, v, max_hops=1)) score += 1 if hit else 0 direct += 1 if any(not loc.hops for loc in hit) else 0 if (score, direct) > (best_score, best_direct): diff --git a/packages/flow/openadapt_flow/generate/verify_contract.py b/packages/flow/openadapt_flow/generate/verify_contract.py index 66db61b83a..d3fb1b7334 100644 --- a/packages/flow/openadapt_flow/generate/verify_contract.py +++ b/packages/flow/openadapt_flow/generate/verify_contract.py @@ -1217,42 +1217,41 @@ def _touched_rows( demos_with = {id(d) for d, _, _ in items} selector: Optional[dict[str, Any]] = None how = "" - if ( + # Reached from the main record by a key column whose target row holds + # located values in single columns (the task's project). Tried first: + # when every demonstration happened to touch the same such row, its + # literal key would excuse that one row for every write, and a + # correct write naming another row would stop. + for p, loc in plan.locations.items(): + if not loc.hops or loc.hops[0].kind != "fk" or loc.hops[0].table != table: + continue + hop = loc.hops[0] + if all( + (a.after or {}).get(hop.column) == (ch.after or {}).get(pk) + for _, a, ch in items + ): + sel = { + x.columns[0]: {"param": q} + for q, x in plan.locations.items() + if len(x.hops) == 1 + and x.hops[0] == hop + and len(x.columns) == 1 + and x.rule == "exact" + } + if sel: + selector = sel + how = f"the {table} row the task's {', '.join(sorted(v['param'] for v in sel.values()))} name" + break + if selector is None and ( len(keys) == 1 and len(demos_with) == len(plan.demos) and len(items) == len(plan.demos) ): + # The same row (by key) in every demonstration: a counter row. key = (items[0][2].after or {}).get(pk) if key is not None: selector = {pk: {"literal": key}} how = f"{table} row {pk}={key}" - if selector is None: - # Reached from the main record by a key column whose target row - # holds located values in single columns. - for p, loc in plan.locations.items(): - if ( - not loc.hops - or loc.hops[0].kind != "fk" - or loc.hops[0].table != table - ): - continue - hop = loc.hops[0] - if all( - (a.after or {}).get(hop.column) == (ch.after or {}).get(pk) - for _, a, ch in items - ): - sel = { - x.columns[0]: {"param": q} - for q, x in plan.locations.items() - if len(x.hops) == 1 - and x.hops[0] == hop - and len(x.columns) == 1 - and x.rule == "exact" - } - if sel: - selector = sel - how = f"the {table} row the task's {', '.join(sorted(v['param'] for v in sel.values()))} name" - break if selector is None or len(items) != len({id(d) for d, _, _ in items}): plan.unexcused.setdefault(table, set()).update(cols) continue diff --git a/packages/flow/tests/test_generate_verify_contract_attacks_r2.py b/packages/flow/tests/test_generate_verify_contract_attacks_r2.py index ee0590f66d..1b66789bca 100644 --- a/packages/flow/tests/test_generate_verify_contract_attacks_r2.py +++ b/packages/flow/tests/test_generate_verify_contract_attacks_r2.py @@ -429,3 +429,49 @@ def check(role: str) -> list[str]: assert any("TRIGGER on table public.t" in p for p in ro2), ro2 assert any("CREATE on schema public" in p for p in ro2), ro2 assert check("ro3") == [] + + +# -- #1675 item 1: a row every demonstration happened to share ----------------- + + +def test_a_shared_parent_row_is_named_by_the_task_not_by_its_key( + gv: dict[str, Any], +) -> None: + """Both set_status demonstrations were in Help Desk, so they touched the + same project row. The excuse names the row the task's project names, + not that row's literal key.""" + rows = gv["report"]["kinds"]["set_status"]["touched_rows"] + projects = [r for r in rows if r["table"] == "projects"] + assert projects and all(not r["by_key"] for r in projects), rows + assert all("literal" not in s for r in projects for s in r["selector"].values()), ( + rows + ) + + +def test_correct_status_update_in_another_project_confirmed( + gv: dict[str, Any], +) -> None: + app = gv["app"] + iid = app.create_issue( + {"project": "Billing Portal", "subject": "S1 other", "status": "New"} + ) + v = {"project": "Billing Portal", "subject": "S1 other", "status": "Resolved"} + out = _run(gv, "set_status", v, lambda: app.save(iid, {"status": "Resolved"})) + assert out.startswith("CONFIRMED"), out + + +def test_status_update_that_touches_another_project_stops( + gv: dict[str, Any], +) -> None: + """The excuse is for the task's project only: the same bookkeeping on + a project the task doesn't name is collateral.""" + app = gv["app"] + iid = _fresh(app, "S2 other") + v = {"project": "Help Desk", "subject": "S2 other", "status": "Resolved"} + + def write() -> None: + app.save(iid, {"status": "Resolved"}) + app.q("UPDATE projects SET updated_on=now() WHERE name='Billing Portal'") + + out = _run(gv, "set_status", v, write) + assert not out.startswith("CONFIRMED"), out diff --git a/packages/flow/tests/test_generate_verify_contract_false_stops.py b/packages/flow/tests/test_generate_verify_contract_false_stops.py new file mode 100644 index 0000000000..105941677b --- /dev/null +++ b/packages/flow/tests/test_generate_verify_contract_false_stops.py @@ -0,0 +1,182 @@ +# ruff: noqa: F811 (pytest fixtures imported from the generator test, then used as arguments) +"""False stops of generated contracts left open by the #1672 reviews (#1675). + +The round-2 tracker (one audit row per changed field, no version column) on +a throwaway PostgreSQL, with a kind that changes one or two fields per +demonstration. Correct writes must be CONFIRMED; the wrong writes beside +them must still stop. Records are made up. +""" + +from __future__ import annotations + +import json +import os +from typing import Any + +import pytest +import yaml + +psycopg2 = pytest.importorskip("psycopg2") + +from openadapt_flow.generate import verify_contract as gen # noqa: E402 +from tests.test_generate_verify_contract import BOT, _run, pg # noqa: E402,F401 +from tests.test_generate_verify_contract_attacks_r2 import ( # noqa: E402 + KINDS_V, + SCHEMA_V, + AppV, +) + +KINDS_P = { + "kinds": [ + *KINDS_V["kinds"], + {"kind": "edit_plan", "parameters": ["project", "subject", "done_ratio", "due_date"], + "examples": [ + {"values": {"project": "Billing Portal", "subject": "Refund button missing", + "done_ratio": 30, "due_date": "2026-11-30"}}, + {"values": {"project": "Help Desk", "subject": "Printer jams", "done_ratio": 50, + "due_date": None}}, + ]}, + ] +} # fmt: skip + + +@pytest.fixture(scope="module") +def gp(pg: Any) -> dict[str, Any]: + root, port = pg["root"], pg["port"] + admin = psycopg2.connect( + host="127.0.0.1", port=port, user="admin", dbname="postgres" + ) + admin.autocommit = True + admin.cursor().execute("CREATE DATABASE trackerp") + admin.close() + app = AppV(port, "trackerp") + with app.con() as c, c.cursor() as cur: + cur.execute( + "DO $$BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='oa_reader') " + "THEN CREATE ROLE oa_reader LOGIN PASSWORD 'reader-pw'; END IF; END$$" + ) + cur.execute(SCHEMA_V) + os.environ["OA_GEN_TEST_PW"] = "reader-pw" + work = root / "workp" + work.mkdir() + read = {"sql_driver": "psycopg2", + "sql_connect_args": {"host": "127.0.0.1", "port": port, "user": "oa_reader", "dbname": "trackerp"}, + "sql_password_env": "OA_GEN_TEST_PW"} # fmt: skip + (work / "read.yaml").write_text(yaml.safe_dump(read)) + (work / "kinds.yaml").write_text(yaml.safe_dump(KINDS_P, allow_unicode=True)) + common = ["--read", str(work / "read.yaml")] + + def demo(kind: str, values: dict[str, Any], write: Any) -> None: + assert gen.main(["snapshot", *common, "--out", str(work / "before.json")]) == 0 + write() + assert gen.main(["record", *common, "--before", str(work / "before.json"), "--kind", kind, + "--values", json.dumps(values), "--demos", str(work / "demos")]) == 0 # fmt: skip + + for k in KINDS_P["kinds"]: + for ex in k["examples"]: + v = ex["values"] + if k["kind"] == "create_issue": + demo("create_issue", v, lambda v=v: app.create_issue(v)) + else: + sets = {p: v[p] for p in k["parameters"][2:] if v.get(p) is not None} + demo( + k["kind"], + v, + lambda v=v, s=sets: app.save( + app.find(v["project"], v["subject"]), s + ), + ) + argv = ["propose", *common, "--kinds", str(work / "kinds.yaml"), "--demos", str(work / "demos"), + "--out", str(work / "out"), "--actor-param", f"bot_user_id={BOT}"] # fmt: skip + assert gen.main(argv) == 0 + report = json.loads((work / "out" / "report.json").read_text()) + assert "audits" in report["kinds"]["edit_plan"]["log_candidates"], report + # The person's answers: the proposal, plus the audit table as a log (one + # row per changed field, so the count differs between demonstrations). + answers = {q["key"]: q["proposal"] for q in report["questions"]} + bk = answers["bookkeeping"] + bk = {"accept": True} if bk == "accept" else dict(bk) + bk["logs_add"] = ["audits"] + answers["bookkeeping"] = bk + (work / "answers.yaml").write_text(yaml.safe_dump(answers)) + assert gen.main([*argv, "--answers", str(work / "answers.yaml")]) == 0 + report = json.loads((work / "out" / "report.json").read_text()) + return {"app": app, "out": work / "out", "report": report, "read": read} + + +def _fresh(app: AppV, subject: str) -> int: + return app.create_issue( + {"project": "Billing Portal", "subject": subject, "status": "New"} + ) + + +# -- item 2: a kind whose demonstrations audited different numbers of fields --- + + +def test_a_kind_auditing_one_or_two_fields_is_an_update(gp: dict[str, Any]) -> None: + """One demonstration changed two fields (two audit rows), the other one. + The issue is still the main record: a project id equal to an issue id + by chance doesn't hide the values the save put in the issue.""" + k = gp["report"]["kinds"]["edit_plan"] + assert k["mode"] == "update" and not k["refused"], k + assert k["emitted"] > 0, k + + +def test_correct_two_field_plan_edit_confirmed(gp: dict[str, Any]) -> None: + app = gp["app"] + iid = _fresh(app, "P1 two") + v = {"project": "Billing Portal", "subject": "P1 two", "done_ratio": 70, + "due_date": "2026-12-15"} # fmt: skip + out = _run( + gp, + "edit_plan", + v, + lambda: app.save(iid, {"done_ratio": 70, "due_date": "2026-12-15"}), + ) + assert out.startswith("CONFIRMED"), out + + +def test_correct_one_field_plan_edit_confirmed(gp: dict[str, Any]) -> None: + app = gp["app"] + iid = _fresh(app, "P2 one") + v = {"project": "Billing Portal", "subject": "P2 one", "done_ratio": 40, + "due_date": None} # fmt: skip + out = _run(gp, "edit_plan", v, lambda: app.save(iid, {"done_ratio": 40})) + assert out.startswith("CONFIRMED"), out + + +def test_plan_edit_with_the_wrong_ratio_stops(gp: dict[str, Any]) -> None: + app = gp["app"] + iid = _fresh(app, "P3 wrong") + v = {"project": "Billing Portal", "subject": "P3 wrong", "done_ratio": 40, + "due_date": None} # fmt: skip + out = _run(gp, "edit_plan", v, lambda: app.save(iid, {"done_ratio": 60})) + assert not out.startswith("CONFIRMED"), out + + +def test_plan_edit_that_also_changes_the_status_stops(gp: dict[str, Any]) -> None: + app = gp["app"] + iid = _fresh(app, "P4 extra") + v = {"project": "Billing Portal", "subject": "P4 extra", "done_ratio": 40, + "due_date": None} # fmt: skip + out = _run( + gp, + "edit_plan", + v, + lambda: app.save(iid, {"done_ratio": 40, "status": "Resolved"}), + ) + assert not out.startswith("CONFIRMED"), out + + +def test_plan_edit_that_loses_another_users_save_stops(gp: dict[str, Any]) -> None: + app = gp["app"] + iid = _fresh(app, "P5 lost") + v = {"project": "Billing Portal", "subject": "P5 lost", "done_ratio": 40, + "due_date": None} # fmt: skip + + def write() -> None: + app.save(iid, {"status": "In Progress"}, user=2) + app.save(iid, {"done_ratio": 40, "status": "New"}) + + out = _run(gp, "edit_plan", v, write) + assert not out.startswith("CONFIRMED"), out From 63b63a313c8aa4d08fb2efb076eef09b208aeed8 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 20:03:21 -0800 Subject: [PATCH 08/21] identity: read a dialog target's band on the dialog alone (OpenEMR duplicate dialog) (#1615) * identity: read the band of a target inside a dialog on the dialog alone (#1347) The recorder stores the box of the topmost dialog holding a clicked target (PlaywrightBackend.dialog_region_at). The compiler reads that step's identity band on the frame blanked outside the box and stores Anchor.dialog_region. Replay blanks the live frame outside the live dialog the same way, and fails the identity check closed when no dialog holds the resolved point. Also: the throughput harness retries its form setup once after a navigation race destroys the evaluate context (#1348). * identity: address the dialog-scope review Scope to modal dialogs only (dialog:modal, aria-modal, alertdialog, a shown Bootstrap modal) and treat an opacity-0 dialog as gone. Require the box to hold the probed point. Mask the extra identity looks' fresh frames with the same dialog box; give the signal quorum the whole frame. Heal re-reads a dialog target's band on the dialog and moves dialog_region with the point. The render-scaled view maps the probe. The harness retry tolerates a frame that detaches while it waits. * identity: a dialog target's band is the whole dialog The row of a dialog button often holds nothing but the other buttons, so on OpenEMR's duplicate dialog the scoped row band was too short to arm identity and clinical-write would refuse the save. Read every line of the dialog instead (its result text and count included), at compile and replay, with no row re-read, and skip the row census for a dialog target. * throughput harness: --backend, to replay with the backend the bundle was recorded with A bundle the demo's record command makes carries the OpenEMR adapter's form field identity; replayed through the base Playwright backend, its first step reads only the field label and halts. * benchmark: identity at OpenEMR's duplicate dialog, before and after #1347 On the OCR band path the main bundle refused the confirm step at 8 of 8 form scroll positions behind the dialog; the dialog-scoped bundle verified at all 8 and still refused a dialog that lists a possible duplicate. 26-item worklists: 26/26 correct on both. * Register the dialog identity measurement and add its capability row * fix: preserve dialog row identity (untested WIP) Signed-off-by: abrichr * test: pin dialog row and mask refusal regressions Signed-off-by: abrichr * fix: keep dialog identity checks generic and typed Signed-off-by: abrichr * test: separate modal row text from button crops Signed-off-by: abrichr * fix: publish repair hold metadata atomically Signed-off-by: abrichr --------- Signed-off-by: abrichr --- CAPABILITIES.md | 1 + llms-full.txt | 1 + .../dialog_identity_2026-10-28/README.md | 45 +++ .../batch-after-branch.json | 61 +++ .../batch-before-main.json | 61 +++ .../listed-duplicate.json | 16 + .../scroll-ocr-tier.json | 58 +++ .../scroll-openemr-adapter.json | 58 +++ .../scroll_identity.py | 61 +++ .../benchmark/openemr_throughput/harness.py | 34 +- .../backends/playwright_backend.py | 120 +++++- .../flow/openadapt_flow/compiler/compile.py | 78 +++- packages/flow/openadapt_flow/ir.py | 31 +- packages/flow/openadapt_flow/recorder.py | 27 ++ packages/flow/openadapt_flow/repair/hold.py | 14 +- packages/flow/openadapt_flow/runtime/heal.py | 42 ++- .../flow/openadapt_flow/runtime/identity.py | 51 +++ .../openadapt_flow/runtime/render_scale.py | 13 + .../flow/openadapt_flow/runtime/replayer.py | 110 +++++- packages/flow/public-artifacts.json | 5 + packages/flow/tests/test_bundle_schema_v2.py | 28 ++ .../test_dialog_identity_live_openemr.py | 103 +++++ .../flow/tests/test_dialog_row_identity.py | 354 ++++++++++++++++++ .../flow/tests/test_dialog_scoped_identity.py | 301 +++++++++++++++ packages/flow/tests/test_heal.py | 53 +++ packages/flow/tests/test_recorder.py | 20 + packages/flow/tests/test_repair_hold.py | 81 ++++ 27 files changed, 1798 insertions(+), 29 deletions(-) create mode 100644 packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/README.md create mode 100644 packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-after-branch.json create mode 100644 packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-before-main.json create mode 100644 packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/listed-duplicate.json create mode 100644 packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-ocr-tier.json create mode 100644 packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-openemr-adapter.json create mode 100644 packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll_identity.py create mode 100644 packages/flow/tests/test_dialog_identity_live_openemr.py create mode 100644 packages/flow/tests/test_dialog_row_identity.py create mode 100644 packages/flow/tests/test_dialog_scoped_identity.py diff --git a/CAPABILITIES.md b/CAPABILITIES.md index c934e587b1..eedf385b1a 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -230,6 +230,7 @@ improvising or retrying when something is off. | One program that switches surfaces, tabs and popups mid-run | Released ([#1197](https://github.com/OpenAdaptAI/OpenAdapt/pull/1197)) | Real Chromium; Windows switch on a mock | [#1197](https://github.com/OpenAdaptAI/OpenAdapt/pull/1197) | | Halt when a workflow qualified with page structure runs on a pixels-only surface | Released | Synthetic | `openadapt-flow qualify` | | Stop before acting when another shown row has the same identity; when the rows carry an app-owned record id (a `data-*` id, the row link), act on the row with the recorded id; on a surface that cannot count the rows, stop before any step that leads into a write | Released ([#1504](https://github.com/OpenAdaptAI/OpenAdapt/pull/1504)) | Real browser (headless Chromium worklist with identical rows) | `runtime/identity_census.py`; `PlaywrightBackend.identity_candidates_at` | +| Read a dialog target's identity on the topmost modal dialog alone, not the page behind it, and stop when no dialog holds the target | On main ([#1615](https://github.com/OpenAdaptAI/OpenAdapt/pull/1615)) | Real app (local OpenEMR, synthetic data): at the duplicate dialog with the form scrolled to 8 positions behind it, 0 of 8 false stops, against 8 of 8 before; a dialog listing a possible duplicate still stops | `PlaywrightBackend.dialog_region_at`, `Anchor.dialog_region`; [dialog_identity_2026-10-28](packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/README.md) | | Lift look-alike identity stops with a complete known-ID list | Released | Synthetic (rendered record list) | `openadapt-flow known-ids` | | Take extra looks before stopping on a look-alike record ID: fresh frames, the known-ID list, and a select-and-copy of the ID where the surface allows it | Released ([#1333](https://github.com/OpenAdaptAI/OpenAdapt/pull/1333)) | Synthetic (dense record list in a browser) | `runtime.identity_copy_look` in deployment.yaml; [IDENTITY_LOOKS.md](packages/flow/benchmark/identity_looks/IDENTITY_LOOKS.md) | | Check an icon-only click target's hover response (highlight, tooltip) before clicking on a pixel surface, and re-check the target and identity regions on a fresh frame right before every pixel click | Released ([#1334](https://github.com/OpenAdaptAI/OpenAdapt/pull/1334)) | Synthetic (in-process kiosk) | `Recorder(capture_hover=True)`; [hover_identity](packages/flow/benchmark/hover_identity/README.md) | diff --git a/llms-full.txt b/llms-full.txt index ec09e0ba23..ec3b480a19 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -633,6 +633,7 @@ improvising or retrying when something is off. | One program that switches surfaces, tabs and popups mid-run | Released ([#1197](https://github.com/OpenAdaptAI/OpenAdapt/pull/1197)) | Real Chromium; Windows switch on a mock | [#1197](https://github.com/OpenAdaptAI/OpenAdapt/pull/1197) | | Halt when a workflow qualified with page structure runs on a pixels-only surface | Released | Synthetic | `openadapt-flow qualify` | | Stop before acting when another shown row has the same identity; when the rows carry an app-owned record id (a `data-*` id, the row link), act on the row with the recorded id; on a surface that cannot count the rows, stop before any step that leads into a write | Released ([#1504](https://github.com/OpenAdaptAI/OpenAdapt/pull/1504)) | Real browser (headless Chromium worklist with identical rows) | `runtime/identity_census.py`; `PlaywrightBackend.identity_candidates_at` | +| Read a dialog target's identity on the topmost modal dialog alone, not the page behind it, and stop when no dialog holds the target | On main ([#1615](https://github.com/OpenAdaptAI/OpenAdapt/pull/1615)) | Real app (local OpenEMR, synthetic data): at the duplicate dialog with the form scrolled to 8 positions behind it, 0 of 8 false stops, against 8 of 8 before; a dialog listing a possible duplicate still stops | `PlaywrightBackend.dialog_region_at`, `Anchor.dialog_region`; [dialog_identity_2026-10-28](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/README.md) | | Lift look-alike identity stops with a complete known-ID list | Released | Synthetic (rendered record list) | `openadapt-flow known-ids` | | Take extra looks before stopping on a look-alike record ID: fresh frames, the known-ID list, and a select-and-copy of the ID where the surface allows it | Released ([#1333](https://github.com/OpenAdaptAI/OpenAdapt/pull/1333)) | Synthetic (dense record list in a browser) | `runtime.identity_copy_look` in deployment.yaml; [IDENTITY_LOOKS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/identity_looks/IDENTITY_LOOKS.md) | | Check an icon-only click target's hover response (highlight, tooltip) before clicking on a pixel surface, and re-check the target and identity regions on a fresh frame right before every pixel click | Released ([#1334](https://github.com/OpenAdaptAI/OpenAdapt/pull/1334)) | Synthetic (in-process kiosk) | `Recorder(capture_hover=True)`; [hover_identity](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/hover_identity/README.md) | diff --git a/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/README.md b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/README.md new file mode 100644 index 0000000000..015a689d68 --- /dev/null +++ b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/README.md @@ -0,0 +1,45 @@ +# Identity at OpenEMR's duplicate dialog, before and after #1347 + +Measured on 2026-10-28 on the pinned local OpenEMR 8.0.0.3 fixture, with +synthetic patients only. One referral was recorded once with the #1347 engine, +so the confirm click carries `dialog_region`. It was compiled twice with +`scripts/openemr_local_demo.py`: +- with `main` (490d0dc40) as the baseline; +- with this branch, which reads the band on the dialog alone. + +The OCR band was the full-width strip around the button, so it read a label of +the form behind the dialog. "Before" means `main` and "after" means this branch. + +## The identity check at the confirm step, form scrolled behind the dialog + +`scroll_identity.py` opens the real duplicate dialog with the New Patient form +scrolled to several positions behind it. It runs the replayer's identity check +for the confirm step of each bundle and saves nothing. + +| Path | Positions | `main` false stops | Branch false stops | +|---|---|---|---| +| OCR band (base Playwright backend, as in #1347) | 8 | 8 | 0 | +| OpenEMR adapter's structured identity | 8 | 0 | 0 | + +- **Before.** On the OCR path, `main` read "Title: Cancel", "DOB: Cancel" or "License/ID: Cancel", whichever label sat beside the dialog, and refused every time. The recording had read "County:". +- **After.** The branch read the dialog's own text and verified every time. +- **Adapter path.** With the adapter, the structured tier decides, so neither bundle reads the band. + +Files: `scroll-ocr-tier.json` and `scroll-openemr-adapter.json`. + +## Fail closed on a listed duplicate + +`listed-duplicate.json`: the dialog for a patient who matches a seeded chart lists that chart. Both bundles refuse at both positions. The branch refuses because the dialog's listed row is not the empty result it recorded. + +## Worklist batches + +`batch-before-main.json` and `batch-after-branch.json` are throughput harness runs (`--items 26 --workers 1 --speed standard --backend openemr`). + +| Bundle | Correct | Wrong or extra charts | Stops | +|---|---|---|---| +| `main` | 26/26 | 0 | 0 | +| Branch | 26/26 | 0 | 0 | + +The `engine_sha` in these files is the harness checkout, not the engine. The baseline ran `main`'s `openadapt_flow`. + +In these batches the form behind the dialog sat where it was recorded, so the baseline didn't hit the stop. The batches show that the branch holds no correct save. The scroll table above shows the stop itself. diff --git a/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-after-branch.json b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-after-branch.json new file mode 100644 index 0000000000..bfa36a89c6 --- /dev/null +++ b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-after-branch.json @@ -0,0 +1,61 @@ +{ + "tag": "after-branch", + "workers": 1, + "items": 26, + "window": 0, + "halt_at": null, + "profile": false, + "engine_sha": "4bb59503add4816e015a877dffe8f685527645d8", + "speed_profile": "standard", + "native_fill": false, + "page_quiet_ms": 100, + "ocr_threads": null, + "engine_dirty": false, + "bundle_digest": "93f8127d4af367500fc86ddd0bafa0d27b6e5764818f8999ab5c4abc56ec2973", + "baseline": "a1b14e0031d1", + "seed_status": 201, + "setup_s": 6.03, + "started": "2026-10-28 20:24:40", + "processes": false, + "wall_s": 492.285, + "pool_wall_s": 492.28, + "sign_in_s": { + "oa_clerk1": 2.24 + }, + "counts": { + "completed": 26 + }, + "stop": null, + "items_run": 26, + "saved_correct": 26, + "wrong_or_extra_charts": 0, + "not_saved": 0, + "new_charts_total": 26, + "entries_per_hour": 190.1, + "written_after_failed_readback": {}, + "per_worker": [ + { + "name": "oa_clerk1", + "items": 26, + "states": { + "completed": 26 + }, + "busy_s": 489.9 + } + ], + "load1_mean": 23.04, + "load1_max": 90.04, + "ncpu": 12, + "gate": { + "threshold": 40.0, + "waited_s": 15.0, + "gave_up": false, + "load_at_start": [ + 8.98681640625, + 15.39794921875, + 37.7783203125 + ] + }, + "valid": true, + "ended": "2026-10-28 20:33:09" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-before-main.json b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-before-main.json new file mode 100644 index 0000000000..72c4802b2a --- /dev/null +++ b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-before-main.json @@ -0,0 +1,61 @@ +{ + "tag": "before-main", + "workers": 1, + "items": 26, + "window": 0, + "halt_at": null, + "profile": false, + "engine_sha": "4bb59503add4816e015a877dffe8f685527645d8", + "speed_profile": "standard", + "native_fill": false, + "page_quiet_ms": 100, + "ocr_threads": null, + "engine_dirty": false, + "bundle_digest": "6750b6dc15e5ddf1fb836c10e52f6e0577caf3cba89d39e9103049b8d3f86aa8", + "baseline": "a1b14e0031d1", + "seed_status": 201, + "setup_s": 7.79, + "started": "2026-10-28 20:15:00", + "processes": false, + "wall_s": 540.018, + "pool_wall_s": 540.009, + "sign_in_s": { + "oa_clerk1": 2.79 + }, + "counts": { + "completed": 26 + }, + "stop": null, + "items_run": 26, + "saved_correct": 26, + "wrong_or_extra_charts": 0, + "not_saved": 0, + "new_charts_total": 26, + "entries_per_hour": 173.3, + "written_after_failed_readback": {}, + "per_worker": [ + { + "name": "oa_clerk1", + "items": 26, + "states": { + "completed": 26 + }, + "busy_s": 537.1 + } + ], + "load1_mean": 10.65, + "load1_max": 16.03, + "ncpu": 12, + "gate": { + "threshold": 40.0, + "waited_s": 15.0, + "gave_up": false, + "load_at_start": [ + 15.7275390625, + 52.67236328125, + 65.37109375 + ] + }, + "valid": true, + "ended": "2026-10-28 20:24:17" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/listed-duplicate.json b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/listed-duplicate.json new file mode 100644 index 0000000000..c86a87cfd0 --- /dev/null +++ b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/listed-duplicate.json @@ -0,0 +1,16 @@ +[ + { + "scroll": 0, + "main": "mismatch", + "main_observed": "Who Cancel", + "branch": "mismatch", + "branch_observed": "Hits Name Preferred Name Birth Name DOB Birth Sex External ID Previous Names 2 D" + }, + { + "scroll": 600, + "main": "mismatch", + "main_observed": "Cancel", + "branch": "mismatch", + "branch_observed": "Hits Name Preferred Name Birth Name DOB Birth Sex External ID Previous Names 2 D" + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-ocr-tier.json b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-ocr-tier.json new file mode 100644 index 0000000000..0bdcfbcc03 --- /dev/null +++ b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-ocr-tier.json @@ -0,0 +1,58 @@ +[ + { + "scroll": 0, + "main": "mismatch", + "main_observed": "Title: Cancel", + "branch": "verified", + "branch_observed": "Preferred Birth Birth External Previous Hits Name Name Name DOB Sex ID Names No " + }, + { + "scroll": 150, + "main": "mismatch", + "main_observed": "DOB: Cancel", + "branch": "verified", + "branch_observed": "Preferred Birth Birth External Previous Hits Name Name Name DOB Sex ID Names No " + }, + { + "scroll": 300, + "main": "mismatch", + "main_observed": "License/ID: Cancel", + "branch": "verified", + "branch_observed": "Preferred Birth Birth External Previous Hits Name Name Name DOB Sex ID Names No " + }, + { + "scroll": 450, + "main": "mismatch", + "main_observed": "License/ID: Cancel", + "branch": "verified", + "branch_observed": "Preferred Birth Birth External Previous Hits Name Name Name DOB Sex ID Names No " + }, + { + "scroll": 600, + "main": "mismatch", + "main_observed": "License/ID: Cancel", + "branch": "verified", + "branch_observed": "Preferred Birth Birth External Previous Hits Name Name Name DOB Sex ID Names No " + }, + { + "scroll": 750, + "main": "mismatch", + "main_observed": "License/ID: Cancel", + "branch": "verified", + "branch_observed": "Preferred Birth Birth External Previous Hits Name Name Name DOB Sex ID Names No " + }, + { + "scroll": 900, + "main": "mismatch", + "main_observed": "License/ID: Cancel", + "branch": "verified", + "branch_observed": "Preferred Birth Birth External Previous Hits Name Name Name DOB Sex ID Names No " + }, + { + "scroll": 1100, + "main": "mismatch", + "main_observed": "License/ID: Cancel", + "branch": "verified", + "branch_observed": "Preferred Birth Birth External Previous Hits Name Name Name DOB Sex ID Names No " + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-openemr-adapter.json b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-openemr-adapter.json new file mode 100644 index 0000000000..ed0658a267 --- /dev/null +++ b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-openemr-adapter.json @@ -0,0 +1,58 @@ +[ + { + "scroll": 0, + "main": "verified", + "main_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New ", + "branch": "verified", + "branch_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New " + }, + { + "scroll": 150, + "main": "verified", + "main_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New ", + "branch": "verified", + "branch_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New " + }, + { + "scroll": 300, + "main": "verified", + "main_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New ", + "branch": "verified", + "branch_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New " + }, + { + "scroll": 450, + "main": "verified", + "main_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New ", + "branch": "verified", + "branch_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New " + }, + { + "scroll": 600, + "main": "verified", + "main_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New ", + "branch": "verified", + "branch_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New " + }, + { + "scroll": 750, + "main": "verified", + "main_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New ", + "branch": "verified", + "branch_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New " + }, + { + "scroll": 900, + "main": "verified", + "main_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New ", + "branch": "verified", + "branch_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New " + }, + { + "scroll": 1100, + "main": "verified", + "main_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New ", + "branch": "verified", + "branch_observed": "{\"form_path\":\"/interface/new/new_comprehensive.php\",\"label\":\"Confirm Create New " + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll_identity.py b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll_identity.py new file mode 100644 index 0000000000..76ae9e44af --- /dev/null +++ b/packages/flow/benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll_identity.py @@ -0,0 +1,61 @@ +"""Identity verdict at OpenEMR's confirm step, form behind scrolled to N positions.""" + +import json +import os +import sys +from pathlib import Path + +from benchmark.openemr_throughput import harness +from openadapt_flow.ir import Resolution, Workflow +from openadapt_flow.runtime import Replayer + +demo = harness._demo_module() +fx = harness.make_fixture( + Path(os.path.expanduser("~/oa/scratch/throughput/fixture/state")) +) +M = Path(os.path.expanduser("~/oa/scratch/real-app-edge-cases")) +bundles = { + name: (M / f"bundle-{name}", Workflow.load(M / f"bundle-{name}")) + for name in ("main", "branch") +} +spec_cls = __import__( + "openadapt_flow.benchmark.openemr_local", fromlist=["SyntheticPatientSpec"] +).SyntheticPatientSpec +params = spec_cls().params() +scrolls = [int(v) for v in sys.argv[1].split(",")] +rows = [] +for y in scrolls: + backend, close = demo.OpenEMRPlaywrightBackend.launch(fx.ui_base_url, headless=True) + try: + demo._preauthenticate_browser(backend, fx) + page = backend.page + form = demo._form_frame(page) + form.locator("#form_fname").fill(os.environ.get("P_FNAME", "Wilhelmina")) + form.locator("#form_lname").fill(os.environ.get("P_LNAME", "Okonkwo-Hale")) + form.locator("#form_DOB").fill(os.environ.get("P_DOB", "1961-04-09")) + form.locator("#form_DOB").press("Escape") + form.locator("#form_sex").select_option(label="Female") + form.evaluate(f"() => window.scrollTo(0, {y})") + form.locator("#create").evaluate("el => el.click()") + point = demo._center(demo._confirm_create_locator(page)) + png = backend.screenshot() + row = {"scroll": y} + for name, (bdir, wf) in bundles.items(): + step = wf.steps[-1] + rep = Replayer(demo._BasePlaywrightBackend(page)) + res = Resolution( + rung="structural", point=point, confidence=1.0, elapsed_ms=0.0 + ) + check = rep._verify_identity(step, res, png, params, wf, bdir) + row[name] = check.status + row[name + "_observed"] = check.observed[:80] + rows.append(row) + print(json.dumps(row), flush=True) + finally: + close() +out = M / os.environ.get("PERTURB_OUT", "perturb.json") +out.write_text(json.dumps(rows, indent=1)) +for name in ("main", "branch"): + print( + name, "false stops:", sum(r[name] != "verified" for r in rows), "of", len(rows) + ) diff --git a/packages/flow/benchmark/openemr_throughput/harness.py b/packages/flow/benchmark/openemr_throughput/harness.py index fcf9a2e817..d67108689c 100644 --- a/packages/flow/benchmark/openemr_throughput/harness.py +++ b/packages/flow/benchmark/openemr_throughput/harness.py @@ -274,7 +274,29 @@ def __init__(self, page: Any) -> None: self.page = page def open_form(page: Any) -> None: + """Open the blank form, once more if a navigation raced the setup (#1348). + + Sign-in or the previous item's chart can still be navigating the page + when a worker claims its next item; an ``evaluate`` then loses its + execution context. That is a setup race, not an engine failure: wait + for the page to load and run the setup again, once. + """ + try: + _open_form(page) + except Exception as exc: # noqa: BLE001 - only the navigation race retries + if "Execution context was destroyed" not in str(exc): + raise + page.wait_for_load_state("load", timeout=60_000) + for frame in page.frames: + try: + frame.wait_for_load_state("load", timeout=60_000) + except Exception: # noqa: BLE001 - a frame may detach meanwhile + continue + _open_form(page) + + def _open_form(page: Any) -> None: """Start each item where the recording started: no chart open, blank form.""" + page.wait_for_load_state("load", timeout=60_000) if page.evaluate( "() => !!(window.app_view_model && app_view_model.application_data.patient())" ): @@ -306,7 +328,14 @@ def run_item(page: Any, idx: int, row: dict[str, Any], item_dir: Path, pipeline: if not session: session.append(fixture.token_session("oracle")) oracle = OpenEMRPatientOracle(fixture.api_base_url, session[0], spec) - backend = demo._BasePlaywrightBackend(page) + # Replay with the backend the bundle was recorded with: a bundle the + # demo's ``record`` made carries the OpenEMR adapter's field identity. + backend_cls = ( + demo.OpenEMRPlaywrightBackend + if opts.get("backend") == "openemr" + else demo._BasePlaywrightBackend + ) + backend = backend_cls(page) prof = None if opts.get("profile"): from benchmark.openemr_throughput.profiler import RunProfiler @@ -398,6 +427,7 @@ def run(a: argparse.Namespace) -> dict[str, Any]: "window": a.window, "profile": a.profile, "keep_pngs": a.keep_pngs, "page_quiet_ms": a.page_quiet_ms, "ocr_threads": a.ocr_threads, "speed": a.speed, + "backend": a.backend, } if a.processes: from openadapt_flow.runtime.worklist_pool import ProcessWorker @@ -495,6 +525,8 @@ def main() -> None: r.add_argument("--window", type=int, default=0) r.add_argument("--halt-at", type=int, default=None) r.add_argument("--profile", action="store_true") + r.add_argument("--backend", choices=["base", "openemr"], default="base", + help="the browser backend the bundle was recorded with") r.add_argument("--speed", choices=["standard", "fast"], default="standard", help="named speed profile (openadapt_flow.runtime.speed)") r.add_argument("--ocr-threads", type=int, default=None, diff --git a/packages/flow/openadapt_flow/backends/playwright_backend.py b/packages/flow/openadapt_flow/backends/playwright_backend.py index 90e629b093..6efeb3f683 100644 --- a/packages/flow/openadapt_flow/backends/playwright_backend.py +++ b/packages/flow/openadapt_flow/backends/playwright_backend.py @@ -1344,7 +1344,12 @@ def _fill_iso_date(target: Any, text: str) -> bool: ? Array.from(document.getElementsByTagName(level.tagName)).filter( (candidate) => !tag || candidate.getElementsByTagName(tag).length > 0) : Array.from(document.querySelectorAll('tr, [role="row"], li, [role="listitem"]')); - const pool = new Set(siblings.concat(pageWide)); + const dialog = el.closest( + 'dialog:modal, [aria-modal="true"], [role="alertdialog"], .modal.show' + ); + const pool = new Set(siblings.concat(pageWide).filter( + (candidate) => !dialog || dialog.contains(candidate) + )); const result = { record_key: recordKey(row, el), incomplete: pool.size > cap, @@ -1675,6 +1680,82 @@ def _fill_iso_date(target: Any, text: str) -> bool: return null; }""" +# The active modal dialog holding the element under a viewport point. Walks the +# hit-tested chain from the top document down through same-origin frames; the +# first dialog met is the outermost one, and hit testing already follows the +# stacking order, so a dialog painted over another wins. The box is the +# dialog's first element (from the dialog down toward the hit) that does not +# cover the viewport, so a Bootstrap ``.modal`` overlay yields its +# ``.modal-dialog``. Answers ``{status: "dialog", box}``, ``{status: "none"}``, +# or ``{status: "unknown"}`` when a frame on the chain is cross-origin or +# transformed and the chain cannot be followed. +_DIALOG_AT_JS = r"""([x, y]) => { + // Modal dialogs only: a popover or date picker with role="dialog" leaves + // the page behind live, and its rows stay identity evidence. + const SELECTOR = 'dialog:modal, [aria-modal="true"], [role="alertdialog"], ' + + '.modal.show'; + const visible = (el, win) => { + const style = win.getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden') return false; + if (parseFloat(style.opacity) < 0.05) return false; + const r = el.getBoundingClientRect(); + return r.width > 0 && r.height > 0; + }; + let doc = document; + let win = window; + let ox = 0, oy = 0; + let px = x, py = y; + for (let depth = 0; depth < 9; depth++) { + const hit = doc.elementFromPoint(px, py); + if (!hit) return {status: 'none'}; + const dialog = hit.closest(SELECTOR); + if (dialog && visible(dialog, win)) { + const chain = []; + for (let node = hit; node; node = node.parentElement) { + chain.push(node); + if (node === dialog) break; + } + const vw = win.innerWidth, vh = win.innerHeight; + let box = null; + for (let i = chain.length - 1; i >= 0; i--) { + const r = chain[i].getBoundingClientRect(); + if (r.width <= 0 || r.height <= 0) continue; + if (r.width >= 0.95 * vw && r.height >= 0.95 * vh) continue; + box = r; + break; + } + if (!box) box = dialog.getBoundingClientRect(); + const left = Math.max(0, box.left), top = Math.max(0, box.top); + const right = Math.min(vw, box.right), bottom = Math.min(vh, box.bottom); + if (right <= left || bottom <= top) return {status: 'unknown'}; + return {status: 'dialog', box: [ + Math.floor(left + ox), Math.floor(top + oy), + Math.ceil(right - left), Math.ceil(bottom - top), + ]}; + } + const tag = hit.localName; + if (tag !== 'iframe' && tag !== 'frame') return {status: 'none'}; + let child; + try { child = hit.contentDocument; } catch (_) { child = null; } + if (!child || !hit.contentWindow) return {status: 'unknown'}; + const r = hit.getBoundingClientRect(); + if (!(hit.offsetWidth > 0 && hit.offsetHeight > 0)) return {status: 'unknown'}; + if (Math.abs(r.width / hit.offsetWidth - 1) > 0.01 || + Math.abs(r.height / hit.offsetHeight - 1) > 0.01) { + return {status: 'unknown'}; + } + const style = win.getComputedStyle(hit); + const dx = r.left + hit.clientLeft + (parseFloat(style.paddingLeft) || 0); + const dy = r.top + hit.clientTop + (parseFloat(style.paddingTop) || 0); + ox += dx; oy += dy; + px -= dx; py -= dy; + doc = child; + win = hit.contentWindow; + } + return {status: 'unknown'}; +}""" + + _PROJECT_FRAME_POINT_JS = r"""(el, point) => { const [px, py] = point; if (document.elementFromPoint(px, py) !== el) return null; @@ -2367,6 +2448,43 @@ def structured_text_at(self, x: int, y: int) -> Optional[str]: return None return result or None + def dialog_region_at( + self, x: int, y: int + ) -> tuple[str, Optional[tuple[int, int, int, int]]]: + """The box of the active dialog holding viewport pixel (x, y). + + Returns ``("dialog", (x, y, w, h))`` when the element under the point + sits in an open modal dialog (a modal ``dialog``, ``aria-modal``, an + ``alertdialog``, or a shown Bootstrap modal), the outermost one along the frame + chain; ``("none", None)`` when the chain was followed and no dialog + holds the point; ``("unknown", None)`` when it could not be followed + (a cross-origin or transformed frame, an evaluation error). Callers + that need a dialog treat ``unknown`` as a failed check. Never raises. + """ + try: + result = self.page.evaluate(_DIALOG_AT_JS, [int(x), int(y)]) + except Exception: + return ("unknown", None) + if not isinstance(result, dict): + return ("unknown", None) + status = result.get("status") + if status == "none": + return ("none", None) + box = result.get("box") + if ( + status == "dialog" + and isinstance(box, list) + and len(box) == 4 + and all(isinstance(v, (int, float)) for v in box) + and box[2] > 0 + and box[3] > 0 + ): + bx, by, bw, bh = (int(v) for v in box) + if not (bx <= int(x) < bx + bw and by <= int(y) < by + bh): + return ("unknown", None) + return ("dialog", (bx, by, bw, bh)) + return ("unknown", None) + def identity_candidates_at(self, x: int, y: int) -> Optional[dict[str, Any]]: """Census of the rows that show the same record identity as (x, y). diff --git a/packages/flow/openadapt_flow/compiler/compile.py b/packages/flow/openadapt_flow/compiler/compile.py index 5a423898ca..e4ce15e8f0 100644 --- a/packages/flow/openadapt_flow/compiler/compile.py +++ b/packages/flow/openadapt_flow/compiler/compile.py @@ -83,6 +83,7 @@ context_region_from_lines, coverage, identifier_text_from_lines, + mask_outside, ) from openadapt_flow.vision import imgops from openadapt_flow.vision.hashing import phash_distance, phash_png @@ -1599,6 +1600,26 @@ def _clamp_to_frame(region: Region, frame_size: tuple[int, int]) -> Optional[Reg return (x0, y0, x1 - x0, y1 - y0) +def _event_dialog_region( + event: dict[str, Any], frame_size: tuple[int, int] +) -> Optional[Region]: + """The recorded dialog box holding the click target, clamped to the frame. + + The recorder stores ``dialog_region`` on a pointer event whose target sat + in an open dialog (``PlaywrightBackend.dialog_region_at``). A malformed + value or one off the frame is ignored: the band then stays full width. + """ + value = event.get("dialog_region") + if not isinstance(value, (list, tuple)) or len(value) != 4: + return None + if not all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in value): + return None + x, y, w, h = (int(v) for v in value) + if w <= 0 or h <= 0: + return None + return _clamp_to_frame((x, y, w, h), frame_size) + + def _marked_identifier_region(value: object, *, source: str) -> Optional[Region]: """Validate an explicitly marked identifier region from the recording. @@ -1687,6 +1708,8 @@ def _emit_identifier_crop( crop_region: Region, click: Point, reference_date: Optional[date], + scope: Optional[Region] = None, + dialog_row: bool = True, ) -> tuple[Optional[str], Optional[Region], Optional[str]]: """Emit the pixel identifier crop for one click step, or say why not. @@ -1738,10 +1761,11 @@ def _emit_identifier_crop( band_box = context_region_from_lines( frame_lines, exclude_region=crop_region, - band=band_region(click, crop_region[3], frame_size), - point=click, + band=scope or band_region(click, crop_region[3], frame_size), + point=None if scope is not None and not dialog_row else click, min_confidence=MIN_OCR_CONFIDENCE, reference_date=reference_date, + scope=scope, ) region = _clamp_to_frame(band_box, frame_size) if band_box else None if region is None: @@ -2487,19 +2511,51 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: # (a table row's discriminative name column sits outside the # 160x64 template — see runtime.identity). Verified against the # live band before every click at replay time. + # A target inside a dialog: read its identity band on the frame + # with everything outside the dialog blanked, so labels on the + # page behind it are not frozen into the identity (#1347). + dialog_region = _event_dialog_region( + event, (frame.shape[1], frame.shape[0]) + ) + identity_lines = frame_lines + if dialog_region is not None: + masked = mask_outside(before_png, dialog_region) + identity_lines = ( + cached_lines(i, "dialog", masked) if masked is not None else [] + ) + # Keep the clicked row inside the dialog. Only a positively + # identified standalone control may fall back to the whole dialog + # when its row has too little text to arm identity. + dialog_row = True + context_band = dialog_region or band_region( + click, crop_region[3], (frame.shape[1], frame.shape[0]) + ) context_text = context_from_lines( - frame_lines, + identity_lines, exclude_region=crop_region, - band=band_region( - click, crop_region[3], (frame.shape[1], frame.shape[0]) - ), + band=context_band, # Row refinement: record only the click point's OWN text # row, matching what replay-time verification reads (the # 64px band spans 2-3 rows of a dense table). point=click, min_confidence=MIN_OCR_CONFIDENCE, reference_date=reference_date, + scope=dialog_region, ) + if ( + context_text is None + and dialog_region is not None + and event.get("dialog_row") is False + ): + dialog_row = False + context_text = context_from_lines( + identity_lines, + exclude_region=crop_region, + band=context_band, + min_confidence=MIN_OCR_CONFIDENCE, + reference_date=reference_date, + scope=dialog_region, + ) # Structured identity (DOM / a11y text of the clicked row), # captured by the recorder when the recording backend exposed it # (openadapt_flow.backend.IdentityBackend). Stored alongside the @@ -2555,10 +2611,12 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: ), context_text=context_text, structured_identity=structured_identity, - frame_lines=frame_lines, + frame_lines=identity_lines, crop_region=crop_region, click=click, reference_date=reference_date, + scope=dialog_region, + dialog_row=dialog_row, ) ) # The OCR fallback must compare like-for-like evidence. Derive the @@ -2570,7 +2628,7 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: cropped_context: Optional[str] = None if identifier_region is not None: cropped_context = identifier_text_from_lines( - frame_lines, + identity_lines, region=identifier_region, min_confidence=MIN_OCR_CONFIDENCE, reference_date=reference_date, @@ -2632,6 +2690,8 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: landmarks=anchor_landmarks, identifier_crop=identifier_crop_rel, identifier_region=identifier_region, + dialog_region=dialog_region, + dialog_row=dialog_row if dialog_region is not None else None, ) drag_end_anchor: Optional[Anchor] = None if kind == "drag": @@ -2687,7 +2747,7 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: unarmed_reason: Optional[str] = None if not identity_armed: unarmed_reason = _identity_unarmed_reason( - frame_lines, + identity_lines, band=band_region( click, crop_region[3], diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 494fa4722f..b42973ba60 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -545,6 +545,27 @@ class Anchor(BaseModel): " tiers compare like-for-like. Set iff `identifier_crop` is set." ), ) + dialog_region: Optional[Region] = Field( + default=None, + description=( + "Box of the active dialog (the topmost modal) that held the target" + " in the recorded frame, when it sat in one. The context band was" + " cut to it, so text of the page behind the dialog is not identity" + " evidence. Replay cuts the live band to the live dialog's box," + " and fails the identity check closed when no dialog holds the" + " resolved point. None outside a dialog and on bundles compiled" + " before this capability." + ), + ) + dialog_row: Optional[bool] = Field( + default=None, + description=( + "True reads the target's own row inside dialog_region. False permits" + " the whole dialog when teach time positively found a standalone" + " control with no usable row text; replay must confirm no record" + " row again. None conservatively uses the target's row." + ), + ) form_identity: Optional[FormIdentity] = Field( default=None, description=( @@ -563,6 +584,10 @@ class Anchor(BaseModel): def _serialize_compatible(self, handler: Any) -> dict[str, Any]: """Leave unset census metadata out, so older bundle bytes stay equal.""" data: dict[str, Any] = handler(self) + if self.dialog_region is None: + data.pop("dialog_region", None) + if self.dialog_row is None: + data.pop("dialog_row", None) if self.record_key is None: data.pop("record_key", None) if self.structured_row is None: @@ -4151,11 +4176,13 @@ class IdentityCheck(BaseModel): "look-alike identifier (runtime.identity_looks), in order." ), ) - look_halt: Optional[Literal["identity_look_changed_screen"]] = Field( + look_halt: Optional[ + Literal["identity_look_changed_screen", "identity_dialog_mask_failed"] + ] = Field( default=None, description=( "Set when an extra look changed the screen outside the " - "identifier it read. The step halts." + "identifier it read, or could not mask its dialog. The step halts." ), ) candidate_count: Optional[int] = Field( diff --git a/packages/flow/openadapt_flow/recorder.py b/packages/flow/openadapt_flow/recorder.py index 896fc0b013..50200123bb 100644 --- a/packages/flow/openadapt_flow/recorder.py +++ b/packages/flow/openadapt_flow/recorder.py @@ -769,6 +769,17 @@ def _record(self, event: dict[str, Any], act: Callable[[], None]) -> None: # scrolls out of a virtualized list, the text check # alone would pass on it: replay halts before a write. event = {**event, "structured_ambiguous": True} + # The active dialog holding the target, when it sits in one: the + # compiler cuts the identity band to its box, so a label of the + # page behind the dialog is not frozen into the identity. + dialog = self._dialog_region_at(int(event["x"]), int(event["y"])) + if dialog is not None: + event = {**event, "dialog_region": list(dialog)} + # A whole-dialog fallback is valid only for a control that + # positively has no record row, never for unreadable row text. + census = self._identity_census_at(int(event["x"]), int(event["y"])) + if census is not None: + event = {**event, "dialog_row": census.get("row") is not False} # The part of the form the target sits in (a legend, an accordion # or tab header, a heading), for the program's named sections. # Display only: the compiler never puts it on a step. @@ -1141,6 +1152,22 @@ def _structured_identity_at(self, x: int, y: int) -> Optional[str]: except Exception: return None + def _dialog_region_at(self, x: int, y: int) -> Optional[tuple[int, ...]]: + """Box of the active dialog holding (x, y), if the backend can say. + + Backends MAY expose ``dialog_region_at`` (the Playwright backend + does). None when the point is in no dialog, the backend cannot tell, + or the probe failed: the band then stays full width, as before. + """ + getter = getattr(self._backend, "dialog_region_at", None) + if getter is None: + return None + try: + status, box = getter(int(x), int(y)) + except Exception: + return None + return tuple(box) if status == "dialog" and box else None + def _identity_census_at(self, x: int, y: int) -> Optional[dict[str, Any]]: """The row census under (x, y), if the backend exposes one. diff --git a/packages/flow/openadapt_flow/repair/hold.py b/packages/flow/openadapt_flow/repair/hold.py index 5fb85ad699..07fad0125f 100644 --- a/packages/flow/openadapt_flow/repair/hold.py +++ b/packages/flow/openadapt_flow/repair/hold.py @@ -100,6 +100,8 @@ def hold_for_repair( ended = "timeout" stopping: list[int] = [] previous = None + staged_hold: Path | None = None + published = False if threading.current_thread() is threading.main_thread(): # The parent ending the hold (SIGTERM) must still clean up and let the # caller close the browser, so turn it into a normal end of the hold. @@ -111,7 +113,10 @@ def hold_for_repair( os.chmod(sock_path, 0o600) server.listen(1) deadline = time.monotonic() + timeout_s - fd = os.open(hold_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + fd, staged_name = tempfile.mkstemp( + prefix=f".{HOLD_FILE}.", suffix=".tmp", dir=run_dir + ) + staged_hold = Path(staged_name) with os.fdopen(fd, "w") as handle: json.dump( { @@ -123,6 +128,8 @@ def hold_for_repair( }, handle, ) + os.replace(staged_hold, hold_file) + published = True while True: remaining = deadline - time.monotonic() if stopping: @@ -164,7 +171,10 @@ def hold_for_repair( if previous is not None: signal.signal(signal.SIGTERM, previous) server.close() - hold_file.unlink(missing_ok=True) + if staged_hold is not None: + staged_hold.unlink(missing_ok=True) + if published: + hold_file.unlink(missing_ok=True) shutil.rmtree(private, ignore_errors=True) return ended diff --git a/packages/flow/openadapt_flow/runtime/heal.py b/packages/flow/openadapt_flow/runtime/heal.py index 4d3f9c9a38..2b0ae05c74 100644 --- a/packages/flow/openadapt_flow/runtime/heal.py +++ b/packages/flow/openadapt_flow/runtime/heal.py @@ -109,6 +109,8 @@ def _recontext( frame: tuple[int, int], *, identifier_region: Region | None = None, + dialog_region: Region | None = None, + dialog_row: bool = True, ) -> str | None: """Re-derive the anchor's identity context band from the live frame. @@ -139,11 +141,12 @@ def _recontext( return identity_mod.context_from_lines( lines, exclude_region=region, - band=identity_mod.band_region(click_point, region[3], frame), + band=dialog_region or identity_mod.band_region(click_point, region[3], frame), # Same row refinement the compiler applies: record only the click # point's own text row, matching what verification reads back. - point=click_point, + point=None if dialog_region is not None and not dialog_row else click_point, reference_date=date.today(), + scope=dialog_region, ) @@ -203,6 +206,15 @@ def build_heal_event( iw, ih, ) + translated_dialog_region = None + if old_anchor.dialog_region is not None: + dx, dy, dw, dh = old_anchor.dialog_region + translated_dialog_region = ( + dx + resolution.point[0] - old_anchor.click_point[0], + dy + resolution.point[1] - old_anchor.click_point[1], + dw, + dh, + ) if ( old_anchor.context_text is None and old_anchor.identity_template is not None @@ -213,13 +225,24 @@ def build_heal_event( # identity evidence. Do not add live plaintext to the healed anchor. new_context = None else: - new_context = _recontext( - vision, - frame_png, - new_region, - resolution.point, - frame, - identifier_region=translated_identifier_region, + context_png: bytes | None = frame_png + if translated_dialog_region is not None: + # A target in a dialog keeps its band read on the dialog alone, as + # the compiler read it; replay masks the live frame the same way. + context_png = identity_mod.mask_outside(frame_png, translated_dialog_region) + new_context = ( + _recontext( + vision, + context_png, + new_region, + resolution.point, + frame, + identifier_region=translated_identifier_region, + dialog_region=translated_dialog_region, + dialog_row=old_anchor.dialog_row is not False, + ) + if context_png is not None + else None ) new_anchor = old_anchor.model_copy( update={ @@ -228,6 +251,7 @@ def build_heal_event( "ocr_text": new_text if new_text is not None else old_anchor.ocr_text, "context_text": new_context, "identifier_region": translated_identifier_region, + "dialog_region": translated_dialog_region, } ) event = HealEvent( diff --git a/packages/flow/openadapt_flow/runtime/identity.py b/packages/flow/openadapt_flow/runtime/identity.py index ed4f680a71..b151351b1d 100644 --- a/packages/flow/openadapt_flow/runtime/identity.py +++ b/packages/flow/openadapt_flow/runtime/identity.py @@ -401,6 +401,20 @@ def regions_intersect(a: Region, b: Region) -> bool: _intersects = regions_intersect +def centered_in(region: Region, scope: Region) -> bool: + """Whether ``region``'s center lies inside ``scope``. + + Scopes the identity band to the active dialog: a line belongs to the + dialog when its center is inside the dialog's box, so a form label beside + the dialog is not identity evidence. A line the dialog's edge only clips + is judged by where most of it sits. + """ + x, y, w, h = region + sx, sy, sw, sh = scope + cx, cy = x + w // 2, y + h // 2 + return sx <= cx < sx + sw and sy <= cy < sy + sh + + def lines_near_point(lines: Iterable[Any], point_y: int) -> list[Any]: """Filter OCR lines to the single text row containing ``point_y``. @@ -473,6 +487,7 @@ def context_from_lines( point: Optional[Point] = None, min_confidence: float = 0.5, reference_date: Optional[date] = None, + scope: Optional[Region] = None, ) -> Optional[str]: """Extract the context-band text from full-frame OCR lines. @@ -497,6 +512,8 @@ def context_from_lines( reference_date: The recording date, enabling the near/far date split; when None every date-bearing line is dropped (heal-time re-extraction has no recording date). + scope: The active dialog's box when the target sits in one; lines + centered outside it (the page behind the dialog) are dropped. Returns: The joined band text, or None when the surviving text is too short @@ -509,6 +526,7 @@ def context_from_lines( point=point, min_confidence=min_confidence, reference_date=reference_date, + scope=scope, ) if not kept: return None @@ -563,6 +581,7 @@ def _kept_context_lines( point: Optional[Point], min_confidence: float, reference_date: Optional[date], + scope: Optional[Region] = None, ) -> list[tuple[int, Any, str]]: """The confident, in-band, non-excluded, non-volatile identity lines. @@ -580,6 +599,8 @@ def _kept_context_lines( center_y = ly + lh // 2 if not (band_y <= center_y < band_y + band_h): continue + if scope is not None and not centered_in(line.region, scope): + continue if _intersects(line.region, exclude_region): continue if is_volatile_line(text, reference_date=reference_date): @@ -599,6 +620,7 @@ def context_region_from_lines( point: Optional[Point] = None, min_confidence: float = 0.5, reference_date: Optional[date] = None, + scope: Optional[Region] = None, ) -> Optional[Region]: """Bounding box of the identity lines :func:`context_from_lines` keeps. @@ -618,6 +640,7 @@ def context_region_from_lines( point=point, min_confidence=min_confidence, reference_date=reference_date, + scope=scope, ) if not kept: return None @@ -2528,6 +2551,34 @@ def crop_region(frame_png: bytes, region: Region) -> Optional[bytes]: return buf.tobytes() if ok else None +def mask_outside(frame_png: bytes, region: Region) -> Optional[bytes]: + """Blank every pixel of a PNG frame outside ``region`` (x, y, w, h). + + The identity band of a target inside a dialog is read on this frame, at + record and at replay time, so the page behind the dialog cannot reach the + OCR at all: a label beside the dialog is neither read nor merged into a + line with the dialog's own text. The fill is flat white, which OCR reads + as no text. Returns None when the frame cannot be decoded or the region + is empty. + """ + import cv2 + import numpy as np + + img = cv2.imdecode(np.frombuffer(frame_png, np.uint8), cv2.IMREAD_COLOR) + if img is None: + return None + h, w = img.shape[:2] + x0, y0 = max(0, int(region[0])), max(0, int(region[1])) + x1 = min(w, int(region[0]) + int(region[2])) + y1 = min(h, int(region[1]) + int(region[3])) + if x1 <= x0 or y1 <= y0: + return None + out = np.full_like(img, 255) + out[y0:y1, x0:x1] = img[y0:y1, x0:x1] + ok, buf = cv2.imencode(".png", out) + return buf.tobytes() if ok else None + + def upscale_crop(frame_png: bytes, region: Region, factor: int = 2) -> Optional[bytes]: """Crop ``region`` from a frame and upscale it (cubic) for a re-OCR. diff --git a/packages/flow/openadapt_flow/runtime/render_scale.py b/packages/flow/openadapt_flow/runtime/render_scale.py index a3f6818cb3..f011cd490d 100644 --- a/packages/flow/openadapt_flow/runtime/render_scale.py +++ b/packages/flow/openadapt_flow/runtime/render_scale.py @@ -692,6 +692,18 @@ def method( return method +def _dialog_method(name: str) -> Callable[..., Any]: + """Probe the live dialog at the live point; return its box in the view.""" + + def method(self: RenderScaledBackend, x: int, y: int) -> Any: + status, box = getattr(self._inner, name)(*self._transform.to_live(x, y)) + if status != "dialog" or box is None: + return status, None + return status, self._transform.region_to_recorded(box) + + return method + + def _passthrough(name: str) -> property: return property(lambda self: getattr(self._inner, name)) @@ -718,6 +730,7 @@ def _passthrough(name: str) -> property: "select_option": _hash_method, "select_option_guarded": _hash_method, "arm_remote_frame_contract": _regions_method, + "dialog_region_at": _dialog_method, "take_dialogs": _passthrough, "arm_dialog": _passthrough, "transport": _passthrough, diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index ef03ef4ca8..0e9e10f3d3 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -11000,10 +11000,15 @@ def _identity_gate_error( "on the fresh frame — refusing to act; run aborted" ) elif check.look_halt is not None: + reason = ( + "the dialog could not be masked" + if check.look_halt == "identity_dialog_mask_failed" + else "the screen changed outside the identifier" + ) error = ( f"Step '{step.id}' ({step.intent}) took an extra look at a " - "look-alike identifier and the screen changed outside the " - f"identifier ({check.look_halt}) — refusing to act" + f"look-alike identifier and {reason} " + f"({check.look_halt}) — refusing to act" ) result.safety_halt = True elif check.known_id_halt is not None: @@ -15882,6 +15887,43 @@ def _compare_direct_signal_text( parameter_names, ) + def _dialog_scoped_frame( + self, resolution: Resolution, before_png: bytes + ) -> Union[bytes, IdentityCheck]: + """The step's frame with everything outside the live dialog blanked. + + Returns a ``mismatch`` :class:`IdentityCheck` instead when no dialog + holds the resolved point, the backend cannot locate one, or the frame + cannot be masked: a target recorded inside a dialog is not the same + target outside one. + """ + probe = getattr(self.backend, "dialog_region_at", None) + status, box = "unknown", None + if callable(probe): + try: + status, box = probe(int(resolution.point[0]), int(resolution.point[1])) + except Exception: + status, box = "unknown", None + masked = ( + identity_mod.mask_outside(before_png, box) + if status == "dialog" and box is not None + else None + ) + if masked is not None: + self._identity_dialog_box = box + return masked + observed = ( + "no dialog holds the target" + if status == "none" + else "the dialog holding the target could not be located" + ) + return IdentityCheck( + status="mismatch", + mode="context", + expected="the target inside the dialog it was recorded in", + observed=observed, + ) + @timed_span("identity") def _verify_identity( self, @@ -15935,6 +15977,32 @@ def _verify_identity( """ anchor = step.anchor assert anchor is not None + # The live dialog box this check reads in, for the extra looks it may + # take on fresh frames (see _look_again). None outside a dialog. + self._identity_dialog_box = None + unscoped_png = before_png + if anchor.dialog_region is not None: + # The target was recorded inside a dialog and its identity band + # was read on the dialog alone. Read the live band the same way, + # on the live topmost dialog, so the page behind it is not + # evidence (#1347). No dialog at the point, or one that cannot be + # located, fails the check closed. + scoped = self._dialog_scoped_frame(resolution, before_png) + if isinstance(scoped, IdentityCheck): + return scoped + before_png = scoped + if anchor.dialog_row is False: + # Whole-dialog text cannot identify a row's Select button. + # Reconfirm the recorded non-row role before any tier runs. + if ( + self._structured_census(step, resolution.point).verdict + != "not_a_row" + ): + return IdentityCheck( + status="unreadable", + mode="context", + observed="the dialog control's non-row role could not be confirmed", + ) project = getattr(workflow, "qualification", None) if project is not None: identity_policy = project.identity_policies.get(step.id) @@ -15942,10 +16010,12 @@ def _verify_identity( identity_policy is not None and identity_policy.enforcement.value == "signal_quorum" ): + # Quorum signals name their own regions (a banner outside + # the dialog is one), so they read the whole frame. quorum = self._verify_signal_quorum( step=step, resolution=resolution, - before_png=before_png, + before_png=unscoped_png, params=runtime_params_for_gui(params), workflow=workflow, bundle_dir=bundle_dir, @@ -16137,13 +16207,22 @@ def _ocr_census( tmpl = anchor.identity_template if not (anchor.context_text or (tmpl is not None and tmpl.tokens)): return census_mod.CensusDecision("cannot_count") + if anchor.dialog_row is False and anchor.dialog_region is not None: + # Count through the live row probe; the dialog itself proves + # nothing about whether its target belongs to a record row. + return self._structured_census(step, resolution.point) today = date.today() + dialog_box = self.__dict__.get("_identity_dialog_box") try: lines = [ line for line in self.vision.ocr(before_png, region=None) if line.text.strip() and not identity_mod.is_volatile_line(line.text, reference_date=today) + and ( + dialog_box is None + or identity_mod.centered_in(line.region, dialog_box) + ) ] except Exception: return census_mod.CensusDecision("cannot_count") @@ -16679,6 +16758,11 @@ def _verify_identity_ocr( band = identity_mod.band_region( resolution.point, anchor.region[3], self.backend.viewport ) + # Dialog pixels are scoped by _verify_identity. Row refinement still + # applies unless compile positively identified a standalone control. + dialog_box: Optional[Region] = self.__dict__.get("_identity_dialog_box") + if dialog_box is not None: + band = dialog_box # The recorded band was extracted EXCLUDING the target's own crop # (labels are mutable evidence the ladder heals through) and # excluding volatile lines. The live band must mirror both, or @@ -16728,7 +16812,8 @@ def attempt( line.text, reference_date=today ) ] - lines = identity_mod.lines_near_point(lines, point_y) + if dialog_box is None or anchor.dialog_row is not False: + lines = identity_mod.lines_near_point(lines, point_y) if row_lines is not None: row_lines.extend(lines) observed = " ".join(line.text.strip() for line in lines) @@ -16766,7 +16851,9 @@ def native_attempt( # translated identifier region. OCRing the crop itself can # change segmentation/glyph recognition and false-halt a # healthy screen. - None if identifier_region is not None else live_region, + None + if identifier_region is not None or dialog_box is not None + else live_region, resolution.point[1], exclude, scoped_identifier=identifier_region is not None, @@ -16860,6 +16947,7 @@ def with_looks(final: IdentityCheck) -> IdentityCheck: check.status == "verified" or identity_read_contradicts(check) or identifier_region is not None + or dialog_box is not None ): # A read of a different record is final: the row re-read below # must never overrule it into "verified". @@ -17361,6 +17449,18 @@ def _look_again( for _ in range(looks_mod.FRAME_LOOKS): time.sleep(self.poll_interval_s) png = self.backend.screenshot() + box = self.__dict__.get("_identity_dialog_box") + if box is not None: + # Read the fresh frame in the same dialog as the first look. + masked = identity_mod.mask_outside(png, box) + if masked is None: + return first.model_copy( + update={ + "status": "unreadable", + "look_halt": "identity_dialog_mask_failed", + } + ) + png = masked frames.append((read(png), sha256_bytes(png))) if frames[-1][0].status == "mismatch": break # final; no further looks diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 9efad21e89..2b23af02aa 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -421,6 +421,11 @@ "benchmark/openemr_local/verify.yaml", "benchmark/openemr_referral_timing/figures.json", "benchmark/openemr_referral_timing/runs.csv", + "benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-after-branch.json", + "benchmark/openemr_throughput/dialog_identity_2026-10-28/batch-before-main.json", + "benchmark/openemr_throughput/dialog_identity_2026-10-28/listed-duplicate.json", + "benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-ocr-tier.json", + "benchmark/openemr_throughput/dialog_identity_2026-10-28/scroll-openemr-adapter.json", "benchmark/openimis_claims/deployment.eligibility.yaml", "benchmark/openimis_claims/environment.lock.json", "benchmark/openimis_claims/submit_binding_probe.json", diff --git a/packages/flow/tests/test_bundle_schema_v2.py b/packages/flow/tests/test_bundle_schema_v2.py index 20953da05e..ae254c499e 100644 --- a/packages/flow/tests/test_bundle_schema_v2.py +++ b/packages/flow/tests/test_bundle_schema_v2.py @@ -1092,3 +1092,31 @@ def test_a_read_spec_and_grade_floor_are_sealed_content(tmp_path): ) content = bv._workflow_content(read) assert content["steps"][0]["read"]["field"]["name"] == "amount" + + +@pytest.mark.parametrize("shape", ["linear", "program"]) +def test_dialog_row_evidence_preserves_frozen_legacy_seal_and_is_tamper_evident( + tmp_path, shape +): + b = _write_bundle_dir(tmp_path) + wf = _structural_workflow(shape) + old_digest = _write_synthetic_pre_field_bundle(b, wf, shape, encrypted=False) + assert old_digest == _SYNTHETIC_LEGACY_DIGESTS[shape] + raw = json.loads((b / "workflow.json").read_text()) + assert "dialog_row" not in _anchor_payload(raw, shape) + loaded = Workflow.load(b, verify_integrity=True) + anchor = ( + loaded.steps[0].anchor + if shape == "linear" + else loaded.program.states["s1"].step.anchor + ) + assert anchor.dialog_row is None + for evidence in (True, False): + anchor.dialog_row = evidence + assert ( + bv.compute_content_digest(loaded, loaded.manifest.file_hashes) != old_digest + ) + _anchor_payload(raw, shape)["dialog_row"] = evidence + (b / "workflow.json").write_text(json.dumps(raw)) + with pytest.raises(bv.BundleIntegrityError): + Workflow.load(b, verify_integrity=True) diff --git a/packages/flow/tests/test_dialog_identity_live_openemr.py b/packages/flow/tests/test_dialog_identity_live_openemr.py new file mode 100644 index 0000000000..6832360c89 --- /dev/null +++ b/packages/flow/tests/test_dialog_identity_live_openemr.py @@ -0,0 +1,103 @@ +"""Live check of #1347 on a real OpenEMR: the duplicate dialog's identity band. + +OpenEMR's duplicate check opens "Confirm Create New Patient" in a modal over +the New Patient form. The full-width OCR band around that button also read a +label of the form behind the modal, which one depended on how far the form was +scrolled, so a correct save was refused every 9 to 17 items. This test opens +the real dialog with the form scrolled to several positions and checks that the +band read on the dialog alone (the frame blanked outside +``PlaywrightBackend.dialog_region_at``) is the same every time and holds no +form label. + +Bring up the throughput fixture first (see +``benchmark/openemr_throughput/README.md``), then:: + + OPENEMR_LIVE_STATE=~/oa/scratch/throughput/fixture/state \\ + THROUGHPUT_UI_PORT=9911 THROUGHPUT_API_PORT=9910 \\ + OPENADAPT_FLOW_BROWSER=bundled \\ + PYTHONPATH=../types:. python -m pytest tests/test_dialog_identity_live_openemr.py -v + +Skipped unless ``OPENEMR_LIVE_STATE`` is set. It saves nothing: each dialog is +left unconfirmed and its browser closed. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.skipif( + not os.environ.get("OPENEMR_LIVE_STATE"), + reason="live OpenEMR: set OPENEMR_LIVE_STATE to the fixture state directory", +) + +# Form scroll positions behind the dialog. 0 leaves "County:" beside the +# dialog's button row in the pinned image; the others move other labels there. +SCROLLS = (0, 250, 600, 900) + + +def _open_dialog(demo, fixture, scroll_y: int): + backend, close = demo.OpenEMRPlaywrightBackend.launch( + fixture.ui_base_url, headless=True + ) + demo._preauthenticate_browser(backend, fixture) + page = backend.page + form = demo._form_frame(page) + form.locator("#form_fname").fill("Wilhelmina") + form.locator("#form_lname").fill("Okonkwo-Hale") + form.locator("#form_DOB").fill("1961-04-09") + form.locator("#form_DOB").press("Escape") # close the date picker + form.locator("#form_sex").select_option(label="Female") + form.locator("#create").scroll_into_view_if_needed() + form.evaluate(f"() => window.scrollTo(0, {int(scroll_y)})") + form.locator("#create").evaluate("el => el.click()") + confirm = demo._confirm_create_locator(page) + return backend, close, demo._center(confirm) + + +def test_duplicate_dialog_band_ignores_the_form_behind() -> None: + from benchmark.openemr_throughput import harness + from openadapt_flow.runtime import identity as identity_mod + from openadapt_flow.vision.ocr import ocr + + demo = harness._demo_module() + fixture = harness.make_fixture(Path(os.environ["OPENEMR_LIVE_STATE"]).expanduser()) + full_bands: list[str] = [] + dialog_bands: list[str] = [] + for scroll_y in SCROLLS: + backend, close, point = _open_dialog(demo, fixture, scroll_y) + try: + status, box = backend.dialog_region_at(*point) + assert status == "dialog", (scroll_y, status) + assert box is not None + frame = backend.screenshot() + viewport = backend.viewport + band = identity_mod.band_region(point, 64, viewport) + # The button's own crop is excluded, as the compiler excludes it. + exclude = (point[0] - 80, point[1] - 32, 160, 64) + + def read(png: bytes, scope=None) -> str: + return ( + identity_mod.context_from_lines( + ocr(png), + exclude_region=exclude, + band=band, + point=point, + scope=scope, + ) + or "" + ) + + full_bands.append(read(frame)) + masked = identity_mod.mask_outside(frame, box) + assert masked is not None + dialog_bands.append(read(masked, box)) + finally: + close() + print("full-width bands:", full_bands) + print("dialog bands:", dialog_bands) + assert len(set(dialog_bands)) == 1, dialog_bands + for label in ("County", "Emergency", "Contact", "Phone", "Address"): + assert label not in dialog_bands[0], dialog_bands[0] diff --git a/packages/flow/tests/test_dialog_row_identity.py b/packages/flow/tests/test_dialog_row_identity.py new file mode 100644 index 0000000000..8a4cb9c026 --- /dev/null +++ b/packages/flow/tests/test_dialog_row_identity.py @@ -0,0 +1,354 @@ +"""A modal scopes identity to its contents without merging its record rows.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass + +import cv2 +import numpy as np +import pytest + +from openadapt_flow.compiler import compile_recording +from openadapt_flow.ir import ( + ActionKind, + Anchor, + IdentityCheck, + Resolution, + Step, + Workflow, +) +from openadapt_flow.runtime import identity as identity_mod +from openadapt_flow.runtime.replayer import Replayer + +DIALOG = (200, 50, 600, 300) + + +@dataclass +class Line: + text: str + region: tuple[int, int, int, int] + confidence: float = 0.99 + + +def frame(): + ok, encoded = cv2.imencode(".png", np.zeros((400, 1000, 3), np.uint8)) + assert ok + return encoded.tobytes() + + +def resolution(y): + return Resolution(rung="template", point=(730, y), confidence=1, elapsed_ms=1) + + +def step(context, *, dialog_row=None): + return Step( + id="select", + intent="select the recorded record", + action=ActionKind.CLICK, + anchor=Anchor( + template="unused.png", + region=(650, 118, 160, 64), + click_point=(730, 150), + context_text=context, + dialog_region=DIALOG, + **({"dialog_row": dialog_row} if dialog_row is not None else {}), + ), + ) + + +class Backend: + viewport = (1000, 400) + + def __init__(self, census=None): + self.census = census + + def dialog_region_at(self, x, y): + return "dialog", DIALOG + + def identity_candidates_at(self, x, y): + return self.census + + def screenshot(self): + return frame() + + +class Vision: + def __init__(self, lines): + self.lines = lines + + def ocr(self, png, region=None): + return self.lines + + +def check(lines, anchor_step, *, y=150, census=None): + replayer = Replayer(Backend(census), vision=Vision(lines), poll_interval_s=0) + return replayer._verify_identity( + anchor_step, resolution(y), frame(), {}, Workflow(name="dialog") + ) + + +def test_grok_wrong_row_probe_keeps_the_clicked_row_inside_dialog(): + # Independent regression for the review probe: every visible token is the + # same when resolving Jane's button one row down to Ann's button. + lines = [ + Line("Select patient", (220, 60, 160, 20)), + Line("Jane Lindqvist", (220, 140, 160, 20)), + Line("Room 7 East", (450, 140, 120, 20)), + Line("Select", (700, 140, 60, 20)), + Line("Ann Wuttke", (220, 240, 160, 20)), + Line("Room 9 West", (450, 240, 120, 20)), + Line("Select", (700, 240, 60, 20)), + Line("Cancel", (700, 310, 60, 20)), + ] + recorded = identity_mod.context_from_lines( + lines, + exclude_region=(650, 118, 160, 64), + band=DIALOG, + point=(730, 150), + scope=DIALOG, + ) + assert recorded == "Jane Lindqvist Room 7 East" + assert check(lines, step(recorded)).status == "verified" + wrong = check(lines, step(recorded), y=250) + assert wrong.status == "mismatch", wrong + assert "Ann Wuttke" in wrong.observed + assert "Jane Lindqvist" not in wrong.observed + + +def test_legacy_whole_dialog_context_cannot_verify_the_wrong_row(): + # Bundles compiled before row refinement have the entire modal's text. + # Replaying one must fail closed rather than accept another row's button. + lines = [ + Line("Select patient", (220, 60, 160, 20)), + Line("Jane Lindqvist Room 7 East", (220, 140, 350, 20)), + Line("Select", (700, 140, 60, 20)), + Line("Ann Wuttke Room 9 West", (220, 240, 350, 20)), + Line("Select", (700, 240, 60, 20)), + ] + legacy_context = identity_mod.context_from_lines( + lines, exclude_region=(650, 118, 160, 64), band=DIALOG, scope=DIALOG + ) + assert legacy_context is not None + wrong = check(lines, step(legacy_context), y=250) + assert wrong.status != "verified", wrong + + +def test_dialog_census_stops_identical_rows(): + lines = [Line("Jane Lindqvist Room 7 East", (220, y, 350, 20)) for y in (140, 240)] + result = check(lines, step("Jane Lindqvist Room 7 East")) + assert result.status == "abstain" + assert result.ambiguity_halt == "identical_candidates" + assert result.candidate_count == 2 + + +def test_dialog_census_ignores_rows_behind_the_modal(): + lines = [ + Line("Jane Lindqvist Room 7 East", (220, 140, 350, 20)), + Line("Jane Lindqvist Room 7 East", (10, 365, 350, 20)), + ] + result = check(lines, step("Jane Lindqvist Room 7 East")) + assert result.status == "verified" + assert result.candidate_count == 1 + + +@pytest.mark.parametrize( + "census", + [None, {"row": True, "uncountable": True}, {"candidates": [{"self": True}]}], +) +def test_whole_dialog_requires_positive_live_nonrow_census(census): + result = check( + [Line("No matching records were found", (220, 90, 350, 20))], + step("No matching records were found", dialog_row=False), + census=census, + ) + assert result.status == "unreadable" + + +def test_standalone_confirmation_can_read_dialog_result(): + result = check( + [Line("No matching records were found", (220, 90, 350, 20))], + step("No matching records were found", dialog_row=False), + census={"row": False}, + ) + assert result.status == "verified" + assert result.census_status == "not_a_row" + + +def test_fresh_look_mask_failure_never_reads_unscoped_frame(monkeypatch): + replayer = Replayer(Backend(), vision=Vision([]), poll_interval_s=0) + replayer._identity_dialog_box = DIALOG + monkeypatch.setattr(identity_mod, "mask_outside", lambda *args: None) + reads = [] + first = IdentityCheck( + status="abstain", abstain_reason="glyph_confusable_identifier" + ) + result = replayer._look_again( + step("C0X1234 Jane Lindqvist"), + first, + None, + lambda png: reads.append(png) or IdentityCheck(status="verified"), + {}, + Workflow(name="dialog"), + None, + frame(), + ) + assert reads == [] + assert result.status == "unreadable" + assert result.look_halt == "identity_dialog_mask_failed" + + +def test_unset_dialog_row_preserves_old_anchor_serialization(): + assert "dialog_row" not in step("Jane Lindqvist").anchor.model_dump() + + +@pytest.fixture +def live_dialog_page(): + from playwright.sync_api import sync_playwright + + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + with sync_playwright() as pw: + browser = pw.chromium.launch(headless=True) + page = browser.new_page(viewport={"width": 1000, "height": 500}) + yield page, PlaywrightBackend(page) + browser.close() + + +def _compile_page(page, backend, target, tmp_path, *, known_nonrow=False): + box = page.locator(target).bounding_box() + point = (int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2)) + status, dialog = backend.dialog_region_at(*point) + assert status == "dialog" + png = backend.screenshot() + recording = tmp_path / "recording" + (recording / "frames").mkdir(parents=True) + for suffix in ("before", "after"): + (recording / "frames" / f"0000_{suffix}.png").write_bytes(png) + event = { + "i": 0, + "kind": "click", + "x": point[0], + "y": point[1], + "t": 1.0, + "dialog_region": list(dialog), + } + if known_nonrow: + census = backend.identity_candidates_at(*point) + assert census == {"row": False, "top_level": True} + event["dialog_row"] = False + (recording / "events.jsonl").write_text(json.dumps(event) + "\n") + (recording / "meta.json").write_text( + json.dumps( + { + "id": "dialog-row-regression", + "created_at": "2026-11-04T00:00:00+00:00", + "viewport": [1000, 500], + "params": {}, + } + ) + ) + workflow = compile_recording(recording, tmp_path / "bundle", name="dialog-row") + anchor_step = next(s for s in workflow.steps if s.anchor is not None) + return workflow, anchor_step, point + + +@pytest.mark.parametrize("duplicate", [False, True]) +def test_chromium_compile_and_ocr_refuse_wrong_dialog_row( + live_dialog_page, tmp_path, duplicate +): + page, backend = live_dialog_page + second = "Jane Lindqvist Room 7 East" if duplicate else "Ann Wuttke Room 9 West" + page.set_content( + """

Jane Lindqvist Room 7 East

Select record

+ + +
Jane Lindqvist Room 7 East
""" + + second + + """
""" + ) + workflow, anchor_step, point = _compile_page(page, backend, "#first", tmp_path) + assert anchor_step.identity_armed is True, anchor_step.identity_unarmed_reason + assert anchor_step.anchor.dialog_row is True + box = page.locator("#second").bounding_box() + wrong_point = (int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2)) + replayer = Replayer(backend, poll_interval_s=0) + result = replayer._verify_identity( + anchor_step, + Resolution(rung="template", point=wrong_point, confidence=1, elapsed_ms=1), + backend.screenshot(), + {}, + workflow, + tmp_path / "bundle", + ) + assert result.status in ("mismatch", "abstain"), result + if duplicate: + assert result.ambiguity_halt == "identical_candidates" + assert result.candidate_count == 2 + else: + correct = replayer._verify_identity( + anchor_step, + Resolution(rung="template", point=point, confidence=1, elapsed_ms=1), + backend.screenshot(), + {}, + workflow, + tmp_path / "bundle", + ) + assert correct.status == "verified", correct + + +@pytest.mark.parametrize("target", ["#confirm", "#cancel"]) +def test_chromium_compile_standalone_buttons_keep_dialog_fallback( + live_dialog_page, tmp_path, target +): + page, backend = live_dialog_page + page.set_content("""

County: Emergency Contact:

+

Check existing records

No matching records were found.

+ + """) + workflow, anchor_step, point = _compile_page( + page, backend, target, tmp_path, known_nonrow=True + ) + assert anchor_step.anchor.dialog_row is False + result = Replayer(backend, poll_interval_s=0)._verify_identity( + anchor_step, + Resolution(rung="template", point=point, confidence=1, elapsed_ms=1), + backend.screenshot(), + {}, + workflow, + tmp_path / "bundle", + ) + assert result.status == "verified", result + assert result.census_status == "not_a_row" + + +def test_chromium_short_record_row_never_falls_back_to_whole_dialog( + live_dialog_page, tmp_path +): + page, backend = live_dialog_page + page.set_content("""

Select existing record

+ +
Ann
Bob
 + """) + _, anchor_step, _ = _compile_page(page, backend, "#first", tmp_path) + assert anchor_step.anchor.dialog_row is True + assert anchor_step.anchor.context_text is None + assert anchor_step.anchor.identity_template is None + + +def test_chromium_structured_census_counts_only_modal_rows(live_dialog_page): + page, backend = live_dialog_page + page.set_content("""
Jane Lindqvist
+ +
Jane Lindqvist
Jane Lindqvist
 + """) + box = page.locator("#first").bounding_box() + census = backend.identity_candidates_at( + int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + ) + assert len(census["candidates"]) == 2, census diff --git a/packages/flow/tests/test_dialog_scoped_identity.py b/packages/flow/tests/test_dialog_scoped_identity.py new file mode 100644 index 0000000000..bd20398de4 --- /dev/null +++ b/packages/flow/tests/test_dialog_scoped_identity.py @@ -0,0 +1,301 @@ +"""The identity band of a target inside a dialog is read on the dialog alone. + +OpenEMR's "Confirm Create New Patient" sits in a Bootstrap modal over the New +Patient form. The OCR identity band is a full-width strip, so it also read a +form label beside the modal: "County:" in one run, "Emergency Contact:" in +another, depending on where the form behind happened to be scrolled. The check +then refused a correct save every 9 to 17 items (#1347). + +The recorder now stores the box of the topmost dialog holding the target, the +compiler reads the band on the frame with everything outside that box blanked, +and replay does the same on the live dialog. No dialog at the point fails the +check closed. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from types import SimpleNamespace + +import cv2 +import numpy as np +import pytest + +from openadapt_flow.compiler.compile import _event_dialog_region +from openadapt_flow.ir import Anchor, IdentityCheck +from openadapt_flow.runtime import identity as identity_mod +from openadapt_flow.runtime.replayer import Replayer + +# A form behind a Bootstrap-style modal: the overlay covers the viewport and +# the visible box is its .modal-dialog. The dialog's buttons live in a +# same-origin iframe, as OpenEMR's dlgopen dialogs do. +_PAGE_HTML = """ +
+
County:
+
Emergency Contact:
+
Home Phone:
+
+ + +""" + + +@pytest.fixture(scope="module") +def page_backend(): + sync = pytest.importorskip("playwright.sync_api") + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + with sync.sync_playwright() as pw: + try: + browser = pw.chromium.launch(headless=True) + except Exception as exc: # pragma: no cover - no browser installed + pytest.skip(f"chromium unavailable: {exc}") + page = browser.new_page( + viewport={"width": 1000, "height": 400}, device_scale_factor=1 + ) + page.set_content(_PAGE_HTML, wait_until="load") + page.frame_locator("#body").locator("#confirm").wait_for() + yield page, PlaywrightBackend(page) + browser.close() + + +def _center(page, selector: str, frame: str | None = None) -> tuple[int, int]: + js = ( + "el => { const r = el.getBoundingClientRect();" + " return [r.x + r.width / 2, r.y + r.height / 2]; }" + ) + if frame is None: + x, y = page.eval_on_selector(selector, js) + return int(x), int(y) + fx, fy = page.eval_on_selector( + frame, "el => { const r = el.getBoundingClientRect(); return [r.x, r.y]; }" + ) + x, y = page.frame_locator(frame).locator(selector).evaluate(js) + return int(fx + x), int(fy + y) + + +def test_dialog_box_is_the_visible_dialog_not_the_overlay(page_backend) -> None: + page, backend = page_backend + status, box = backend.dialog_region_at(*_center(page, "#confirm", "#body")) + assert status == "dialog" + assert box == (300, 60, 400, 200) + + +def test_point_on_the_overlay_outside_the_box_reads_the_dialog(page_backend) -> None: + page, backend = page_backend + # The overlay itself is the dialog element; every child box covers less + # than the viewport, so the overlay's own rect is the scope. + status, box = backend.dialog_region_at(100, 300) + assert status == "dialog" + assert box == (0, 0, 1000, 400) + + +def test_fading_out_modal_is_not_a_dialog(page_backend) -> None: + page, backend = page_backend + page.evaluate("document.querySelector('.modal').style.opacity = '0'") + try: + status, _ = backend.dialog_region_at(*_center(page, "#confirm", "#body")) + assert status == "none" + finally: + page.evaluate("document.querySelector('.modal').style.opacity = '1'") + + +def test_non_modal_popover_is_not_a_dialog(page_backend) -> None: + page, backend = page_backend + page.evaluate("document.querySelector('.modal').style.display = 'none'") + page.evaluate( + "document.body.insertAdjacentHTML('beforeend', '
' + "
')" + ) + try: + assert backend.dialog_region_at(*_center(page, "#yes")) == ("none", None) + finally: + page.evaluate("document.getElementById('pop').remove()") + page.evaluate("document.querySelector('.modal').style.display = 'block'") + + +def test_no_dialog_open_is_none(page_backend) -> None: + page, backend = page_backend + page.evaluate("document.querySelector('.modal').style.display = 'none'") + try: + assert backend.dialog_region_at(*_center(page, "#outside")) == ("none", None) + finally: + page.evaluate("document.querySelector('.modal').style.display = 'block'") + + +@dataclass +class _Line: + text: str + region: tuple[int, int, int, int] + confidence: float = 0.99 + + +def test_band_scope_drops_the_label_behind_the_dialog() -> None: + lines = [ + _Line("County:", (20, 160, 70, 20)), + _Line("Confirm Create New Patient", (310, 160, 220, 20)), + _Line("Cancel", (560, 160, 60, 20)), + ] + band = identity_mod.band_region((420, 170), 64, (1000, 400)) + exclude = (0, 0, 1, 1) + full = identity_mod.context_from_lines( + lines, exclude_region=exclude, band=band, point=(420, 170) + ) + scoped = identity_mod.context_from_lines( + lines, + exclude_region=exclude, + band=band, + point=(420, 170), + scope=(300, 60, 400, 200), + ) + assert full == "County: Confirm Create New Patient Cancel" + assert scoped == "Confirm Create New Patient Cancel" + + +def _png(img: np.ndarray) -> bytes: + ok, buf = cv2.imencode(".png", img) + assert ok + return buf.tobytes() + + +def test_mask_outside_keeps_only_the_dialog_pixels() -> None: + img = np.zeros((40, 60, 3), np.uint8) + out = identity_mod.mask_outside(_png(img), (10, 5, 20, 10)) + assert out is not None + got = cv2.imdecode(np.frombuffer(out, np.uint8), cv2.IMREAD_COLOR) + assert (got[5:15, 10:30] == 0).all() + got[5:15, 10:30] = 255 + assert (got == 255).all() + assert identity_mod.mask_outside(_png(img), (100, 100, 5, 5)) is None + + +def test_event_dialog_region_is_validated_and_clamped() -> None: + assert _event_dialog_region( + {"dialog_region": [300, 60, 400, 200]}, (1000, 400) + ) == ( + 300, + 60, + 400, + 200, + ) + assert _event_dialog_region( + {"dialog_region": [900, 300, 400, 200]}, (1000, 400) + ) == ( + 900, + 300, + 100, + 100, + ) + assert _event_dialog_region({}, (1000, 400)) is None + assert _event_dialog_region({"dialog_region": [1, 2, 3]}, (1000, 400)) is None + assert _event_dialog_region({"dialog_region": [0, 0, 0, 5]}, (1000, 400)) is None + assert _event_dialog_region({"dialog_region": [True, 0, 5, 5]}, (1000, 400)) is None + + +def test_anchor_without_dialog_serializes_as_before() -> None: + anchor = Anchor(template="t.png", region=(0, 0, 10, 10), click_point=(5, 5)) + assert "dialog_region" not in anchor.model_dump() + scoped = anchor.model_copy(update={"dialog_region": (1, 2, 3, 4)}) + assert scoped.model_dump()["dialog_region"] == (1, 2, 3, 4) + + +class _Probe: + def __init__(self, answer): + self.answer = answer + + def dialog_region_at(self, x, y): + if isinstance(self.answer, Exception): + raise self.answer + return self.answer + + +def _scoped(backend, png: bytes): + replayer = Replayer.__new__(Replayer) + replayer.backend = backend + return Replayer._dialog_scoped_frame( + replayer, SimpleNamespace(point=(400, 150)), png + ) + + +@pytest.mark.parametrize( + "backend", + [ + _Probe(("none", None)), + _Probe(("unknown", None)), + _Probe(RuntimeError("page closed")), + SimpleNamespace(), # a backend that cannot see dialogs + ], +) +def test_no_live_dialog_fails_the_identity_check_closed(backend) -> None: + result = _scoped(backend, _png(np.zeros((400, 1000, 3), np.uint8))) + assert isinstance(result, IdentityCheck) + assert result.status == "mismatch" + + +def test_live_dialog_masks_the_frame() -> None: + result = _scoped( + _Probe(("dialog", (300, 60, 400, 200))), + _png(np.zeros((400, 1000, 3), np.uint8)), + ) + assert isinstance(result, bytes) + got = cv2.imdecode(np.frombuffer(result, np.uint8), cv2.IMREAD_COLOR) + assert (got[60:260, 300:700] == 0).all() + assert (got[:60] == 255).all() + + +def test_render_scaled_view_maps_the_dialog_box_back() -> None: + from openadapt_flow.runtime import render_scale as rs + + class Inner: + viewport = (1250, 500) + + def screenshot(self): + return b"" + + def dialog_region_at(self, x, y): + assert (x, y) == (500, 200) + return ("dialog", (375, 75, 500, 250)) + + transform = rs.RenderTransform( + scale=1.25, + offset=(0.0, 0.0), + recorded_viewport=(1000, 400), + live_viewport=(1250, 500), + landmarks_used=5, + landmarks_located=5, + max_residual_px=1.0, + ) + view = rs.render_scaled_backend(Inner(), transform) + assert view.dialog_region_at(400, 160) == ("dialog", (300, 60, 400, 200)) + + +def test_dialog_band_is_the_whole_dialog_not_the_button_row() -> None: + # OpenEMR's duplicate dialog: the button row holds only the other button, + # so the band is every line of the dialog, the result text included. + dialog = (202, 12, 876, 500) + lines = [ + _Line("1 - 0 of 0", (996, 52, 60, 16)), + _Line("No matches were found.", (227, 175, 160, 40)), + _Line("Confirm Create New Patient", (500, 270, 200, 20)), + _Line("Cancel", (730, 270, 50, 20)), + _Line("County:", (110, 262, 60, 18)), # the form behind + ] + exclude = (520, 247, 160, 64) # the confirm button's crop + text = identity_mod.context_from_lines( + lines, exclude_region=exclude, band=dialog, point=None, scope=dialog + ) + assert text is not None + assert "No matches were found." in text and "Cancel" in text + assert "County" not in text and "Confirm" not in text diff --git a/packages/flow/tests/test_heal.py b/packages/flow/tests/test_heal.py index 2edd9cbec4..ce0b96289a 100644 --- a/packages/flow/tests/test_heal.py +++ b/packages/flow/tests/test_heal.py @@ -476,3 +476,56 @@ def test_heal_recontext_keeps_dob_lines_and_drops_clocks(bundle, run_dir): heal = report.results[0].heal assert heal is not None assert heal.new_anchor.context_text == "Jane Sample DOB 1980-01-01" + + +def test_heal_keeps_a_dialog_target_band_inside_the_dialog(): + """A healed dialog step re-reads its band on the dialog alone (#1347).""" + import cv2 + import numpy as np + + step = ocr_anchored_step() + assert step.anchor is not None + step.anchor.dialog_region = (60, 40, 180, 120) + old_click = step.anchor.click_point + frames: list[bytes] = [] + + class RecordingVision(FakeVision): + def ocr(self, png, region=None): + frames.append(png) + return super().ocr(png, region=region) + + vision = RecordingVision() + resolution = Resolution( + rung="template_global", + point=(old_click[0] + 10, old_click[1] + 5), + confidence=0.99, + elapsed_ms=1.0, + ) + event, _crop = build_heal_event( + step, resolution, (100, 140, 50, 20), make_png(VIEWPORT, (0, 0, 0)), vision + ) + assert event.new_anchor.dialog_region == (70, 45, 180, 120) + full = cv2.imdecode(np.frombuffer(frames[-1], np.uint8), cv2.IMREAD_COLOR) + assert (full[:45] == 255).all() and (full[45:165, 70:250] == 0).all() + + +def test_heal_dialog_mask_failure_never_recontexts_unmasked_frame(monkeypatch): + from openadapt_flow.runtime import identity as identity_mod + + step = ocr_anchored_step() + step.anchor.dialog_region = (60, 40, 180, 120) + monkeypatch.setattr(identity_mod, "mask_outside", lambda *args: None) + resolution = Resolution( + rung="template_global", + point=step.anchor.click_point, + confidence=0.99, + elapsed_ms=1, + ) + event, _ = build_heal_event( + step, + resolution, + step.anchor.region, + make_png(VIEWPORT, (0, 0, 0)), + FakeVision(), + ) + assert event.new_anchor.context_text is None diff --git a/packages/flow/tests/test_recorder.py b/packages/flow/tests/test_recorder.py index 2f0e0426d9..3e68cfd122 100644 --- a/packages/flow/tests/test_recorder.py +++ b/packages/flow/tests/test_recorder.py @@ -306,3 +306,23 @@ def test_recorder_omits_structural_keys_on_plain_backend( "pages_after", ): assert key not in event + + +@pytest.mark.parametrize( + "census, expected", [({"row": False}, False), ({"row": True}, True), (None, None)] +) +def test_dialog_records_nonrow_evidence_without_structured_identity( + tmp_path, census, expected +): + class DialogBackend(FakeBackend): + def dialog_region_at(self, x, y): + return "dialog", (0, 0, 500, 500) + + def identity_candidates_at(self, x, y): + return census + + rec = Recorder(DialogBackend(), tmp_path / "rec") + rec.click(100, 100) + event = _read_events(rec.finish())[0] + assert event.get("dialog_row") is expected + assert "structured_identity" not in event diff --git a/packages/flow/tests/test_repair_hold.py b/packages/flow/tests/test_repair_hold.py index 761046e5d9..6651e8d663 100644 --- a/packages/flow/tests/test_repair_hold.py +++ b/packages/flow/tests/test_repair_hold.py @@ -77,6 +77,87 @@ def test_times_out_and_cleans_up(tmp_path: Path) -> None: assert not Path(hold["socket"]).parent.exists() +def test_hold_is_published_only_after_serialization( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + writing = threading.Event() + resume = threading.Event() + dump = json.dump + result: dict = {} + errors: list[Exception] = [] + + def paused_dump(payload, handle): + handle.write('{"socket":') + handle.flush() + writing.set() + assert resume.wait(10) + handle.seek(0) + handle.truncate() + dump(payload, handle) + + def serve() -> None: + try: + result["ended"] = hold_for_repair(FakeBackend(), tmp_path, 10) + except Exception as exc: + errors.append(exc) + + monkeypatch.setattr(json, "dump", paused_dump) + thread = threading.Thread(target=serve) + thread.start() + hold_path = tmp_path / HOLD_FILE + try: + assert writing.wait(5) + assert not hold_path.exists() + staged = list(tmp_path.iterdir()) + assert len(staged) == 1 + assert staged[0].stat().st_mode & 0o777 == 0o600 + resume.set() + deadline = time.monotonic() + 5 + while not hold_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + hold = json.loads(hold_path.read_text()) + assert hold_path.stat().st_mode & 0o777 == 0o600 + assert request(hold, {"op": "release"}) == {"ok": True} + finally: + resume.set() + thread.join(15) + assert not thread.is_alive() + assert errors == [] + assert result["ended"] == "released" + assert list(tmp_path.iterdir()) == [] + assert not Path(hold["socket"]).parent.exists() + + +@pytest.mark.parametrize("failure", ["serialization", "publication"]) +@pytest.mark.parametrize("existing", [False, True]) +def test_failed_publication_cleans_staging_and_preserves_existing_hold( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str, existing: bool +) -> None: + import os + + hold_path = tmp_path / HOLD_FILE + previous = "previous hold authority" + if existing: + hold_path.write_text(previous) + + def failed_dump(payload, handle): + handle.write('{"socket":') + raise OSError("serialization interrupted") + + def failed_replace(source, target): + raise OSError("publication interrupted") + + if failure == "serialization": + monkeypatch.setattr(json, "dump", failed_dump) + else: + monkeypatch.setattr(os, "replace", failed_replace) + with pytest.raises(OSError, match=f"{failure} interrupted"): + hold_for_repair(FakeBackend(), tmp_path, 10) + if existing: + assert hold_path.read_text() == previous + assert list(tmp_path.iterdir()) == ([hold_path] if existing else []) + + def test_sigterm_ends_the_hold_and_cleans_up(tmp_path: Path) -> None: import os import signal From de6076b45d9d2efb75e797f345d2a0c3aa82ebc3 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 20:13:29 -0800 Subject: [PATCH 09/21] fix: synchronize worklist stop and readback fixtures (#1930) Signed-off-by: abrichr --- packages/flow/tests/test_worklist_pool.py | 100 ++++++++++++++++++++-- 1 file changed, 91 insertions(+), 9 deletions(-) diff --git a/packages/flow/tests/test_worklist_pool.py b/packages/flow/tests/test_worklist_pool.py index 3b813bf6dc..5e08c6dd53 100644 --- a/packages/flow/tests/test_worklist_pool.py +++ b/packages/flow/tests/test_worklist_pool.py @@ -107,10 +107,15 @@ def test_forced_halt_on_one_worker_stops_the_others(tmp_path: Path) -> None: halted_idx: list[int] = [] w1_items: list[int] = [] + stopped = threading.Event() + peers_ready = threading.Event() + peer_claims: dict[str, int] = {} + peer_lock = threading.Lock() def halting(idx: int) -> ItemRun: w1_items.append(idx) if len(w1_items) == 2: # w1's second item halts, whatever its index + assert peers_ready.wait(10), "peers never obtained their in-flight claims" halted_idx.append(idx) return ItemRun( state=HALTED, @@ -122,9 +127,31 @@ def halting(idx: int) -> ItemRun: return ItemRun(state=VERIFIED) w1 = FakeWorker("w1", halting, delay_s=0.01) - peers = [FakeWorker(f"w{i}", delay_s=0.03) for i in (2, 3, 4)] + + def finish_after_stop(idx: int, peer: str) -> ItemRun: + # A delay cannot guarantee which worker obtains the next SQLite + # claim. Establish real in-flight peers, then let STOP release them. + with peer_lock: + assert peer not in peer_claims + assert not stopped.is_set() + peer_claims[peer] = idx + if len(peer_claims) == 3: + peers_ready.set() + assert stopped.wait(10), "the halting worker never committed STOP" + return ItemRun(VERIFIED) + + peers = [ + FakeWorker( + f"w{i}", lambda idx, i=i: finish_after_stop(idx, f"w{i}"), delay_s=0.03 + ) + for i in (2, 3, 4) + ] report = run_worklist_pool( - _rows(100), [w1, *peers], run_dir=tmp_path, policy=_policy(4) + _rows(100), + [w1, *peers], + run_dir=tmp_path, + policy=_policy(4), + on_event=lambda msg: stopped.set() if msg.startswith("STOP (") else None, ) assert not report.success @@ -133,6 +160,11 @@ def halting(idx: int) -> ItemRun: assert "identity check failed" in report.stop["reason"] states = {item["idx"]: item["state"] for item in report.items} assert states[halted_idx[0]] == HALTED + assert set(peer_claims) == {"w2", "w3", "w4"} + for item in report.items: + if item["idx"] in peer_claims.values(): + assert item["state"] == VERIFIED + assert item["claimed_at"] < report.stop["at"] <= item["finished_at"] # Far fewer than 100 items ran: claims stopped everywhere. ran = [idx for w in (w1, *peers) for idx in w.ran] assert len(ran) < 40 @@ -257,20 +289,34 @@ def test_only_the_claiming_worker_records_a_result(tmp_path: Path) -> None: class PipelinedWorker(FakeWorker): - """Read-back runs on a side thread and resolves after ``readback_s``.""" + """Read-back runs on a side thread, optionally held until a later write.""" def __init__( - self, name: str, fail_idx: Optional[int], readback_s: float, **kw: Any + self, + name: str, + fail_idx: Optional[int], + readback_s: float, + *, + hold_failure_until_next_write: bool = False, + **kw: Any, ) -> None: super().__init__(name, **kw) self.fail_idx = fail_idx self.readback_s = readback_s + self.hold_failure_until_next_write = hold_failure_until_next_write + self._next_write = threading.Event() self.max_outstanding = 0 self._outstanding = 0 self._lock = threading.Lock() def run_item(self, idx: int, row: dict[str, Any], item_dir: Path) -> ItemRun: self.ran.append(idx) + if ( + self.hold_failure_until_next_write + and self.fail_idx is not None + and idx == self.fail_idx + 1 + ): + self._next_write.set() time.sleep(self.delay_s) fut: Future = Future() with self._lock: @@ -278,9 +324,16 @@ def run_item(self, idx: int, row: dict[str, Any], item_dir: Path) -> ItemRun: self.max_outstanding = max(self.max_outstanding, self._outstanding) def readback() -> None: - time.sleep(self.readback_s) + ready = True + if idx == self.fail_idx and self.hold_failure_until_next_write: + ready = self._next_write.wait(10) + else: + time.sleep(self.readback_s) with self._lock: self._outstanding -= 1 + if not ready: + fut.set_exception(RuntimeError("the later write never started")) + return if idx == self.fail_idx: fut.set_result( (RECONCILIATION_REQUIRED, {"halt_reason": "saved record differs"}) @@ -296,7 +349,13 @@ def readback() -> None: def test_pipelined_readback_failure_names_items_written_after( tmp_path: Path, window: int ) -> None: - w = PipelinedWorker("w1", fail_idx=3, readback_s=0.05, delay_s=0.01) + w = PipelinedWorker( + "w1", + fail_idx=3, + readback_s=0.05, + delay_s=0.01, + hold_failure_until_next_write=True, + ) report = run_worklist_pool( _rows(20), [w], run_dir=tmp_path, policy=_policy(1, readback_window=window) ) @@ -311,6 +370,8 @@ def test_pipelined_readback_failure_names_items_written_after( # The items written after it still got their own read-back. for idx in after: assert states[idx] == VERIFIED + later = next(item for item in report.items if item["idx"] == 4) + assert later["claimed_at"] < report.stop["at"] assert max(w.ran) <= 3 + window md = (tmp_path / POOL_REPORT_MD).read_text() assert "Read-back of item 3 failed" in md @@ -358,15 +419,22 @@ def test_cookie_fingerprint_tells_sessions_apart() -> None: def make_process_fake( - name: str, fail_idx: int = -1, pipelined: bool = False, instant: bool = False + name: str, + fail_idx: int = -1, + pipelined: bool = False, + instant: bool = False, + hold_failure_until_next_write: bool = False, ) -> Any: """Factory the child process imports (see ProcessWorker).""" + next_write = threading.Event() class _W(FakeWorker): def open(self) -> None: self.thread = threading.get_ident() def run_item(self, idx: int, row: dict[str, Any], item_dir: Path) -> ItemRun: + if hold_failure_until_next_write and idx == fail_idx + 1: + next_write.set() item_dir.mkdir(parents=True, exist_ok=True) (item_dir / "ran_in_pid").write_text(str(__import__("os").getpid())) time.sleep(0.02) @@ -386,7 +454,12 @@ def run_item(self, idx: int, row: dict[str, Any], item_dir: Path) -> ItemRun: fut: Future = Future() def later() -> None: - time.sleep(0.05) + if idx == fail_idx and hold_failure_until_next_write: + if not next_write.wait(10): + fut.set_exception(RuntimeError("the later write never started")) + return + else: + time.sleep(0.05) fut.set_result( (RECONCILIATION_REQUIRED, {"halt_reason": "differs"}) if idx == fail_idx @@ -542,7 +615,12 @@ def test_process_worker_halt_and_pipelined_failure(tmp_path: Path) -> None: ProcessWorker( "q", "tests.test_worklist_pool:make_process_fake", - {"name": "q", "fail_idx": 3, "pipelined": True}, + { + "name": "q", + "fail_idx": 3, + "pipelined": True, + "hold_failure_until_next_write": True, + }, ) ] report = run_worklist_pool( @@ -550,6 +628,10 @@ def test_process_worker_halt_and_pipelined_failure(tmp_path: Path) -> None: ) assert report.stop is not None and report.stop["idx"] == 3 assert report.written_after_failed_readback["3"] == [4] + later = next(item for item in report.items if item["idx"] == 4) + assert later["state"] == VERIFIED + assert later["claimed_at"] < report.stop["at"] + assert (tmp_path / "b" / "items" / "00004" / "ran_in_pid").is_file() def test_function_worker_reads_reports(tmp_path: Path) -> None: From ff5257cdb81208cdc4a410f5de60bd108afe3bf5 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 20:47:44 -0800 Subject: [PATCH 10/21] chore(release): openadapt 1.20.0 on openadapt-flow 2.6.0 (#1850) * chore(release): prepare openadapt 1.20.0 on Flow 2.6 Signed-off-by: abrichr * chore: refresh published platform components for launcher draft Signed-off-by: abrichr * chore: synchronize launcher candidate lock version Signed-off-by: abrichr * chore(release): lock launcher against published Flow 2.6.0 Signed-off-by: abrichr --------- Signed-off-by: abrichr --- CHANGELOG.md | 19 ++ docs/platform-compatibility-report.md | 50 ++--- openadapt/cli.py | 25 ++- platform-manifest.json | 239 ++++++++++++----------- pyproject.toml | 25 ++- tests/test_cli_shell_hints.py | 4 +- tests/test_cli_smoke.py | 40 ++-- tests/test_deploy_preflight_readiness.py | 2 +- tests/test_flow_floor.py | 8 +- tests/test_verify_command.py | 27 +++ uv.lock | 53 ++--- 11 files changed, 267 insertions(+), 225 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 114bbdeb65..b58aa349d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ +## v1.20.0 (2026-11-04) + +### Requirements + +- Require `openadapt-flow>=2.6.0,<3.0.0` in the base install and every Flow + extra. `openadapt doctor` and `openadapt deploy` report older Flow releases + as unsupported. This supplies the persistent owed-recheck API used by + ledger listing. + +### Features + +- `openadapt verify ledger list` shows provisional confirmed writes whose + rechecks are still owed, with their due time and actor. Owed rechecks remain + visible after the server restarts and disappear when the recheck completes. + An empty ledger reports that no held writes, open writes or owed rechecks + remain. +- The platform manifest and compatibility report select published component + releases, including Desktop 0.25.1. + ## v1.19.0 (2026-11-02) ### Breaking Changes diff --git a/docs/platform-compatibility-report.md b/docs/platform-compatibility-report.md index 2c1b4199d6..3278ecc665 100644 --- a/docs/platform-compatibility-report.md +++ b/docs/platform-compatibility-report.md @@ -2,7 +2,7 @@ -- Generated: `2026-11-03T00:23:35+00:00` +- Generated: `2026-11-04T22:39:52+00:00` - Selection: `explicit-published` - Compatibility: **dependency-compatible** - Signature: `unsigned (signing infrastructure pending)` @@ -11,22 +11,22 @@ | Role | Package | Version | Release source reference | Published artifacts | |---|---|---:|---|---| -| launcher | `openadapt` | `1.17.0` | [`v1.17.0`](https://github.com/OpenAdaptAI/OpenAdapt/tree/v1.17.0)
commit `e022d97ee8b3c3992f4a4cb6e522f1532fef0834`
tree `2e0efeaa337a26d3425320ffa8f9aa3ac31d3f9c` | `openadapt-1.17.0-py3-none-any.whl` `c33c069c8cd9c270562200339b631fb77423b5b7f24364b5f0a6a0730e5ca8b8`
`openadapt-1.17.0.tar.gz` `03926f94f4d9d37eeb8c6c9cbdc94516d43c0ace971f2282b3310d77d73b6547` | -| flow | `openadapt-flow` | `2.5.0` | [`openadapt-flow-v2.5.0`](https://github.com/OpenAdaptAI/OpenAdapt/tree/openadapt-flow-v2.5.0/packages/flow)
commit `5a3bf49a0bf2d662430386295a1c34036c064d58`
tree `cab536c90e79fcfe887b8876e884ec2ed651f23e` | `openadapt_flow-2.5.0-py3-none-any.whl` `1dbc5d42d103cfa8a452d95805dde60c7d460035ec4d81df5cc642a32f118339`
`openadapt_flow-2.5.0.tar.gz` `54d99e77976ee7d9e6bc171b2ea148497528ead77b226fdf6fd4713b37c689b3` | +| launcher | `openadapt` | `1.19.0` | [`v1.19.0`](https://github.com/OpenAdaptAI/OpenAdapt/tree/v1.19.0)
commit `1a64acd47bf6e60de20b7ac97993bd23122eb697`
tree `4ff3db2dc38334cd3957e66f89c95ea292abd9b8` | `openadapt-1.19.0-py3-none-any.whl` `0a6963404d5b2a83aa83f8701c017838a8d09878128264b729064515b722a8b4`
`openadapt-1.19.0.tar.gz` `fc513011804644133ba8c93d41979cf2136829316839afb870e5092079551887` | +| flow | `openadapt-flow` | `2.6.0` | [`openadapt-flow-v2.6.0`](https://github.com/OpenAdaptAI/OpenAdapt/tree/openadapt-flow-v2.6.0/packages/flow)
commit `1c832365d11e9dbe2a166485b13f7b8effd6d8d0`
tree `954b03e36d350803bec59046ffaf56c02c3d556b` | `openadapt_flow-2.6.0-py3-none-any.whl` `43718301bdb2579b1a78c60dfc025e8d60f2be9a3bc4172894c4194e0600fa9a`
`openadapt_flow-2.6.0.tar.gz` `4f1b2b9e28ec666c0e50ef2593d985ba387efc42bfb9b8f9f24148f6c8085112` | | capture | `openadapt-capture` | `1.2.2` | [`v1.2.2`](https://github.com/OpenAdaptAI/openadapt-capture/tree/v1.2.2)
commit `06ade22948c12dcc6608bd54b5cca5c9c306e77c`
tree `b0a3b5b53f23536e01314f36aa46fd5b2d8baba6` | `openadapt_capture-1.2.2-py3-none-any.whl` `550fffe50990fa8431d332bdd5fe4755976d83baa7ad62553f50ad9e4ddcd1ae`
`openadapt_capture-1.2.2.tar.gz` `6f4298d3daadf9cced8994207e34b3116a79209ef792e98f778a0770e2f3406c` | | privacy | `openadapt-privacy` | `1.1.2` | [`v1.1.2`](https://github.com/OpenAdaptAI/openadapt-privacy/tree/v1.1.2)
commit `9ae96d8ed73ea5b9e445d0604b5db48788936a04`
tree `fe59ca5f04fc99d16424e9228dcd797b05115cb8` | `openadapt_privacy-1.1.2-py3-none-any.whl` `6e542b2fea44bdf8872430687946ec805fef22dc7d228a56a1e65871beebe47a`
`openadapt_privacy-1.1.2.tar.gz` `d90287edaa13470955f15d5f6eeb37b075678aeb55de324f8b8207682009bcc2` | | types | `openadapt-types` | `0.24.0` | [`openadapt-types-v0.24.0`](https://github.com/OpenAdaptAI/OpenAdapt/tree/openadapt-types-v0.24.0/packages/types)
commit `f2f04c5b432d88b262d46103a542b0e2c3dcc8a9`
tree `38ccf341ec74a0dd0facc9f64343c27644a7ca7d` | `openadapt_types-0.24.0-py3-none-any.whl` `8025e8a4dc9235fd81d4b24ae7fa7e544120abdb8d4ee7df160e4f50d9230d75`
`openadapt_types-0.24.0.tar.gz` `16dcd1da1c2c78c08d650a19eb9cc439474f996812a87198b316d51bfac443db` | -| desktop | `openadapt-desktop` | `0.16.0` | [`v0.16.0`](https://github.com/OpenAdaptAI/openadapt-desktop/tree/v0.16.0)
commit `7c59750d37be09a2871cd1bada8cb0bce946d363`
tree `9a1307794df6f2c77ee3dab90973845f3d08ee53` | `openadapt_desktop-0.16.0-py3-none-any.whl` `06e608a934deef56c93ee1ca8de511d6daba08459626e8245bd0db4c391e64e9`
`openadapt_desktop-0.16.0.tar.gz` `15f0c59a4d481328da76f3e5aa6a29ab8cdced8ab183c97346d69b4c14c996ff` | +| desktop | `openadapt-desktop` | `0.25.1` | [`openadapt-desktop-v0.25.1`](https://github.com/OpenAdaptAI/OpenAdapt/tree/openadapt-desktop-v0.25.1/packages/desktop)
commit `b21a86d4a263f2bd8c4e8037e81f09457e998bdb`
tree `77acb2dfaab10e413f0d348ce98fbff9e24da752` | `openadapt_desktop-0.25.1-py3-none-any.whl` `5f55ecffe173d0f7740cbbf011c19b763a495d9e1b37054f4f6ccf16a07a7be2`
`openadapt_desktop-0.25.1.tar.gz` `5a9c22996b76a0efbcfa87d3710d88611ab2feb772de284d7233b9d38b04241b` | | agent | `openadapt-agent` | `2.2.1` | [`openadapt-agent-v2.2.1`](https://github.com/OpenAdaptAI/OpenAdapt/tree/openadapt-agent-v2.2.1/packages/agent)
commit `8d0758b9687744b07355c6512b6a57942bec4bb4`
tree `5300b0849929dd3051c57640ba6fcd74ac483728` | `openadapt_agent-2.2.1-py3-none-any.whl` `847499c4091144cb0c1bb1d2731ef064ae9cd844867c2b603acf573c6e12928a`
`openadapt_agent-2.2.1.tar.gz` `859e6b7d464ab813aba85af7f1f227ac9afcdb60112bc564719a82d3766fb733` | ## Runtime selections | Unit | Version | Selected or locked OpenAdapt packages | |---|---:|---| -| launcher_environment | `1.17.0` | `openadapt==1.17.0`
`openadapt-flow==2.5.0` | -| customer_runner | `2.5.0` | `openadapt-flow==2.5.0`
`openadapt-types==0.24.0` | -| desktop_sidecar | `0.16.0` | `openadapt-capture==1.2.1`
`openadapt-flow==1.34.0`
`openadapt-privacy==1.0.0`
`openadapt-types==0.10.0` | -| agent_bridge | `2.2.1` | `openadapt-agent==2.2.1`
`openadapt-flow==2.5.0` | +| launcher_environment | `1.19.0` | `openadapt==1.19.0`
`openadapt-flow==2.6.0` | +| customer_runner | `2.6.0` | `openadapt-flow==2.6.0`
`openadapt-types==0.24.0` | +| desktop_sidecar | `0.25.1` | `openadapt-capture==1.3.0`
`openadapt-flow==2.4.1`
`openadapt-privacy==1.1.3`
`openadapt-types==0.24.0` | +| agent_bridge | `2.2.1` | `openadapt-agent==2.2.1`
`openadapt-flow==2.6.0` | ## Selected-platform dependency compatibility @@ -34,33 +34,33 @@ These edges show whether each selected integration version satisfies its publish | Scope | Source | Target | Constraint | Selected | Activation | |---|---|---|---|---:|---| -| desktop-sidecar-lock | `openadapt-desktop==0.16.0` | `openadapt-capture` | `>=1.2.1` | `1.2.1` | `always` | -| desktop-sidecar-lock | `openadapt-desktop==0.16.0` | `openadapt-flow` | `==1.34.0` | `1.34.0` | `extra == "build"` | -| desktop-sidecar-lock | `openadapt-desktop==0.16.0` | `openadapt-privacy` | `>=1.0.0` | `1.0.0` | `always` | -| desktop-sidecar-lock | `openadapt-desktop==0.16.0` | `openadapt-types` | `==0.10.0` | `0.10.0` | `always` | -| selected-platform | `openadapt-agent==2.2.1` | `openadapt-flow` | `<3,>=2.2.0` | `2.5.0` | `always`
`extra == "tutorial"` | +| desktop-sidecar-lock | `openadapt-desktop==0.25.1` | `openadapt-capture` | `>=1.3.0` | `1.3.0` | `always` | +| desktop-sidecar-lock | `openadapt-desktop==0.25.1` | `openadapt-flow` | `==2.4.1` | `2.4.1` | `extra == "build"` | +| desktop-sidecar-lock | `openadapt-desktop==0.25.1` | `openadapt-privacy` | `>=1.0.0` | `1.1.3` | `always` | +| desktop-sidecar-lock | `openadapt-desktop==0.25.1` | `openadapt-types` | `<0.25.0,>=0.24.0` | `0.24.0` | `always` | +| selected-platform | `openadapt-agent==2.2.1` | `openadapt-flow` | `<3,>=2.2.0` | `2.6.0` | `always`
`extra == "tutorial"` | | selected-platform | `openadapt-agent==2.2.1` | `openadapt-privacy` | `>=1.1.2` | `1.1.2` | `extra == "repair-screens"` | | selected-platform | `openadapt-capture==1.2.2` | `openadapt-privacy` | `>=0.1.0` | `1.1.2` | `extra == "all"`
`extra == "privacy"` | -| selected-platform | `openadapt-flow==2.5.0` | `openadapt-capture` | `>=1.2.0` | `1.2.2` | `extra == "capture"` | -| selected-platform | `openadapt-flow==2.5.0` | `openadapt-privacy` | `>=1.0.0` | `1.1.2` | `extra == "privacy"` | -| selected-platform | `openadapt-flow==2.5.0` | `openadapt-types` | `<0.25.0,>=0.24.0` | `0.24.0` | `always` | -| selected-platform | `openadapt==1.17.0` | `openadapt-agent` | `<3,>=2.1.0` | `2.2.1` | `python_version < "3.13"`
`python_version < "3.13" and extra == "agent"`
`python_version < "3.13" and extra == "all"` | -| selected-platform | `openadapt==1.17.0` | `openadapt-capture` | `<2.0.0,>=1.2.0` | `1.2.2` | `extra == "all"`
`extra == "capture"`
`extra == "core"` | -| selected-platform | `openadapt==1.17.0` | `openadapt-flow` | `<3.0.0,>=2.0.0` | `2.5.0` | `(sys_platform == "darwin" and python_version < "3.13") and extra == "all"`
`(sys_platform == "darwin" and python_version < "3.13") and extra == "macos"`
`(sys_platform == "linux" and python_version < "3.13") and extra == "all"`
`(sys_platform == "linux" and python_version < "3.13") and extra == "linux"`
`python_version < "3.13"`
`python_version < "3.13" and extra == "all"`
`python_version < "3.13" and extra == "browser"`
`python_version < "3.13" and extra == "capture"`
`python_version < "3.13" and extra == "core"`
`python_version < "3.13" and extra == "flow"`
`python_version < "3.13" and extra == "privacy"`
`python_version < "3.13" and extra == "rdp"`
`python_version < "3.13" and extra == "windows"` | +| selected-platform | `openadapt-flow==2.6.0` | `openadapt-capture` | `>=1.2.0` | `1.2.2` | `extra == "capture"` | +| selected-platform | `openadapt-flow==2.6.0` | `openadapt-privacy` | `>=1.0.0` | `1.1.2` | `extra == "privacy"` | +| selected-platform | `openadapt-flow==2.6.0` | `openadapt-types` | `<0.25.0,>=0.24.0` | `0.24.0` | `always` | +| selected-platform | `openadapt==1.19.0` | `openadapt-agent` | `<3,>=2.1.0` | `2.2.1` | `python_version < "3.13"`
`python_version < "3.13" and extra == "agent"`
`python_version < "3.13" and extra == "all"` | +| selected-platform | `openadapt==1.19.0` | `openadapt-capture` | `<2.0.0,>=1.2.0` | `1.2.2` | `extra == "all"`
`extra == "capture"`
`extra == "core"` | +| selected-platform | `openadapt==1.19.0` | `openadapt-flow` | `<3.0.0,>=2.5.0` | `2.6.0` | `(sys_platform == "darwin" and python_version < "3.13") and extra == "all"`
`(sys_platform == "darwin" and python_version < "3.13") and extra == "macos"`
`(sys_platform == "linux" and python_version < "3.13") and extra == "all"`
`(sys_platform == "linux" and python_version < "3.13") and extra == "linux"`
`python_version < "3.13"`
`python_version < "3.13" and extra == "all"`
`python_version < "3.13" and extra == "browser"`
`python_version < "3.13" and extra == "capture"`
`python_version < "3.13" and extra == "core"`
`python_version < "3.13" and extra == "flow"`
`python_version < "3.13" and extra == "privacy"`
`python_version < "3.13" and extra == "rdp"`
`python_version < "3.13" and extra == "windows"` | ## Cross-component schemas | Contract | Owner | Accepted versions | Exact source | |---|---|---|---| -| `workflow_bundle` | `flow` | `schema_version` [1, 2] | [`packages/flow/openadapt_flow/bundle_validation.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/bundle_validation.py) `63532fb8e49f25a8aba8dccca505362a9293b8db9cbc0f2904dd8f9a6e23fea8`
[`packages/flow/openadapt_flow/ir.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/ir.py) `efb7fdce6fa625f631901389ca767b016f4db3111caa2905321ff3da675e376d` | +| `workflow_bundle` | `flow` | `schema_version` [1, 2] | [`packages/flow/openadapt_flow/bundle_validation.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/bundle_validation.py) `63532fb8e49f25a8aba8dccca505362a9293b8db9cbc0f2904dd8f9a6e23fea8`
[`packages/flow/openadapt_flow/ir.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/ir.py) `509e9ef8666ec65496b4b7afdd0b2bc8af4ba0663e64a4079ce1813950bf028e` | | `capture_structural_observation` | `capture` | `openadapt.capture.structural-observation/v` [1] | [`openadapt_capture/structural.py`](https://github.com/OpenAdaptAI/openadapt-capture/blob/06ade22948c12dcc6608bd54b5cca5c9c306e77c/openadapt_capture/structural.py) `8f8bce37148a3357ace147c46a7c45e6d5501b70622b24b1ac2490f2def304fc` | | `control_overlay_frame` | `types` | `openadapt.control-overlay-frame/v` [1, 2] | [`packages/types/openadapt_types/control_overlay.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/f2f04c5b432d88b262d46103a542b0e2c3dcc8a9/packages/types/openadapt_types/control_overlay.py) `3aa01f89b43e12478c9e2ff1a9e61689397370691d15247ded9dcd5b05242790`
[`packages/types/openadapt_types/control_overlay_tracking.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/f2f04c5b432d88b262d46103a542b0e2c3dcc8a9/packages/types/openadapt_types/control_overlay_tracking.py) `af8d934f0572c5f4e9c63ba720d4a1d0e73c8cdf3f7b87cc4562c01a8a551755` | | `control_overlay_timeline` | `types` | `openadapt.control-overlay-timeline/v` [1, 2] | [`packages/types/openadapt_types/control_overlay.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/f2f04c5b432d88b262d46103a542b0e2c3dcc8a9/packages/types/openadapt_types/control_overlay.py) `3aa01f89b43e12478c9e2ff1a9e61689397370691d15247ded9dcd5b05242790`
[`packages/types/openadapt_types/control_overlay_tracking.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/f2f04c5b432d88b262d46103a542b0e2c3dcc8a9/packages/types/openadapt_types/control_overlay_tracking.py) `af8d934f0572c5f4e9c63ba720d4a1d0e73c8cdf3f7b87cc4562c01a8a551755` | | `human_decision_task` | `types` | `openadapt.human-decision-task/v` [1, 2] | [`packages/types/openadapt_types/human_decision.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/f2f04c5b432d88b262d46103a542b0e2c3dcc8a9/packages/types/openadapt_types/human_decision.py) `0bb39a3575f511e10d01d92858b340982adee2f0fc20d2409441dbc0b2c55c4f` | | `human_decision_receipt` | `types` | `openadapt.human-decision-receipt/v` [1] | [`packages/types/openadapt_types/human_decision.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/f2f04c5b432d88b262d46103a542b0e2c3dcc8a9/packages/types/openadapt_types/human_decision.py) `0bb39a3575f511e10d01d92858b340982adee2f0fc20d2409441dbc0b2c55c4f` | -| `human_decision_relay` | `flow` | `openadapt.human-decision-relay/v` [2] | [`packages/flow/openadapt_flow/console/decision_relay.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/console/decision_relay.py) `d5320404d4342dbb52be1f2ead1cdb5cf0e22877feaaa1e47974b301df697292` | -| `remote_decision_projection` | `flow` | `openadapt.remote-decision-projection/v` [1] | [`packages/flow/openadapt_flow/console/human_decisions.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/console/human_decisions.py) `02651ad8888a5c17bfff6fb47165ba861193243dc0f7c951f8b94f544ca19477` | -| `runtime_validation` | `flow` | `openadapt.runtime-validation/v` [1, 2, 3] | [`packages/flow/openadapt_flow/runtime_validation.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/runtime_validation.py) `a5a220f570c5ee5769ac2f21d8d140bb69373d4de2ccc0ebfd08540122883f0e` | +| `human_decision_relay` | `flow` | `openadapt.human-decision-relay/v` [2] | [`packages/flow/openadapt_flow/console/decision_relay.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/console/decision_relay.py) `d5320404d4342dbb52be1f2ead1cdb5cf0e22877feaaa1e47974b301df697292` | +| `remote_decision_projection` | `flow` | `openadapt.remote-decision-projection/v` [1] | [`packages/flow/openadapt_flow/console/human_decisions.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/console/human_decisions.py) `02651ad8888a5c17bfff6fb47165ba861193243dc0f7c951f8b94f544ca19477` | +| `runtime_validation` | `flow` | `openadapt.runtime-validation/v` [1, 2, 3] | [`packages/flow/openadapt_flow/runtime_validation.py`](https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/runtime_validation.py) `a5a220f570c5ee5769ac2f21d8d140bb69373d4de2ccc0ebfd08540122883f0e` | ## Host operating systems @@ -82,5 +82,5 @@ Compatibility and delivery facts do not assign lifecycle state. Use the signed P ## Qualification evidence - [openadapt-public-status](https://openadapt.ai/status.json): Canonical machine-readable release, capability, and substrate qualification status, maintained in openadapt-web (public/status.json). -- [flow-public-demo-evidence-packs](https://github.com/OpenAdaptAI/OpenAdapt/tree/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/public-demo/evidence-packs): Versioned public evidence packs with per-pack manifest files, replay evidence, and fault-case results. -- [flow-effectbench-task-pack](https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/benchmark/effectbench/task_pack/manifest.json): Effect-verification benchmark task pack manifest used by the flow qualification harness. +- [flow-public-demo-evidence-packs](https://github.com/OpenAdaptAI/OpenAdapt/tree/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/public-demo/evidence-packs): Versioned public evidence packs with per-pack manifest files, replay evidence, and fault-case results. +- [flow-effectbench-task-pack](https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/benchmark/effectbench/task_pack/manifest.json): Effect-verification benchmark task pack manifest used by the flow qualification harness. diff --git a/openadapt/cli.py b/openadapt/cli.py index cb1306fc9e..28bb728ebb 100644 --- a/openadapt/cli.py +++ b/openadapt/cli.py @@ -849,9 +849,9 @@ def _warn_about_launcher_conflicts() -> None: _SECRET_REFERENCE = re.compile(r"^(?:env:[A-Z][A-Z0-9_]*|keychain:[^/\s]+/[^/\s]+)$") # Flow 1.33 and 1.34 probe four X11 libraries in the wrong case, so on Linux # they refuse to download Chromium on hosts that have them. 1.35.0 fixed it. -# Launcher 1.19 requires Flow 2.5: `openadapt shadow` forwards to its shadow -# command, and `openadapt verify serve` offers its recheck. -_SUPPORTED_FLOW_RANGE = ">=2.5.0,<3.0.0" +# Launcher 1.20 requires Flow 2.6: ledger listing uses its persistent +# owed-recheck API. +_SUPPORTED_FLOW_RANGE = ">=2.6.0,<3.0.0" _RDP_INSTALL_COMMAND = _pip_install("openadapt[rdp]") _CHROMIUM_SYSTEM_LIBS_COMMAND = ( f"{shlex.quote(sys.executable)} -m playwright install-deps chromium" @@ -861,7 +861,7 @@ def _warn_about_launcher_conflicts() -> None: def _supported_flow_version(value: str) -> bool: """Return whether a stable Flow version is in the launcher's supported range.""" parsed = _release_tuple(value) - return parsed is not None and (2, 5, 0) <= parsed < (3, 0, 0) + return parsed is not None and (2, 6, 0) <= parsed < (3, 0, 0) # The Python modules each native or remote Flow backend imports when its @@ -1350,12 +1350,23 @@ def verify_ledger() -> None: @verify_ledger.command("list") @_LEDGER_OPTION def verify_ledger_list(ledger_dir: Optional[str]) -> None: - """Show held and open writes: key, state, when it began, outcome.""" + """Show held and open writes: key, state, when it began, outcome. + + Also each provisional CONFIRMED whose recheck is still owed, with its due + time, so a recheck isn't lost when the server that holds it restarts. + """ ledger = _write_ledger(ledger_dir) rows = ledger.unresolved() - if not rows: - click.echo("No held or open writes.") + owed = ledger.rechecks_owed() + if not rows and not owed: + click.echo("No held or open writes. No rechecks owed.") return + for key, attempt in owed: + click.echo( + f"{key} recheck-owed began {attempt.begun.get('at')} " + f"due {attempt.recheck_owed} by {attempt.begun.get('actor_kind')}:" + f"{attempt.begun.get('actor_id')}" + ) for key, attempt in rows: outcome = (attempt.resolved or {}).get("outcome", "-") state = "review" if attempt.review else ledger.state(attempt) diff --git a/platform-manifest.json b/platform-manifest.json index a59c049229..751c5b7899 100644 --- a/platform-manifest.json +++ b/platform-manifest.json @@ -1,7 +1,7 @@ { "manifest_kind": "openadapt-platform-release-manifest", "schema_version": "2.0.0", - "generated_at": "2026-11-03T00:23:35+00:00", + "generated_at": "2026-11-04T22:39:52+00:00", "generation": { "files": [ { @@ -19,9 +19,9 @@ "component_versions": { "agent": "2.2.1", "capture": "1.2.2", - "desktop": "0.16.0", - "flow": "2.5.0", - "launcher": "1.17.0", + "desktop": "0.25.1", + "flow": "2.6.0", + "launcher": "1.19.0", "privacy": "1.1.2", "types": "0.24.0" } @@ -29,7 +29,7 @@ "components": { "launcher": { "package": "openadapt", - "version": "1.17.0", + "version": "1.19.0", "source": "pypi", "requires_python": ">=3.10", "requires": { @@ -43,7 +43,7 @@ ">=0.1.0" ], "openadapt-flow": [ - "<3.0.0,>=2.0.0" + "<3.0.0,>=2.5.0" ], "openadapt-grounding": [ ">=0.1.0" @@ -115,109 +115,109 @@ }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": null, "marker": "python_version < \"3.13\" and extra == \"all\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": null, "marker": "python_version < \"3.13\" and extra == \"flow\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "browser", "marker": "python_version < \"3.13\" and extra == \"all\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "browser", "marker": "python_version < \"3.13\" and extra == \"browser\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "browser,hosted", "marker": "python_version < \"3.13\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "capture", "marker": "python_version < \"3.13\" and extra == \"all\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "capture", "marker": "python_version < \"3.13\" and extra == \"capture\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "capture", "marker": "python_version < \"3.13\" and extra == \"core\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "linux", "marker": "(sys_platform == \"linux\" and python_version < \"3.13\") and extra == \"all\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "linux", "marker": "(sys_platform == \"linux\" and python_version < \"3.13\") and extra == \"linux\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "macos", "marker": "(sys_platform == \"darwin\" and python_version < \"3.13\") and extra == \"all\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "macos", "marker": "(sys_platform == \"darwin\" and python_version < \"3.13\") and extra == \"macos\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "privacy", "marker": "python_version < \"3.13\" and extra == \"all\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "privacy", "marker": "python_version < \"3.13\" and extra == \"privacy\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "rdp", "marker": "python_version < \"3.13\" and extra == \"all\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "rdp", "marker": "python_version < \"3.13\" and extra == \"rdp\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "windows", "marker": "python_version < \"3.13\" and extra == \"all\"" }, { "package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", + "requires": "<3.0.0,>=2.5.0", "extras": "windows", "marker": "python_version < \"3.13\" and extra == \"windows\"" }, @@ -284,29 +284,29 @@ ], "provenance": { "repository": "OpenAdaptAI/OpenAdapt", - "release_ref": "v1.17.0", - "commit": "e022d97ee8b3c3992f4a4cb6e522f1532fef0834", - "tree": "2e0efeaa337a26d3425320ffa8f9aa3ac31d3f9c", - "url": "https://github.com/OpenAdaptAI/OpenAdapt/tree/v1.17.0" + "release_ref": "v1.19.0", + "commit": "1a64acd47bf6e60de20b7ac97993bd23122eb697", + "tree": "4ff3db2dc38334cd3957e66f89c95ea292abd9b8", + "url": "https://github.com/OpenAdaptAI/OpenAdapt/tree/v1.19.0" }, "artifacts": [ { "type": "bdist_wheel", - "filename": "openadapt-1.17.0-py3-none-any.whl", - "url": "https://files.pythonhosted.org/packages/55/aa/1184cad0892f2531fe54d24b808a3b76fafc87b77ccd865435ce2387ebef/openadapt-1.17.0-py3-none-any.whl", - "sha256": "c33c069c8cd9c270562200339b631fb77423b5b7f24364b5f0a6a0730e5ca8b8" + "filename": "openadapt-1.19.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/b4/23/925da0c3b52dde4e0eb26fed95c4c60d7e32bfe6c6678b05e85c9d9eb6ab/openadapt-1.19.0-py3-none-any.whl", + "sha256": "0a6963404d5b2a83aa83f8701c017838a8d09878128264b729064515b722a8b4" }, { "type": "sdist", - "filename": "openadapt-1.17.0.tar.gz", - "url": "https://files.pythonhosted.org/packages/19/8d/23723916889acfa5149e25c6eadf4aef0ca015aac72e0a9bf96a95eb4e57/openadapt-1.17.0.tar.gz", - "sha256": "03926f94f4d9d37eeb8c6c9cbdc94516d43c0ace971f2282b3310d77d73b6547" + "filename": "openadapt-1.19.0.tar.gz", + "url": "https://files.pythonhosted.org/packages/b4/12/e9eedc624f4451f88e423591876b753f3365714510866fb2cdb0382e5788/openadapt-1.19.0.tar.gz", + "sha256": "fc513011804644133ba8c93d41979cf2136829316839afb870e5092079551887" } ] }, "flow": { "package": "openadapt-flow", - "version": "2.5.0", + "version": "2.6.0", "source": "pypi", "requires_python": "<3.13,>=3.10", "requires": { @@ -351,24 +351,24 @@ ], "provenance": { "repository": "OpenAdaptAI/OpenAdapt", - "release_ref": "openadapt-flow-v2.5.0", - "commit": "5a3bf49a0bf2d662430386295a1c34036c064d58", - "tree": "cab536c90e79fcfe887b8876e884ec2ed651f23e", - "url": "https://github.com/OpenAdaptAI/OpenAdapt/tree/openadapt-flow-v2.5.0/packages/flow", + "release_ref": "openadapt-flow-v2.6.0", + "commit": "1c832365d11e9dbe2a166485b13f7b8effd6d8d0", + "tree": "954b03e36d350803bec59046ffaf56c02c3d556b", + "url": "https://github.com/OpenAdaptAI/OpenAdapt/tree/openadapt-flow-v2.6.0/packages/flow", "source_root": "packages/flow" }, "artifacts": [ { "type": "bdist_wheel", - "filename": "openadapt_flow-2.5.0-py3-none-any.whl", - "url": "https://files.pythonhosted.org/packages/7d/7b/76f2027c37fd059c0c1409cd6ca6c995da13fc5891c40b138f9b0cf1362f/openadapt_flow-2.5.0-py3-none-any.whl", - "sha256": "1dbc5d42d103cfa8a452d95805dde60c7d460035ec4d81df5cc642a32f118339" + "filename": "openadapt_flow-2.6.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/8f/a8/4cdb822b1245b47cbc80a27d30ab285e0239e1bc5d69b2077ce5495dbb93/openadapt_flow-2.6.0-py3-none-any.whl", + "sha256": "43718301bdb2579b1a78c60dfc025e8d60f2be9a3bc4172894c4194e0600fa9a" }, { "type": "sdist", - "filename": "openadapt_flow-2.5.0.tar.gz", - "url": "https://files.pythonhosted.org/packages/b3/9e/c4b87cfdba0b1ae03d57062bad41686cb9b3affd46e901ee1749fcf1e5b8/openadapt_flow-2.5.0.tar.gz", - "sha256": "54d99e77976ee7d9e6bc171b2ea148497528ead77b226fdf6fd4713b37c689b3" + "filename": "openadapt_flow-2.6.0.tar.gz", + "url": "https://files.pythonhosted.org/packages/7c/e6/dbe17ef0adf7c0f66e762eb30d49bdf3cb665fe43b400b6b7e6ae1b3c590/openadapt_flow-2.6.0.tar.gz", + "sha256": "4f1b2b9e28ec666c0e50ef2593d985ba387efc42bfb9b8f9f24148f6c8085112" } ] }, @@ -479,33 +479,33 @@ }, "desktop": { "package": "openadapt-desktop", - "version": "0.16.0", + "version": "0.25.1", "source": "pypi", "requires_python": ">=3.11", "requires": { "openadapt-capture": [ - ">=1.2.1" + ">=1.3.0" ], "openadapt-flow": [ - "==1.34.0" + "==2.4.1" ], "openadapt-privacy": [ ">=1.0.0" ], "openadapt-types": [ - "==0.10.0" + "<0.25.0,>=0.24.0" ] }, "dependency_constraints": [ { "package": "openadapt-capture", - "requires": ">=1.2.1", + "requires": ">=1.3.0", "extras": null, "marker": null }, { "package": "openadapt-flow", - "requires": "==1.34.0", + "requires": "==2.4.1", "extras": "browser,console", "marker": "extra == \"build\"" }, @@ -517,30 +517,31 @@ }, { "package": "openadapt-types", - "requires": "==0.10.0", + "requires": "<0.25.0,>=0.24.0", "extras": null, "marker": null } ], "provenance": { - "repository": "OpenAdaptAI/openadapt-desktop", - "release_ref": "v0.16.0", - "commit": "7c59750d37be09a2871cd1bada8cb0bce946d363", - "tree": "9a1307794df6f2c77ee3dab90973845f3d08ee53", - "url": "https://github.com/OpenAdaptAI/openadapt-desktop/tree/v0.16.0" + "repository": "OpenAdaptAI/OpenAdapt", + "release_ref": "openadapt-desktop-v0.25.1", + "commit": "b21a86d4a263f2bd8c4e8037e81f09457e998bdb", + "tree": "77acb2dfaab10e413f0d348ce98fbff9e24da752", + "url": "https://github.com/OpenAdaptAI/OpenAdapt/tree/openadapt-desktop-v0.25.1/packages/desktop", + "source_root": "packages/desktop" }, "artifacts": [ { "type": "bdist_wheel", - "filename": "openadapt_desktop-0.16.0-py3-none-any.whl", - "url": "https://files.pythonhosted.org/packages/46/15/a08ec0d22de26b8ad7d38a6496ea18eaebb669f3910b7472821d851e209d/openadapt_desktop-0.16.0-py3-none-any.whl", - "sha256": "06e608a934deef56c93ee1ca8de511d6daba08459626e8245bd0db4c391e64e9" + "filename": "openadapt_desktop-0.25.1-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/9c/fd/02310af2344b394cbc72901d73607b9816950e2246ce59d09e869bae916e/openadapt_desktop-0.25.1-py3-none-any.whl", + "sha256": "5f55ecffe173d0f7740cbbf011c19b763a495d9e1b37054f4f6ccf16a07a7be2" }, { "type": "sdist", - "filename": "openadapt_desktop-0.16.0.tar.gz", - "url": "https://files.pythonhosted.org/packages/82/6b/cd9bf7928765a71dab9f58b557499e05058c9ecee3ef30fd12963bd95729/openadapt_desktop-0.16.0.tar.gz", - "sha256": "15f0c59a4d481328da76f3e5aa6a29ab8cdced8ab183c97346d69b4c14c996ff" + "filename": "openadapt_desktop-0.25.1.tar.gz", + "url": "https://files.pythonhosted.org/packages/1e/b3/f93c118e93d03ff41f3c4db961cae256d7bbd157542e870d3afb36b8dd0f/openadapt_desktop-0.25.1.tar.gz", + "sha256": "5a9c22996b76a0efbcfa87d3710d88611ab2feb772de284d7233b9d38b04241b" } ] }, @@ -604,51 +605,51 @@ "runtime_units": { "launcher_environment": { "kind": "launcher-environment", - "version": "1.17.0", + "version": "1.19.0", "source_component": "launcher", "entry_point": "openadapt", "requirements": { - "openadapt": "==1.17.0", - "openadapt-flow": "<3.0.0,>=2.0.0" + "openadapt": "==1.19.0", + "openadapt-flow": "<3.0.0,>=2.5.0" }, "selected": { - "openadapt": "1.17.0", - "openadapt-flow": "2.5.0" + "openadapt": "1.19.0", + "openadapt-flow": "2.6.0" } }, "customer_runner": { "kind": "customer-controlled-runner", - "version": "2.5.0", + "version": "2.6.0", "source_component": "flow", "entry_point": "openadapt-flow", "requirements": { - "openadapt-flow": "==2.5.0", + "openadapt-flow": "==2.6.0", "openadapt-types": "<0.25.0,>=0.24.0" }, "selected": { - "openadapt-flow": "2.5.0", + "openadapt-flow": "2.6.0", "openadapt-types": "0.24.0" } }, "desktop_sidecar": { "kind": "frozen-desktop-sidecar", - "version": "0.16.0", + "version": "0.25.1", "source_component": "desktop", - "source_ref": "v0.16.0", - "lock_url": "https://raw.githubusercontent.com/OpenAdaptAI/openadapt-desktop/v0.16.0/uv.lock", - "lock_sha256": "9c2a57f02b96a7b505fba2b1cde0519968bd5744cd790cdb78bf86ce60008079", + "source_ref": "openadapt-desktop-v0.25.1", + "lock_url": "https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/openadapt-desktop-v0.25.1/packages/desktop/uv.lock", + "lock_sha256": "7c06835915ca3314cf7f40630c4ac45a7e8329e3565c3dc96bf251fa845903bb", "binary": "openadapt-engine", "requirements": { - "openadapt-flow": "==1.34.0", - "openadapt-capture": ">=1.2.1", + "openadapt-flow": "==2.4.1", + "openadapt-capture": ">=1.3.0", "openadapt-privacy": ">=1.0.0", - "openadapt-types": "==0.10.0" + "openadapt-types": "<0.25.0,>=0.24.0" }, "resolved": { - "openadapt-capture": "1.2.1", - "openadapt-flow": "1.34.0", - "openadapt-privacy": "1.0.0", - "openadapt-types": "0.10.0" + "openadapt-capture": "1.3.0", + "openadapt-flow": "2.4.1", + "openadapt-privacy": "1.1.3", + "openadapt-types": "0.24.0" } }, "agent_bridge": { @@ -662,7 +663,7 @@ }, "selected": { "openadapt-agent": "2.2.1", - "openadapt-flow": "2.5.0" + "openadapt-flow": "2.6.0" } } }, @@ -671,11 +672,11 @@ "scope": "desktop-sidecar-lock", "source_role": "desktop", "source_package": "openadapt-desktop", - "source_version": "0.16.0", + "source_version": "0.25.1", "target_role": "capture", "target_package": "openadapt-capture", - "requires": ">=1.2.1", - "resolved": "1.2.1", + "requires": ">=1.3.0", + "resolved": "1.3.0", "activations": [ { "extras": null, @@ -687,11 +688,11 @@ "scope": "desktop-sidecar-lock", "source_role": "desktop", "source_package": "openadapt-desktop", - "source_version": "0.16.0", + "source_version": "0.25.1", "target_role": "flow", "target_package": "openadapt-flow", - "requires": "==1.34.0", - "resolved": "1.34.0", + "requires": "==2.4.1", + "resolved": "2.4.1", "activations": [ { "extras": "browser,console", @@ -703,11 +704,11 @@ "scope": "desktop-sidecar-lock", "source_role": "desktop", "source_package": "openadapt-desktop", - "source_version": "0.16.0", + "source_version": "0.25.1", "target_role": "privacy", "target_package": "openadapt-privacy", "requires": ">=1.0.0", - "resolved": "1.0.0", + "resolved": "1.1.3", "activations": [ { "extras": null, @@ -719,11 +720,11 @@ "scope": "desktop-sidecar-lock", "source_role": "desktop", "source_package": "openadapt-desktop", - "source_version": "0.16.0", + "source_version": "0.25.1", "target_role": "types", "target_package": "openadapt-types", - "requires": "==0.10.0", - "resolved": "0.10.0", + "requires": "<0.25.0,>=0.24.0", + "resolved": "0.24.0", "activations": [ { "extras": null, @@ -739,7 +740,7 @@ "target_role": "flow", "target_package": "openadapt-flow", "requires": "<3,>=2.2.0", - "resolved": "2.5.0", + "resolved": "2.6.0", "activations": [ { "extras": null, @@ -791,7 +792,7 @@ "scope": "selected-platform", "source_role": "flow", "source_package": "openadapt-flow", - "source_version": "2.5.0", + "source_version": "2.6.0", "target_role": "capture", "target_package": "openadapt-capture", "requires": ">=1.2.0", @@ -807,7 +808,7 @@ "scope": "selected-platform", "source_role": "flow", "source_package": "openadapt-flow", - "source_version": "2.5.0", + "source_version": "2.6.0", "target_role": "privacy", "target_package": "openadapt-privacy", "requires": ">=1.0.0", @@ -823,7 +824,7 @@ "scope": "selected-platform", "source_role": "flow", "source_package": "openadapt-flow", - "source_version": "2.5.0", + "source_version": "2.6.0", "target_role": "types", "target_package": "openadapt-types", "requires": "<0.25.0,>=0.24.0", @@ -839,7 +840,7 @@ "scope": "selected-platform", "source_role": "launcher", "source_package": "openadapt", - "source_version": "1.17.0", + "source_version": "1.19.0", "target_role": "agent", "target_package": "openadapt-agent", "requires": "<3,>=2.1.0", @@ -863,7 +864,7 @@ "scope": "selected-platform", "source_role": "launcher", "source_package": "openadapt", - "source_version": "1.17.0", + "source_version": "1.19.0", "target_role": "capture", "target_package": "openadapt-capture", "requires": "<2.0.0,>=1.2.0", @@ -887,11 +888,11 @@ "scope": "selected-platform", "source_role": "launcher", "source_package": "openadapt", - "source_version": "1.17.0", + "source_version": "1.19.0", "target_role": "flow", "target_package": "openadapt-flow", - "requires": "<3.0.0,>=2.0.0", - "resolved": "2.5.0", + "requires": "<3.0.0,>=2.5.0", + "resolved": "2.6.0", "activations": [ { "extras": null, @@ -979,19 +980,19 @@ ], "minimum": 1, "maximum": 2, - "source_commit": "5a3bf49a0bf2d662430386295a1c34036c064d58", + "source_commit": "1c832365d11e9dbe2a166485b13f7b8effd6d8d0", "source_files": [ { "path": "packages/flow/openadapt_flow/bundle_validation.py", - "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/bundle_validation.py", + "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/bundle_validation.py", "git_blob": "78c30442341699d497d934699154a72fe213791d", "sha256": "63532fb8e49f25a8aba8dccca505362a9293b8db9cbc0f2904dd8f9a6e23fea8" }, { "path": "packages/flow/openadapt_flow/ir.py", - "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/ir.py", - "git_blob": "fd235efbebf639c3f047eb0ca7421115733cfb85", - "sha256": "efb7fdce6fa625f631901389ca767b016f4db3111caa2905321ff3da675e376d" + "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/ir.py", + "git_blob": "494fa4722f0c76705b3e09f982c68938c5358e9d", + "sha256": "509e9ef8666ec65496b4b7afdd0b2bc8af4ba0663e64a4079ce1813950bf028e" } ] }, @@ -1108,11 +1109,11 @@ ], "minimum": 2, "maximum": 2, - "source_commit": "5a3bf49a0bf2d662430386295a1c34036c064d58", + "source_commit": "1c832365d11e9dbe2a166485b13f7b8effd6d8d0", "source_files": [ { "path": "packages/flow/openadapt_flow/console/decision_relay.py", - "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/console/decision_relay.py", + "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/console/decision_relay.py", "git_blob": "297d2f71727f83496221091a4237d704c25665d6", "sha256": "d5320404d4342dbb52be1f2ead1cdb5cf0e22877feaaa1e47974b301df697292" } @@ -1126,11 +1127,11 @@ ], "minimum": 1, "maximum": 1, - "source_commit": "5a3bf49a0bf2d662430386295a1c34036c064d58", + "source_commit": "1c832365d11e9dbe2a166485b13f7b8effd6d8d0", "source_files": [ { "path": "packages/flow/openadapt_flow/console/human_decisions.py", - "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/console/human_decisions.py", + "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/console/human_decisions.py", "git_blob": "caaf87396aca4770eace5eead3fdf6515e31fbfc", "sha256": "02651ad8888a5c17bfff6fb47165ba861193243dc0f7c951f8b94f544ca19477" } @@ -1146,11 +1147,11 @@ ], "minimum": 1, "maximum": 3, - "source_commit": "5a3bf49a0bf2d662430386295a1c34036c064d58", + "source_commit": "1c832365d11e9dbe2a166485b13f7b8effd6d8d0", "source_files": [ { "path": "packages/flow/openadapt_flow/runtime_validation.py", - "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/openadapt_flow/runtime_validation.py", + "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/openadapt_flow/runtime_validation.py", "git_blob": "fd6121209e50abcab41bb8b70f95ff2647d9303f", "sha256": "a5a220f570c5ee5769ac2f21d8d140bb69373d4de2ccc0ebfd08540122883f0e" } @@ -1170,7 +1171,7 @@ ">=0.1.0" ], "openadapt-flow": [ - ">=2.5.0,<3.0.0" + ">=2.6.0,<3.0.0" ], "openadapt-grounding": [ ">=0.1.0" @@ -1236,10 +1237,10 @@ "repository": "OpenAdaptAI/OpenAdapt", "path": "packages/flow/public-demo/evidence-packs", "description": "Versioned public evidence packs with per-pack manifest files, replay evidence, and fault-case results.", - "source_commit": "5a3bf49a0bf2d662430386295a1c34036c064d58", + "source_commit": "1c832365d11e9dbe2a166485b13f7b8effd6d8d0", "source_object_type": "tree", "source_object_sha": "2eac2252f62349e873b461866546dd0295169a19", - "url": "https://github.com/OpenAdaptAI/OpenAdapt/tree/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/public-demo/evidence-packs" + "url": "https://github.com/OpenAdaptAI/OpenAdapt/tree/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/public-demo/evidence-packs" }, { "id": "flow-effectbench-task-pack", @@ -1248,10 +1249,10 @@ "repository": "OpenAdaptAI/OpenAdapt", "path": "packages/flow/benchmark/effectbench/task_pack/manifest.json", "description": "Effect-verification benchmark task pack manifest used by the flow qualification harness.", - "source_commit": "5a3bf49a0bf2d662430386295a1c34036c064d58", + "source_commit": "1c832365d11e9dbe2a166485b13f7b8effd6d8d0", "source_object_type": "blob", "source_object_sha": "1c568d2289342e10e3556c5c46cd08a0e370ac84", - "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/5a3bf49a0bf2d662430386295a1c34036c064d58/packages/flow/benchmark/effectbench/task_pack/manifest.json" + "url": "https://github.com/OpenAdaptAI/OpenAdapt/blob/1c832365d11e9dbe2a166485b13f7b8effd6d8d0/packages/flow/benchmark/effectbench/task_pack/manifest.json" } ], "signature": { diff --git a/pyproject.toml b/pyproject.toml index a59fbe1195..42a50f5d4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openadapt" -version = "1.19.0" +version = "1.20.0" description = "Enters approved information into the apps your team already uses, checks that it saved, and stops for a person when something doesn't match." # The package description is README.md with its lifecycle block swapped for a # link to the live record; see hatch_build.py. @@ -59,10 +59,9 @@ dependencies = [ # The quickstart's terminal view (MIT). Pure Python; the engine doesn't # depend on it. Gated like the engine: without it the quickstart can't run. "rich>=13.0.0; python_version < '3.13'", - # Every Flow floor is 2.5.0: `openadapt shadow` forwards to Flow's shadow - # command, and `openadapt verify serve` offers its recheck, which 2.4 and - # earlier lack (tests/test_flow_floor.py). - "openadapt-flow[browser,hosted]>=2.5.0,<3.0.0; python_version < '3.13'", + # Every Flow floor is 2.6.0: ledger listing uses its persistent owed-recheck + # API (tests/test_flow_floor.py). + "openadapt-flow[browser,hosted]>=2.6.0,<3.0.0; python_version < '3.13'", # Day-1 partner kit: `pip install openadapt` must expose the # `openadapt-agent` command without a second package. 2.1.0 adds Seal checks; # `--bundles`; the bundle-free tutorial `serve --allow-run` needs a later @@ -77,7 +76,7 @@ browser = [ # launcher now selects this Flow contract because the bundled quickstart # uses the browser. OpenAdapt requests Chromium on first browser use after # the host dependency check passes. - "openadapt-flow[browser]>=2.5.0,<3.0.0; python_version < '3.13'", + "openadapt-flow[browser]>=2.6.0,<3.0.0; python_version < '3.13'", ] # Local human desktop recording. Flow owns the supported capture adapter # contract; the direct floor prevents an already-installed pre-1.0 Capture @@ -85,22 +84,22 @@ browser = [ # separately provisioned executable, not a Python-package dependency. capture = [ "openadapt-capture>=1.2.0,<2.0.0", - "openadapt-flow[capture]>=2.5.0,<3.0.0; python_version < '3.13'", + "openadapt-flow[capture]>=2.6.0,<3.0.0; python_version < '3.13'", ] # Replay substrate dependencies stay separate from capture: recording observes # the human on the local interactive desktop, while replay may target a WAA # agent, a native app, or a remote framebuffer. windows = [ - "openadapt-flow[windows]>=2.5.0,<3.0.0; python_version < '3.13'", + "openadapt-flow[windows]>=2.6.0,<3.0.0; python_version < '3.13'", ] macos = [ - "openadapt-flow[macos]>=2.5.0,<3.0.0; sys_platform == 'darwin' and python_version < '3.13'", + "openadapt-flow[macos]>=2.6.0,<3.0.0; sys_platform == 'darwin' and python_version < '3.13'", ] linux = [ - "openadapt-flow[linux]>=2.5.0,<3.0.0; sys_platform == 'linux' and python_version < '3.13'", + "openadapt-flow[linux]>=2.6.0,<3.0.0; sys_platform == 'linux' and python_version < '3.13'", ] rdp = [ - "openadapt-flow[rdp]>=2.5.0,<3.0.0; python_version < '3.13'", + "openadapt-flow[rdp]>=2.6.0,<3.0.0; python_version < '3.13'", ] ml = [ "openadapt-ml>=0.2.0", @@ -118,13 +117,13 @@ retrieval = [ "openadapt-retrieval>=0.1.0", ] privacy = [ - "openadapt-flow[privacy]>=2.5.0,<3.0.0; python_version < '3.13'", + "openadapt-flow[privacy]>=2.6.0,<3.0.0; python_version < '3.13'", ] # `flow` now ships in the base install (see `dependencies` above). This # extra is kept for backward compatibility so `pip install openadapt[flow]` # and the `all` bundle still resolve. flow = [ - "openadapt-flow>=2.5.0,<3.0.0; python_version < '3.13'", + "openadapt-flow>=2.6.0,<3.0.0; python_version < '3.13'", ] # `openadapt-agent` also ships in the base install. Keep the extra so # `pip install openadapt[agent]` still resolves. diff --git a/tests/test_cli_shell_hints.py b/tests/test_cli_shell_hints.py index 129718ff3c..4dd431f04e 100644 --- a/tests/test_cli_shell_hints.py +++ b/tests/test_cli_shell_hints.py @@ -95,12 +95,12 @@ def test_deploy_console_hint_is_double_quoted(monkeypatch, supported_python): "importlib.util.find_spec", lambda name, *args: object() if name in present else None, ) - monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.5.1") + monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.6.1") result = CliRunner().invoke(cli_main, ["deploy"]) assert result.exit_code == 0, result.output - assert '"openadapt-flow[console]==2.5.1"' in result.output + assert '"openadapt-flow[console]==2.6.1"' in result.output assert not _UNQUOTED_SPEC.search(result.output), result.output diff --git a/tests/test_cli_smoke.py b/tests/test_cli_smoke.py index 91f17f2afc..0b08f91c71 100644 --- a/tests/test_cli_smoke.py +++ b/tests/test_cli_smoke.py @@ -380,29 +380,29 @@ def test_launcher_flow_and_substrate_extras_metadata(): assert ( metadata["dependencies"].count( - f"openadapt-flow[browser,hosted]>=2.5.0,<3.0.0; {supported}" + f"openadapt-flow[browser,hosted]>=2.6.0,<3.0.0; {supported}" ) == 1 ) assert ( metadata["dependencies"].count(f"openadapt-agent>=2.1.0,<3; {supported}") == 1 ) - assert extras["flow"] == [f"openadapt-flow>=2.5.0,<3.0.0; {supported}"] + assert extras["flow"] == [f"openadapt-flow>=2.6.0,<3.0.0; {supported}"] assert extras["agent"] == [f"openadapt-agent>=2.1.0,<3; {supported}"] - assert extras["browser"] == [f"openadapt-flow[browser]>=2.5.0,<3.0.0; {supported}"] - assert extras["privacy"] == [f"openadapt-flow[privacy]>=2.5.0,<3.0.0; {supported}"] + assert extras["browser"] == [f"openadapt-flow[browser]>=2.6.0,<3.0.0; {supported}"] + assert extras["privacy"] == [f"openadapt-flow[privacy]>=2.6.0,<3.0.0; {supported}"] assert extras["capture"] == [ "openadapt-capture>=1.2.0,<2.0.0", - f"openadapt-flow[capture]>=2.5.0,<3.0.0; {supported}", + f"openadapt-flow[capture]>=2.6.0,<3.0.0; {supported}", ] - assert extras["windows"] == [f"openadapt-flow[windows]>=2.5.0,<3.0.0; {supported}"] + assert extras["windows"] == [f"openadapt-flow[windows]>=2.6.0,<3.0.0; {supported}"] assert extras["macos"] == [ - f"openadapt-flow[macos]>=2.5.0,<3.0.0; sys_platform == 'darwin' and {supported}" + f"openadapt-flow[macos]>=2.6.0,<3.0.0; sys_platform == 'darwin' and {supported}" ] assert extras["linux"] == [ - f"openadapt-flow[linux]>=2.5.0,<3.0.0; sys_platform == 'linux' and {supported}" + f"openadapt-flow[linux]>=2.6.0,<3.0.0; sys_platform == 'linux' and {supported}" ] - assert extras["rdp"] == [f"openadapt-flow[rdp]>=2.5.0,<3.0.0; {supported}"] + assert extras["rdp"] == [f"openadapt-flow[rdp]>=2.6.0,<3.0.0; {supported}"] assert extras["all"] == [ "openadapt[browser,core,grounding,retrieval,privacy,flow,windows,rdp,agent]", "openadapt[macos]; sys_platform == 'darwin'", @@ -433,7 +433,7 @@ def test_released_flow_x11_sonames_are_case_corrected(monkeypatch): ) -@pytest.mark.parametrize("version", ["2.5.0", "2.5.1", "2.99.0"]) +@pytest.mark.parametrize("version", ["2.6.0", "2.6.1", "2.99.0"]) def test_launcher_accepts_flow_versions_in_the_supported_range(version): assert _supported_flow_version(version) @@ -533,7 +533,7 @@ def test_deploy_native_backend_fails_without_its_platform_extra( monkeypatch, backend, host, modules, install ): _host(monkeypatch, host, {"openadapt_flow"}) - monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.5.0") + monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.6.0") result = CliRunner().invoke(cli_main, ["deploy", "--backend", backend]) @@ -550,7 +550,7 @@ def test_deploy_native_backend_passes_with_its_platform_extra( monkeypatch, backend, host, modules, install ): _host(monkeypatch, host, {"openadapt_flow"} | modules) - monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.5.0") + monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.6.0") result = CliRunner().invoke(cli_main, ["deploy", "--backend", backend]) @@ -690,7 +690,7 @@ def test_deploy_preflight_composes_existing_flow_interfaces_without_secrets( else None ), ) - monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.5.0") + monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.6.0") monkeypatch.setenv("BYOC_CONNECTOR_TOKEN", "connector-token-value") result = CliRunner().invoke( cli_main, @@ -715,13 +715,13 @@ def test_deploy_base_hosted_install_gives_conditional_console_setup(monkeypatch) "importlib.util.find_spec", lambda name: object() if name in {"openadapt_flow", "aardwolf"} else None, ) - monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.5.0") + monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.6.0") result = CliRunner().invoke(cli_main, ["deploy", "--backend", "rdp"]) assert result.exit_code == 0, result.output assert "optional local operator console is not installed" in result.output - assert 'python -m pip install "openadapt-flow[console]==2.5.0"' in result.output + assert 'python -m pip install "openadapt-flow[console]==2.6.0"' in result.output assert "openadapt flow console --bundles" not in result.output assert "Re-run this preflight" in result.output @@ -736,13 +736,13 @@ def test_deploy_console_requires_openadapt_types(monkeypatch): else None ), ) - monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.5.0") + monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.6.0") result = CliRunner().invoke(cli_main, ["deploy", "--backend", "rdp"]) assert result.exit_code == 0, result.output assert "optional local operator console is not installed" in result.output - assert 'python -m pip install "openadapt-flow[console]==2.5.0"' in result.output + assert 'python -m pip install "openadapt-flow[console]==2.6.0"' in result.output assert "openadapt flow console --bundles" not in result.output @@ -767,7 +767,7 @@ def test_deploy_preflight_fails_without_web_runtime(monkeypatch): "importlib.util.find_spec", lambda name: object() if name == "openadapt_flow" else None, ) - monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.5.0") + monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.6.0") result = CliRunner().invoke(cli_main, ["deploy", "--backend", "web"]) @@ -785,7 +785,7 @@ def test_deploy_preflight_fails_without_rdp_transport(monkeypatch): "importlib.util.find_spec", lambda name: object() if name == "openadapt_flow" else None, ) - monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.5.0") + monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.6.0") result = CliRunner().invoke(cli_main, ["deploy", "--backend", "rdp"]) @@ -807,7 +807,7 @@ def test_deploy_preflight_fails_for_unsupported_flow(monkeypatch, flow_version): assert result.exit_code != 0 assert "[UNSUPPORTED]" in result.output - assert ">=2.5.0,<3.0.0" in result.output + assert ">=2.6.0,<3.0.0" in result.output assert "Preflight passed" not in result.output diff --git a/tests/test_deploy_preflight_readiness.py b/tests/test_deploy_preflight_readiness.py index d1eef428ad..12ff5ac351 100644 --- a/tests/test_deploy_preflight_readiness.py +++ b/tests/test_deploy_preflight_readiness.py @@ -29,7 +29,7 @@ def _deploy(monkeypatch, args, installed=READY_WEB, host=None): "importlib.util.find_spec", lambda name: object() if name in installed else None, ) - monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.5.0") + monkeypatch.setattr("importlib.metadata.version", lambda _name: "2.6.0") return CliRunner().invoke(cli_main, ["deploy", *args]) diff --git a/tests/test_flow_floor.py b/tests/test_flow_floor.py index 751df09bcb..21c03b77c4 100644 --- a/tests/test_flow_floor.py +++ b/tests/test_flow_floor.py @@ -24,10 +24,10 @@ import tomli as tomllib ROOT = Path(__file__).resolve().parents[1] -# Launcher 1.19 requires Flow 2.5 (`openadapt shadow` and the recheck that -# `openadapt verify serve` offers), so 1.x and 2.0 to 2.4 are below the floor. -BELOW_FLOOR = ["1.29.0", "1.35.1", "2.0.0", "2.2.0", "2.4.1"] -ACCEPTED = "2.5.0" +# Launcher 1.20 requires Flow 2.6's persistent owed-recheck API. +# Keep every earlier rejection case, including the former 2.5 floor. +BELOW_FLOOR = ["1.29.0", "1.35.1", "2.0.0", "2.2.0", "2.4.1", "2.5.0", "2.5.1"] +ACCEPTED = "2.6.0" def _flow_requirements() -> list[Requirement]: diff --git a/tests/test_verify_command.py b/tests/test_verify_command.py index 088b6e00c1..9ce361a041 100644 --- a/tests/test_verify_command.py +++ b/tests/test_verify_command.py @@ -103,3 +103,30 @@ def test_verify_ledger_lists_and_releases_a_held_write(tmp_path) -> None: assert released.exit_code == 0, released.output listed = runner.invoke(main, ["verify", "ledger", "list", "--ledger", str(folder)]) assert "No held or open writes." in listed.output + + +def test_verify_ledger_lists_a_recheck_owed_until_it_runs(tmp_path) -> None: + ledger_mod = pytest.importorskip("openadapt_flow.write_ledger") + folder = tmp_path / "ledger" + ledger = ledger_mod.Ledger(ledger_mod.FileLedger(folder)) + key = "d" * 64 + due = "2026-01-01T00:02:00Z" + ledger.begun(key, "write-2", contract="c" * 64, actor_kind="agent", actor_id="bot") + ledger.resolved( + key, + "write-2", + outcome="CONFIRMED", + retry_safe=False, + delivered=True, + result={"outcome": "CONFIRMED", "provisional": True, "recheck_due_at": due}, + ) + runner = CliRunner() + listed = runner.invoke(main, ["verify", "ledger", "list", "--ledger", str(folder)]) + assert listed.exit_code == 0, listed.output + assert key in listed.output and "recheck-owed" in listed.output + assert f"due {due}" in listed.output + + ledger.rechecked(key, "write-2", outcome="CONFIRMED") + listed = runner.invoke(main, ["verify", "ledger", "list", "--ledger", str(folder)]) + assert "recheck-owed" not in listed.output + assert "No rechecks owed." in listed.output diff --git a/uv.lock b/uv.lock index 9b636a0e1b..48a0fc1748 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -7,10 +7,10 @@ resolution-markers = [ "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform != 'win32'", "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform == 'darwin'", "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform == 'linux'", "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version < '3.11' and sys_platform == 'win32'", ] @@ -525,7 +525,7 @@ name = "build" version = "1.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "(python_full_version >= '3.13' and os_name == 'nt' and sys_platform == 'darwin') or (python_full_version >= '3.13' and os_name == 'nt' and sys_platform == 'linux') or (os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "colorama", marker = "(python_full_version >= '3.13' and os_name == 'nt') or (os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux')" }, { name = "importlib-metadata", marker = "python_full_version < '3.10.2'" }, { name = "packaging" }, { name = "pyproject-hooks" }, @@ -3132,6 +3132,7 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version < '3.11' and sys_platform == 'win32'", ] dependencies = [ @@ -3167,21 +3168,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b6/ca/862b1e7a639460f0ca25fd5b6135fb42cf9deea86d398a92e44dfda2279d/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2b9233c4947907fd1818d0e581c049c41ccc39b2856cc942ff6d26317cee145", size = 17394184, upload-time = "2025-10-22T03:47:08.127Z" }, ] -[[package]] -name = "onnxruntime" -version = "1.24.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", -] -dependencies = [ - { name = "flatbuffers" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, -] - [[package]] name = "onnxruntime" version = "1.27.0" @@ -3245,7 +3231,7 @@ wheels = [ [[package]] name = "openadapt" -version = "1.19.0" +version = "1.20.0" source = { editable = "." } dependencies = [ { name = "click" }, @@ -3335,15 +3321,15 @@ requires-dist = [ { name = "openadapt-agent", marker = "python_full_version < '3.13' and extra == 'agent'", specifier = ">=2.1.0,<3" }, { name = "openadapt-capture", marker = "extra == 'capture'", specifier = ">=1.2.0,<2.0.0" }, { name = "openadapt-evals", marker = "extra == 'evals'", specifier = ">=0.1.0" }, - { name = "openadapt-flow", marker = "python_full_version < '3.13' and extra == 'flow'", specifier = ">=2.5.0,<3.0.0" }, - { name = "openadapt-flow", extras = ["browser"], marker = "python_full_version < '3.13' and extra == 'browser'", specifier = ">=2.5.0,<3.0.0" }, - { name = "openadapt-flow", extras = ["browser", "hosted"], marker = "python_full_version < '3.13'", specifier = ">=2.5.0,<3.0.0" }, - { name = "openadapt-flow", extras = ["capture"], marker = "python_full_version < '3.13' and extra == 'capture'", specifier = ">=2.5.0,<3.0.0" }, - { name = "openadapt-flow", extras = ["linux"], marker = "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'linux'", specifier = ">=2.5.0,<3.0.0" }, - { name = "openadapt-flow", extras = ["macos"], marker = "python_full_version < '3.13' and sys_platform == 'darwin' and extra == 'macos'", specifier = ">=2.5.0,<3.0.0" }, - { name = "openadapt-flow", extras = ["privacy"], marker = "python_full_version < '3.13' and extra == 'privacy'", specifier = ">=2.5.0,<3.0.0" }, - { name = "openadapt-flow", extras = ["rdp"], marker = "python_full_version < '3.13' and extra == 'rdp'", specifier = ">=2.5.0,<3.0.0" }, - { name = "openadapt-flow", extras = ["windows"], marker = "python_full_version < '3.13' and extra == 'windows'", specifier = ">=2.5.0,<3.0.0" }, + { name = "openadapt-flow", marker = "python_full_version < '3.13' and extra == 'flow'", specifier = ">=2.6.0,<3.0.0" }, + { name = "openadapt-flow", extras = ["browser"], marker = "python_full_version < '3.13' and extra == 'browser'", specifier = ">=2.6.0,<3.0.0" }, + { name = "openadapt-flow", extras = ["browser", "hosted"], marker = "python_full_version < '3.13'", specifier = ">=2.6.0,<3.0.0" }, + { name = "openadapt-flow", extras = ["capture"], marker = "python_full_version < '3.13' and extra == 'capture'", specifier = ">=2.6.0,<3.0.0" }, + { name = "openadapt-flow", extras = ["linux"], marker = "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'linux'", specifier = ">=2.6.0,<3.0.0" }, + { name = "openadapt-flow", extras = ["macos"], marker = "python_full_version < '3.13' and sys_platform == 'darwin' and extra == 'macos'", specifier = ">=2.6.0,<3.0.0" }, + { name = "openadapt-flow", extras = ["privacy"], marker = "python_full_version < '3.13' and extra == 'privacy'", specifier = ">=2.6.0,<3.0.0" }, + { name = "openadapt-flow", extras = ["rdp"], marker = "python_full_version < '3.13' and extra == 'rdp'", specifier = ">=2.6.0,<3.0.0" }, + { name = "openadapt-flow", extras = ["windows"], marker = "python_full_version < '3.13' and extra == 'windows'", specifier = ">=2.6.0,<3.0.0" }, { name = "openadapt-grounding", marker = "extra == 'grounding'", specifier = ">=0.1.0" }, { name = "openadapt-ml", marker = "extra == 'ml'", specifier = ">=0.2.0" }, { name = "openadapt-retrieval", marker = "extra == 'retrieval'", specifier = ">=0.1.0" }, @@ -3437,7 +3423,7 @@ wheels = [ [[package]] name = "openadapt-flow" -version = "2.5.0" +version = "2.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, @@ -3452,9 +3438,9 @@ dependencies = [ { name = "rapidocr-onnxruntime" }, { name = "tomli", marker = "python_full_version < '3.11' or python_full_version >= '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b3/9e/c4b87cfdba0b1ae03d57062bad41686cb9b3affd46e901ee1749fcf1e5b8/openadapt_flow-2.5.0.tar.gz", hash = "sha256:54d99e77976ee7d9e6bc171b2ea148497528ead77b226fdf6fd4713b37c689b3", size = 31358408, upload-time = "2026-11-03T00:16:39.944Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/e6/dbe17ef0adf7c0f66e762eb30d49bdf3cb665fe43b400b6b7e6ae1b3c590/openadapt_flow-2.6.0.tar.gz", hash = "sha256:4f1b2b9e28ec666c0e50ef2593d985ba387efc42bfb9b8f9f24148f6c8085112", size = 33981672, upload-time = "2026-11-04T22:21:34.154Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/7b/76f2027c37fd059c0c1409cd6ca6c995da13fc5891c40b138f9b0cf1362f/openadapt_flow-2.5.0-py3-none-any.whl", hash = "sha256:1dbc5d42d103cfa8a452d95805dde60c7d460035ec4d81df5cc642a32f118339", size = 4335676, upload-time = "2026-11-03T00:16:37.35Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a8/4cdb822b1245b47cbc80a27d30ab285e0239e1bc5d69b2077ce5495dbb93/openadapt_flow-2.6.0-py3-none-any.whl", hash = "sha256:43718301bdb2579b1a78c60dfc025e8d60f2be9a3bc4172894c4194e0600fa9a", size = 4390728, upload-time = "2026-11-04T22:21:30.764Z" }, ] [package.optional-dependencies] @@ -4828,8 +4814,7 @@ version = "1.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, - { name = "onnxruntime", version = "1.23.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'darwin') or (python_full_version >= '3.13' and sys_platform == 'darwin') or (python_full_version < '3.11' and sys_platform == 'linux') or (python_full_version >= '3.13' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.13' and sys_platform == 'win32')" }, - { name = "onnxruntime", version = "1.24.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32') or (python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "onnxruntime", version = "1.23.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.13'" }, { name = "onnxruntime", version = "1.27.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, { name = "opencv-python" }, { name = "pillow" }, From 9ec50e43b7121448dc9a3698f5524b2996784dbf Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 20:52:00 -0800 Subject: [PATCH 11/21] Add generic phase timing observations and coverage analysis (#1931) * feat: collect explicit benchmark timing observations Signed-off-by: abrichr * fix: narrow optional timing values before comparison Signed-off-by: abrichr * test: preserve timing coverage and outer phase diagnostics Signed-off-by: abrichr * fix: bind untimed observation fields by name Signed-off-by: abrichr * ci: check the generic benchmark timing contract Signed-off-by: abrichr * fix: retain missing named phase coverage explicitly Signed-off-by: abrichr --------- Signed-off-by: abrichr --- .github/workflows/packages.yml | 6 +- .../flow/benchmark/phase_latency/__init__.py | 1 + .../flow/benchmark/phase_latency/collector.py | 469 ++++++++++++++++++ packages/flow/tests/test_phase_latency.py | 374 ++++++++++++++ 4 files changed, 848 insertions(+), 2 deletions(-) create mode 100644 packages/flow/benchmark/phase_latency/__init__.py create mode 100644 packages/flow/benchmark/phase_latency/collector.py create mode 100644 packages/flow/tests/test_phase_latency.py diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index c8e4aa0e12..48e9ba0828 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -357,7 +357,7 @@ jobs: # every public data/config asset listed in public-artifacts.json. run: python scripts/check_release_consistency.py - name: Ruff lint - run: ruff check openadapt_flow benchmark/harness + run: ruff check openadapt_flow benchmark/harness benchmark/phase_latency - name: No app-specific code in the engine # App knowledge lives in profiles, contracts and fixtures (CONTRIBUTING: # "App knowledge lives in data"); the denylist is scripts/generality.toml. @@ -376,9 +376,11 @@ jobs: ruff check --isolated --select E9 --target-version py310 . uv run --no-project --python 3.10 python -m compileall -q -x '(^|/)(\.venv|node_modules|build|dist)/' . - name: Ruff format check - run: ruff format --check openadapt_flow tests benchmark/harness + run: ruff format --check openadapt_flow tests benchmark/harness benchmark/phase_latency - name: Mypy run: mypy + - name: Phase timing contract types + run: mypy --follow-imports=silent --ignore-missing-imports benchmark/phase_latency/collector.py - name: Mypy (strict, safety path only) run: | mypy \ diff --git a/packages/flow/benchmark/phase_latency/__init__.py b/packages/flow/benchmark/phase_latency/__init__.py new file mode 100644 index 0000000000..915081269d --- /dev/null +++ b/packages/flow/benchmark/phase_latency/__init__.py @@ -0,0 +1 @@ +"""Generic numeric timing contracts; measurements remain private.""" diff --git a/packages/flow/benchmark/phase_latency/collector.py b/packages/flow/benchmark/phase_latency/collector.py new file mode 100644 index 0000000000..e415208090 --- /dev/null +++ b/packages/flow/benchmark/phase_latency/collector.py @@ -0,0 +1,469 @@ +"""Numeric-only timing observations; no fixture control or artifact publishing. + +Step spans are exclusive. Outer intervals are inclusive and remain a separate +basis. Uncovered time is unattributed, never inferred engine overhead. +""" + +from __future__ import annotations + +import math +import re +import statistics +import time +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from typing import Any, Callable, Iterator, Mapping, Sequence + +from openadapt_flow.runtime.step_timing import SPANS + +STEP_GROUPS = { + "settle_before": "settle", + "settle_after": "settle", + "identity": "identity", + "actuate": "actions", + "postcondition": "verification", + "readback": "verification", + "resolve": "resolution", + "screenshot": "capture_ocr", + "ocr": "capture_ocr", + "artifact_write": "evidence", + "other": "unattributed_step", +} +OUTER_PHASES = frozenset( + { + "navigation", + "session", + "authentication", + "admission", + "replay", + "oracle", + "finalization", + "engine_bookkeeping", + } +) +OUTCOMES = frozenset( + {"VERIFIED", "HALTED", "FAILED", "COMPLETED_UNVERIFIED", "ROLLED_BACK"} +) + + +def _number(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + ) + + +@dataclass(frozen=True, order=True) +class Trial: + """Opaque app label and a preregistered matrix slot; no task values.""" + + app: str + mode: str + block: int + profiled: bool + + def __post_init__(self) -> None: + if not re.fullmatch(r"app_[a-z0-9]+", self.app): + raise ValueError("use an opaque app label") + if ( + self.mode not in {"cold", "warm"} + or type(self.block) is not int + or self.block < 1 + or type(self.profiled) is not bool + ): + raise ValueError("invalid matrix slot") + + +@dataclass(frozen=True) +class Observation: + trial: Trial + attempt: int + source_head: str + wall_ms: float | None + status: str + outcome: str + oracle: str + step_count: int + measured_steps: int + step_exclusive_ms: dict[str, float] + step_issues: tuple[tuple[int, str], ...] + outer_inclusive_ms: dict[str, float] + outer_overlap_ms: float | None + unattributed_outer_ms: float | None + load_samples: tuple[dict[str, float], ...] + load_missing: bool + human_active_s: float | None + human_wait_s: float | None + + @property + def verified(self) -> bool: + return ( + self.status == "completed" + and self.outcome == "VERIFIED" + and self.oracle == "pass" + ) + + def to_dict(self) -> dict[str, Any]: + return {"schema": "openadapt.phase-observation/v1", **asdict(self)} + + +def untimed_failure( + trial: Trial, + *, + attempt: int, + source_head: str, + status: str, + human_active_s: float | None = None, + human_wait_s: float | None = None, +) -> Observation: + """Retain a started attempt even when setup or its clock produced no data.""" + if ( + type(attempt) is not int + or attempt < 1 + or not re.fullmatch(r"[0-9a-f]{40}", source_head) + ): + raise ValueError("invalid attempt provenance") + if status not in {"setup_error", "timeout", "error"}: + raise ValueError("an untimed attempt cannot claim completion") + if any(v is not None and not _number(v) for v in (human_active_s, human_wait_s)): + raise ValueError("invalid human-time observation") + return Observation( + trial=trial, + attempt=attempt, + source_head=source_head, + wall_ms=None, + status=status, + outcome="unknown", + oracle="unrun", + step_count=0, + measured_steps=0, + step_exclusive_ms={}, + step_issues=((-1, "missing_outer_clock"),), + outer_inclusive_ms={}, + outer_overlap_ms=None, + unattributed_outer_ms=None, + load_samples=(), + load_missing=True, + human_active_s=human_active_s, + human_wait_s=human_wait_s, + ) + + +def _steps( + report: Mapping[str, Any] | None, +) -> tuple[int, int, dict[str, float], tuple[tuple[int, str], ...]]: + results = report.get("results") if report is not None else None + if not isinstance(results, list) or not results: + return 0, 0, {}, ((-1, "missing_steps"),) + totals: dict[str, float] = {} + issues: list[tuple[int, str]] = [] + measured = 0 + known = set(SPANS) | {"other"} + for i, result in enumerate(results): + if not isinstance(result, Mapping) or not isinstance( + result.get("timings"), Mapping + ): + issues.append((i, "missing_timings")) + continue + spans = result["timings"] + elapsed = result.get("elapsed_ms") + if not _number(elapsed) or any(not _number(v) for v in spans.values()): + issues.append((i, "invalid_duration")) + continue + if any(k not in known or k not in STEP_GROUPS for k in spans): + issues.append((i, "unknown_span")) + continue + # StepTimer rounds each span to 0.1 ms. Never renormalize a mismatch. + if abs(sum(spans.values()) - elapsed) > 0.05 * len(spans) + 0.05: + issues.append((i, "span_elapsed_mismatch")) + continue + measured += 1 + for span, duration in spans.items(): + group = STEP_GROUPS[span] + totals[group] = totals.get(group, 0.0) + duration + return len(results), measured, totals, tuple(issues) + + +class AttemptClock: + """Monotonic outer clock. A finished attempt cannot be reused.""" + + def __init__(self, *, clock: Callable[[], float] = time.perf_counter) -> None: + self._clock = clock + self._start = clock() + if ( + isinstance(self._start, bool) + or not isinstance(self._start, (int, float)) + or not math.isfinite(self._start) + ): + raise ValueError("invalid monotonic clock") + self._last = self._start + self._intervals: list[tuple[str, float, float]] = [] + self._loads: list[dict[str, float]] = [] + self._open = 0 + self._finished = False + + def _now(self) -> float: + if self._finished: + raise ValueError("attempt already finished") + now = self._clock() + if ( + isinstance(now, bool) + or not isinstance(now, (int, float)) + or not math.isfinite(now) + or now < self._last + ): + raise ValueError("clock did not remain monotonic") + self._last = now + return (now - self._start) * 1000.0 + + @contextmanager + def span(self, phase: str) -> Iterator[None]: + if phase not in OUTER_PHASES: + raise ValueError("unknown outer phase") + start = self._now() + self._open += 1 + try: + yield + finally: + self._intervals.append((phase, start, self._now())) + self._open -= 1 + + def load(self, *, load1: float, cpu_percent: float, available_bytes: int) -> None: + if ( + not all(_number(v) for v in (load1, cpu_percent, available_bytes)) + or cpu_percent > 100 + or type(available_bytes) is not int + ): + raise ValueError("invalid numeric resource observation") + self._loads.append( + { + "at_ms": self._now(), + "load1": load1, + "cpu_percent": cpu_percent, + "available_bytes": available_bytes, + } + ) + + def finish( + self, + trial: Trial, + *, + attempt: int, + source_head: str, + status: str, + oracle: str, + report: Mapping[str, Any] | None = None, + human_active_s: float | None = None, + human_wait_s: float | None = None, + ) -> Observation: + if self._open: + raise ValueError("outer phase still open") + if ( + type(attempt) is not int + or attempt < 1 + or not re.fullmatch(r"[0-9a-f]{40}", source_head) + ): + raise ValueError("invalid attempt provenance") + if status not in { + "completed", + "setup_error", + "timeout", + "error", + } or oracle not in {"pass", "fail", "unknown", "unrun"}: + raise ValueError("invalid attempt verdict") + if any( + v is not None and not _number(v) for v in (human_active_s, human_wait_s) + ): + raise ValueError("invalid human-time observation") + wall = self._now() + self._finished = True + inclusive: dict[str, float] = {} + union_end = 0.0 + covered = 0.0 + for phase, start, end in sorted(self._intervals, key=lambda row: row[1]): + inclusive[phase] = inclusive.get(phase, 0.0) + end - start + covered += max(0.0, end - max(start, union_end)) + union_end = max(union_end, end) + step_count, measured, totals, issues = _steps(report) + outcome = report.get("execution_outcome") if report is not None else None + outcome = ( + outcome if isinstance(outcome, str) and outcome in OUTCOMES else "unknown" + ) + points = [0.0, *(row["at_ms"] for row in self._loads), wall] + missing_load = not self._loads or any( + b - a > 2000.0 for a, b in zip(points, points[1:]) + ) + return Observation( + trial, + attempt, + source_head, + wall, + status, + outcome, + oracle, + step_count, + measured, + totals, + issues, + inclusive, + max(0.0, sum(inclusive.values()) - covered), + max(0.0, wall - covered), + tuple(self._loads), + missing_load, + human_active_s, + human_wait_s, + ) + + +def analyze(plan: Sequence[Trial], rows: Sequence[Observation]) -> dict[str, Any]: + """Keep all attempts; pair only unretried, independently verified slots.""" + planned = set(plan) + if len(planned) != len(plan): + raise ValueError("duplicate planned slot") + if len({r.attempt for r in rows}) != len(rows) or any( + r.trial not in planned for r in rows + ): + raise ValueError("duplicate attempt or unplanned observation") + if any(r.wall_ms is not None and not _number(r.wall_ms) for r in rows): + raise ValueError("invalid authoritative wall duration") + if len({r.source_head for r in rows}) > 1: + raise ValueError("profiling pairs require a single frozen source") + by_slot = {slot: [r for r in rows if r.trial == slot] for slot in plan} + pairs: list[dict[str, Any]] = [] + pair_keys = {(t.app, t.mode, t.block) for t in plan} + for app, mode, block in sorted(pair_keys): + arms = [ + by_slot.get(Trial(app, mode, block, state), []) for state in (False, True) + ] + if all( + len(a) == 1 and a[0].verified and a[0].wall_ms is not None for a in arms + ): + off, on = arms[0][0].wall_ms, arms[1][0].wall_ms + if off is not None and on is not None: + pairs.append( + {"app": app, "mode": mode, "block": block, "delta_ms": on - off} + ) + + def distribution(values: list[float | None]) -> dict[str, Any]: + measured_values = [v for v in values if v is not None] + return { + "n": len(measured_values), + "median_ms": statistics.median(measured_values) + if measured_values + else None, + "min_ms": min(measured_values) if measured_values else None, + "max_ms": max(measured_values) if measured_values else None, + } + + groups: list[dict[str, Any]] = [] + for app, mode, profiled in sorted({(t.app, t.mode, t.profiled) for t in plan}): + observed = [ + r + for r in rows + if (r.trial.app, r.trial.mode, r.trial.profiled) == (app, mode, profiled) + ] + good = [r for r in observed if r.verified] + valid = [r for r in good if not r.step_issues] + costs: list[dict[str, Any]] = [] + for phase in sorted(set(STEP_GROUPS.values()) - {"unattributed_step"}): + values = [ + r.step_exclusive_ms[phase] + for r in valid + if phase in r.step_exclusive_ms + ] + costs.append( + { + "phase": phase, + "observed_median_ms": statistics.median(values) if values else None, + "covered_attempts": len(values), + "missing_verified_attempts": len(good) - len(values), + } + ) + costs.sort( + key=lambda row: ( + row["observed_median_ms"] is None, + -(row["observed_median_ms"] or 0), + ) + ) + outer_costs = [] + for phase in sorted(OUTER_PHASES): + outer_values = [ + r.outer_inclusive_ms[phase] + for r in good + if phase in r.outer_inclusive_ms + ] + outer_costs.append( + { + "phase": phase, + "observed_median_ms": statistics.median(outer_values) + if outer_values + else None, + "covered_attempts": len(outer_values), + "missing_verified_attempts": len(good) - len(outer_values), + } + ) + groups.append( + { + "app": app, + "mode": mode, + "profiled": profiled, + "planned_slots": sum( + (t.app, t.mode, t.profiled) == (app, mode, profiled) for t in plan + ), + "attempts": len(observed), + "verified_attempts": len(good), + "all_attempt_wall": distribution([r.wall_ms for r in observed]), + "verified_wall": distribution([r.wall_ms for r in good]), + "phase_costs": costs, + "outer_phase_costs": outer_costs, + "outer_basis": "inclusive_not_additive_with_step_spans", + } + ) + + def human(field: str) -> dict[str, Any]: + values = [getattr(r, field) for r in rows] + known = sum(v for v in values if v is not None) / 60.0 + unknown = sum(v is None for v in values) + return { + "known_minutes": known, + "total_minutes": None if unknown else known, + "unknown_attempts": unknown, + "scope": "recorded_attempts_only", + } + + return { + "schema": "openadapt.phase-analysis/v1", + "planned_slots": len(plan), + "started_attempts": len(rows), + "unrun_slots": [asdict(t) for t in plan if not by_slot[t]], + "retried_slots": sum(len(v) > 1 for v in by_slot.values()), + "verified_attempts": sum(r.verified for r in rows), + "statuses": { + status: sum(r.status == status for r in rows) + for status in ("completed", "setup_error", "timeout", "error") + }, + "outcomes": { + outcome: sum(r.outcome == outcome for r in rows) + for outcome in sorted(OUTCOMES | {"unknown"}) + }, + "oracle_results": { + value: sum(r.oracle == value for r in rows) + for value in ("pass", "fail", "unknown", "unrun") + }, + "timing_incomplete_attempts": sum(bool(r.step_issues) for r in rows), + "wall_missing_attempts": sum(r.wall_ms is None for r in rows), + "outer_overlap_attempts": sum( + r.outer_overlap_ms is not None and r.outer_overlap_ms > 0 for r in rows + ), + "load_missing_attempts": sum(r.load_missing for r in rows), + "paired_slots": len(pair_keys), + "eligible_pairs": len(pairs), + "excluded_pairs": len(pair_keys) - len(pairs), + "paired_profiler_delta_ms": pairs, + "human_active": human("human_active_s"), + "human_wait": human("human_wait_s"), + "groups": groups, + "interpretation": "observed_exclusive_cost_not_achieved_saving", + } diff --git a/packages/flow/tests/test_phase_latency.py b/packages/flow/tests/test_phase_latency.py new file mode 100644 index 0000000000..6335aa8d30 --- /dev/null +++ b/packages/flow/tests/test_phase_latency.py @@ -0,0 +1,374 @@ +"""Synthetic contracts only; no app, account, network or latency benchmark.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from benchmark.phase_latency.collector import ( + STEP_GROUPS, + AttemptClock, + Trial, + analyze, + untimed_failure, +) +from openadapt_flow.runtime import step_timing + +HEAD = "a" * 40 + + +class Clock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + def spend(self, ms: float) -> None: + self.now += ms / 1000 + + +def report(timings=None, elapsed=100): + if timings is None: + timings = {"identity": 20, "actuate": 30, "postcondition": 40, "other": 10} + return { + "execution_outcome": "VERIFIED", + "results": [{"elapsed_ms": elapsed, "timings": timings}], + } + + +def observation( + trial=None, *, attempt=1, wall=100, oracle="pass", status="completed", data=None +): + clock = Clock() + timer = AttemptClock(clock=clock) + clock.spend(wall) + return timer.finish( + trial or Trial("app_a", "cold", 1, False), + attempt=attempt, + source_head=HEAD, + status=status, + oracle=oracle, + report=data or report(), + ) + + +def test_reuse_actual_exclusive_step_timer(monkeypatch): + clock = Clock() + monkeypatch.setattr(step_timing, "time", SimpleNamespace(perf_counter=clock)) + timer, previous = step_timing.start_step() + try: + timer.enter("actuate") + clock.spend(10) + timer.enter("settle") + clock.spend(20) + timer.exit() + clock.spend(30) + timer.exit() + clock.spend(40) + spans = timer.result_ms(100) + finally: + step_timing.end_step(previous) + row = observation(data=report(spans)) + assert row.step_exclusive_ms == { + "actions": 40, + "settle": 20, + "unattributed_step": 40, + } + assert row.measured_steps == row.step_count == 1 + assert not row.step_issues + assert set(STEP_GROUPS) == set(step_timing.SPANS) | {"other"} + + +def test_outer_clock_overlaps_and_gaps_are_separate_from_steps(): + clock = Clock() + timer = AttemptClock(clock=clock) + clock.spend(100) + with timer.span("replay"): + clock.spend(50) + with timer.span("navigation"): + clock.spend(200) + clock.spend(50) + clock.spend(100) + row = timer.finish( + Trial("app_a", "warm", 1, True), + attempt=1, + source_head=HEAD, + status="completed", + oracle="pass", + report=report(), + ) + assert row.wall_ms == pytest.approx(500) + assert row.outer_inclusive_ms == pytest.approx({"replay": 300, "navigation": 200}) + assert row.outer_overlap_ms == pytest.approx(200) + assert row.unattributed_outer_ms == pytest.approx(200) + assert "engine_bookkeeping" not in row.outer_inclusive_ms + assert "engine_overhead" not in json.dumps(row.to_dict()) + group = analyze([row.trial], [row])["groups"][0] + outer = {item["phase"]: item for item in group["outer_phase_costs"]} + assert outer["navigation"]["observed_median_ms"] == pytest.approx(200) + assert outer["navigation"]["covered_attempts"] == 1 + assert outer["engine_bookkeeping"]["observed_median_ms"] is None + assert outer["engine_bookkeeping"]["missing_verified_attempts"] == 1 + assert group["outer_basis"] == "inclusive_not_additive_with_step_spans" + + +@pytest.mark.parametrize( + ("data", "kind"), + [ + ({"results": []}, "missing_steps"), + ({"results": [{"elapsed_ms": 10, "timings": None}]}, "missing_timings"), + (report({}, 10), "span_elapsed_mismatch"), + (report({"identity": -1}, 10), "invalid_duration"), + (report({"identity": float("nan")}, 10), "invalid_duration"), + (report({"identity": float("inf")}, 10), "invalid_duration"), + (report({"identity": True}, 1), "invalid_duration"), + (report({"identity": 10}, "10"), "invalid_duration"), + (report({"private-payload-sentinel": 10}, 10), "unknown_span"), + (report({"identity": 20}, 10), "span_elapsed_mismatch"), + ], +) +def test_incomplete_timing_is_not_zero_or_engine_cost(data, kind): + row = observation(data=data) + assert row.measured_steps == 0 + assert row.step_exclusive_ms == {} + assert row.step_issues[0][1] == kind + assert "private-payload-sentinel" not in json.dumps(row.to_dict()) + summary = analyze([row.trial], [row]) + assert summary["started_attempts"] == 1 + assert summary["timing_incomplete_attempts"] == 1 + assert all( + p["observed_median_ms"] is None for p in summary["groups"][0]["phase_costs"] + ) + + +def test_step_rounding_budget_is_not_renormalized(): + row = observation( + data=report({"identity": 33.3, "actuate": 33.3, "other": 33.3}, 100) + ) + assert not row.step_issues + assert sum(row.step_exclusive_ms.values()) == pytest.approx(99.9) + row = observation(data=report({"identity": 30, "actuate": 30}, 100)) + assert row.step_issues == ((0, "span_elapsed_mismatch"),) + + +def test_raw_report_values_and_unknown_verdict_are_not_exported(): + data = report() + data.update( + patient="private-payload-sentinel", + url="private-url-sentinel", + execution_outcome={"bad": "private-payload-sentinel"}, + ) + row = observation(data=data) + assert row.outcome == "unknown" + assert not row.verified + encoded = json.dumps(row.to_dict(), allow_nan=False) + assert "private-payload-sentinel" not in encoded + assert "private-url-sentinel" not in encoded + + +def test_independent_oracle_failure_never_counts_as_verified(): + row = observation(oracle="fail") + summary = analyze([row.trial], [row]) + assert row.outcome == "VERIFIED" + assert not row.verified + assert summary["verified_attempts"] == 0 + assert summary["oracle_results"]["fail"] == 1 + assert summary["excluded_pairs"] == 1 + + +def test_retries_and_failures_remain_in_denominator_and_cannot_form_pair(): + off = Trial("app_a", "warm", 1, False) + on = Trial("app_a", "warm", 1, True) + rows = [ + observation(off, attempt=1, status="setup_error", oracle="unrun"), + observation(off, attempt=2), + observation(on, attempt=3), + ] + summary = analyze([off, on], rows) + assert summary["planned_slots"] == 2 + assert summary["started_attempts"] == 3 + assert summary["retried_slots"] == 1 + assert summary["statuses"]["setup_error"] == 1 + assert summary["verified_attempts"] == 2 + assert summary["eligible_pairs"] == 0 + assert summary["excluded_pairs"] == 1 + + +def test_pairs_use_same_workload_mode_and_keep_on_off_difference(): + plan = [ + Trial(app, mode, 1, state) + for app in ("app_a", "app_b") + for mode in ("cold", "warm") + for state in (False, True) + ] + rows = [ + observation( + t, attempt=i + 1, wall=(100 if t.app == "app_a" else 1000) + 5 * t.profiled + ) + for i, t in enumerate(plan) + ] + summary = analyze(plan, rows) + assert summary["eligible_pairs"] == 4 + assert [ + p["delta_ms"] for p in summary["paired_profiler_delta_ms"] + ] == pytest.approx([5] * 4) + assert len(summary["groups"]) == 8 + assert "phase_costs" not in summary + assert sorted( + {g["all_attempt_wall"]["median_ms"] for g in summary["groups"]} + ) == pytest.approx([100, 105, 1000, 1005]) + + +def test_all_sixty_unrun_slots_are_visible(): + plan = [ + Trial(app, mode, block, state) + for app in ("app_a", "app_b", "app_c") + for mode in ("cold", "warm") + for block in range(1, 6) + for state in (False, True) + ] + summary = analyze(plan, []) + assert summary["planned_slots"] == 60 + assert summary["started_attempts"] == 0 + assert len(summary["unrun_slots"]) == 60 + assert summary["eligible_pairs"] == 0 + + +def test_unknown_human_time_is_not_zero_total(): + row = observation() + summary = analyze([row.trial], [row]) + assert summary["human_active"] == { + "known_minutes": 0, + "total_minutes": None, + "unknown_attempts": 1, + "scope": "recorded_attempts_only", + } + known = replace(row, human_active_s=120, human_wait_s=60) + summary = analyze([known.trial], [known]) + assert summary["human_active"]["total_minutes"] == 2 + assert summary["human_wait"]["total_minutes"] == 1 + + +def test_resource_sampling_gaps_are_explicit(): + clock = Clock() + timer = AttemptClock(clock=clock) + timer.load(load1=1, cpu_percent=20, available_bytes=4096) + clock.spend(2000) + timer.load(load1=2, cpu_percent=30, available_bytes=4000) + clock.spend(2000) + row = timer.finish( + Trial("app_a", "cold", 1, False), + attempt=1, + source_head=HEAD, + status="completed", + oracle="pass", + report=report(), + ) + assert not row.load_missing + assert row.load_samples[1]["at_ms"] == 2000 + assert observation(wall=4001).load_missing + + +def test_exception_span_is_retained_and_open_span_cannot_finish(): + clock = Clock() + timer = AttemptClock(clock=clock) + with pytest.raises(RuntimeError): + with timer.span("session"): + clock.spend(20) + raise RuntimeError("synthetic failure") + row = timer.finish( + Trial("app_a", "cold", 1, False), + attempt=1, + source_head=HEAD, + status="setup_error", + oracle="unrun", + ) + assert row.outer_inclusive_ms == {"session": 20} + assert row.outcome == "unknown" + with pytest.raises(ValueError, match="finished"): + timer.load(load1=0, cpu_percent=0, available_bytes=1) + timer = AttemptClock(clock=clock) + with timer.span("replay"): + with pytest.raises(ValueError, match="still open"): + timer.finish( + Trial("app_a", "cold", 1, False), + attempt=2, + source_head=HEAD, + status="completed", + oracle="pass", + ) + + +def test_invalid_source_plan_and_attempt_identity_refuse_analysis(): + row = observation() + with pytest.raises(ValueError, match="planned"): + analyze([row.trial, row.trial], [row]) + with pytest.raises(ValueError, match="attempt"): + analyze([row.trial], [row, row]) + with pytest.raises(ValueError, match="unplanned"): + analyze([], [row]) + on = Trial("app_a", "cold", 1, True) + with pytest.raises(ValueError, match="frozen source"): + analyze( + [row.trial, on], + [row, replace(observation(on, attempt=2), source_head="b" * 40)], + ) + + +@pytest.mark.parametrize("bad", [float("nan"), float("inf"), True, "0"]) +def test_invalid_clock_refuses_instead_of_reporting_time(bad): + with pytest.raises(ValueError): + AttemptClock(clock=lambda: bad) + + +def test_regressing_clock_refuses_instead_of_clamping(): + clock = Clock() + timer = AttemptClock(clock=clock) + clock.now = -1 + with pytest.raises(ValueError, match="monotonic"): + with timer.span("replay"): + pass + + +def test_clock_or_setup_failure_stays_started_with_unknown_wall_time(): + off = Trial("app_a", "cold", 1, False) + on = Trial("app_a", "cold", 1, True) + failed = untimed_failure(off, attempt=1, source_head=HEAD, status="error") + summary = analyze([off, on], [failed, observation(on, attempt=2)]) + assert summary["started_attempts"] == 2 + assert summary["unrun_slots"] == [] + assert summary["wall_missing_attempts"] == 1 + assert summary["statuses"]["error"] == 1 + assert summary["eligible_pairs"] == 0 + assert summary["groups"][0]["all_attempt_wall"]["n"] == 0 + assert failed.wall_ms is None + assert failed.unattributed_outer_ms is None + with pytest.raises(ValueError, match="cannot claim"): + untimed_failure(off, attempt=3, source_head=HEAD, status="completed") + known = untimed_failure( + off, attempt=4, source_head=HEAD, status="setup_error", human_active_s=60 + ) + assert analyze([off], [known])["human_active"]["total_minutes"] == 1 + + +def test_unobserved_step_phase_is_missing_but_explicit_zero_is_retained(): + row = observation() + phases = { + item["phase"]: item + for item in analyze([row.trial], [row])["groups"][0]["phase_costs"] + } + assert phases["settle"]["observed_median_ms"] is None + assert phases["settle"]["covered_attempts"] == 0 + assert phases["settle"]["missing_verified_attempts"] == 1 + row = observation(data=report({"identity": 0.0, "other": 100.0})) + phases = { + item["phase"]: item + for item in analyze([row.trial], [row])["groups"][0]["phase_costs"] + } + assert phases["identity"]["observed_median_ms"] == 0.0 + assert phases["identity"]["covered_attempts"] == 1 + assert phases["identity"]["missing_verified_attempts"] == 0 From 7553066054a7cf8e150c1f874e198fed3e47f45d Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 21:39:39 -0800 Subject: [PATCH 12/21] fix: wait for the WPF census fixture to render (#1940) Signed-off-by: abrichr --- packages/flow/tests/_native_app_startup.py | 105 ++++++ .../flow/tests/test_native_app_startup.py | 298 ++++++++++++++++++ .../tests/test_native_census_real_apps.py | 163 ++++++---- 3 files changed, 508 insertions(+), 58 deletions(-) create mode 100644 packages/flow/tests/_native_app_startup.py create mode 100644 packages/flow/tests/test_native_app_startup.py diff --git a/packages/flow/tests/_native_app_startup.py b/packages/flow/tests/_native_app_startup.py new file mode 100644 index 0000000000..1bcc8d3ed5 --- /dev/null +++ b/packages/flow/tests/_native_app_startup.py @@ -0,0 +1,105 @@ +"""Owned-process startup checks for the native census test fixture.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from time import monotonic, sleep +from typing import Any, Callable, NoReturn, Optional + + +class NativeAppStartupError(AssertionError): + """The owned fixture did not prove that its rendered window is ready.""" + + +def _tail(path: Path, limit: int = 4096) -> str: + try: + with path.open("rb") as stream: + stream.seek(0, 2) + stream.seek(max(0, stream.tell() - limit)) + return stream.read(limit).decode("utf-8", errors="replace") + except OSError as exc: + return f"" + + +def wait_for_rendered_items( + proc: subprocess.Popen, + ready_path: Path, + expected_rows: int, + find_items: Callable[[int], Optional[list[Any]]], + stdout_path: Path, + stderr_path: Path, + *, + timeout_s: float, +) -> list[Any]: + """Wait for the owned render signal and UIA items within one deadline.""" + started = monotonic() + deadline = started + timeout_s + phase = "ContentRendered signal" + signal: Optional[dict[str, Any]] = None + + def fail(reason: str) -> NoReturn: + raise NativeAppStartupError( + f"native fixture startup: {reason}; pid={proc.pid}; " + f"exit={proc.poll()}; phase={phase}; " + f"elapsed={monotonic() - started:.3f}s; " + f"ready={ready_path.exists()}; " + f"stdout_tail={_tail(stdout_path)!r}; " + f"stderr_tail={_tail(stderr_path)!r}" + ) + + while True: + if proc.poll() is not None: + fail("owned child exited") + if monotonic() >= deadline: + fail("overall startup deadline exceeded") + if signal is None and ready_path.exists(): + try: + with ready_path.open("rb") as stream: + raw = stream.read(4097) + if len(raw) > 4096: + raise ValueError("oversized signal") + candidate = json.loads(raw) + if ( + not isinstance(candidate, dict) + or set(candidate) != {"event", "pid", "hwnd", "rows"} + or candidate["event"] != "content_rendered" + or type(candidate["pid"]) is not int + or candidate["pid"] != proc.pid + or type(candidate["hwnd"]) is not int + or candidate["hwnd"] <= 0 + or type(candidate["rows"]) is not int + or candidate["rows"] != expected_rows + ): + raise ValueError("signal does not identify the owned fixture") + signal = candidate + except (OSError, ValueError, TypeError): + fail("invalid ContentRendered signal") + if signal is not None: + phase = "owned window UIA items" + try: + items = find_items(signal["hwnd"]) + except Exception as exc: + fail(f"UIA discovery failed ({type(exc).__name__}: {str(exc)[:400]})") + if proc.poll() is not None: + fail("owned child exited during UIA discovery") + if monotonic() >= deadline: + fail("overall startup deadline exceeded") + if items is not None and len(items) == expected_rows: + return items + sleep(min(0.1, max(0.0, deadline - monotonic()))) + + +def stop_owned_process(proc: subprocess.Popen, timeout_s: float = 5.0) -> None: + """Reap only the process returned by this fixture's Popen call.""" + if proc.poll() is None: + try: + proc.terminate() + except ProcessLookupError: + pass + try: + proc.wait(timeout=timeout_s) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=timeout_s) diff --git a/packages/flow/tests/test_native_app_startup.py b/packages/flow/tests/test_native_app_startup.py new file mode 100644 index 0000000000..47e5574993 --- /dev/null +++ b/packages/flow/tests/test_native_app_startup.py @@ -0,0 +1,298 @@ +"""Portable controls for the native fixture's readiness and process lifecycle.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests import _native_app_startup as startup + + +class Clock: + def __init__(self): + self.now = 0.0 + + def monotonic(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +class Process: + pid = 1234 + + def __init__(self): + self.code = None + + def poll(self): + return self.code + + +def _signal(path: Path, **changes): + signal = {"event": "content_rendered", "pid": 1234, "hwnd": 5678, "rows": 2} + signal.update(changes) + path.write_text(json.dumps(signal)) + + +@pytest.fixture +def fixture_state(tmp_path, monkeypatch): + clock = Clock() + monkeypatch.setattr(startup, "monotonic", clock.monotonic) + monkeypatch.setattr(startup, "sleep", clock.sleep) + stdout = tmp_path / "stdout.log" + stderr = tmp_path / "stderr.log" + stdout.write_text("") + stderr.write_text("") + return clock, Process(), tmp_path / "ready.json", stdout, stderr + + +def _wait(state, find_items, timeout_s=120): + _, proc, ready, stdout, stderr = state + return startup.wait_for_rendered_items( + proc, ready, 2, find_items, stdout, stderr, timeout_s=timeout_s + ) + + +def test_render_signal_can_arrive_after_old_title_poll_bound( + fixture_state, monkeypatch +): + clock, _, ready, _, _ = fixture_state + calls = [] + + def sleep(seconds): + clock.now += seconds + if clock.now >= 35 and not ready.exists(): + _signal(ready) + + monkeypatch.setattr(startup, "sleep", sleep) + result = _wait(fixture_state, lambda hwnd: calls.append(hwnd) or ["a", "b"]) + assert result == ["a", "b"] and calls == [5678] + assert 35 <= clock.now < 36 # simulated seconds, not a WPF timing claim + + +@pytest.mark.parametrize( + "changes", + [ + {"pid": 999}, + {"pid": True}, + {"hwnd": 0}, + {"hwnd": "5678"}, + {"rows": 1}, + {"rows": True}, + {"event": "loaded"}, + {"extra": 1}, + ], +) +def test_signal_must_identify_owned_rendered_fixture(fixture_state, changes): + _signal(fixture_state[2], **changes) + with pytest.raises(startup.NativeAppStartupError, match="invalid ContentRendered"): + _wait(fixture_state, lambda hwnd: pytest.fail("untrusted HWND was used")) + + +@pytest.mark.parametrize("raw", [b"{broken", b"[]", b"x" * 4097]) +def test_malformed_or_oversized_signal_fails_without_discovery(fixture_state, raw): + fixture_state[2].write_bytes(raw) + with pytest.raises(startup.NativeAppStartupError, match="invalid ContentRendered"): + _wait(fixture_state, lambda hwnd: pytest.fail("invalid signal was used")) + + +def test_ready_but_exited_process_is_not_accepted(fixture_state): + _, proc, ready, _, stderr = fixture_state + _signal(ready) + proc.code = 7 + stderr.write_text("startup failed") + with pytest.raises(startup.NativeAppStartupError, match="exit=7.*startup failed"): + _wait(fixture_state, lambda hwnd: pytest.fail("dead process was used")) + + +def test_exit_during_discovery_is_not_accepted(fixture_state): + _, proc, ready, _, _ = fixture_state + _signal(ready) + + def discover(hwnd): + proc.code = 8 + return ["a", "b"] + + with pytest.raises(startup.NativeAppStartupError, match="exited during UIA"): + _wait(fixture_state, discover) + + +def test_signal_and_items_share_one_deadline(fixture_state, monkeypatch): + clock, _, ready, _, _ = fixture_state + + def sleep(seconds): + clock.now += seconds + if clock.now >= 0.8 and not ready.exists(): + _signal(ready) + + monkeypatch.setattr(startup, "sleep", sleep) + with pytest.raises(startup.NativeAppStartupError, match="phase=owned window UIA"): + _wait(fixture_state, lambda hwnd: ["only one row"], timeout_s=1) + assert 1 <= clock.now < 1.1 + + +def test_slow_discovery_cannot_return_after_deadline(fixture_state): + clock, _, ready, _, _ = fixture_state + _signal(ready) + + def discover(hwnd): + clock.now = 2 + return ["a", "b"] + + with pytest.raises(startup.NativeAppStartupError, match="deadline exceeded"): + _wait(fixture_state, discover, timeout_s=1) + + +def test_discovery_failure_contains_owned_diagnostics(fixture_state): + _signal(fixture_state[2]) + + def discover(hwnd): + raise ValueError("wrong process for rendered HWND") + + with pytest.raises(startup.NativeAppStartupError, match="wrong process.*pid=1234"): + _wait(fixture_state, discover) + + +@pytest.mark.parametrize("mode", ["ready", "exit", "hang"]) +def test_real_owned_child_readiness_exit_and_cleanup(tmp_path, mode): + script = tmp_path / "child.py" + ready = tmp_path / "ready.json" + stdout_path, stderr_path = tmp_path / "stdout.log", tmp_path / "stderr.log" + script.write_text( + "import json, os, pathlib, sys, time\n" + "mode, ready = sys.argv[1:]\n" + "print('owned stdout', flush=True)\n" + "print('owned stderr', file=sys.stderr, flush=True)\n" + "if mode == 'exit': sys.exit(9)\n" + "if mode == 'ready':\n" + " time.sleep(0.05)\n" + " path = pathlib.Path(ready)\n" + " temporary = path.with_suffix('.tmp')\n" + " temporary.write_text(json.dumps(dict(event='content_rendered', " + "pid=os.getpid(), hwnd=5678, rows=2)))\n" + " temporary.replace(path)\n" + "time.sleep(60)\n" + ) + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + proc = subprocess.Popen( + [sys.executable, str(script), mode, str(ready)], + stdout=stdout, + stderr=stderr, + ) + try: + if mode == "ready": + assert startup.wait_for_rendered_items( + proc, + ready, + 2, + lambda hwnd: ["a", "b"], + stdout_path, + stderr_path, + timeout_s=5, + ) == ["a", "b"] + else: + reason = "owned child exited" if mode == "exit" else "deadline exceeded" + with pytest.raises( + startup.NativeAppStartupError, match=reason + ) as error: + startup.wait_for_rendered_items( + proc, + ready, + 2, + lambda hwnd: pytest.fail("not ready"), + stdout_path, + stderr_path, + timeout_s=1, + ) + assert "owned stdout" in str(error.value) + assert "owned stderr" in str(error.value) + if mode == "exit": + assert "exit=9" in str(error.value) + finally: + startup.stop_owned_process(proc) + assert proc.poll() is not None + + +def test_diagnostic_tails_are_bounded(tmp_path): + path = tmp_path / "stderr" + path.write_bytes(b"prefix" + b"x" * 5000 + b"end") + tail = startup._tail(path) + assert len(tail) == 4096 and tail.endswith("end") and "prefix" not in tail + + +def test_cleanup_escalates_only_owned_process(): + events = [] + + class Stubborn(Process): + def terminate(self): + events.append("terminate") + + def wait(self, timeout): + events.append(("wait", timeout)) + if self.code is None: + raise subprocess.TimeoutExpired("owned fixture", timeout) + return self.code + + def kill(self): + events.append("kill") + self.code = -9 + + proc = Stubborn() + startup.stop_owned_process(proc, timeout_s=0.5) + assert events == ["terminate", ("wait", 0.5), "kill", ("wait", 0.5)] + + +@pytest.mark.parametrize("identity", ["owned", "other_pid", "other_title"]) +def test_wpf_fixture_uses_signalled_handle_and_owned_window( + tmp_path, monkeypatch, identity +): + from tests import test_native_census_real_apps as native + + calls = [] + items = [SimpleNamespace(ControlTypeName="ListItemControl") for _ in range(2)] + listing = SimpleNamespace(Exists=lambda *args: True, GetChildren=lambda: items) + window = SimpleNamespace( + ProcessId=1234 if identity != "other_pid" else 999, + Name=native.TITLE if identity != "other_title" else "other window", + ListControl=lambda **kwargs: listing, + ) + auto = SimpleNamespace(ControlFromHandle=lambda hwnd: calls.append(hwnd) or window) + + class Owned(Process): + def terminate(self): + self.code = -15 + + def wait(self, timeout): + return self.code + + proc = Owned() + + def spawn(argv, stdout, stderr): + assert all(flag in argv for flag in ("-STA", "-NonInteractive", "-NoProfile")) + _signal(Path(argv[argv.index("-ReadyPath") + 1])) + return proc + + monkeypatch.setattr(native, "sys", SimpleNamespace(platform="win32")) + monkeypatch.setenv("OPENADAPT_TEST_WINDOWS_UIA", "1") + monkeypatch.setattr( + native, "pytest", SimpleNamespace(importorskip=lambda name: auto) + ) + monkeypatch.setattr(native, "subprocess", SimpleNamespace(Popen=spawn)) + fixture = native.wpf_worklist.__wrapped__(tmp_path) + try: + _, start = next(fixture) + if identity == "owned": + assert start([{}, {}]) == items + else: + with pytest.raises(startup.NativeAppStartupError, match="owned worklist"): + start([{}, {}]) + assert calls == [5678] + finally: + fixture.close() + assert proc.poll() == -15 diff --git a/packages/flow/tests/test_native_census_real_apps.py b/packages/flow/tests/test_native_census_real_apps.py index e14b769007..e82397c336 100644 --- a/packages/flow/tests/test_native_census_real_apps.py +++ b/packages/flow/tests/test_native_census_real_apps.py @@ -20,6 +20,7 @@ import subprocess import sys import time +from contextlib import ExitStack from pathlib import Path from typing import Any, Optional @@ -27,6 +28,7 @@ from openadapt_flow.runtime import identity_template as itmpl from openadapt_flow.runtime.identity_census import CensusDecision, decide +from tests._native_app_startup import stop_owned_process, wait_for_rendered_items pytestmark = pytest.mark.timeout(180) @@ -60,34 +62,53 @@ def _wait(predicate, timeout_s: float = 30.0) -> Any: # -- Windows (WPF + UI Automation) --------------------------------------------- _WPF_APP = r""" -param([string]$RowsJson) -Add-Type -AssemblyName PresentationFramework -$rows = $RowsJson | ConvertFrom-Json -$window = New-Object System.Windows.Window -$window.Title = '__TITLE__' -$window.Width = 520 -$window.Height = 360 -$window.Left = 40 -$window.Top = 40 -$list = New-Object System.Windows.Controls.ListBox -foreach ($row in $rows) { - $item = New-Object System.Windows.Controls.ListBoxItem - $panel = New-Object System.Windows.Controls.StackPanel - $panel.Orientation = 'Horizontal' - foreach ($cell in $row.cells) { - $text = New-Object System.Windows.Controls.TextBlock - $text.Text = $cell - $text.Margin = '0,0,24,0' - [void]$panel.Children.Add($text) +param([string]$RowsJson, [string]$ReadyPath) +$ErrorActionPreference = 'Stop' +try { + Add-Type -AssemblyName PresentationFramework + $rows = $RowsJson | ConvertFrom-Json + $window = New-Object System.Windows.Window + $window.Title = '__TITLE__' + $window.Width = 520 + $window.Height = 360 + $window.Left = 40 + $window.Top = 40 + $list = New-Object System.Windows.Controls.ListBox + foreach ($row in $rows) { + $item = New-Object System.Windows.Controls.ListBoxItem + $panel = New-Object System.Windows.Controls.StackPanel + $panel.Orientation = 'Horizontal' + foreach ($cell in $row.cells) { + $text = New-Object System.Windows.Controls.TextBlock + $text.Text = $cell + $text.Margin = '0,0,24,0' + [void]$panel.Children.Add($text) + } + $item.Content = $panel + if ($row.id) { + [System.Windows.Automation.AutomationProperties]::SetAutomationId($item, $row.id) + } + [void]$list.Items.Add($item) } - $item.Content = $panel - if ($row.id) { - [System.Windows.Automation.AutomationProperties]::SetAutomationId($item, $row.id) - } - [void]$list.Items.Add($item) + $window.Content = $list + $window.add_ContentRendered({ + if ([System.IO.File]::Exists($ReadyPath)) { return } + $helper = [System.Windows.Interop.WindowInteropHelper]::new($window) + $signal = @{ + event = 'content_rendered' + pid = $PID + hwnd = $helper.Handle.ToInt64() + rows = $list.Items.Count + } + $temporary = $ReadyPath + '.tmp' + [System.IO.File]::WriteAllText($temporary, ($signal | ConvertTo-Json -Compress)) + [System.IO.File]::Move($temporary, $ReadyPath) + }) + [void]$window.ShowDialog() +} catch { + [Console]::Error.WriteLine($_.Exception.ToString()) + exit 1 } -$window.Content = $list -[void]$window.ShowDialog() """.replace("__TITLE__", TITLE) @@ -100,43 +121,69 @@ def wpf_worklist(tmp_path: Path): script.write_text(_WPF_APP, encoding="utf-8") procs: list[subprocess.Popen] = [] - def start(rows: list[dict[str, Any]]) -> list[Any]: - proc = subprocess.Popen( - [ - "powershell", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-File", - str(script), - "-RowsJson", - json.dumps(rows), - ] - ) - procs.append(proc) - window = _wait( - lambda: ( - auto.WindowControl(searchDepth=1, Name=TITLE) - if auto.WindowControl(searchDepth=1, Name=TITLE).Exists(0, 0) - else None + with ExitStack() as owned: + + def start(rows: list[dict[str, Any]]) -> list[Any]: + prefix = tmp_path / f"worklist-{len(procs)}" + ready = prefix.with_suffix(".ready.json") + stdout_path, stderr_path = ( + prefix.with_suffix(".stdout"), + prefix.with_suffix(".stderr"), ) - ) - listing = window.ListControl(searchDepth=3) - return _wait( - lambda: ( + stdout = owned.enter_context(stdout_path.open("wb")) + stderr = owned.enter_context(stderr_path.open("wb")) + proc = subprocess.Popen( [ - c - for c in listing.GetChildren() - if c.ControlTypeName == "ListItemControl" + "powershell", + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-STA", + "-ExecutionPolicy", + "Bypass", + "-File", + str(script), + "-RowsJson", + json.dumps(rows), + "-ReadyPath", + str(ready), + ], + stdout=stdout, + stderr=stderr, + ) + procs.append(proc) + owned.callback(stop_owned_process, proc) + + def items_from_owned_window(hwnd: int) -> Optional[list[Any]]: + window = auto.ControlFromHandle(hwnd) + if window is None: + return None + if window.ProcessId != proc.pid or window.Name != TITLE: + raise ValueError( + "rendered HWND does not identify the owned worklist" + ) + listing = window.ListControl(searchDepth=3) + if not listing.Exists(0, 0): + return None + return [ + child + for child in listing.GetChildren() + if child.ControlTypeName == "ListItemControl" ] - if listing.Exists(0, 0) - else None + + # One startup budget inside the existing 180s test limit leaves + # 60s for the unchanged census assertions and owned cleanup. + return wait_for_rendered_items( + proc, + ready, + len(rows), + items_from_owned_window, + stdout_path, + stderr_path, + timeout_s=120.0, ) - ) - yield auto, start - for proc in procs: - proc.kill() + yield auto, start def _first_cell_center(item: Any) -> tuple[int, int]: From 2b5bb1707b2e7e4e0397d2ab716c38ae54f4f7a4 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 21:42:17 -0800 Subject: [PATCH 13/21] flow: API steps for multipart saves, the clicked button's value, and script-sent saves (#1851) * flow: API steps for multipart saves, the clicked button's value, and script-sent saves * flow: types and formatting for the new API step shapes * benchmark: API-step generalization harness for five apps (Redmine, WordPress, NetBox, Dolibarr, Trac) * flow: real-app test of the new API step shapes on the lab fixtures (opt-in) * llms-full: refresh for the API-step capabilities * fix: bind API form shapes to native submission evidence Signed-off-by: abrichr * fix: preserve refusal wording and settle synthetic form capture Signed-off-by: abrichr * fix: compare form destination with transport URL construction Signed-off-by: abrichr * benchmark: generate the NetBox fixture's superuser token per run; pin the old synthetic literal's exact gitleaks finding --------- Signed-off-by: abrichr --- .gitleaksignore | 4 + CAPABILITIES.md | 3 +- llms-full.txt | 3 +- .../benchmark/api_generalization/PROTOCOL.md | 138 +++++++ .../benchmark/api_generalization/__init__.py | 0 .../api_generalization/apps/__init__.py | 1 + .../benchmark/api_generalization/apps/doli.py | 58 +++ .../apps/dolibarr_contact.py | 68 +++ .../apps/dolibarr_thirdparty.py | 62 +++ .../benchmark/api_generalization/apps/nb.py | 78 ++++ .../api_generalization/apps/netbox_ip.py | 93 +++++ .../api_generalization/apps/redmine.py | 124 ++++++ .../apps/trac.Containerfile | 8 + .../benchmark/api_generalization/apps/trac.py | 120 ++++++ .../apps/wordpress_category.py | 66 +++ .../apps/wordpress_comment.py | 65 +++ .../benchmark/api_generalization/apps/wp.py | 76 ++++ .../benchmark/api_generalization/common.py | 330 +++++++++++++++ .../api_generalization/counterfactual.py | 48 +++ .../benchmark/api_generalization/harness.py | 340 +++++++++++++++ .../benchmark/api_generalization/proxy.py | 149 +++++++ .../flow/benchmark/api_generalization/run.sh | 8 + packages/flow/docs/API_TIER.md | 62 ++- packages/flow/docs/NETWORK_CAPTURE.md | 5 +- .../openadapt_flow/compiler/api_proposal.py | 210 +++++++++- packages/flow/openadapt_flow/ir.py | 38 +- .../flow/openadapt_flow/network_capture.py | 81 +++- .../runtime/actuators/form_eval.py | 176 ++++++-- .../runtime/actuators/web_form.py | 296 ++++++++++++-- .../flow/tests/test_api_shape_boundaries.py | 387 ++++++++++++++++++ packages/flow/tests/test_api_step_shapes.py | 362 ++++++++++++++++ .../tests/test_api_step_shapes_fixtures.py | 84 ++++ 32 files changed, 3443 insertions(+), 100 deletions(-) create mode 100644 packages/flow/benchmark/api_generalization/PROTOCOL.md create mode 100644 packages/flow/benchmark/api_generalization/__init__.py create mode 100644 packages/flow/benchmark/api_generalization/apps/__init__.py create mode 100644 packages/flow/benchmark/api_generalization/apps/doli.py create mode 100644 packages/flow/benchmark/api_generalization/apps/dolibarr_contact.py create mode 100644 packages/flow/benchmark/api_generalization/apps/dolibarr_thirdparty.py create mode 100644 packages/flow/benchmark/api_generalization/apps/nb.py create mode 100644 packages/flow/benchmark/api_generalization/apps/netbox_ip.py create mode 100644 packages/flow/benchmark/api_generalization/apps/redmine.py create mode 100644 packages/flow/benchmark/api_generalization/apps/trac.Containerfile create mode 100644 packages/flow/benchmark/api_generalization/apps/trac.py create mode 100644 packages/flow/benchmark/api_generalization/apps/wordpress_category.py create mode 100644 packages/flow/benchmark/api_generalization/apps/wordpress_comment.py create mode 100644 packages/flow/benchmark/api_generalization/apps/wp.py create mode 100644 packages/flow/benchmark/api_generalization/common.py create mode 100644 packages/flow/benchmark/api_generalization/counterfactual.py create mode 100644 packages/flow/benchmark/api_generalization/harness.py create mode 100644 packages/flow/benchmark/api_generalization/proxy.py create mode 100755 packages/flow/benchmark/api_generalization/run.sh create mode 100644 packages/flow/tests/test_api_shape_boundaries.py create mode 100644 packages/flow/tests/test_api_step_shapes.py create mode 100644 packages/flow/tests/test_api_step_shapes_fixtures.py diff --git a/.gitleaksignore b/.gitleaksignore index b61b5d9e4c..f1ecf70645 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -60,3 +60,7 @@ e8ec96725c4451aa64cb97230c18d04b55d98503:packages/flow/tests/e2e/test_mcp_teach_ # test_network_capture_redact_json.py; the next commit builds them at run time. 52502adf407793c211fb3c0ac5aba37556206d8c:packages/flow/tests/test_network_capture_redact_json.py:generic-api-key:17 52502adf407793c211fb3c0ac5aba37556206d8c:packages/flow/tests/test_network_capture_redact_json.py:generic-api-key:146 +# The API-step generalization harness (PR #1851) first started its local NetBox +# container with a fixed synthetic superuser token (a run of hex digits, never a +# credential for any real system); the harness now generates it per run. +9922cee8dbbbaa4494b1c69ec6c71e4899305de6:packages/flow/benchmark/api_generalization/apps/nb.py:generic-api-key:60 diff --git a/CAPABILITIES.md b/CAPABILITIES.md index eedf385b1a..6b585caab9 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -100,7 +100,7 @@ monorepo and installable from source. | Learns branching programs from several demonstrations, from repeated halts, and from many background runs. | `induce`, `teach-branch`, [SHADOW_MINING.md](packages/capture/docs/SHADOW_MINING.md), [LEARN_FROM_MANY_RUNS.md](packages/flow/docs/LEARN_FROM_MANY_RUNS.md) | | Chains programs across surfaces, and one program can switch surfaces, tabs and popups mid-run. | `compose`, `process`, [#1197](https://github.com/OpenAdaptAI/OpenAdapt/pull/1197) | | Uses page structure first on web, Windows, macOS and Linux, with pixels as the fallback for remote desktops. Benchmarks labelled "vision-only" ran a deliberately pixel-only arm. | [RESOLUTION_LADDER.md](packages/flow/docs/RESOLUTION_LADDER.md) | -| Replaces a single HTML form submission with the request the app itself sends, compiled from the network traffic captured while recording, then reads the record back. JSON or GraphQL requests, multi-step wizards and governed runs still use the screen steps. | `api`, [API_TIER.md](packages/flow/docs/API_TIER.md) | +| Replaces a single HTML form submission with the request the app itself sends, compiled from the network traffic captured while recording, then reads the record back. URL-encoded and file-less multipart saves, the clicked button's value, and saves the page's script sends to another endpoint are covered. JSON or GraphQL requests, uploads, multi-step wizards and governed runs still use the screen steps. | `api`, [API_TIER.md](packages/flow/docs/API_TIER.md) | | Builds the record check itself. After a save, the app usually reads the record back to show it; `scaffold-verifier` finds that read in the captured traffic or a read-only database, builds the check from it and tries to break it before a person reviews it. No API or SQL needs to be written by hand. | [EFFECT_KIT.md](packages/flow/docs/EFFECT_KIT.md#a-record-check-built-from-the-apps-own-reads-scaffold-verifier) | | Checks a save on desktop, RDP and Citrix apps with no API at all, by looking the record up in the app's own screens in a separate session. | [EFFECT_KIT.md](packages/flow/docs/EFFECT_KIT.md#separate-session-desktop-read-back-desktop_readback) | | Keeps pixel-path runs working across display scaling from 100% to 175% and font smoothing on or off, and stops rather than saves when the record changed. | [benchmark/pixel_scale](packages/flow/benchmark/pixel_scale/README.md) | @@ -244,6 +244,7 @@ improvising or retrying when something is off. | Capture a recording's network traffic and compile a clean save into an API step, with the same record check | Released ([#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269)) | Synthetic patients on a local OpenEMR test copy | network capture, `ApiBinding` | | Fall back from the API step to page structure when the API is unavailable, before anything is sent | Released ([#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269)) | Synthetic | [#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269) | | Run a form's own script before an API save instead of refusing the form: field handlers, `onsubmit`, submit listeners and custom validate.js rules (OpenEMR's `pastDate`) run on the run's values in a scratch copy of the session with every write blocked, and the save is sent only when the page lets the submit through and would send the same body | On main | Synthetic (headless Chromium against a local test page) | [API_TIER.md](packages/flow/docs/API_TIER.md#what-happens-at-run-time), `runtime/actuators/form_eval.py` | +| Propose API steps for more save shapes: multipart saves with no file (encoded as Chromium encodes them), the clicked submit button's `name=value`, empty file inputs, forms with no `action`, and saves the page's script sends to another endpoint (run through the page's own script before every save); success addresses keep a record's path with numbers and hashes generalized | This PR | Headless Chromium against local test pages, and recordings from Redmine, WordPress, NetBox, Dolibarr and Trac fixtures | [API_TIER.md](packages/flow/docs/API_TIER.md#what-the-compiler-checks) | | Declare the other writes in an API step's span as an ordered sequence: the page's own background requests (another document's script, loaded before the form, to an endpoint the form doesn't name) are listed and not sent; a write a step caused is sent in its recorded place before the save, read back by that step's record check, and a failure stops the run with nothing after it sent and nothing retried | On main | Synthetic | [API_TIER.md](packages/flow/docs/API_TIER.md#other-writes-in-the-span) | | Approve an API step's exact request under per-item approval: one digest binds the request bytes, the record and the record check, and a changed request is not sent | Released ([#1355](https://github.com/OpenAdaptAI/OpenAdapt/pull/1355)) | Synthetic | [API_TIER.md](packages/flow/docs/API_TIER.md#under-an-approval-mode) | | Run an API step in a durable run: checkpoint the span after the save is sent, never resend or redo a save that may have landed, and reconcile it by reading the record back | Released ([#1358](https://github.com/OpenAdaptAI/OpenAdapt/pull/1358)) | Synthetic | [API_TIER.md](packages/flow/docs/API_TIER.md#in-a-durable-run) | diff --git a/llms-full.txt b/llms-full.txt index ec3b480a19..86de7479c0 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -503,7 +503,7 @@ monorepo and installable from source. | Learns branching programs from several demonstrations, from repeated halts, and from many background runs. | `induce`, `teach-branch`, [SHADOW_MINING.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/capture/docs/SHADOW_MINING.md), [LEARN_FROM_MANY_RUNS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/LEARN_FROM_MANY_RUNS.md) | | Chains programs across surfaces, and one program can switch surfaces, tabs and popups mid-run. | `compose`, `process`, [#1197](https://github.com/OpenAdaptAI/OpenAdapt/pull/1197) | | Uses page structure first on web, Windows, macOS and Linux, with pixels as the fallback for remote desktops. Benchmarks labelled "vision-only" ran a deliberately pixel-only arm. | [RESOLUTION_LADDER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/RESOLUTION_LADDER.md) | -| Replaces a single HTML form submission with the request the app itself sends, compiled from the network traffic captured while recording, then reads the record back. JSON or GraphQL requests, multi-step wizards and governed runs still use the screen steps. | `api`, [API_TIER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/API_TIER.md) | +| Replaces a single HTML form submission with the request the app itself sends, compiled from the network traffic captured while recording, then reads the record back. URL-encoded and file-less multipart saves, the clicked button's value, and saves the page's script sends to another endpoint are covered. JSON or GraphQL requests, uploads, multi-step wizards and governed runs still use the screen steps. | `api`, [API_TIER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/API_TIER.md) | | Builds the record check itself. After a save, the app usually reads the record back to show it; `scaffold-verifier` finds that read in the captured traffic or a read-only database, builds the check from it and tries to break it before a person reviews it. No API or SQL needs to be written by hand. | [EFFECT_KIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/EFFECT_KIT.md#a-record-check-built-from-the-apps-own-reads-scaffold-verifier) | | Checks a save on desktop, RDP and Citrix apps with no API at all, by looking the record up in the app's own screens in a separate session. | [EFFECT_KIT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/EFFECT_KIT.md#separate-session-desktop-read-back-desktop_readback) | | Keeps pixel-path runs working across display scaling from 100% to 175% and font smoothing on or off, and stops rather than saves when the record changed. | [benchmark/pixel_scale](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_scale/README.md) | @@ -647,6 +647,7 @@ improvising or retrying when something is off. | Capture a recording's network traffic and compile a clean save into an API step, with the same record check | Released ([#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269)) | Synthetic patients on a local OpenEMR test copy | network capture, `ApiBinding` | | Fall back from the API step to page structure when the API is unavailable, before anything is sent | Released ([#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269)) | Synthetic | [#1269](https://github.com/OpenAdaptAI/OpenAdapt/pull/1269) | | Run a form's own script before an API save instead of refusing the form: field handlers, `onsubmit`, submit listeners and custom validate.js rules (OpenEMR's `pastDate`) run on the run's values in a scratch copy of the session with every write blocked, and the save is sent only when the page lets the submit through and would send the same body | On main | Synthetic (headless Chromium against a local test page) | [API_TIER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/API_TIER.md#what-happens-at-run-time), `runtime/actuators/form_eval.py` | +| Propose API steps for more save shapes: multipart saves with no file (encoded as Chromium encodes them), the clicked submit button's `name=value`, empty file inputs, forms with no `action`, and saves the page's script sends to another endpoint (run through the page's own script before every save); success addresses keep a record's path with numbers and hashes generalized | This PR | Headless Chromium against local test pages, and recordings from Redmine, WordPress, NetBox, Dolibarr and Trac fixtures | [API_TIER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/API_TIER.md#what-the-compiler-checks) | | Declare the other writes in an API step's span as an ordered sequence: the page's own background requests (another document's script, loaded before the form, to an endpoint the form doesn't name) are listed and not sent; a write a step caused is sent in its recorded place before the save, read back by that step's record check, and a failure stops the run with nothing after it sent and nothing retried | On main | Synthetic | [API_TIER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/API_TIER.md#other-writes-in-the-span) | | Approve an API step's exact request under per-item approval: one digest binds the request bytes, the record and the record check, and a changed request is not sent | Released ([#1355](https://github.com/OpenAdaptAI/OpenAdapt/pull/1355)) | Synthetic | [API_TIER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/API_TIER.md#under-an-approval-mode) | | Run an API step in a durable run: checkpoint the span after the save is sent, never resend or redo a save that may have landed, and reconcile it by reading the record back | Released ([#1358](https://github.com/OpenAdaptAI/OpenAdapt/pull/1358)) | Synthetic | [API_TIER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/API_TIER.md#in-a-durable-run) | diff --git a/packages/flow/benchmark/api_generalization/PROTOCOL.md b/packages/flow/benchmark/api_generalization/PROTOCOL.md new file mode 100644 index 0000000000..d6dcb51291 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/PROTOCOL.md @@ -0,0 +1,138 @@ +# Protocol: does the API step generalize beyond the OpenEMR referral? + +## Question + +The API step ([docs/API_TIER.md](../../docs/API_TIER.md)) replaces one HTML +form save with the request the app's own page sends, then reads the record +back. Its evidence so far is one workflow on one app (the OpenEMR referral). +This experiment asks whether the same compiler and runtime, unchanged, cover +other apps that save records through server-rendered HTML forms, and whether +they stay safe when they do. + +## Frozen engine + +The compiler and runtime are frozen at one `main` commit taken after the six +review fixes merged (JSON redaction, provenance-based parameter binding, URL +identifiers, refusing spans with other writes, HTML validation, hashing the +full replaced-step semantics). The commit is recorded in +`results/engine.json` and in every `compile.json` and `runs.csv` row +(`engine_sha`). Every run checks `git diff -- openadapt_flow` is +empty. No compiler or runtime code changes during the experiment; bugs found +are logged in the report and the experiment continues. + +## Apps and workflows + +Five unrelated open-source apps, each run locally in a container as an +external service (no app source is copied into this repository), with +synthetic data only. One or two save workflows each, chosen as the +ordinary "create a record" task of each app, not by whether the API step +would cover it: + +| App (license) | Kind | Workflow | Adapter | +| --- | --- | --- | --- | +| Redmine 6.0 (GPL-2.0) | ticketing, Rails | new issue | `apps/redmine.py` | +| WordPress 6.6 (GPL-2.0) | CMS, PHP | visitor posts a comment | `apps/wordpress_comment.py` | +| WordPress 6.6 | CMS, PHP | editor adds a category | `apps/wordpress_category.py` | +| NetBox 4.1 (Apache-2.0) | inventory, Django | add an IP address | `apps/netbox_ip.py` | +| Dolibarr 20 (GPL-3.0) | ERP/CRM, PHP | new contact | `apps/dolibarr_contact.py` | +| Dolibarr 20 | ERP/CRM, PHP | new customer (third party) | `apps/dolibarr_thirdparty.py` | +| Trac 1.6 (BSD-3-Clause) | ticketing, Python | new ticket | `apps/trac.py` | + +ERPNext was considered and left out: its desk saves records as JSON over +XHR, not through a server-rendered form, so it is outside the population +this question is about. Odoo is in the same position. + +## Procedure per workflow + +1. **Record** (`harness.py record`): a scripted person signs in (not + recorded), network capture starts, the form page opens, and the person + fills the form and clicks save through `Recorder` on the generic + `PlaywrightBackend`. The judge must confirm the demonstration saved the + right record, or the recording is redone. +2. **Compile** (`harness.py compile`): `compile_recording`; the person marks + the fields that may be blank as optional parameters; the save step gets + the record check (`Effect` list: exactly one record matched by the unique + field, and each typed field read back); then `propose_api_step` + (`openadapt-flow api propose`). When an API step is proposed, the harness + admits it under the name `benchmark-fixture` (`openadapt-flow api admit` + in use). `compile.json` keeps the proposal, or the reasons for none. +3. **Matrix** (`harness.py matrix`): each case below runs in both arms, the + arm order alternating by case. Each run gets a fresh browser context, + signs in, opens the form, and runs the bundle with + `Replayer(api_steps=False)` (arm `gui`) or `Replayer(api_steps=True)` + (arm `api`). Values carry a per-run tag so repeated runs never collide + with an app's own duplicate rules. +4. **Timing** (`harness.py timing`), only where an API step ran: normal + cases, both arms interleaved, in a quiet window agreed with the + coordinator, with the machine's 1-minute load logged per run. + +One app fixture runs at a time. Disk is checked before each app; the +experiment stops below 15 GiB free. + +## Cases + +| Case | Kind | Condition | The app should end up with | +| --- | --- | --- | --- | +| N1-N3 | normal | three ordinary value sets | one new record with the values | +| B1 | blank | optional fields left blank | one new record, blanks stored empty | +| D1 | dup_values | the same text in several fields | one new record with the values | +| C1 | changed_default | after recording, the app's default for a field the person never touches changes (Redmine default priority, NetBox custom-field default, Dolibarr company country, Trac default priority) | one new record holding the **new** default, as a person's save would | +| S1 | expired_session | the server ends the session after the form is open (session rows or files deleted) | no record | +| F1 | fault_reject | the proxy answers the save with 503 without forwarding it | no record | +| F2 | fault_partial | the proxy forwards the save, the app commits, the proxy answers 502 | one record (no second one) | +| F3 | fault_duplicate | the proxy sends the save to the app twice | two records exist (the fixture made the second); the run must not report success | +| V1 | validation | a value the server rejects | no record | + +Faults are injected by `proxy.py`, a reverse proxy in front of each app that +both arms go through, on the first request that is the save (path, and for +Trac, a body without the auto-preview's `preview=`). `C1` is not applicable +to forms with no app-configurable default the person leaves alone (the +WordPress comment and category forms, the Dolibarr third-party form). `S1` +is not applicable to the anonymous WordPress comment. + +## Scoring + +The judge (`common.judge`) reads the whole record table before and after +each run through the app's database, with its own query, separate from the +record check the run itself uses. A run is: + +- **correct** when the app holds what it should and the run's own claim + matches: success exactly when the expected write happened, or + `RECONCILIATION_REQUIRED` after a lost response (F2), and never success + when the fixture duplicated the save (F3); +- an **incorrect write** when the app holds something the run should not + have caused: a wrong value, an extra record, a changed or deleted record + (the fixture's own duplicate in F3 is not counted against the run); +- a **false success** when the run reported success and the judge disagrees; +- a **missed write** when nothing wrong was written but an expected record + is missing (an honest stop). + +## Measures + +Per app and overall: coverage (workflows with an API step proposed and +admitted against those refused, with the compiler's reasons); incorrect +writes and false successes; correct refusals and fallbacks to the GUI steps +(`api_fallback` on the run); manual setup minutes per app (adapter writing +and fixture bring-up, from the session log); and, where the API step ran, +engine time against the GUI steps. + +## Counterfactual arm (labelled, not a product result) + +Development compiles on `main` 3cb70fa6 showed every workflow refused +before any proposal; three of them (WordPress comment, Dolibarr contact, +Trac) only because the clicked submit button's own `name=value` is in the +saved request and the compiler skips submit buttons when it reads the form. +To find what would block next, and to exercise the frozen **runtime** on new +apps at all, `counterfactual.py` copies those recordings with only that one +pair removed from the captured save, and the same frozen compiler runs on +the copy. Those bundles run as arm `api_cf`. Their numbers describe the +runtime on these apps under that one input edit; they are not coverage of +the shipped compiler and are reported apart. + +## Known before the protocol was finalized + +The development compiles above (on 3cb70fa6, before the review fixes) and a +development matrix on the Trac counterfactual bundle (used to fix the +harness: preview posts, required blank parameters, Trac validation rule) +were seen before this protocol was finalized. Their outputs are not part of +the results. diff --git a/packages/flow/benchmark/api_generalization/__init__.py b/packages/flow/benchmark/api_generalization/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/flow/benchmark/api_generalization/apps/__init__.py b/packages/flow/benchmark/api_generalization/apps/__init__.py new file mode 100644 index 0000000000..8f3b0f70fc --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/__init__.py @@ -0,0 +1 @@ +"""One adapter per app (see common.AppBase).""" diff --git a/packages/flow/benchmark/api_generalization/apps/doli.py b/packages/flow/benchmark/api_generalization/apps/doli.py new file mode 100644 index 0000000000..d9f1af5139 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/doli.py @@ -0,0 +1,58 @@ +"""Dolibarr 20 (ERP/CRM, PHP, MariaDB): the shared fixture. + +Runs as an external service from the project's image; nothing from +Dolibarr (GPL-3.0) is copied here. Synthetic companies and contacts. +""" + +from __future__ import annotations + +import subprocess +from typing import Any + +from benchmark.api_generalization.common import AppBase, mysql_rows, podman + +POD = "apigen-doli" +DB_C, APP_C = "apigen-doli-db", "apigen-doli-app" +APP_IMAGE = "docker.io/dolibarr/dolibarr:20.0.2" +DB_IMAGE = "docker.io/library/mariadb@sha256:efb4959ef2c835cd735dbc388eb9ad6aab0c78dd64febcd51bc17481111890c4" +DB_PASS = "synthetic-db-pass" # throwaway local container +ADMIN_PASS = "synthetic-admin-pass" # throwaway local container +UPSTREAM, PROXY = 9851, 9852 + + +def dsql(query: str) -> list[dict[str, Any]]: + return mysql_rows(DB_C, "root", DB_PASS, "dolibarr", query) + + +class DolibarrBase(AppBase): + upstream_port = UPSTREAM + proxy_port = PROXY + + def up(self) -> None: + if subprocess.run(["podman", "pod", "exists", POD]).returncode == 0: + podman("pod", "start", POD) + self.wait_http("/") + return + podman("pod", "create", "--name", POD, "-p", f"127.0.0.1:{UPSTREAM}:80") + podman("run", "-d", "--pod", POD, "--name", DB_C, "-e", f"MARIADB_ROOT_PASSWORD={DB_PASS}", + "-e", "MARIADB_DATABASE=dolibarr", DB_IMAGE) + podman("run", "-d", "--pod", POD, "--name", APP_C, "-e", "DOLI_DB_HOST=127.0.0.1", "-e", "DOLI_DB_NAME=dolibarr", + "-e", "DOLI_DB_USER=root", "-e", f"DOLI_DB_PASSWORD={DB_PASS}", "-e", "DOLI_ADMIN_LOGIN=admin", + "-e", f"DOLI_ADMIN_PASSWORD={ADMIN_PASS}", "-e", "DOLI_INSTALL_AUTO=1", "-e", "DOLI_INIT_DEMO=0", + "-e", f"DOLI_URL_ROOT=http://127.0.0.1:{PROXY}", "-e", "DOLI_COMPANY_NAME=Synthetic Supplies", + "-e", "DOLI_ENABLE_MODULES=Societe", APP_IMAGE) + self.wait_http("/", timeout_s=900) + + def down(self) -> None: + podman("pod", "stop", POD, check=False) + + def login(self, page: Any) -> None: + page.goto(self.base + "/index.php") + page.fill("#username", "admin") + page.fill("#password", ADMIN_PASS) + page.click("input[type=submit]") + page.wait_for_load_state("load") + + def expire_session(self, page: Any) -> None: + # PHP keeps sessions in files; removing them ends every session. + podman("exec", APP_C, "sh", "-c", "rm -f /var/lib/php/sessions/* /tmp/sess_* 2>/dev/null; true") diff --git a/packages/flow/benchmark/api_generalization/apps/dolibarr_contact.py b/packages/flow/benchmark/api_generalization/apps/dolibarr_contact.py new file mode 100644 index 0000000000..1f32b87c13 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/dolibarr_contact.py @@ -0,0 +1,68 @@ +"""Dolibarr: a sales assistant adds a contact (Third parties > New contact/address).""" + +from __future__ import annotations + +from typing import Any + +from benchmark.api_generalization.apps.doli import DolibarrBase, dsql +from benchmark.api_generalization.common import ExecSqlVerifier, rec_click, rec_text +from openadapt_flow.runtime.effects.effect import Effect, EffectKind, ValueExpr + +QUERY = ("SELECT rowid AS id, lastname, firstname, poste, email, phone AS phone_pro FROM llx_socpeople") +JUDGE = ("SELECT p.rowid AS id, p.lastname, p.firstname, p.poste, p.email, p.phone, p.fk_soc, p.civility, " + "COALESCE(c.code, '') AS country, p.priv, p.statut, p.entity FROM llx_socpeople p " + "LEFT JOIN llx_c_country c ON c.rowid = p.fk_pays ORDER BY p.rowid") + + +class App(DolibarrBase): + name = "dolibarr_contact" + workflow = "new-contact" + form_path = "/contact/card.php?action=create" + save_path_regex = r"^/contact/card\.php$" + unique_param = "lastname" + normal_values = [ + dict(lastname="Okafor", firstname="Chidi", poste="Purchasing lead", email="c.okafor@example.invalid", phone_pro="555-0161"), + dict(lastname="Lindgren", firstname="Maja", poste="Accounts payable", email="m.lindgren@example.invalid", phone_pro="555-0162"), + dict(lastname="Tanaka", firstname="Hiro", poste="Site manager", email="h.tanaka@example.invalid", phone_pro="555-0163"), + ] + blank_values = dict(lastname="Moreau", firstname="", poste="", email="", phone_pro="") + dup_values = dict(lastname="Quinn", firstname="Quinn", poste="Quinn", email="quinn@example.invalid", phone_pro="555-0164") + invalid_values = dict(lastname="Bad Mail", firstname="Ivo", poste="Buyer", email="ivo.at.example.invalid", phone_pro="555-0165") + invalid_note = "email without @ (the server rejects it)" + default_column = "country" + changed_default_value = "FR" + + def record(self, rec: Any, page: Any, v: dict[str, str]) -> None: + rec_text(rec, page, "#lastname", v["lastname"], "lastname") + rec_text(rec, page, "#firstname", v["firstname"], "firstname") + rec_text(rec, page, "#title", v["poste"], "poste") + rec_text(rec, page, "#phone_pro", v["phone_pro"], "phone_pro") + rec_text(rec, page, "#email", v["email"], "email") + rec_click(rec, page, "input[name=add]") + + def change_default(self) -> None: + # The company's own country is the new contact's default country. + dsql("INSERT INTO llx_const (name, entity, value, type, visible) " + "VALUES ('MAIN_INFO_SOCIETE_COUNTRY', 1, '1:FR:France', 'chaine', 0)") + + def restore_default(self) -> None: + dsql("DELETE FROM llx_const WHERE name = 'MAIN_INFO_SOCIETE_COUNTRY'") + + def rows(self) -> list[dict[str, Any]]: + return dsql(JUDGE) + + def verifier(self) -> Any: + return ExecSqlVerifier(dsql, QUERY, timeout_s=10.0) + + def expect_fields(self, v: dict[str, str]) -> dict[str, str]: + return {"lastname": v["lastname"], "firstname": v["firstname"], "poste": v["poste"], + "email": v["email"], "phone": v["phone_pro"], "country": ""} + + def effects(self) -> list[Effect]: + match = {"lastname": ValueExpr(param="lastname")} + out = [Effect(kind=EffectKind.RECORD_WRITTEN, match=match, expected_count=1, risk="reversible", + probe="exactly one contact with this last name", timeout_s=8.0)] + for f in ("firstname", "poste", "email", "phone_pro"): + out.append(Effect(kind=EffectKind.FIELD_EQUALS, match=match, field=f, value=ValueExpr(param=f), + risk="reversible", probe=f"contact {f} read-back", timeout_s=8.0)) + return out diff --git a/packages/flow/benchmark/api_generalization/apps/dolibarr_thirdparty.py b/packages/flow/benchmark/api_generalization/apps/dolibarr_thirdparty.py new file mode 100644 index 0000000000..71b68de402 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/dolibarr_thirdparty.py @@ -0,0 +1,62 @@ +"""Dolibarr: a sales assistant creates a customer (Third parties > New third party).""" + +from __future__ import annotations + +from typing import Any + +from benchmark.api_generalization.apps.doli import DolibarrBase, dsql +from benchmark.api_generalization.common import ( + ExecSqlVerifier, + rec_click, + rec_select2_click, + rec_text, +) +from openadapt_flow.runtime.effects.effect import Effect, EffectKind, ValueExpr + +QUERY = "SELECT rowid AS id, nom AS name, town, email, phone FROM llx_societe" +JUDGE = ("SELECT rowid AS id, nom AS name, town, email, phone, client, status, fk_typent, entity " + "FROM llx_societe ORDER BY rowid") + + +class App(DolibarrBase): + name = "dolibarr_thirdparty" + workflow = "new-customer" + form_path = "/societe/card.php?action=create" + save_path_regex = r"^/societe/card\.php$" + unique_param = "name" + normal_values = [ + dict(name="Harbor Office Supply", town="Exampleton", email="orders@harbor.example.invalid", phone="555-0171"), + dict(name="Northgate Catering", town="Sampleville", email="hello@northgate.example.invalid", phone="555-0172"), + dict(name="Blue Fern Cleaning", town="Testford", email="info@bluefern.example.invalid", phone="555-0173"), + ] + blank_values = dict(name="Plain Name Traders", town="", email="", phone="") + dup_values = dict(name="Mirror Town", town="Mirror Town", email="mirror@example.invalid", phone="555-0174") + invalid_values = dict(name="Bad Mail Traders", town="Testford", email="no-at-sign.example.invalid", phone="555-0175") + invalid_note = "email without @ (the server rejects it)" + + def record(self, rec: Any, page: Any, v: dict[str, str]) -> None: + rec_text(rec, page, "#name", v["name"], "name") + rec_select2_click(rec, page, "customerprospect", "Customer") + rec_text(rec, page, "#town", v["town"], "town") + rec_text(rec, page, "#phone", v["phone"], "phone") + rec_text(rec, page, "#email", v["email"], "email") + rec_click(rec, page, "input[name=save]") + + def rows(self) -> list[dict[str, Any]]: + return dsql(JUDGE) + + def verifier(self) -> Any: + return ExecSqlVerifier(dsql, QUERY, timeout_s=10.0) + + def expect_fields(self, v: dict[str, str]) -> dict[str, str]: + return {"name": v["name"], "town": v["town"], "email": v["email"], "phone": v["phone"], + "client": "1", "status": "1"} + + def effects(self) -> list[Effect]: + match = {"name": ValueExpr(param="name")} + out = [Effect(kind=EffectKind.RECORD_WRITTEN, match=match, expected_count=1, risk="reversible", + probe="exactly one third party with this name", timeout_s=8.0)] + for f in ("town", "email", "phone"): + out.append(Effect(kind=EffectKind.FIELD_EQUALS, match=match, field=f, value=ValueExpr(param=f), + risk="reversible", probe=f"third party {f} read-back", timeout_s=8.0)) + return out diff --git a/packages/flow/benchmark/api_generalization/apps/nb.py b/packages/flow/benchmark/api_generalization/apps/nb.py new file mode 100644 index 0000000000..125b67f086 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/nb.py @@ -0,0 +1,78 @@ +"""NetBox 4.1 (network inventory, Django, PostgreSQL): the shared fixture. + +Runs as an external service from the community image (NetBox is +Apache-2.0); nothing is copied here. Synthetic tenants and addresses +(192.0.2.0/24 and 198.51.100.0/24 are documentation ranges). +""" + +from __future__ import annotations + +import csv +import io +import secrets +import subprocess +from typing import Any + +from benchmark.api_generalization.common import AppBase, podman + +POD = "apigen-nb" +PG_C, REDIS_C, NB_C = "apigen-nb-pg", "apigen-nb-redis", "apigen-nb-app" +NB_IMAGE = "docker.io/netboxcommunity/netbox:v4.1.11" +PG_IMAGE = "docker.io/library/postgres:16-alpine" +REDIS_IMAGE = "ffb4a863fae9" # docker.io/library/redis (alpine), already on this machine +DB_PASS = "synthetic-db-pass" # throwaway local container +ADMIN_PASS = "synthetic-admin-pass" # throwaway local container +UPSTREAM, PROXY = 9841, 9842 +SEED = """ +from extras.models import CustomField +from core.models import ObjectType +from ipam.models import IPAddress +cf, _ = CustomField.objects.get_or_create(name='owner_team', defaults=dict(type='text', label='Owner team', default='Network Ops')) +cf.default = 'Network Ops'; cf.save() +cf.object_types.set([ObjectType.objects.get_for_model(IPAddress)]) +print('seeded') +""" + + +def nbsql(query: str) -> list[dict[str, Any]]: + out = podman("exec", "-i", PG_C, "psql", "-U", "netbox", "-d", "netbox", "--csv", "-v", "ON_ERROR_STOP=1", "-f", "-", + input=query) + return list(csv.DictReader(io.StringIO(out))) + + +class NetBoxBase(AppBase): + upstream_port = UPSTREAM + proxy_port = PROXY + + def up(self) -> None: + if subprocess.run(["podman", "pod", "exists", POD]).returncode == 0: + podman("pod", "start", POD) + self.wait_http("/login/") + return + podman("pod", "create", "--name", POD, "-p", f"127.0.0.1:{UPSTREAM}:8080") + podman("run", "-d", "--pod", POD, "--name", PG_C, "-e", "POSTGRES_USER=netbox", "-e", f"POSTGRES_PASSWORD={DB_PASS}", + "-e", "POSTGRES_DB=netbox", PG_IMAGE) + podman("run", "-d", "--pod", POD, "--name", REDIS_C, REDIS_IMAGE) + podman("run", "-d", "--pod", POD, "--name", NB_C, + "-e", "DB_HOST=127.0.0.1", "-e", "DB_NAME=netbox", "-e", "DB_USER=netbox", "-e", f"DB_PASSWORD={DB_PASS}", + "-e", "REDIS_HOST=127.0.0.1", "-e", "REDIS_CACHE_HOST=127.0.0.1", + "-e", "SECRET_KEY=synthetic-test-only-secret-key-0123456789abcdefghijklmnopqrstuvwxyz", + "-e", "SKIP_SUPERUSER=false", "-e", "SUPERUSER_NAME=admin", "-e", "SUPERUSER_EMAIL=admin@example.invalid", + "-e", f"SUPERUSER_PASSWORD={ADMIN_PASS}", "-e", f"SUPERUSER_API_TOKEN={secrets.token_hex(20)}", + NB_IMAGE) + self.wait_http("/login/", timeout_s=900) + print(podman("exec", "-i", NB_C, "/opt/netbox/venv/bin/python", "/opt/netbox/netbox/manage.py", "shell", + input=SEED, timeout=300)) + + def down(self) -> None: + podman("pod", "stop", POD, check=False) + + def login(self, page: Any) -> None: + page.goto(self.base + "/login/") + page.fill("#id_username", "admin") + page.fill("#id_password", ADMIN_PASS) + page.click("button[type=submit]") + page.wait_for_load_state("load") + + def expire_session(self, page: Any) -> None: + nbsql("DELETE FROM django_session") diff --git a/packages/flow/benchmark/api_generalization/apps/netbox_ip.py b/packages/flow/benchmark/api_generalization/apps/netbox_ip.py new file mode 100644 index 0000000000..b14ba60e4b --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/netbox_ip.py @@ -0,0 +1,93 @@ +"""NetBox: an engineer records a new IP address (IPAM > IP Addresses > Add).""" + +from __future__ import annotations + +import random +from typing import Any + +from benchmark.api_generalization.apps.nb import NB_C, NetBoxBase, nbsql +from benchmark.api_generalization.common import ( + Case, + ExecSqlVerifier, + podman, + rec_click, + rec_text, +) +from openadapt_flow.runtime.effects.effect import Effect, EffectKind, ValueExpr + +QUERY = ("SELECT id, host(address) || '/' || masklen(address) AS address, dns_name, description " + "FROM ipam_ipaddress") +JUDGE = ("SELECT id, host(address) || '/' || masklen(address) AS address, status, dns_name, description, " + "COALESCE(custom_field_data->>'owner_team', '') AS owner_team, role, tenant_id, vrf_id, comments " + "FROM ipam_ipaddress ORDER BY id") + + +def set_cf_default(value: str) -> None: + podman("exec", "-i", NB_C, "/opt/netbox/venv/bin/python", "/opt/netbox/netbox/manage.py", "shell", + input=f"from extras.models import CustomField\ncf=CustomField.objects.get(name='owner_team'); " + f"cf.default={value!r}; cf.save()\n", timeout=300) + + +class App(NetBoxBase): + name = "netbox_ip" + workflow = "add-ip-address" + form_path = "/ipam/ip-addresses/add/" + save_path_regex = r"^/ipam/ip-addresses/add/$" + unique_param = "address" + normal_values = [ + dict(address="10.77.0.10/24", dns_name="printer-2f.example.invalid", description="Second floor printer"), + dict(address="10.77.0.11/24", dns_name="door-ctl.example.invalid", description="Badge door controller"), + dict(address="10.77.0.12/24", dns_name="kiosk-lobby.example.invalid", description="Lobby kiosk"), + ] + blank_values = dict(address="10.77.0.13/24", dns_name="", description="") + dup_values = dict(address="10.77.0.14/24", dns_name="same.example.invalid", description="same.example.invalid") + invalid_values = dict(address="10.77.0.300/24", dns_name="bad-octet.example.invalid", description="Octet over 255") + invalid_note = "an address with an octet over 255 (the server rejects it)" + default_column = "owner_team" + changed_default_value = "Platform" + + def unique(self, case: Case, tag: str) -> Case: + if case.kind == "validation": + return case + taken = {r["address"] for r in nbsql(QUERY)} + rng = random.Random(tag) + while True: + addr = f"10.77.{rng.randrange(1, 250)}.{rng.randrange(1, 250)}/24" + if addr not in taken: + break + params = dict(case.params, address=addr) + fields = dict(case.expect_fields) + if fields: + fields["address"] = addr + return Case(case.id, case.kind, params, case.expect_new, fields, case.fault, case.note) + + def record(self, rec: Any, page: Any, v: dict[str, str]) -> None: + rec_text(rec, page, "#id_address", v["address"], "address") + rec_text(rec, page, "#id_dns_name", v["dns_name"], "dns_name") + rec_text(rec, page, "#id_description", v["description"], "description") + rec_click(rec, page, "button[name=_create]") + + def change_default(self) -> None: + set_cf_default("Platform") + + def restore_default(self) -> None: + set_cf_default("Network Ops") + + def rows(self) -> list[dict[str, Any]]: + return nbsql(JUDGE) + + def verifier(self) -> Any: + return ExecSqlVerifier(nbsql, QUERY, timeout_s=10.0) + + def expect_fields(self, v: dict[str, str]) -> dict[str, str]: + return {"address": v["address"], "dns_name": v["dns_name"], "description": v["description"], + "status": "active", "owner_team": "Network Ops"} + + def effects(self) -> list[Effect]: + match = {"address": ValueExpr(param="address")} + out = [Effect(kind=EffectKind.RECORD_WRITTEN, match=match, expected_count=1, risk="reversible", + probe="exactly one IP address record with this address", timeout_s=8.0)] + for f in ("dns_name", "description"): + out.append(Effect(kind=EffectKind.FIELD_EQUALS, match=match, field=f, value=ValueExpr(param=f), + risk="reversible", probe=f"IP address {f} read-back", timeout_s=8.0)) + return out diff --git a/packages/flow/benchmark/api_generalization/apps/redmine.py b/packages/flow/benchmark/api_generalization/apps/redmine.py new file mode 100644 index 0000000000..9fa81db390 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/redmine.py @@ -0,0 +1,124 @@ +"""Redmine 6.0 (ticketing, Ruby on Rails, SQLite): a new issue in one project. + +Runs as an external service from the official image; nothing from Redmine +(GPL-2.0) is copied here. Synthetic project, synthetic text. +""" + +from __future__ import annotations + +import json +from typing import Any + +from benchmark.api_generalization.common import ( + AppBase, + ExecSqlVerifier, + podman, + rec_click, + rec_select, + rec_text, +) +from openadapt_flow.runtime.effects.effect import Effect, EffectKind, ValueExpr + +C = "apigen-redmine" +IMAGE = "docker.io/library/redmine:6.0-alpine" +DB = "/usr/src/redmine/sqlite/redmine.db" +ENV = ["-u", "redmine", "-e", "RAILS_ENV=production", "-e", "SECRET_KEY_BASE=synthetic-test-only-0123456789abcdef", + "-e", "REDMINE_LANG=en"] +PASSWORD = "synthetic-admin-pass" # a throwaway local container's admin, not a real credential +SEED = f""" +u = User.find_by_login('admin'); u.password = '{PASSWORD}'; u.password_confirmation = '{PASSWORD}' +u.must_change_passwd = false; u.save! +p = Project.find_by_identifier('helpdesk') || Project.create!(name: 'Synthetic Helpdesk', identifier: 'helpdesk', is_public: false) +p.trackers = Tracker.all; p.enabled_module_names = ['issue_tracking']; p.save! +""" +QUERY = ("SELECT i.id, i.subject, COALESCE(i.description, '') AS description, t.name AS tracker, " + "COALESCE(i.estimated_hours, '') AS estimated_hours, p.name AS priority " + "FROM issues i JOIN trackers t ON t.id = i.tracker_id JOIN enumerations p ON p.id = i.priority_id " + "WHERE i.project_id = 1") +JUDGE = ("SELECT i.*, t.name AS tracker, p.name AS priority, s.name AS status FROM issues i " + "JOIN trackers t ON t.id = i.tracker_id JOIN enumerations p ON p.id = i.priority_id " + "JOIN issue_statuses s ON s.id = i.status_id ORDER BY i.id") + + +def sql(query: str, *, write: bool = False) -> list[dict[str, Any]]: + ro = "false" if write else "true" + script = (f'db=SQLite3::Database.new("{DB}", readonly: {ro}); db.busy_timeout=5000; db.results_as_hash=true; ' + f'r=db.execute(STDIN.read); puts JSON.dump(r.map {{|h| h.reject {{|k,_| k.is_a?(Integer)}} }})') + out = podman("exec", "-i", C, "ruby", "-rsqlite3", "-rjson", "-e", script, input=query) + return json.loads(out or "[]") + + +class App(AppBase): + name = "redmine" + workflow = "new-issue" + upstream_port = 9821 + proxy_port = 9822 + form_path = "/projects/helpdesk/issues/new" + save_path_regex = r"^/projects/helpdesk/issues$" + unique_param = "subject" + normal_values = [ + dict(tracker="Bug", subject="Badge printer jams on second floor", description="Paper feed stalls after ten sheets.", estimated_hours="1.5"), + dict(tracker="Support", subject="Reset shared calendar permissions", description="Front desk cannot see room bookings.", estimated_hours="0.25"), + dict(tracker="Feature", subject="Add export of visitor log", description="Weekly CSV for the facilities team.", estimated_hours="3.5"), + ] + blank_values = dict(tracker="Bug", subject="Kiosk screen flickers", description="", estimated_hours="") + dup_values = dict(tracker="Support", subject="Same words twice", description="Same words twice", estimated_hours="2.5") + invalid_values = dict(tracker="Bug", subject="Hours field holds text", description="Validation case.", estimated_hours="abc") + invalid_note = "estimated hours 'abc' (the server rejects it)" + default_column = "priority" + changed_default_value = "High" + + def up(self) -> None: + podman("run", "-d", "--name", C, "-p", f"127.0.0.1:{self.upstream_port}:3000", + "-e", "REDMINE_SECRET_KEY_BASE=synthetic-test-only-0123456789abcdef", IMAGE) + self.wait_http("/login") + podman("exec", *ENV, C, "bundle", "exec", "rake", "redmine:load_default_data", timeout=600) + podman("exec", "-i", *ENV, C, "sh", "-c", "cat > /tmp/seed.rb && bundle exec rails runner /tmp/seed.rb", + input=SEED, timeout=600) + + def down(self) -> None: + podman("rm", "-f", C, check=False) + + def login(self, page: Any) -> None: + page.goto(self.base + "/login") + page.fill("#username", "admin") + page.fill("#password", PASSWORD) + page.click("#login-submit") + page.wait_for_load_state("load") + + def record(self, rec: Any, page: Any, v: dict[str, str]) -> None: + rec_select(rec, page, "#issue_tracker_id", v["tracker"], "tracker") + page.wait_for_timeout(1500) # the tracker change re-renders the form's attributes + rec_text(rec, page, "#issue_subject", v["subject"], "subject") + rec_text(rec, page, "#issue_description", v["description"], "description") + rec_text(rec, page, "#issue_estimated_hours", v["estimated_hours"], "estimated_hours") + rec_click(rec, page, "input[name=commit]") + + def expire_session(self, page: Any) -> None: + sql("DELETE FROM tokens WHERE action = 'session'", write=True) + + def change_default(self) -> None: + sql("UPDATE enumerations SET is_default = (name = 'High') WHERE type = 'IssuePriority'", write=True) + + def restore_default(self) -> None: + sql("UPDATE enumerations SET is_default = (name = 'Normal') WHERE type = 'IssuePriority'", write=True) + + def rows(self) -> list[dict[str, Any]]: + return [r | {"estimated_hours": "" if r.get("estimated_hours") is None else str(r["estimated_hours"])} + for r in sql(JUDGE)] + + def verifier(self) -> Any: + return ExecSqlVerifier(sql, QUERY, timeout_s=10.0) + + def expect_fields(self, v: dict[str, str]) -> dict[str, str]: + return {"subject": v["subject"], "description": v["description"], "tracker": v["tracker"], + "estimated_hours": v["estimated_hours"], "priority": "Normal"} + + def effects(self) -> list[Effect]: + match = {"subject": ValueExpr(param="subject")} + out = [Effect(kind=EffectKind.RECORD_WRITTEN, match=match, expected_count=1, risk="reversible", + probe="exactly one issue with this subject", timeout_s=8.0)] + for f in ("tracker", "description", "estimated_hours"): + out.append(Effect(kind=EffectKind.FIELD_EQUALS, match=match, field=f, value=ValueExpr(param=f), + risk="reversible", probe=f"issue {f} read-back", timeout_s=8.0)) + return out diff --git a/packages/flow/benchmark/api_generalization/apps/trac.Containerfile b/packages/flow/benchmark/api_generalization/apps/trac.Containerfile new file mode 100644 index 0000000000..267581a531 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/trac.Containerfile @@ -0,0 +1,8 @@ +# Trac 1.6 (BSD-3-Clause) for the API-step generalization benchmark: a synthetic +# local service desk. Throwaway admin password; bind to 127.0.0.1 only. +FROM docker.io/library/python:3.12-slim-bookworm +RUN pip install --no-cache-dir "Trac==1.6" "Jinja2<3.2" "Babel" "setuptools<81" && trac-admin /srv/trac initenv "Synthetic Service Desk" sqlite:db/trac.db \ + && printf 'admin:{SHA}%s\n' "$(python -c 'import hashlib,base64;print(base64.b64encode(hashlib.sha1(b"synthetic-admin-pass").digest()).decode())')" > /srv/trac/htpasswd \ + && trac-admin /srv/trac permission add admin TRAC_ADMIN +EXPOSE 8000 +CMD ["tracd", "-s", "--port", "8000", "--basic-auth", "*,/srv/trac/htpasswd,trac", "/srv/trac"] diff --git a/packages/flow/benchmark/api_generalization/apps/trac.py b/packages/flow/benchmark/api_generalization/apps/trac.py new file mode 100644 index 0000000000..46d236b1b6 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/trac.py @@ -0,0 +1,120 @@ +"""Trac 1.6 (ticketing, Python, SQLite): a new ticket. + +Built from PyPI by ``trac.Containerfile`` (Trac is BSD-3-Clause); nothing is +copied here. Sign-in is HTTP basic auth on /login, which sets Trac's own +session cookie. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from benchmark.api_generalization.common import ( + AppBase, + ExecSqlVerifier, + podman, + rec_click, + rec_select, + rec_text, +) +from openadapt_flow.runtime.effects.effect import Effect, EffectKind, ValueExpr + +C = "apigen-trac" +IMAGE = "localhost/apigen-trac:1.6" +DB = "/srv/trac/db/trac.db" +PASSWORD = "synthetic-admin-pass" # throwaway local container +QUERY = "SELECT id, summary, description, type, keywords FROM ticket" +JUDGE = "SELECT id, summary, description, type, priority, keywords, reporter, component, status, owner FROM ticket ORDER BY id" + + +def tsql(query: str, *, write: bool = False) -> list[dict[str, Any]]: + script = ("import json,sqlite3,sys\n" + f"c=sqlite3.connect('file:{DB}' + ('' if {write!r} else '?mode=ro'), uri=True, timeout=5)\n" + "c.row_factory=sqlite3.Row\n" + "cur=c.execute(sys.argv[1]); rows=[dict(r) for r in cur.fetchall()] if cur.description else []\n" + "c.commit(); print(json.dumps(rows))\n") + return json.loads(podman("exec", C, "python", "-c", script, query) or "[]") + + +class App(AppBase): + name = "trac" + workflow = "new-ticket" + upstream_port = 9871 + proxy_port = 9872 + form_path = "/newticket" + save_path_regex = r"^/newticket$" + unique_param = "summary" + normal_values = [ + dict(type="defect", summary="Label printer skips every other label", description="Seen on the shipping desk unit.", keywords="printer shipping"), + dict(type="task", summary="Rotate the guest wifi passphrase", description="Quarterly rotation for the lobby network.", keywords="wifi"), + dict(type="enhancement", summary="Show room capacity on the booking page", description="Facilities asked for this.", keywords="booking"), + ] + blank_values = dict(type="defect", summary="Projector remote missing", description="", keywords="") + dup_values = dict(type="task", summary="Same words", description="Same words", keywords="Same words") + invalid_values = dict(type="defect", summary="Summary longer than this desk allows: the ticket form takes sixty characters at most", + description="Validation case.", keywords="limits") + invalid_note = "a summary over the desk's 60-character limit (trac.ini max_summary_size; the server rejects it)" + save_body_exclude = r"(^|&)preview=" + default_column = "priority" + changed_default_value = "critical" + + def up(self) -> None: + here = Path(__file__).resolve().parent + if podman("images", "-q", IMAGE).strip() == "": + podman("build", "-t", IMAGE, "-f", str(here / "trac.Containerfile"), str(here), timeout=900) + podman("run", "-d", "--name", C, "-p", f"127.0.0.1:{self.upstream_port}:8000", IMAGE) + podman("exec", C, "trac-admin", "/srv/trac", "config", "set", "ticket", "max_summary_size", "60") + self.wait_http("/") + + def down(self) -> None: + podman("rm", "-f", C, check=False) + + def login(self, page: Any) -> None: + # Basic auth answers the /login challenge once; Trac then signs the + # browser in with its own trac_auth cookie. + import base64 + + auth = base64.b64encode(f"admin:{PASSWORD}".encode()).decode() + page.context.request.get(self.base + "/login", headers={"Authorization": f"Basic {auth}"}) + page.goto(self.base + "/") + page.wait_for_load_state("load") + + def expire_session(self, page: Any) -> None: + tsql("DELETE FROM auth_cookie", write=True) + + def record(self, rec: Any, page: Any, v: dict[str, str]) -> None: + rec_text(rec, page, "#field-summary", v["summary"], "summary") + rec_text(rec, page, "#field-description", v["description"], "description") + rec_select(rec, page, "#field-type", v["type"], "type") + rec_text(rec, page, "#field-keywords", v["keywords"], "keywords") + rec_click(rec, page, "input[name=submit]") + + def _priority(self, value: str) -> None: + podman("exec", C, "trac-admin", "/srv/trac", "config", "set", "ticket", "default_priority", value) + + def change_default(self) -> None: + self._priority("critical") + + def restore_default(self) -> None: + self._priority("major") + + def rows(self) -> list[dict[str, Any]]: + return tsql(JUDGE) + + def verifier(self) -> Any: + return ExecSqlVerifier(tsql, QUERY, timeout_s=10.0) + + def expect_fields(self, v: dict[str, str]) -> dict[str, str]: + return {"summary": v["summary"], "description": v["description"], "type": v["type"], + "keywords": v["keywords"], "priority": "major", "reporter": "admin"} + + def effects(self) -> list[Effect]: + match = {"summary": ValueExpr(param="summary")} + out = [Effect(kind=EffectKind.RECORD_WRITTEN, match=match, expected_count=1, risk="reversible", + probe="exactly one ticket with this summary", timeout_s=8.0)] + for f in ("description", "type", "keywords"): + out.append(Effect(kind=EffectKind.FIELD_EQUALS, match=match, field=f, value=ValueExpr(param=f), + risk="reversible", probe=f"ticket {f} read-back", timeout_s=8.0)) + return out diff --git a/packages/flow/benchmark/api_generalization/apps/wordpress_category.py b/packages/flow/benchmark/api_generalization/apps/wordpress_category.py new file mode 100644 index 0000000000..0c4ff9aba2 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/wordpress_category.py @@ -0,0 +1,66 @@ +"""WordPress: an editor adds a post category (wp-admin, Posts > Categories).""" + +from __future__ import annotations + +from typing import Any + +from benchmark.api_generalization.apps.wp import WordPressBase, wpsql +from benchmark.api_generalization.common import ( + ExecSqlVerifier, + rec_click, + rec_select, + rec_text, +) +from openadapt_flow.runtime.effects.effect import Effect, EffectKind, ValueExpr + +QUERY = ("SELECT t.term_id AS id, t.name, tt.description, COALESCE(p.name, 'None') AS parent " + "FROM wp_terms t JOIN wp_term_taxonomy tt ON tt.term_id = t.term_id " + "LEFT JOIN wp_terms p ON p.term_id = tt.parent WHERE tt.taxonomy = 'category'") +JUDGE = ("SELECT t.term_id AS id, t.name, t.slug, tt.taxonomy, tt.description, tt.parent, " + "COALESCE(p.name, 'None') AS parent_name, tt.count FROM wp_terms t " + "JOIN wp_term_taxonomy tt ON tt.term_id = t.term_id LEFT JOIN wp_terms p ON p.term_id = tt.parent " + "ORDER BY t.term_id") + + +class App(WordPressBase): + name = "wordpress_category" + workflow = "add-category" + form_path = "/wp-admin/edit-tags.php?taxonomy=category" + save_path_regex = r"^/wp-admin/admin-ajax\.php$" + save_body_require = r"(^|&)action=add-tag(&|$)" + unique_param = "name" + normal_values = [ + dict(name="Community Events", parent="None", description="Fairs, meetups and open days."), + dict(name="Road Works", parent="Uncategorized", description="Closures and detours."), + dict(name="Library Hours", parent="None", description="Opening times and holidays."), + ] + blank_values = dict(name="Notices", parent="None", description="") + dup_values = dict(name="Same Text", parent="None", description="Same Text") + invalid_values = dict(name="Uncategorized", parent="None", description="A name that already exists.") + invalid_note = "a category name that already exists at that level (the server rejects it)" + + def record(self, rec: Any, page: Any, v: dict[str, str]) -> None: + rec_text(rec, page, "#tag-name", v["name"], "name") + rec_select(rec, page, "#parent", v["parent"], "parent") + rec_text(rec, page, "#tag-description", v["description"], "description") + rec_click(rec, page, "#submit") + page.wait_for_timeout(1500) + + def rows(self) -> list[dict[str, Any]]: + return wpsql(JUDGE) + + def verifier(self) -> Any: + return ExecSqlVerifier(wpsql, QUERY, timeout_s=10.0) + + def expect_fields(self, v: dict[str, str]) -> dict[str, str]: + return {"name": v["name"], "taxonomy": "category", "description": v["description"], + "parent_name": v["parent"]} + + def effects(self) -> list[Effect]: + match = {"name": ValueExpr(param="name")} + out = [Effect(kind=EffectKind.RECORD_WRITTEN, match=match, expected_count=1, risk="reversible", + probe="exactly one category with this name", timeout_s=8.0)] + for f in ("description", "parent"): + out.append(Effect(kind=EffectKind.FIELD_EQUALS, match=match, field=f, value=ValueExpr(param=f), + risk="reversible", probe=f"category {f} read-back", timeout_s=8.0)) + return out diff --git a/packages/flow/benchmark/api_generalization/apps/wordpress_comment.py b/packages/flow/benchmark/api_generalization/apps/wordpress_comment.py new file mode 100644 index 0000000000..7711b1a20b --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/wordpress_comment.py @@ -0,0 +1,65 @@ +"""WordPress: a visitor posts a comment on a post (public form, no sign-in).""" + +from __future__ import annotations + +from typing import Any + +from benchmark.api_generalization.apps.wp import WordPressBase, wpsql +from benchmark.api_generalization.common import ExecSqlVerifier, rec_click, rec_text +from openadapt_flow.runtime.effects.effect import Effect, EffectKind, ValueExpr + +QUERY = ("SELECT comment_ID AS id, comment_content AS comment, comment_author AS author, " + "comment_author_email AS email, comment_author_url AS url FROM wp_comments WHERE comment_post_ID = 1") +JUDGE = ("SELECT comment_ID AS id, comment_post_ID, comment_parent, comment_content, comment_author, " + "comment_author_email, comment_author_url, comment_approved, comment_type, user_id " + "FROM wp_comments ORDER BY comment_ID") + + +class App(WordPressBase): + name = "wordpress_comment" + workflow = "post-comment" + form_path = "/?p=1" + save_path_regex = r"^/wp-comments-post\.php$" + login_required = False + unique_param = "comment" + normal_values = [ + dict(comment="The parking map on page two helped a lot.", author="Ada Park", email="ada.park@example.invalid", url="https://ada.example.invalid/"), + dict(comment="Could the meeting notes list room numbers?", author="Ravi Osei", email="ravi.osei@example.invalid", url="https://ravi.example.invalid/"), + dict(comment="Thanks for posting the bus timetable.", author="Lena Fischer", email="lena.f@example.invalid", url="https://lena.example.invalid/"), + ] + blank_values = dict(comment="No website for this one.", author="Mo Diallo", email="mo.diallo@example.invalid", url="") + dup_values = dict(comment="Sam Ortiz", author="Sam Ortiz", email="sam.ortiz@example.invalid", url="https://sam.example.invalid/") + invalid_values = dict(comment="This email has no at sign.", author="Kim Vu", email="kim.vu.example.invalid", url="") + invalid_note = "email without @ (the server rejects it)" + + def open_form(self, page: Any) -> None: + super().open_form(page) + page.locator("#comment").scroll_into_view_if_needed() # where a reader starts typing + page.wait_for_timeout(300) + + def record(self, rec: Any, page: Any, v: dict[str, str]) -> None: + rec_text(rec, page, "#comment", v["comment"], "comment") + rec_text(rec, page, "#author", v["author"], "author") + rec_text(rec, page, "#email", v["email"], "email") + rec_text(rec, page, "#url", v["url"], "url") + rec_click(rec, page, "#submit") + + def rows(self) -> list[dict[str, Any]]: + return wpsql(JUDGE) + + def verifier(self) -> Any: + return ExecSqlVerifier(wpsql, QUERY, timeout_s=10.0) + + def expect_fields(self, v: dict[str, str]) -> dict[str, str]: + return {"comment_content": v["comment"], "comment_author": v["author"], + "comment_author_email": v["email"], "comment_author_url": v["url"], + "comment_post_ID": "1", "comment_parent": "0"} + + def effects(self) -> list[Effect]: + match = {"comment": ValueExpr(param="comment")} + out = [Effect(kind=EffectKind.RECORD_WRITTEN, match=match, expected_count=1, risk="reversible", + probe="exactly one comment with this text", timeout_s=8.0)] + for f in ("author", "email", "url"): + out.append(Effect(kind=EffectKind.FIELD_EQUALS, match=match, field=f, value=ValueExpr(param=f), + risk="reversible", probe=f"comment {f} read-back", timeout_s=8.0)) + return out diff --git a/packages/flow/benchmark/api_generalization/apps/wp.py b/packages/flow/benchmark/api_generalization/apps/wp.py new file mode 100644 index 0000000000..35088eb61c --- /dev/null +++ b/packages/flow/benchmark/api_generalization/apps/wp.py @@ -0,0 +1,76 @@ +"""WordPress 6.6 (CMS, PHP, MariaDB): the shared fixture for the WordPress workflows. + +Runs as an external service from the official images; nothing from +WordPress (GPL-2.0) is copied here. The one file written into the container +is this harness's own must-use plugin that turns off WordPress's comment +flood throttle (a test copy posts many comments from one address). +""" + +from __future__ import annotations + +import subprocess +import time +from typing import Any + +from benchmark.api_generalization.common import AppBase, mysql_rows, podman + +POD = "apigen-wp" +DB_C = "apigen-wp-db" +WP_C = "apigen-wp-app" +WP_IMAGE = "docker.io/library/wordpress:6.6-php8.2-apache" +DB_IMAGE = "docker.io/library/mariadb@sha256:efb4959ef2c835cd735dbc388eb9ad6aab0c78dd64febcd51bc17481111890c4" +DB_PASS = "synthetic-db-pass" # throwaway local container +ADMIN_PASS = "synthetic-admin-pass" # throwaway local container +UPSTREAM, PROXY = 9831, 9832 +MU_PLUGIN = " list[dict[str, Any]]: + return mysql_rows(DB_C, "root", DB_PASS, "wordpress", query) + + +class WordPressBase(AppBase): + upstream_port = UPSTREAM + proxy_port = PROXY + + def up(self) -> None: + if subprocess.run(["podman", "pod", "exists", POD]).returncode == 0: + podman("pod", "start", POD) + self.wait_http("/wp-login.php") + return + podman("pod", "create", "--name", POD, "-p", f"127.0.0.1:{UPSTREAM}:80") + podman("run", "-d", "--pod", POD, "--name", DB_C, "-e", f"MARIADB_ROOT_PASSWORD={DB_PASS}", + "-e", "MARIADB_DATABASE=wordpress", DB_IMAGE) + podman("run", "-d", "--pod", POD, "--name", WP_C, "-e", "WORDPRESS_DB_HOST=127.0.0.1", + "-e", "WORDPRESS_DB_USER=root", "-e", f"WORDPRESS_DB_PASSWORD={DB_PASS}", + "-e", "WORDPRESS_DB_NAME=wordpress", WP_IMAGE) + self.wait_http("/wp-admin/install.php") + import urllib.parse + import urllib.request + + body = urllib.parse.urlencode({ + "weblog_title": "Synthetic Community Notes", "user_name": "admin", "admin_password": ADMIN_PASS, + "admin_password2": ADMIN_PASS, "pw_weak": "1", "admin_email": "admin@example.invalid", + "blog_public": "0", "language": ""}).encode() + urllib.request.urlopen(f"http://127.0.0.1:{UPSTREAM}/wp-admin/install.php?step=2", data=body, timeout=120).read() + base = f"http://127.0.0.1:{PROXY}" + wpsql(f"UPDATE wp_options SET option_value='{base}' WHERE option_name IN ('siteurl','home')") + podman("exec", "-i", WP_C, "sh", "-c", + "mkdir -p /var/www/html/wp-content/mu-plugins && cat > /var/www/html/wp-content/mu-plugins/apigen-test-copy.php", + input=MU_PLUGIN) + time.sleep(1) + + def down(self) -> None: + podman("pod", "stop", POD, check=False) + + def login(self, page: Any) -> None: + page.goto(self.base + "/wp-login.php") + page.fill("#user_login", "admin") + page.fill("#user_pass", ADMIN_PASS) + page.click("#wp-submit") + page.wait_for_load_state("load") + + def expire_session(self, page: Any) -> None: + # WordPress keeps each login's session token in usermeta; deleting it + # ends every session on the server. + wpsql("DELETE FROM wp_usermeta WHERE meta_key = 'session_tokens'") diff --git a/packages/flow/benchmark/api_generalization/common.py b/packages/flow/benchmark/api_generalization/common.py new file mode 100644 index 0000000000..8f9183a9f4 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/common.py @@ -0,0 +1,330 @@ +"""Shared pieces: the app adapter shape, the independent judge, the SQL read-back. + +An app adapter (``apps/.py``) gives the harness what it can't know +about an app: how to start it, how a scripted person fills the form, how to +read its records back, and the inputs to try. Everything the API step itself +does comes from the frozen compiler and runtime, unchanged. + +Two read-backs are kept apart on purpose: + +- the **engine's record check** (``Effect`` list on the save step, read by + :class:`ExecSqlVerifier`) is what the run itself uses to confirm the save; +- the **judge** (:func:`judge`) reads the whole record table before and after + each run with its own query, and decides whether the run left the app in + the expected state. A run is scored only by the judge. +""" + +from __future__ import annotations + +import json +import subprocess +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +from openadapt_flow.runtime.effects.sql import SqlRecordVerifier + + +def sh(args: list[str], *, input: Optional[str] = None, timeout: float = 120, check: bool = True) -> str: + r = subprocess.run(args, input=input, capture_output=True, text=True, timeout=timeout) + if check and r.returncode != 0: + raise RuntimeError(f"{' '.join(args[:4])}... failed ({r.returncode}): {r.stderr[-800:]}") + return r.stdout + + +def podman(*args: str, **kw: Any) -> str: + return sh(["podman", *args], **kw) + + +class ExecSqlVerifier(SqlRecordVerifier): + """The engine's record check, read through ``podman exec`` (no DB driver on the host). + + ``fetch`` runs the adapter's fixed read-only SELECT and returns rows as + dicts; any failure reads as unreadable (INDETERMINATE), as in the parent. + """ + + def __init__(self, fetch: Callable[[str], list[dict[str, Any]]], query: str, **kw: Any) -> None: + super().__init__(connect=lambda: None, query=query, **kw) + self._fetch = fetch + + def _fetch_records(self) -> Optional[list[dict[str, Any]]]: + try: + rows = self._fetch(self.query) + except Exception: # noqa: BLE001 - an unreadable record system is INDETERMINATE + return None + return [{k: ("" if v is None else str(v)) for k, v in r.items()} for r in rows] + + +@dataclass +class Case: + """One input or condition for a workflow, and what the app should end up holding.""" + + id: str + kind: str # normal | blank | dup_values | changed_default | expired_session | fault_reject | fault_partial | fault_duplicate | validation + params: dict[str, str] + expect_new: int # records the run should leave (fault_duplicate: what the fixture causes) + expect_fields: dict[str, str] = field(default_factory=dict) # judge columns -> value on each new record + fault: Optional[str] = None # proxy mode + note: str = "" + + +def judge(before: list[dict[str, Any]], after: list[dict[str, Any]], case: Case, key: str = "id") -> dict[str, Any]: + """Compare the full table before and after one run.""" + + b = {str(r[key]): r for r in before} + a = {str(r[key]): r for r in after} + new = [a[k] for k in a if k not in b] + gone = [k for k in b if k not in a] + changed = [k for k in b if k in a and json.dumps(b[k], sort_keys=True, default=str) != json.dumps(a[k], sort_keys=True, default=str)] + mism: list[str] = [] + for r in new: + for col, want in case.expect_fields.items(): + got = "" if r.get(col) is None else str(r.get(col)) + if got != want: + mism.append(f"{col}: want {want!r} got {got!r}") + ok_state = len(new) == case.expect_new and not mism and not gone and not changed + return { + "new_records": len(new), + "field_mismatches": mism, + "collateral": {"deleted": gone, "changed": changed}, + "state_ok": ok_state, + "new_ids": [str(r[key]) for r in new], + } + + +def classify(case: Case, state: dict[str, Any], engine_success: Optional[bool], transaction_outcome: Optional[str]) -> dict[str, Any]: + """Score one run from the judge's view of the app, then the run's own claim. + + - ``incorrect_write``: the app holds something the run should not have + caused: a wrong value, an extra record, a changed or deleted record. + In F3 the fixture's second delivery is not counted against the run. + - ``false_success``: the run reported success and the judge disagrees. + - ``correct``: the app holds what it should and the run's claim matches + (success exactly when the expected write happened, or + RECONCILIATION_REQUIRED after a lost response in F2). In F3 the app + may hold two records (the run must not report success) or one, when + the app itself refuses the second delivery (then the run's claim must + be true either way). + - ``stopped_after_right_write``: the right record was written but the run + did not report success (a safe stop that costs a person a look). + - ``missed_write``: nothing wrong was written but an expected record is + missing (an honest stop). + """ + + collateral = bool(state["collateral"]["deleted"] or state["collateral"]["changed"]) + n = state["new_records"] + mism = bool(state["field_mismatches"]) + if case.kind == "fault_duplicate": + right = n in (1, 2) and not mism and not collateral + incorrect = n > 2 or mism or collateral + correct = right and (not engine_success if n == 2 else True) + false_success = bool(engine_success) and not (right and n == 1) + return {"correct": correct, "incorrect_write": incorrect, "false_success": false_success, + "stopped_after_right_write": right and n == 1 and not engine_success, "missed_write": n == 0} + right = state["state_ok"] + should_succeed = case.expect_new >= 1 + correct = right and bool(engine_success) == should_succeed + if case.kind == "fault_partial" and right and transaction_outcome == "RECONCILIATION_REQUIRED": + correct = True + incorrect = n > case.expect_new or mism or collateral + return {"correct": correct, "incorrect_write": incorrect, + "false_success": bool(engine_success) and not (right and should_succeed), + "stopped_after_right_write": right and should_succeed and not engine_success and not correct, + "missed_write": (not incorrect) and n < case.expect_new} + + +class AppBase: + """Defaults shared by the adapters. Subclasses fill in the app-specific parts.""" + + name = "" + workflow = "" + upstream_port = 0 + proxy_port = 0 + form_path = "" + save_path_regex = "" + save_body_exclude = "" + save_body_require = "" + login_required = True + key = "id" + + @property + def base(self) -> str: + return f"http://127.0.0.1:{self.proxy_port}" + + # lifecycle --------------------------------------------------------------- + def up(self) -> None: + raise NotImplementedError + + def down(self) -> None: + raise NotImplementedError + + def wait_http(self, path: str = "/", timeout_s: float = 600) -> None: + import urllib.request + + deadline = time.time() + timeout_s + while time.time() < deadline: + try: + with urllib.request.urlopen(f"http://127.0.0.1:{self.upstream_port}{path}", timeout=5) as r: + if r.status < 500: + return + except Exception: # noqa: BLE001 + time.sleep(3) + raise RuntimeError(f"{self.name} did not answer on {path}") + + # browser ----------------------------------------------------------------- + def login(self, page: Any) -> None: + raise NotImplementedError + + def open_form(self, page: Any) -> None: + page.goto(self.base + self.form_path) + page.wait_for_load_state("load") + page.wait_for_timeout(800) + + def record(self, rec: Any, page: Any, values: dict[str, str]) -> None: + raise NotImplementedError + + def expire_session(self, page: Any) -> None: + """End the signed-in session on the server (default: drop the browser's cookies).""" + page.context.clear_cookies() + + def change_default(self) -> None: + raise NotImplementedError + + def restore_default(self) -> None: + raise NotImplementedError + + # records ----------------------------------------------------------------- + def rows(self) -> list[dict[str, Any]]: + raise NotImplementedError + + def verifier(self) -> Any: + raise NotImplementedError + + def effects(self) -> list[Any]: + raise NotImplementedError + + # inputs ---------------------------------------------------------------------- + #: The parameter made unique per run (so repeated runs never collide with + #: an app's own duplicate rules), and its judge column. + unique_param = "" + normal_values: list[dict[str, str]] = [] + blank_values: dict[str, str] = {} + dup_values: dict[str, str] = {} + invalid_values: dict[str, str] = {} + invalid_note = "" + #: Judge column whose form default ``change_default`` changes, and the + #: value a person's save holds after the change. + default_column = "" + changed_default_value = "" + + def recording_values(self) -> dict[str, str]: + return dict(self.normal_values[0]) + + def expect_fields(self, values: dict[str, str]) -> dict[str, str]: + raise NotImplementedError + + def unique(self, case: Case, tag: str) -> Case: + if case.kind == "validation": + return case # the invalid value must stay exactly as written + params = dict(case.params) + params[self.unique_param] = f"{params[self.unique_param]} {tag}" + fields = dict(case.expect_fields) + if fields: + col = self.unique_column() + fields[col] = self.expect_fields(params)[col] + return Case(case.id, case.kind, params, case.expect_new, fields, case.fault, case.note) + + def unique_column(self) -> str: + cols = self.expect_fields({k: "\x00" if k == self.unique_param else "" for k in self.normal_values[0]}) + return next(k for k, v in cols.items() if v == "\x00") + + def cases(self) -> list[Case]: + n = self.normal_values + cases = [Case(f"N{i + 1}", "normal", v, 1, self.expect_fields(v)) for i, v in enumerate(n)] + cases.append(Case("B1", "blank", self.blank_values, 1, self.expect_fields(self.blank_values), + note="optional fields left blank")) + cases.append(Case("D1", "dup_values", self.dup_values, 1, self.expect_fields(self.dup_values), + note="the same value in several fields")) + if self.default_column: + ef = self.expect_fields(n[0]) | {self.default_column: self.changed_default_value} + cases.append(Case("C1", "changed_default", n[0], 1, ef, + note=f"the app's default for {self.default_column} changed after recording")) + if self.login_required: + cases.append(Case("S1", "expired_session", n[1], 0, {}, note="session ended on the server before the run")) + cases.append(Case("F1", "fault_reject", n[2], 0, {}, fault="reject", note="503 before the app sees the save")) + cases.append(Case("F2", "fault_partial", n[0], 1, self.expect_fields(n[0]), fault="partial", + note="the app commits, the response is lost (502)")) + cases.append(Case("F3", "fault_duplicate", n[1], 2, self.expect_fields(n[1]), fault="duplicate", + note="the save reaches the app twice")) + cases.append(Case("V1", "validation", self.invalid_values, 0, {}, note=self.invalid_note)) + return cases + + +def center(page: Any, rec: Any, selector: str) -> tuple[int, int]: + """The control's center, scrolling (as a recorded wheel event) until it is in view.""" + + loc = page.locator(selector).first + last_y = None + for _ in range(8): + box = loc.bounding_box() + if box is None: + raise RuntimeError(f"not rendered: {selector}") + h = page.viewport_size["height"] + cy = box["y"] + box["height"] / 2 + if 20 <= cy <= h - 20 and (box["y"] >= 0 and box["y"] + box["height"] <= h or box["y"] == last_y): + return int(box["x"] + box["width"] / 2), int(cy) + last_y = box["y"] + page.mouse.move(640, 400) # the wheel scrolls what is under the pointer; a move is not an event + rec.scroll(0, 300 if cy > h / 2 else -300) + raise RuntimeError(f"could not scroll to {selector}") + + +def rec_text(rec: Any, page: Any, selector: str, value: str, param: str, *, clear: bool = False) -> None: + rec.click(*center(page, rec, selector)) + if clear: + rec.press("Control+a") + rec.type_text(value, param=param) + + +def rec_select(rec: Any, page: Any, selector: str, label: str, param: str) -> None: + """A person picks an option by typing its label into the focused select. + + No Enter: on a closed select, Enter submits the form in Chromium. + """ + + rec.click(*center(page, rec, selector)) + rec.type_text(label, param=param) + + +def rec_click(rec: Any, page: Any, selector: str) -> None: + rec.click(*center(page, rec, selector)) + + +def mysql_rows(container: str, user: str, password: str, db: str, query: str) -> list[dict[str, Any]]: + """Rows from MariaDB/MySQL through the container's own client (tab-separated with a header).""" + + out = podman("exec", "-i", container, "mariadb", f"-u{user}", f"-p{password}", "-B", db, input=query) + lines = out.splitlines() + if not lines: + return [] + cols = lines[0].split("\t") + + def unesc(v: str) -> Any: + if v == "NULL": + return None + return v.replace("\\n", "\n").replace("\\t", "\t").replace("\\0", "\0").replace("\\\\", "\\") + + return [dict(zip(cols, [unesc(v) for v in ln.split("\t")])) for ln in lines[1:]] + + +def rec_select2_click(rec: Any, page: Any, select_id: str, label: str) -> None: + """A select2 widget: open it and click the option with this label (a fixed choice, not a parameter).""" + + rec_click(rec, page, f"#select2-{select_id}-container") + page.wait_for_timeout(300) + opt = page.locator(".select2-results__option", has_text=label) + exact = [i for i in range(opt.count()) if opt.nth(i).inner_text().strip() == label] + if len(exact) != 1: + raise RuntimeError(f"select2 option {label!r}: {len(exact)} exact matches") + box = opt.nth(exact[0]).bounding_box() + rec.click(int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2)) diff --git a/packages/flow/benchmark/api_generalization/counterfactual.py b/packages/flow/benchmark/api_generalization/counterfactual.py new file mode 100644 index 0000000000..4abfb5531f --- /dev/null +++ b/packages/flow/benchmark/api_generalization/counterfactual.py @@ -0,0 +1,48 @@ +"""Counterfactual input edit: drop the clicked submit button's name=value from the captured save. + +Not a product result. The frozen compiler can't explain a submit button's +own name and value (``_FormParser`` skips submit inputs), which refuses every +form whose save button has a name. This copies a recording, removes that one +pair from the captured save request, and asks the same frozen compiler again, +to find the next thing that would block the API step. The compiler is not +changed; only its input is. + + python -m benchmark.api_generalization.counterfactual --app trac --recording --out --field submit +""" + +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path + + +def strip_submit(rec: Path, out: Path, field: str) -> int: + shutil.copytree(rec, out) + lines = (out / "network.jsonl").read_text().splitlines() + n = 0 + fixed = [lines[0]] + for raw in lines[1:]: + d = json.loads(raw) + if d.get("method") == "POST" and d.get("form") is not None: + before = len(d["form"]) + d["form"] = [p for p in d["form"] if p[0] != field] + n += before - len(d["form"]) + fixed.append(json.dumps(d)) + (out / "network.jsonl").write_text("\n".join(fixed) + "\n") + (out / "COUNTERFACTUAL.txt").write_text(f"removed {n} '{field}' pair(s) from captured POST forms\n") + return n + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--recording", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--field", required=True) + a = ap.parse_args() + print(json.dumps({"removed": strip_submit(Path(a.recording), Path(a.out), a.field)})) + + +if __name__ == "__main__": + main() diff --git a/packages/flow/benchmark/api_generalization/harness.py b/packages/flow/benchmark/api_generalization/harness.py new file mode 100644 index 0000000000..1d0eac9cda --- /dev/null +++ b/packages/flow/benchmark/api_generalization/harness.py @@ -0,0 +1,340 @@ +"""Record, compile, propose, admit and run one app's save workflow, two arms. + + python -m benchmark.api_generalization.harness up --app redmine + python -m benchmark.api_generalization.harness record --app redmine --out + python -m benchmark.api_generalization.harness compile --app redmine --recording --out + python -m benchmark.api_generalization.harness matrix --app redmine --bundle --out + python -m benchmark.api_generalization.harness timing --app redmine --bundle --out --reps 10 + python -m benchmark.api_generalization.harness down --app redmine + +``compile`` uses only the frozen compiler: ``compile_recording``, then the +save step's record check (the adapter's ``effects``, as a person adds it), +then ``propose_api_step`` (``openadapt-flow api propose``) and, when there is +a proposal, ``admit_api_step`` under the name ``benchmark-fixture`` so the +matrix can run unattended (``openadapt-flow api admit`` in use). + +Arms: ``gui`` is ``Replayer(api_steps=False)``; ``api`` is +``Replayer(api_steps=True)``. Both use the same bundle, the same record +check, the same proxy and the same judge. +""" + +from __future__ import annotations + +import argparse +import csv +import importlib +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +from benchmark.api_generalization.common import AppBase, Case, classify, judge +from benchmark.api_generalization.proxy import serve + +HERE = Path(__file__).resolve().parent +FLOW = HERE.parents[1] +SETTLE = {"settle_timeout_s": 10.0, "settle_stable_frames": 3, "settle_interval_s": 0.3} +VIEWPORT = {"width": 1280, "height": 800} + + +def load_app(name: str) -> AppBase: + return importlib.import_module(f"benchmark.api_generalization.apps.{name}").App() + + +def engine_sha() -> str: + return subprocess.run(["git", "-C", str(FLOW), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip() + + +def frozen_check() -> dict[str, Any]: + """The runtime and compiler must be the frozen commit's (no local changes).""" + frozen = os.environ.get("APIGEN_FROZEN", "") + diff = subprocess.run( + ["git", "-C", str(FLOW), "diff", "--stat", frozen or "HEAD", "--", "openadapt_flow", "../types"], + capture_output=True, text=True).stdout.strip() + return {"frozen": frozen, "head": engine_sha(), "engine_diff_vs_frozen": diff} + + +def state_dir(app: AppBase) -> Path: + d = Path(os.environ.get("APIGEN_STATE", Path.home() / "oa/scratch/api-generalization")) / app.name + d.mkdir(parents=True, exist_ok=True) + return d + + +def set_fault(app: AppBase, mode: str) -> None: + (state_dir(app) / "mode.json").write_text(json.dumps( + {"mode": mode, "path_regex": app.save_path_regex, "body_exclude": app.save_body_exclude, + "body_require": app.save_body_require})) + + +def proxy_log_len(app: AppBase) -> int: + p = state_dir(app) / "requests.jsonl" + return len(p.read_text().splitlines()) if p.exists() else 0 + + +def proxy_log_since(app: AppBase, n: int) -> list[dict[str, Any]]: + p = state_dir(app) / "requests.jsonl" + return [json.loads(x) for x in p.read_text().splitlines()[n:]] if p.exists() else [] + + +def start_proxy(app: AppBase) -> None: + set_fault(app, "pass") + serve(app.proxy_port, app.upstream_port, state_dir(app)) + + +# -- record ------------------------------------------------------------------------ + + +def cmd_record(a: argparse.Namespace, app: AppBase) -> None: + from playwright.sync_api import sync_playwright + + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.network_capture import NetworkCapture + from openadapt_flow.recorder import Recorder + + out = Path(a.out) + if out.exists(): + raise SystemExit(f"exists: {out}") + start_proxy(app) + values = app.recording_values() + before = app.rows() + pw = sync_playwright().start() + browser = pw.chromium.launch(headless=True) + try: + ctx = browser.new_context(viewport=VIEWPORT, device_scale_factor=1) + page = ctx.new_page() + page.goto(app.base + "/") + if app.login_required: + app.login(page) + capture = NetworkCapture(ctx).start() + app.open_form(page) + rec = Recorder(PlaywrightBackend(page), out, app_url=page.url, network_capture=capture, **SETTLE) + app.record(rec, page, values) + recording = rec.finish() + time.sleep(1.5) + after = app.rows() + case = Case(id="recording", kind="normal", params=values, expect_new=1, + expect_fields=app.expect_fields(values)) + state = judge(before, after, case, app.key) + events = [json.loads(x) for x in (recording / "events.jsonl").read_text().splitlines() if x.strip()] + print(json.dumps({"recording": str(recording), "events": len(events), "exchanges": capture.count, + "judge": state}, indent=1)) + if not state["state_ok"]: + raise SystemExit("the demonstration did not save the expected record; record again") + finally: + browser.close() + pw.stop() + + +# -- compile ----------------------------------------------------------------------- + + +def cmd_compile(a: argparse.Namespace, app: AppBase) -> None: + from openadapt_flow.compiler import compile_recording + from openadapt_flow.compiler.api_proposal import ( + admit_api_step, + describe_api_step, + propose_api_step, + ) + from openadapt_flow.ir import Workflow + + rec, bundle = Path(a.recording), Path(a.out) + t0 = time.time() + compile_recording(rec, bundle, name=f"{app.name}-{app.workflow}") + first = json.loads((bundle / "api_proposal.json").read_text()) if (bundle / "api_proposal.json").exists() else None + wf = Workflow.load(bundle) + # A person marks the fields that may be left blank as optional (compile + # makes every parameter required, and a run refuses "" for a required one). + optional = sorted(k for k, v in app.blank_values.items() if v == "" and k in wf.param_specs) + for k in optional: + wf.param_specs[k].required = False + save = wf.steps[-1] + save.effects = app.effects() + save.risk = "irreversible" + save.risk_explanation = f"the {app.workflow} save" + save.risk_review_required = False + wf, step, reasons = propose_api_step(wf, rec, replace=True) + result: dict[str, Any] = {"app": app.name, "workflow": app.workflow, "steps": len(wf.steps), + "params": sorted(wf.params), "optional_params": optional, "compile_proposal": first, + "proposed": step is not None, "reasons": reasons, + "compile_s": round(time.time() - t0, 1)} | frozen_check() + if step is not None: + wf, admitted = admit_api_step(wf, admitted_by="benchmark-fixture") + result["admitted"] = admitted + result["describe"] = describe_api_step(next(s for s in wf.steps if s.id == step.id)) + result["binding"] = step.api_binding.web_form.model_dump(mode="json") if step.api_binding else None + wf.save(bundle) + (bundle / "apigen-compile.json").write_text(json.dumps(result, indent=1, default=str)) + print(json.dumps({k: v for k, v in result.items() if k != "binding"}, indent=1, default=str)) + + +# -- run --------------------------------------------------------------------------- + +COLS = ["app", "workflow", "bundle_kind", "case", "kind", "arm", "rep", "correct", "incorrect_write", "false_success", "stopped_after_right_write", "missed_write", + "new_records", "field_mismatches", "collateral", "engine_success", "outcome", "transaction_outcome", + "save_actuation", "api_fallback", "halt", "save_posts", "fault", "injected", "error", "wall_s", "setup_s", "engine_s", + "load1", "start_local", "engine_sha"] + + +def _load1() -> float: + return float(subprocess.run(["sysctl", "-n", "vm.loadavg"], capture_output=True, text=True).stdout.split()[1]) + + +def run_one(env: dict[str, Any], app: AppBase, case: Case, arm: str, bundle: Path, out: Path, rep: int = 1) -> dict[str, Any]: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime import Replayer + + out.mkdir(parents=True, exist_ok=True) + res: dict[str, Any] = {"app": app.name, "workflow": app.workflow, "case": case.id, "kind": case.kind, + "arm": arm, "rep": rep, "fault": case.fault, "error": None} + t0 = time.time() + ctx = env["browser"].new_context(viewport=VIEWPORT, device_scale_factor=1) + restore = False + try: + page = ctx.new_page() + page.goto(app.base + "/") + if app.login_required: + app.login(page) + if case.kind == "changed_default": + app.change_default() + restore = True + app.open_form(page) + if case.kind == "expired_session": + app.expire_session(page) + set_fault(app, case.fault or "pass") + before = app.rows() + n_log = proxy_log_len(app) + t_setup = time.time() + rep_ = Replayer(PlaywrightBackend(page), effect_verifier=app.verifier(), api_steps=(arm == "api")).run( + Workflow.load(bundle), params=dict(case.params), bundle_dir=bundle, run_dir=out / "engine") + t_task = time.time() + raw = rep_.model_dump(mode="json") + last = raw["results"][-1] if raw["results"] else {} + res.update(engine_success=raw.get("success"), outcome=str(raw.get("execution_outcome")), + transaction_outcome=raw.get("transaction_outcome"), save_actuation=last.get("actuation"), + api_fallback=next((r["api_fallback"] for r in raw["results"] if r.get("api_fallback")), None), + halt=next(((r["step_id"] + ": " + (r.get("error") or ""))[:240] for r in raw["results"] if r.get("error")), None), + step_errors=[(r["step_id"], (r.get("error") or "")[:300]) for r in raw["results"] if r.get("error")]) + except Exception as exc: # noqa: BLE001 + t_setup = locals().get("t_setup", time.time()) + t_task = time.time() + res["error"] = f"{type(exc).__name__}: {exc}"[:500] + before = locals().get("before") + n_log = locals().get("n_log", proxy_log_len(app)) + time.sleep(1.5) + set_fault(app, "pass") + after = app.rows() + if before is None: + before = after + res["error"] = (res["error"] or "") + " [no before-state]" + state = judge(before, after, case, app.key) + log = proxy_log_since(app, n_log) + posts = [r for r in log if r.get("save")] + res["injected"] = next((r["fault"] for r in log if r.get("fault") not in (None, "pass")), None) + res.update(state) + res["save_posts"] = len(posts) + res.update(classify(case, state, res.get("engine_success"), res.get("transaction_outcome"))) + if restore: + app.restore_default() + ctx.close() + t_end = time.time() + res.update(wall_s=round(t_end - t0, 3), setup_s=round(t_setup - t0, 3), engine_s=round(t_task - t_setup, 3)) + (out / "result.json").write_text(json.dumps(res, indent=1, default=str)) + shutil.rmtree(out / "engine" / "steps", ignore_errors=True) + return res + + +def _env() -> dict[str, Any]: + from playwright.sync_api import sync_playwright + + pw = sync_playwright().start() + return {"pw": pw, "browser": pw.chromium.launch(headless=True)} + + +def _write_row(csv_path: Path, row: dict[str, Any]) -> None: + new = not csv_path.exists() + with open(csv_path, "a", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=COLS, extrasaction="ignore") + if new: + w.writeheader() + w.writerow({k: (json.dumps(v) if isinstance(v, (list, dict)) else v) for k, v in row.items()}) + + +def cmd_matrix(a: argparse.Namespace, app: AppBase) -> None: + out, bundle = Path(a.out), Path(a.bundle) + out.mkdir(parents=True, exist_ok=True) + (out / "engine.json").write_text(json.dumps(frozen_check(), indent=1)) + start_proxy(app) + env = _env() + cases = [c for c in app.cases() if not a.only or c.id in a.only.split(",")] + arms = a.arms.split(",") + try: + for i, case in enumerate(cases): + for arm in (arms if i % 2 == 0 else list(reversed(arms))): + start = time.strftime("%H:%M:%S") + res = run_one(env, app, app.unique(case, f"{case.id}{arm[0]}{a.tag}"), arm, bundle, + out / f"{case.id}-{arm}") + row = res | {"load1": _load1(), "start_local": start, "engine_sha": engine_sha()[:8], + "bundle_kind": a.bundle_kind} + _write_row(out / "runs.csv", row) + print(json.dumps({k: row.get(k) for k in ("case", "arm", "correct", "incorrect_write", "false_success", + "new_records", "engine_success", "save_actuation", + "api_fallback", "save_posts", "engine_s", "error")}), flush=True) + finally: + env["browser"].close() + env["pw"].stop() + + +def cmd_timing(a: argparse.Namespace, app: AppBase) -> None: + out, bundle = Path(a.out), Path(a.bundle) + out.mkdir(parents=True, exist_ok=True) + (out / "engine.json").write_text(json.dumps(frozen_check(), indent=1)) + start_proxy(app) + env = _env() + normal = [c for c in app.cases() if c.kind == "normal"] + try: + for r in range(a.reps): + for arm in (["gui", "api"] if r % 2 == 0 else ["api", "gui"]): + case = app.unique(normal[r % len(normal)], f"t{r:02d}{arm[0]}") + start = time.strftime("%H:%M:%S") + res = run_one(env, app, case, arm, bundle, out / f"r{r:02d}-{arm}", rep=r + 1) + row = res | {"load1": _load1(), "start_local": start, "engine_sha": engine_sha()[:8], + "bundle_kind": a.bundle_kind} + _write_row(out / "runs.csv", row) + print(json.dumps({k: row.get(k) for k in ("rep", "arm", "correct", "save_actuation", "engine_s", + "wall_s", "load1", "error")}), flush=True) + finally: + env["browser"].close() + env["pw"].stop() + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("cmd", choices=["up", "down", "record", "compile", "matrix", "timing", "rows"]) + ap.add_argument("--app", required=True) + ap.add_argument("--out") + ap.add_argument("--recording") + ap.add_argument("--bundle") + ap.add_argument("--only", default="") + ap.add_argument("--arms", default="gui,api") + ap.add_argument("--reps", type=int, default=10) + ap.add_argument("--bundle-kind", default="shipped", choices=["shipped", "counterfactual"]) + ap.add_argument("--tag", default="", help="suffix that keeps a re-run's values unique") + a = ap.parse_args() + app = load_app(a.app) + if a.cmd == "up": + app.up() + elif a.cmd == "down": + app.down() + elif a.cmd == "rows": + print(json.dumps(app.rows()[-5:], indent=1, default=str)) + else: + {"record": cmd_record, "compile": cmd_compile, "matrix": cmd_matrix, "timing": cmd_timing}[a.cmd](a, app) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/flow/benchmark/api_generalization/proxy.py b/packages/flow/benchmark/api_generalization/proxy.py new file mode 100644 index 0000000000..9657a48075 --- /dev/null +++ b/packages/flow/benchmark/api_generalization/proxy.py @@ -0,0 +1,149 @@ +"""A reverse proxy in front of each app that injects faults on the save request. + +Every request from the browser and from the API step goes through this +proxy, so a fault hits both arms at the same place: the app's front door. +The mode applies to the next save, once. A save is a POST whose path matches +``path_regex``, whose body matches ``body_require`` when set (an endpoint +shared by several actions) and does not match ``body_exclude`` (an app whose +page script posts the same form for a preview): + +- ``pass``: forward unchanged. +- ``reject``: answer 503 without forwarding (nothing is written). +- ``partial``: forward, let the app commit, then answer 502 with an empty + body (the write happened, the response was lost). +- ``duplicate``: forward the same request twice and answer with the second + response (an at-least-once retry by something between client and app). + +Each request is logged to ``log_path`` (one JSON line) so the harness can +count how many times a save reached the app. + + python -m benchmark.api_generalization.proxy --listen 9822 --upstream 9821 --state +""" + +from __future__ import annotations + +import argparse +import http.client +import json +import re +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +HOP = {"connection", "keep-alive", "proxy-connection", "transfer-encoding", "te", "trailer", "upgrade"} + + +class Control: + """Mode file shared with the harness (``mode.json`` in the state dir).""" + + def __init__(self, state: Path) -> None: + self.state = state + self.mode_path = state / "mode.json" + self.log_path = state / "requests.jsonl" + self.lock = threading.Lock() + + def take(self, method: str, path: str, body: bytes) -> tuple[str, bool]: + """The fault for this request (consumed if it applies), and whether it is a save.""" + with self.lock: + try: + mode = json.loads(self.mode_path.read_text()) + except (OSError, ValueError): + return "pass", False + text = body.decode("latin-1") + is_save = (method == "POST" and bool(re.search(mode.get("path_regex", "$^"), path)) + and not (mode.get("body_exclude") and re.search(mode["body_exclude"], text)) + and not (mode.get("body_require") and not re.search(mode["body_require"], text))) + if mode.get("mode", "pass") == "pass" or not is_save: + return "pass", is_save + self.mode_path.write_text(json.dumps({**mode, "mode": "pass", "consumed": mode["mode"]})) + return str(mode["mode"]), True + + def log(self, row: dict) -> None: + with self.lock, open(self.log_path, "a") as fh: + fh.write(json.dumps(row) + "\n") + + +def make_handler(upstream_host: str, upstream_port: int, control: Control): + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args) -> None: # quiet + pass + + def _forward(self, body: bytes) -> tuple[int, str, list[tuple[str, str]], bytes]: + conn = http.client.HTTPConnection(upstream_host, upstream_port, timeout=120) + headers = {k: v for k, v in self.headers.items() if k.lower() not in HOP} + headers["Connection"] = "close" + conn.request(self.command, self.path, body=body or None, headers=headers) + r = conn.getresponse() + data = r.read() + out = (r.status, r.reason, [(k, v) for k, v in r.getheaders() if k.lower() not in HOP], data) + conn.close() + return out + + def _reply(self, status: int, reason: str, headers: list[tuple[str, str]], data: bytes) -> None: + self.send_response(status, reason) + for k, v in headers: + if k.lower() != "content-length": + self.send_header(k, v) + self.send_header("Content-Length", str(len(data))) + self.send_header("Connection", "close") + self.end_headers() + if self.command != "HEAD": + self.wfile.write(data) + self.close_connection = True + + def _handle(self) -> None: + n = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(n) if n else b"" + fault, is_save = control.take(self.command, self.path.split("?")[0], body) + t0 = time.time() + status = None + try: + if fault == "reject": + status = 503 + self._reply(503, "Service Unavailable", [("Content-Type", "text/plain")], b"injected fault: reject") + elif fault == "partial": + status, *_ = self._forward(body) + self._reply(502, "Bad Gateway", [("Content-Type", "text/plain")], b"") + status = f"{status}->502" + elif fault == "duplicate": + first, *_ = self._forward(body) + st, reason, headers, data = self._forward(body) + self._reply(st, reason, headers, data) + status = f"{first}+{st}" + else: + st, reason, headers, data = self._forward(body) + self._reply(st, reason, headers, data) + status = st + finally: + control.log({"t": round(t0, 3), "method": self.command, "path": self.path.split("?")[0], + "save": is_save, "fault": fault, "status": status, "ms": round((time.time() - t0) * 1000)}) + + do_GET = do_POST = do_HEAD = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = _handle + + return Handler + + +def serve(listen: int, upstream: int, state: Path, upstream_host: str = "127.0.0.1") -> ThreadingHTTPServer: + state.mkdir(parents=True, exist_ok=True) + srv = ThreadingHTTPServer(("127.0.0.1", listen), make_handler(upstream_host, upstream, Control(state))) + srv.daemon_threads = True + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--listen", type=int, required=True) + ap.add_argument("--upstream", type=int, required=True) + ap.add_argument("--state", required=True) + a = ap.parse_args() + serve(a.listen, a.upstream, Path(a.state)) + while True: + time.sleep(3600) + + +if __name__ == "__main__": + main() diff --git a/packages/flow/benchmark/api_generalization/run.sh b/packages/flow/benchmark/api_generalization/run.sh new file mode 100755 index 0000000000..ffb154a1af --- /dev/null +++ b/packages/flow/benchmark/api_generalization/run.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run the harness with the repository's own packages on the path. +# APIGEN_PY: a Python with playwright and openadapt-flow's dependencies. +set -e +HERE=$(cd "$(dirname "$0")" && pwd) +FLOW=$(cd "$HERE/../.." && pwd) +cd "$FLOW" +PYTHONPATH="$FLOW/../types:$FLOW" exec "${APIGEN_PY:-python}" -m benchmark.api_generalization.harness "$@" diff --git a/packages/flow/docs/API_TIER.md b/packages/flow/docs/API_TIER.md index 3428796a6f..6dc43dc74f 100644 --- a/packages/flow/docs/API_TIER.md +++ b/packages/flow/docs/API_TIER.md @@ -1,10 +1,13 @@ # API steps from a recording's network traffic -**Scope: a single HTML form submission.** An API step repeats one -`application/x-www-form-urlencoded` POST of one HTML form, read again from -the live page, in place of the clicks and keystrokes that filled it. It is -not a general API client: JSON or script-built requests and multi-page -server state stay on the GUI. Other writes during the replaced steps are a +**Scope: a single HTML form submission.** An API step repeats one POST of +one HTML form, read again from the live page, in place of the clicks and +keystrokes that filled it. The POST is `application/x-www-form-urlencoded` +or `multipart/form-data` with no file attached, and either the browser +submits the form or the page's script sends the form's fields to another +endpoint (see [Saves the page's script sends](#saves-the-pages-script-sends)). +It is not a general API client: JSON and GraphQL bodies, uploads and +multi-page server state stay on the GUI. Other writes during the replaced steps are a declared sequence (below): the page's own background requests are listed and left to the page, and a write a step caused is sent in its recorded place before the save and read back. A form whose script @@ -74,6 +77,29 @@ every field it sent: | `select_label` | A select a recorded step typed an option's label into | The option with that label in the live form; refuse if it isn't exactly one | | `session` | A CSRF or session value the server issued | Read from the live form in this session | | `form_default` | The form's own default | Must still equal the recorded default | +| `submitter` | The `name=value` the browser adds for the submit button the person clicked (`commit=Create`) | The live form must have exactly one enabled submit button that sends that value | +| `empty_file` | A file input sent empty in a multipart save | The live form must have exactly the recorded file inputs; the API step never attaches a file | + +The save is sent with the encoding it was recorded with. A multipart save +is encoded the way Chromium encodes a form with no file (each field a part, +an empty file input a part with `filename=""`, line breaks as CRLF), with a +boundary derived from the fields, so the same request always has the same +bytes. A multipart save that attached a file isn't captured +(`body_omitted: "multipart"`), so it isn't proposed. A form with no `action` +posts to its own page and is matched as such. + +The submit button's pair is explained only by exactly one button whose value +is written in the page. An `` without a `value` sends the +browser's own label, which the page doesn't state, so such a pair isn't +explained. When the recording doesn't name the clicked control by `#id` or +`[name]`, the button that sends the pair is the one clicked when the page's +script runs; a recorded click on another control refuses the proposal. + +The save's success signal is the address it sent the browser to. A record's +own address (`/ticket/17`, `/ipam/ip-addresses/3/`) is kept whole with each +number generalized, since the number alone matches almost any URL. A query +value that is a long hash or nonce (a comment's `moderation-hash`) must be +present, with any value. A field is bound by provenance: which recorded step typed into it (the field the click before the typing targeted), never by its value matching a @@ -118,6 +144,22 @@ it sends the save. On-screen checks of steps that made no request (layout and region checks) are screen-only. They are listed for the person who admits the step and aren't repeated. +## Saves the page's script sends + +Some pages don't submit the form. Their script reads the form's fields and +posts them to another endpoint (WordPress's category form posts to +`admin-ajax.php`, not to its `action`). The compiler takes the form whose +controls name every field the request sent, on the last captured page +before the save; two such forms refuse the proposal. The binding names the +form's own action (`form_action`), and the request's endpoint. + +Such a save always runs the page's script first (below): the scratch page's +blocked request to the endpoint is the save compared, and it gets an empty +`200` so the script carries on. The API step sends only when the script +sends exactly the body the API step built. A page that no longer has script +keeps the save on the GUI. There is no redirect to read, so success is the +recorded status and then the record check. + ## Other writes in the span Every other write sent while the replaced steps ran is declared in the @@ -439,9 +481,13 @@ recording. - Web only. A desktop app, an RDP session or a Citrix session has no page requests to capture (`record --capture-network` refuses those backends). -- One form save, posted as `application/x-www-form-urlencoded`. JSON bodies - built by script, multipart uploads, signed or computed fields, and saves - over a websocket aren't proposed. +- One form save, URL-encoded or multipart with no file. JSON and GraphQL + bodies, file uploads, signed or computed fields, and saves over a + websocket aren't proposed. Step writes in the span must be URL-encoded. +- A field the server fills in on every page load (a page-load time, a + per-record code, a date defaulting to today) is a form default that + differs on the next load, so the API step doesn't run and the GUI steps + do. - The server may enforce less than the page. On OpenEMR, the duplicate check and the required fields run only in the browser. The API step carries over the checks it can evaluate (pre-checks, native HTML diff --git a/packages/flow/docs/NETWORK_CAPTURE.md b/packages/flow/docs/NETWORK_CAPTURE.md index 35b8f84b18..9f0cb9f380 100644 --- a/packages/flow/docs/NETWORK_CAPTURE.md +++ b/packages/flow/docs/NETWORK_CAPTURE.md @@ -38,9 +38,10 @@ order the browser finished them: | `status` | yes | HTTP status, or `null` when the request failed | | `failed` | no | `true` when the request failed (no response) | | `request_type` | no | Request content type, without parameters | -| `form` | no | For `application/x-www-form-urlencoded` bodies: `[[name, value], ...]` in the order sent, duplicates kept | +| `form` | no | For `application/x-www-form-urlencoded` bodies, and `multipart/form-data` bodies with no file attached: `[[name, value], ...]` in the order sent, duplicates kept | +| `multipart_files` | no | For a kept multipart body: the names of its file inputs that were sent empty (`[name, ""]` in `form`) | | `body` | no | Any other text body, as text | -| `body_omitted` | no | `"multipart"` when a multipart body was not kept | +| `body_omitted` | no | `"multipart"` when a multipart body was not kept: a file was attached, or the body isn't plain form data | | `request_truncated` | no | `true` when the body was cut at 256 KiB | | `response_type` | no | Response content type, without parameters | | `location` | no | The redirect target, when the response has one | diff --git a/packages/flow/openadapt_flow/compiler/api_proposal.py b/packages/flow/openadapt_flow/compiler/api_proposal.py index d01fbdf688..b892c41168 100644 --- a/packages/flow/openadapt_flow/compiler/api_proposal.py +++ b/packages/flow/openadapt_flow/compiler/api_proposal.py @@ -54,7 +54,7 @@ import urllib.parse as up from datetime import datetime, timezone from pathlib import Path -from typing import Any, Optional +from typing import Any, Literal, Optional from openadapt_flow import network_capture as nc from openadapt_flow import sealed_set as _sealed @@ -73,8 +73,10 @@ from openadapt_flow.runtime.actuators.web_form import ( client_rules, folded_checks_digest, - form_controls, + form_parts, + forms_of, page_scripts, + submitter_button, web_form_effects, ) @@ -271,16 +273,95 @@ def _success_pattern(line: dict[str, Any]) -> Optional[str]: target = m.group(1) if m else None if not target: return None - path = up.urlsplit(target).path.rsplit("/", 1)[-1] - query = up.urlsplit(target).query - pieces = [re.escape(path)] - for name, value in up.parse_qsl(query, keep_blank_values=True): - pieces.append( - re.escape(name) + "=" + (r"\d+" if value.isdigit() else re.escape(value)) - ) + path = up.urlsplit(target).path + leaf = path.rsplit("/", 1)[-1] + if leaf.isdigit() or not leaf: + # A record's own address (``/ticket/17``, ``/ipam/ip-addresses/3/``): + # the record number alone matches almost anything, so the path is + # kept, with each number generalized. + pieces = [ + "/".join( + r"\d+" if seg.isdigit() else re.escape(seg) for seg in path.split("/") + ) + ] + else: + pieces = [re.escape(leaf)] + for name, value in up.parse_qsl(up.urlsplit(target).query, keep_blank_values=True): + if value.isdigit(): + pattern = r"\d+" + elif _is_token(value): + # A value the server makes per record (a hash, a nonce): the + # name must be there, the value can't be known in advance. + pattern = r"[^&#\s]+" + else: + pattern = re.escape(value) + pieces.append(re.escape(name) + "=" + pattern) return r".*?".join(pieces) +def _is_token(value: str) -> bool: + """A long run of letters and digits mixing both, such as a hash.""" + + return ( + len(value) >= 16 + and re.fullmatch(r"[A-Za-z0-9_-]+", value) is not None + and re.search(r"\d", value) is not None + and re.search(r"[A-Za-z]", value) is not None + ) + + +def _button_selector(button: dict[str, Any]) -> str: + """A selector for one submit button of the form being evaluated.""" + + if button.get("id"): + return "#" + button["id"] + sel = f"form[data-openadapt-eval] [name={json.dumps(button['name'])}]" + if button.get("value") is not None and button.get("value") != "": + sel += f"[value={json.dumps(button['value'])}]" + return sel + + +def _script_form( + lines: list[dict[str, Any]], post: dict[str, Any] +) -> tuple[Optional[dict[str, Any]], Optional[dict[str, Any]], Optional[str]]: + """The form whose fields a page script sent to another endpoint. + + The last captured page, before the save, whose form names every field the + save sent (as a control, an empty file input or a submit button). Two + such forms on that page are ambiguous. + """ + + names = {n for n, _ in post["form"]} + best: tuple[Optional[dict[str, Any]], Optional[dict[str, Any]], Optional[str]] = ( + None, + None, + None, + ) + for ln in lines: + if float(ln["t"]) >= float(post["t"]): + break + if not ( + ln["method"] == "GET" + and ln.get("status") == 200 + and ln.get("resource_type") == "document" + and " 1: + best = (None, None, "two forms on the page name every field the save sent") + elif hits: + best = (ln, hits[0], None) + return best + + def _in_page(selector: str, html: str) -> bool: """Whether a recorded ``#id`` or ``[name="..."]`` selector names a control of ``html`` that can start a submit (a `` +""" + + +class Site: + def __init__(self) -> None: + self.html = FORM + self.requests: list[dict[str, Any]] = [] + owner = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: Any) -> None: + pass + + def handle_request(self) -> None: + raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + owner.requests.append( + { + "method": self.command, + "path": self.path, + "content_type": self.headers.get("Content-Type", ""), + "body": raw.decode("utf-8"), + } + ) + body = ( + owner.html if up.urlsplit(self.path).path == "/compose" else "ok" + ).encode() + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = handle_request + do_POST = handle_request + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base = f"http://127.0.0.1:{self.server.server_address[1]}" + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + def writes(self) -> list[dict[str, Any]]: + return [r for r in self.requests if r["method"] == "POST"] + + +@pytest.fixture(scope="module") +def browser(): + with sync_playwright() as playwright: + instance = playwright.chromium.launch(headless=True) + yield instance + instance.close() + + +@pytest.fixture +def site_page(browser): + site = Site() + context = browser.new_context() + page = context.new_page() + page.goto(site.base + "/") + yield site, page + context.close() + site.close() + + +def binding_from_browser(tmp_path: Path, site: Site, page: Any): + """Capture the real normal submit, then propose and test-admit its binding.""" + recording = tmp_path / "recording" + recording.mkdir() + (recording / "meta.json").write_text( + json.dumps({"params": {"title": "Note Alpha"}}) + ) + clock = time.monotonic() + capture = NetworkCapture(page.context).start() + try: + page.goto(site.base + "/compose") + # Finish capturing the form before recording its first input event. + # Playwright may return from goto before requestfinished is delivered. + deadline = time.monotonic() + 3 + while not any( + r.get("method") == "GET" + and r.get("url") == site.base + "/compose" + and r.get("status") == 200 + for r in capture.lines(clock) + ): + assert time.monotonic() < deadline, "the form capture did not finish" + page.wait_for_timeout(10) + page.wait_for_timeout(10) # separate events from millisecond capture stamps + events = [ + { + "i": 0, + "kind": "click", + "t": time.monotonic() - clock, + "structural": { + "selector": "#title", + "role": "textbox", + "name": "Title", + }, + } + ] + page.click("#title") + events.append( + { + "i": 1, + "kind": "type", + "text": "Note Alpha", + "param": "title", + "t": time.monotonic() - clock, + } + ) + page.fill("#title", "Note Alpha") + events.append( + { + "i": 2, + "kind": "click", + "t": time.monotonic() - clock, + "structural": {"selector": "#send", "role": "button", "name": "Send"}, + } + ) + page.click("#send") + page.wait_for_timeout(180) + finally: + capture.stop() + capture.write(recording, clock) + (recording / "events.jsonl").write_text( + "".join(json.dumps(e) + "\n" for e in events) + ) + workflow = Workflow( + name="form-boundaries", + params={"title": "Note Alpha"}, + steps=[ + Step(id="step_000", intent="Focus title", action=ActionKind.CLICK), + Step( + id="step_001", + intent="Type title", + action=ActionKind.TYPE, + param="title", + ), + Step( + id="step_002", + intent="Send", + action=ActionKind.CLICK, + risk="irreversible", + effects=[ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"title": "Note Alpha"}, + expected_count=1, + ) + ], + ), + ], + ) + proposed, step, reasons = propose_api_step(workflow, recording) + assert step is not None, reasons + admitted, identifiers = admit_api_step( + proposed, admitted_by="synthetic-shape-check" + ) + assert identifiers == ["step_002"] + result = admitted.steps[-1].api_binding + assert result is not None and result.admitted() + site.requests.clear() + return result + + +def native_submit(site: Site, page: Any, title: str = "Note Beta") -> None: + page.goto(site.base + "/compose") + page.fill("#title", title) + button = page.locator("#send") + if button.is_enabled(): + button.click(timeout=3000) + else: + with pytest.raises(BrowserTimeout): + button.click(timeout=300) + page.wait_for_timeout(180) + + +@pytest.mark.parametrize("target", ["relative", "absolute", "protocol_relative"]) +def test_an_unbound_transport_destination_is_not_resolved(target, site_page): + site, page = site_page + endpoint = { + "relative": "publish", + "absolute": site.base + "/publish", + "protocol_relative": "//publish", + }[target] + result = PlaywrightSessionTransport(page).evaluate_form( + "/compose", endpoint, [("title", "Note Beta")], submitter="#send" + ) + assert result.status == "unevaluable", result.reason + assert not site.writes() + + +@pytest.mark.parametrize("value", ["7", "68", "907", "", "long-synthetic-value"]) +def test_declared_multipart_secret_is_scrubbed_before_persistence( + value, tmp_path, site_page +): + site, page = site_page + site.html = FORM.replace( + 'method="post"', 'method="post" enctype="multipart/form-data"' + ).replace("
", "
") + elif mode == "first_legend": + site.html = original.replace( + "", "") + elif mode == "required_file": + site.html = original.replace('id="bundle"', 'id="bundle" required') + else: + override = { + "action": 'formaction="/alternate"', + "query": 'formaction="/publish?route=other"', + "method": 'formmethod="get"', + "encoding": 'formenctype="text/plain"', + "same_action": 'formaction="/publish"', + }.get(mode, "") + site.html = original.replace('id="send"', 'id="send" ' + override) + native_submit(site, page) + reference = site.writes() + if mode in ("nested_disabled", "required_file", "method"): + assert not reference + else: + assert len(reference) == 1 + if mode == "action": + assert reference[0]["path"] == "/alternate" + if mode == "query": + assert reference[0]["path"] == "/publish?route=other" + if mode == "encoding": + assert reference[0]["content_type"].startswith("text/plain") + site.requests.clear() + page.goto(site.base + "/") + result = WebFormActuator(PlaywrightSessionTransport(page)).actuate( + binding, {"title": "Note Beta"} + ) + print( + json.dumps( + { + "case": mode, + "browser": reference, + "api": site.writes(), + "status": result.status.value, + } + ) + ) + if allowed: + assert result.status is ActuationStatus.ACTUATED, result.reason + assert ( + len(site.writes()) == 1 and site.writes()[0]["path"] == reference[0]["path"] + ) + actual = site.writes()[0] + if reference[0]["content_type"].startswith("multipart/form-data"): + old = re.search(r"boundary=([^; ]+)", reference[0]["content_type"]) + new = re.search(r"boundary=([^; ]+)", actual["content_type"]) + assert old and new + assert reference[0]["body"].replace(old.group(1), "BOUNDARY") == actual[ + "body" + ].replace(new.group(1), "BOUNDARY") + else: + assert up.parse_qsl(actual["body"], keep_blank_values=True) == up.parse_qsl( + reference[0]["body"], keep_blank_values=True + ) + else: + assert result.status is ActuationStatus.UNAVAILABLE, result.reason + assert not site.writes() + + +@pytest.mark.parametrize( + "channel,remote", [("fetch", False), ("fetch", True), ("submit", True)] +) +def test_actual_script_destination_is_bound(channel, remote, tmp_path, site_page): + site, page = site_page + other = Site() + try: + destination = json.dumps(other.base + "/publish") + send = ( + "fetch(target, {method: 'POST', body: new URLSearchParams(new FormData(f))});" + if channel == "fetch" + else "f.action = target; f.submit();" + ) + site.html = FORM.replace( + "", + """""".replace("DESTINATION", destination).replace("SEND", send), + ) + binding = binding_from_browser(tmp_path, site, page) + title = "Across" if remote else "Note Beta" + native_submit(site, page, title) + if remote: + assert len(other.writes()) == 1 and not site.writes() + else: + assert len(site.writes()) == 1 and not other.writes() + site.requests.clear() + other.requests.clear() + page.goto(site.base + "/") + result = WebFormActuator(PlaywrightSessionTransport(page)).actuate( + binding, {"title": title} + ) + if remote: + assert result.status is ActuationStatus.UNAVAILABLE, result.reason + assert not site.writes() and not other.writes() + else: + assert result.status is ActuationStatus.ACTUATED, result.reason + assert len(site.writes()) == 1 and not other.writes() + finally: + other.close() diff --git a/packages/flow/tests/test_api_step_shapes.py b/packages/flow/tests/test_api_step_shapes.py new file mode 100644 index 0000000000..211990c29d --- /dev/null +++ b/packages/flow/tests/test_api_step_shapes.py @@ -0,0 +1,362 @@ +"""API steps for more save shapes, checked against a real browser. + +The browser is the reference: Chromium submits each form itself, and the API +step must send the same fields, in the same order, with the same encoding. + +- multipart/form-data with no file attached: capture keeps the fields, and + the API step's encoding is byte for byte the browser's (boundary aside); +- the clicked submit button's own ``name=value``; +- a file input left empty; an attached file is never captured or sent; +- a save the page's script sends to another endpoint than the form's action. + +The same shapes on real applications (Redmine, NetBox, Dolibarr, Trac, +WordPress) are in ``test_api_step_shapes_fixtures.py``. +""" + +from __future__ import annotations + +import re +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +import pytest + +from openadapt_flow.compiler.api_proposal import _success_pattern +from openadapt_flow.ir import ApiBinding, WebFormField, WebFormSpec +from openadapt_flow.network_capture import parse_multipart +from openadapt_flow.runtime.actuators import ActuationStatus +from openadapt_flow.runtime.actuators.web_form import ( + PlaywrightSessionTransport, + WebFormActuator, + encode_multipart, + form_parts, + submitter_button, +) + +playwright = pytest.importorskip("playwright.sync_api") + +FORM = """ +
+ + + + + + + +
{script}""" + +SCRIPT_SENT = """ +
+ + + + +
+""" + + +class _App: + def __init__(self) -> None: + self.pages: dict[str, str] = {} + self.posts: list[tuple[str, str, bytes]] = [] + self.loads = 0 + app = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *a: Any) -> None: + pass + + def do_GET(self) -> None: # noqa: N802 + path = self.path.split("?")[0] + if path in app.pages: + app.loads += 1 + body = app.pages[path].replace("{n}", str(app.loads)).encode() + self.send_response(200) + self.send_header("Content-Type", "text/html") + else: + body = b"no" + self.send_response(404) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self) -> None: # noqa: N802 + raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + app.posts.append( + (self.path, self.headers.get("Content-Type") or "", raw) + ) + self.send_response(303) + self.send_header("Location", "/items/17") + self.send_header("Content-Length", "0") + self.end_headers() + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.base = f"http://127.0.0.1:{self.server.server_address[1]}" + + +@pytest.fixture(scope="module") +def browser(): + with playwright.sync_playwright() as pw: + b = pw.chromium.launch(headless=True) + yield b + b.close() + + +@pytest.fixture +def app_page(browser): + app = _App() + app.pages["/new"] = FORM.replace("{script}", "") + app.pages["/"] = "home" + context = browser.new_context() + page = context.new_page() + page.goto(app.base + "/") + yield app, page + context.close() + app.server.shutdown() + + +def _browser_submit( + app: _App, page: Any, subject: str, notes: str +) -> tuple[str, bytes]: + page.goto(app.base + "/new") + page.fill("#subject", subject) + page.fill("#notes", notes) + page.select_option("select[name=kind]", "2") + page.click("input[name=commit]") + page.wait_for_load_state() + _, ctype, raw = app.posts[-1] + return ctype, raw + + +def _binding(**spec: Any) -> ApiBinding: + fields = [ + WebFormField(name="csrf", source="session"), + WebFormField(name="subject", source="param", param="subject"), + WebFormField(name="notes", source="param", param="notes"), + WebFormField(name="kind", source="select_label", param="kind"), + WebFormField(name="attachment", source="empty_file", value=""), + WebFormField(name="commit", source="submitter", value="Create"), + ] + return ApiBinding( + kind="web_form", + method="POST", + url_template="/save", + on_unavailable="gui", + web_form=WebFormSpec( + form_path="/new", + fields=fields, + controls=["csrf", "kind", "notes", "subject"], + encoding="multipart", + submitter="input[name=commit]", + success_pattern=r"/items/\d+", + replaces_steps=["step_001"], + folded_checks_digest="sha256:" + "0" * 64, + **spec, + ), + ) + + +def test_capture_keeps_a_multipart_save_with_no_file(app_page): + app, page = app_page + ctype, raw = _browser_submit(app, page, "Printer jams", "line one\nline two") + parsed = parse_multipart(raw, ctype) + assert parsed is not None + pairs, files = parsed + assert pairs[0][0] == "csrf" and pairs[0][1].startswith("tok-") + assert pairs[1:] == [ + ["subject", "Printer jams"], + ["notes", "line one\r\nline two"], + ["kind", "2"], + ["attachment", ""], + ["commit", "Create"], + ] + assert files == ["attachment"] + + +def test_capture_never_keeps_an_attached_file(app_page, tmp_path): + app, page = app_page + upload = tmp_path / "scan.txt" + upload.write_text("contents") + page.goto(app.base + "/new") + page.set_input_files("input[name=attachment]", str(upload)) + page.click("input[name=commit]") + page.wait_for_load_state() + _, ctype, raw = app.posts[-1] + assert parse_multipart(raw, ctype) is None + + +def test_multipart_encoding_is_the_browsers_byte_for_byte(app_page): + app, page = app_page + ctype, raw = _browser_submit(app, page, "Printer jams", "line one\nline two") + pairs, files = parse_multipart(raw, ctype) + ours, our_type = encode_multipart( + [(k, v.replace("\r\n", "\n")) for k, v in pairs], files + ) + theirs = re.search(r"boundary=(\S+)", ctype).group(1) + mine = re.search(r"boundary=(\S+)", our_type).group(1) + assert ours.encode("utf-8") == raw.replace(theirs.encode(), mine.encode()) + + +def test_api_step_sends_what_the_browser_sends(app_page): + app, page = app_page + _, gui_raw = _browser_submit(app, page, "Printer jams", "two\nlines") + gui_type = app.posts[-1][1] + actuator = WebFormActuator(PlaywrightSessionTransport(page)) + result = actuator.actuate( + _binding(), {"subject": "Printer jams", "notes": "two\nlines", "kind": "Task"} + ) + assert result.status is ActuationStatus.ACTUATED, result.reason + path, api_type, api_raw = app.posts[-1] + assert path == "/save" and api_type.startswith("multipart/form-data") + gui_pairs, gui_files = parse_multipart(gui_raw, gui_type) + api_pairs, api_files = parse_multipart(api_raw, api_type) + # The same fields, order and empty file part; only the session's own + # CSRF value differs (each form read issues a new one). + assert [p for p in api_pairs if p[0] != "csrf"] == [ + p for p in gui_pairs if p[0] != "csrf" + ] + assert api_files == gui_files == ["attachment"] + + +def test_a_changed_submit_button_keeps_the_save_on_the_gui(app_page): + app, page = app_page + app.pages["/new"] = FORM.replace('value="Create">', 'value="Save">').replace( + "{script}", "" + ) + actuator = WebFormActuator(PlaywrightSessionTransport(page)) + result = actuator.actuate( + _binding(), {"subject": "x", "notes": "y", "kind": "Task"} + ) + assert result.status is ActuationStatus.UNAVAILABLE + assert "submit button" in result.reason + assert not app.posts + + +def test_a_new_file_input_keeps_the_save_on_the_gui(app_page): + app, page = app_page + app.pages["/new"] = FORM.replace( + '', + '', + ).replace("{script}", "") + result = WebFormActuator(PlaywrightSessionTransport(page)).actuate( + _binding(), {"subject": "x", "notes": "y", "kind": "Task"} + ) + assert result.status is ActuationStatus.UNAVAILABLE + assert "file inputs changed" in result.reason + assert not app.posts + + +def test_the_button_pair_needs_one_button_with_that_value(): + form = form_parts(FORM.replace("{script}", ""), "/save") + assert submitter_button(form, "commit", "Create") is not None + assert submitter_button(form, "commit", "Create and continue") is None + no_value = form_parts( + '
', "/save" + ) + # An without a value sends the browser's own label. + assert submitter_button(no_value, "go", "Submit") is None + + +def test_a_form_without_an_action_posts_to_its_page(): + html = '
' + assert form_parts(html, "/ipam/add/") is None + assert form_parts(html, "/ipam/add/", "/ipam/add/")["controls"].keys() == {"a"} + + +def _script_binding(extra_fields: list[WebFormField] = ()) -> ApiBinding: + return ApiBinding( + kind="web_form", + method="POST", + url_template="/ajax", + on_unavailable="gui", + web_form=WebFormSpec( + form_path="/tags", + form_action="/edit", + fields=[ + WebFormField(name="action", source="form_default", value="add-tag"), + WebFormField(name="nonce", source="session"), + WebFormField(name="tag-name", source="param", param="name"), + *extra_fields, + ], + controls=["action", "nonce", "tag-name"], + client_scripts=["inline script"], + submitter="#submit", + replaces_steps=["step_001"], + folded_checks_digest="sha256:" + "0" * 64, + ), + ) + + +def test_a_script_sent_save_runs_the_script_first(app_page): + app, page = app_page + app.pages["/tags"] = SCRIPT_SENT.format(extra="") + actuator = WebFormActuator(PlaywrightSessionTransport(page)) + spec = _script_binding().web_form + # The live page's own script list, so the run checks it the same way. + from openadapt_flow.runtime.actuators.web_form import client_rules, page_scripts + + html = app.pages["/tags"].replace("{n}", "1") + scripts = page_scripts( + html, "/edit", ["action", "nonce", "tag-name"], client_rules(html), "/tags" + ) + binding = _script_binding() + binding.web_form.client_scripts = scripts + assert spec.form_action == "/edit" + result = actuator.actuate(binding, {"name": "Community"}) + assert result.status is ActuationStatus.ACTUATED, result.reason + page_script = [e for e in actuator.log if e["stage"] == "page_script"] + assert page_script and page_script[0]["status"] == "valid" + assert [(p, t.split(";")[0]) for p, t, _ in app.posts] == [ + ("/ajax", "application/x-www-form-urlencoded") + ] + + +def test_a_script_that_sends_another_field_keeps_the_save_on_the_gui(app_page): + app, page = app_page + app.pages["/tags"] = SCRIPT_SENT.format(extra="body.append('screen', 'edit');") + from openadapt_flow.runtime.actuators.web_form import client_rules, page_scripts + + html = app.pages["/tags"].replace("{n}", "1") + binding = _script_binding() + binding.web_form.client_scripts = page_scripts( + html, "/edit", ["action", "nonce", "tag-name"], client_rules(html), "/tags" + ) + result = WebFormActuator(PlaywrightSessionTransport(page)).actuate( + binding, {"name": "Community"} + ) + assert result.status is ActuationStatus.UNAVAILABLE + assert "screen" in result.reason + assert not app.posts + + +@pytest.mark.parametrize( + "location, good, bad", + [ + ("http://h/ticket/1", "http://h/ticket/42", "http://h/newticket?x=1"), + ("/ipam/ip-addresses/3/", "/ipam/ip-addresses/9/", "/ipam/ip-addresses/add/"), + ( + "/?p=1&unapproved=2&moderation-hash=e995d4ec71c477832a7f6b9e", + "/?p=1&unapproved=7&moderation-hash=0123abcdef4567890abc", + "/?p=1", + ), + ( + "/contact/card.php?id=1", + "/contact/card.php?id=8", + "/contact/card.php?action=create", + ), + ], +) +def test_success_patterns_generalize_record_numbers_and_hashes(location, good, bad): + pattern = _success_pattern({"location": location}) + assert re.search(pattern, good) + assert not re.search(pattern, bad) diff --git a/packages/flow/tests/test_api_step_shapes_fixtures.py b/packages/flow/tests/test_api_step_shapes_fixtures.py new file mode 100644 index 0000000000..fadb264a86 --- /dev/null +++ b/packages/flow/tests/test_api_step_shapes_fixtures.py @@ -0,0 +1,84 @@ +"""API step save shapes on real applications (opt-in, needs the lab fixtures). + +Each workflow is recorded fresh on its fixture, compiled, and then run with +the API step on: once normally, and once with the save refused by the +fixture's proxy (503, nothing forwarded). The app's own database is the +judge (``benchmark/api_generalization``, PROTOCOL.md). + +- Every run must be correct: the expected record and nothing else after the + normal run, and no claimed success after the refused save. +- When the API step is proposed and admitted, the normal run saves through + it (``actuation: api``) with one save request, unless a value the server + fills in on every page load (a load time, a per-record code) differs, in + which case it says so and the GUI steps save. +- When it isn't proposed, the reason must be one of the known ones. + +Run with the fixtures up (``benchmark/api_generalization/run.sh up --app X``): + + APIGEN_FIXTURES=redmine,trac pytest tests/test_api_step_shapes_fixtures.py +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +import pytest + +APPS = [a for a in os.environ.get("APIGEN_FIXTURES", "").split(",") if a] + +pytestmark = pytest.mark.skipif( + not APPS, reason="set APIGEN_FIXTURES to the running lab fixtures" +) + +#: Why each workflow may stay on the GUI, from the fields its form sends. +KNOWN_REFUSALS = ( + "form's default changed", # a per-load value: _init_time, a customer code + "not a recorded value, a form default or a session value", +) + + +@pytest.mark.parametrize("name", APPS) +def test_api_step_on_a_real_app(name: str, tmp_path: Path) -> None: + from benchmark.api_generalization import harness + + os.environ.setdefault("APIGEN_STATE", str(tmp_path / "state")) + app = harness.load_app(name) + rec, bundle = tmp_path / "rec", tmp_path / "bundle" + harness.cmd_record(argparse.Namespace(out=str(rec)), app) + harness.cmd_compile(argparse.Namespace(recording=str(rec), out=str(bundle)), app) + compiled = json.loads((bundle / "apigen-compile.json").read_text()) + + env = harness._env() + try: + cases = {c.kind: c for c in app.cases()} + normal = harness.run_one( + env, app, app.unique(cases["normal"], "fxn"), "api", bundle, tmp_path / "n" + ) + rejected = harness.run_one( + env, + app, + app.unique(cases["fault_reject"], "fxr"), + "api", + bundle, + tmp_path / "r", + ) + finally: + env["browser"].close() + env["pw"].stop() + + assert normal["correct"], normal + assert not normal["incorrect_write"] and not normal["false_success"], normal + assert not rejected["false_success"] and not rejected["incorrect_write"], rejected + if not compiled["proposed"]: + assert any(r in " ".join(compiled["reasons"]) for r in KNOWN_REFUSALS), compiled + return + assert compiled.get("admitted"), compiled + if normal["save_actuation"] == "api": + assert normal["save_posts"] == 1, normal + else: + # Proposed from one page load, but a value the server fills in on + # every load differs at run time: the GUI steps saved instead. + assert any(r in str(normal["api_fallback"]) for r in KNOWN_REFUSALS), normal From 8b783a1f0ec276e9efbdf4790a17fd6ed28c0a0a Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 22:01:06 -0800 Subject: [PATCH 14/21] fix: frame lab SSH scripts and preflight rollback helpers (#1942) Signed-off-by: abrichr --- .../flow/scripts/desktop/foreign_window.ps1 | 17 ++++ .../scripts/desktop/qualify_lab_rollback.py | 26 ++++-- packages/flow/scripts/desktop/winrm_lab.py | 50 +++++++++-- .../tests/test_qualify_lab_rollback_setup.py | 90 +++++++++++++++++++ packages/flow/tests/test_winrm_lab.py | 86 +++++++++++++++++- 5 files changed, 251 insertions(+), 18 deletions(-) create mode 100644 packages/flow/scripts/desktop/foreign_window.ps1 create mode 100644 packages/flow/tests/test_qualify_lab_rollback_setup.py diff --git a/packages/flow/scripts/desktop/foreign_window.ps1 b/packages/flow/scripts/desktop/foreign_window.ps1 new file mode 100644 index 0000000000..efba066f43 --- /dev/null +++ b/packages/flow/scripts/desktop/foreign_window.ps1 @@ -0,0 +1,17 @@ +# Lab fault: open Notepad as a topmost window over the target app, from the +# signed-in user's own session (a scheduled task from another session cannot +# take the foreground). Used by qualify_lab_native.py. +Add-Type -Namespace OALab -Name Win -MemberDefinition @' +[DllImport("user32.dll")] public static extern bool SetWindowPos(System.IntPtr h, System.IntPtr after, int x, int y, int cx, int cy, uint flags); +'@ +Start-Process notepad.exe +# Windows 11 Notepad is a packaged app: the started process is a stub, so +# find the real window by process name. +$hwnd = [System.IntPtr]::Zero +for ($i = 0; $i -lt 75 -and $hwnd -eq [System.IntPtr]::Zero; $i++) { + Start-Sleep -Milliseconds 200 + $w = Get-Process notepad -ErrorAction SilentlyContinue | Where-Object { $_.MainWindowHandle -ne 0 } | Select-Object -First 1 + if ($w) { $hwnd = $w.MainWindowHandle } +} +# HWND_TOPMOST = -1, SWP_SHOWWINDOW = 0x40 +[void][OALab.Win]::SetWindowPos($hwnd, [System.IntPtr](-1), 0, 0, 1024, 720, 0x40) diff --git a/packages/flow/scripts/desktop/qualify_lab_rollback.py b/packages/flow/scripts/desktop/qualify_lab_rollback.py index 871f212395..31f4d0ea8e 100644 --- a/packages/flow/scripts/desktop/qualify_lab_rollback.py +++ b/packages/flow/scripts/desktop/qualify_lab_rollback.py @@ -22,8 +22,8 @@ Every verdict is checked again against the guest's SQLite file. The lab database is a test copy only these runs write (``exclusive``). -Usage (from packages/flow, with the lab ``.env`` loaded and ``winrm_lab`` on -the path):: +Usage (from packages/flow, with an explicitly configured lab profile and +``winrm_lab`` on the path):: python scripts/desktop/qualify_lab_rollback.py --transport native \\ --out ~/oa/scratch/winlab/runs/rollback-native-1 --evidence ev.json @@ -465,6 +465,21 @@ def covered_reset() -> None: } +def _required_helpers(directories: tuple[Path, ...]) -> dict[str, Path]: + """Resolve every required helper before any guest deployment begins.""" + helpers = {} + missing = [] + for name in ("pn_db.py", "patient_notes.ps1", "foreign_window.ps1"): + local = next((d / name for d in directories if (d / name).is_file()), None) + if local is None: + missing.append(name) + else: + helpers[name] = local + if missing: + raise FileNotFoundError("missing required lab helpers: " + ", ".join(missing)) + return helpers + + def run_trial(transport: str, out: Path, row: dict[str, Any]) -> None: from openadapt_flow.adapters.desktop_recorder import record_desktop_demo @@ -477,12 +492,12 @@ def run_trial(transport: str, out: Path, row: dict[str, Any]) -> None: ).stdout.strip(), } ) + tools = Path(winrm_lab.__file__).resolve().parent + helpers = _required_helpers((HERE, tools)) lab = winrm_lab.WinRMLab() prefix = _use_host(lab) row["host"] = prefix.split("_")[1].lower() # winlab | winlap - tools = Path(winrm_lab.__file__).resolve().parent - for name in ("pn_db.py", "patient_notes.ps1", "foreign_window.ps1"): - local = next(p for p in (HERE / name, tools / name) if p.exists()) + for name, local in helpers.items(): lab.push_file(local, rf"{GUEST}\{name}") rdp = _rdp_backend(prefix) if transport == "rdp" else None if rdp is not None: @@ -578,7 +593,6 @@ def main() -> int: parser.add_argument("--out", type=Path, required=True) parser.add_argument("--evidence", type=Path, required=True) args = parser.parse_args() - winrm_lab.load_env_file(Path.home() / "oa/src/OpenAdapt/.env") args.out.mkdir(parents=True, exist_ok=True) row: dict[str, Any] = {} try: diff --git a/packages/flow/scripts/desktop/winrm_lab.py b/packages/flow/scripts/desktop/winrm_lab.py index a7d981b9dd..4264aad35d 100644 --- a/packages/flow/scripts/desktop/winrm_lab.py +++ b/packages/flow/scripts/desktop/winrm_lab.py @@ -36,6 +36,7 @@ from __future__ import annotations import base64 +import math import os import re import subprocess @@ -59,7 +60,8 @@ ) -# Run over SSH: read the whole script from stdin and run it as one block. Like +# Run over SSH: read a length-framed script from stdin as one block. Windows +# OpenSSH does not always deliver EOF, so ReadToEnd can wait indefinitely. Like # ``powershell -Command``, exit 1 when the script's last statement failed. _SSH_BOOTSTRAP = base64.b64encode( ( @@ -67,7 +69,12 @@ "[Console]::InputEncoding=[Text.Encoding]::UTF8; " "[Console]::OutputEncoding=[Text.Encoding]::UTF8; " "$global:OAFlowLastOk=$true; " - '$s=[Console]::In.ReadToEnd() + "`n`$global:OAFlowLastOk=`$?"; ' + "$n=0; $header=[Console]::In.ReadLine(); " + "if (-not [int]::TryParse($header,[ref]$n) -or $n -le 0) { exit 2 }; " + "$b=New-Object char[] $n; $r=0; " + "while ($r -lt $n) { " + "$k=[Console]::In.Read($b,$r,$n-$r); if ($k -le 0) { exit 2 }; $r+=$k }; " + '$s=(-join $b) + "`n`$global:OAFlowLastOk=`$?"; ' "& ([scriptblock]::Create($s)); " "if (-not $global:OAFlowLastOk) { exit 1 }" ).encode("utf-16-le") @@ -209,24 +216,42 @@ def _ssh_base(self) -> list[str]: f"{self.user}@{self.host}", ] - def _ssh(self, remote_command: str, stdin: bytes = b"") -> Result: + def _ssh( + self, + remote_command: str, + stdin: bytes = b"", + *, + timeout_s: float = 60.0, + operation: str = "PowerShell command", + ) -> Result: + if not math.isfinite(timeout_s) or timeout_s <= 0: + raise ValueError("SSH timeout must be finite and positive") try: proc = subprocess.run( self._ssh_base() + [remote_command], input=stdin, capture_output=True, - timeout=600, + timeout=timeout_s, ) except subprocess.TimeoutExpired: # The exception's own text repeats the command line. - raise WinRMLabError(f"ssh to {self.host} timed out") from None + raise WinRMLabError( + f"{operation} over SSH to {self.host} timed out after {timeout_s:g}s" + ) from None return Result( proc.returncode, proc.stdout.decode("utf-8", "replace"), _clixml_text(proc.stderr.decode("utf-8", "replace")), ) - def run_ps(self, script: str, *, check: bool = True) -> Result: + def run_ps( + self, + script: str, + *, + check: bool = True, + timeout_s: float = 60.0, + operation: str = "PowerShell command", + ) -> Result: if self._session is not None: raw = self._session.run_ps(script) result = Result( @@ -238,9 +263,12 @@ def run_ps(self, script: str, *, check: bool = True) -> Result: # The script goes on stdin, so neither its length nor what it # holds (an agent token) is on a command line. Progress records # would arrive as CLIXML on stderr. + units = len(script.encode("utf-16-le")) // 2 result = self._ssh( f"powershell -NoProfile -NonInteractive -EncodedCommand {_SSH_BOOTSTRAP}", - stdin=script.encode("utf-8"), + stdin=f"{units}\n{script}".encode("utf-8"), + timeout_s=timeout_s, + operation=operation, ) if check and result.status != 0: raise WinRMLabError( @@ -384,7 +412,7 @@ def launch_interactive(self, command_line: str, *, name: str = "OAFlowLab") -> N Register-ScheduledTask -TaskName '{task}' -Action $action -Principal $principal -Settings $settings -Force | Out-Null Start-ScheduledTask -TaskName '{task}' """ - self.run_ps(script) + self.run_ps(script, timeout_s=30.0, operation="launch interactive task") def stop_interactive(self, name: str = "OAFlowLab") -> None: task = name.replace("'", "") @@ -393,6 +421,8 @@ def stop_interactive(self, name: str = "OAFlowLab") -> None: f"Unregister-ScheduledTask -TaskName '{task}' -Confirm:$false " "-ErrorAction SilentlyContinue", check=False, + timeout_s=30.0, + operation="stop interactive task", ) def wait_port(self, port: int, *, timeout_s: float = 60.0) -> bool: @@ -427,7 +457,9 @@ def attach_console(self) -> None: Unregister-ScheduledTask -TaskName OAFlowConsole -Confirm:$false }} """ - self.run_ps(script, check=False) + self.run_ps( + script, check=False, timeout_s=30.0, operation="attach desktop console" + ) def launch_agent( self, diff --git a/packages/flow/tests/test_qualify_lab_rollback_setup.py b/packages/flow/tests/test_qualify_lab_rollback_setup.py new file mode 100644 index 0000000000..da8e19bc18 --- /dev/null +++ b/packages/flow/tests/test_qualify_lab_rollback_setup.py @@ -0,0 +1,90 @@ +"""Rollback lab setup refuses missing helpers before contacting a guest.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +DESKTOP = Path(__file__).resolve().parents[1] / "scripts" / "desktop" +sys.path.insert(0, str(DESKTOP)) +spec = importlib.util.spec_from_file_location( + "qualify_lab_rollback_setup", DESKTOP / "qualify_lab_rollback.py" +) +rollback = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rollback) + + +def test_missing_helpers_refuse_before_lab_construction(tmp_path, monkeypatch): + scripts = tmp_path / "scripts" + tools = tmp_path / "tools" + scripts.mkdir() + tools.mkdir() + (scripts / "pn_db.py").write_text("# synthetic helper\n") + monkeypatch.setattr(rollback, "HERE", scripts) + monkeypatch.setattr(rollback.winrm_lab, "__file__", str(tools / "winrm_lab.py")) + + def must_not_contact_guest(): + pytest.fail("lab was constructed before all helpers were checked") + + monkeypatch.setattr(rollback.winrm_lab, "WinRMLab", must_not_contact_guest) + with pytest.raises(FileNotFoundError) as err: + rollback.run_trial("native", tmp_path / "out", {}) + assert "patient_notes.ps1" in str(err.value) + assert "foreign_window.ps1" in str(err.value) + + +def test_helper_search_preserves_precedence_and_requires_files(tmp_path): + scripts = tmp_path / "scripts" + tools = tmp_path / "tools" + scripts.mkdir() + tools.mkdir() + for name in ("pn_db.py", "patient_notes.ps1", "foreign_window.ps1"): + (tools / name).write_text("synthetic\n") + (scripts / "pn_db.py").write_text("preferred\n") + (scripts / "foreign_window.ps1").mkdir() + helpers = rollback._required_helpers((scripts, tools)) + assert helpers == { + "pn_db.py": scripts / "pn_db.py", + "patient_notes.ps1": tools / "patient_notes.ps1", + "foreign_window.ps1": tools / "foreign_window.ps1", + } + + +def test_checked_in_helpers_are_complete(): + assert set(rollback._required_helpers((DESKTOP,))) == { + "pn_db.py", + "patient_notes.ps1", + "foreign_window.ps1", + } + + +def test_main_does_not_load_a_shared_environment_file(tmp_path, monkeypatch): + out = tmp_path / "trial" + evidence = tmp_path / "evidence.json" + monkeypatch.setattr( + sys, + "argv", + [ + "qualify", + "--transport", + "native", + "--out", + str(out), + "--evidence", + str(evidence), + ], + ) + + def forbidden_read(path): + pytest.fail("the harness tried to read a shared credential file") + + monkeypatch.setattr(rollback.winrm_lab, "load_env_file", forbidden_read) + monkeypatch.setattr( + rollback, "run_trial", lambda transport, out, row: row.update(accepted=True) + ) + assert rollback.main() == 0 + assert json.loads(evidence.read_text())[-1]["accepted"] is True diff --git a/packages/flow/tests/test_winrm_lab.py b/packages/flow/tests/test_winrm_lab.py index 2b09fe685a..ae2d0bd483 100644 --- a/packages/flow/tests/test_winrm_lab.py +++ b/packages/flow/tests/test_winrm_lab.py @@ -72,10 +72,12 @@ def fake_run(argv, **kw): argv, kw = calls[0] assert argv[:3] == ["ssh", "-i", "/k"] and "user@10.0.0.2" in argv # The script travels on stdin, never on the command line. - assert kw["input"] == b"Write-Output 'secret-token'" + assert kw["input"] == b"27\nWrite-Output 'secret-token'" assert not any("secret-token" in a for a in argv) bootstrap = base64.b64decode(argv[-1].rsplit(" ", 1)[1]).decode("utf-16-le") - assert "ReadToEnd" in bootstrap + assert "ReadToEnd" not in bootstrap + assert "ReadLine" in bootstrap and ".Read($b,$r,$n-$r)" in bootstrap + assert "if ($k -le 0) { exit 2 }" in bootstrap def test_ssh_timeout_names_only_the_host(monkeypatch): @@ -84,6 +86,7 @@ def test_ssh_timeout_names_only_the_host(monkeypatch): monkeypatch.setenv("OAFLOW_WINLAP_SSH_KEY", "/k") def fake_run(argv, **kw): + assert kw["timeout"] == 60 raise subprocess.TimeoutExpired(argv, 600) monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) @@ -101,7 +104,9 @@ def test_ssh_run_cmd_quotes_for_powershell(monkeypatch): scripts = [] def fake_run(argv, **kw): - scripts.append(kw["input"].decode()) + units, script = kw["input"].decode().split("\n", 1) + assert int(units) == len(script.encode("utf-16-le")) // 2 + scripts.append(script) return subprocess.CompletedProcess(argv, 0, b"", b"") monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) @@ -109,6 +114,81 @@ def fake_run(argv, **kw): assert scripts == ["& 'tool.exe' 'a&b' 'it''s'; exit $LASTEXITCODE"] +def test_ssh_frame_counts_utf16_units_and_keeps_payload_off_argv(monkeypatch): + monkeypatch.setenv("OAFLOW_WINLAP_HOST", "192.0.2.1") + monkeypatch.setenv("OAFLOW_WINLAP_USER", "operator") + monkeypatch.setenv("OAFLOW_WINLAP_SSH_KEY", "/synthetic/key") + calls = [] + + def fake_run(argv, **kw): + calls.append((argv, kw)) + return subprocess.CompletedProcess(argv, 0, b"ok", b"") + + monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) + script = "Write-Output 'synthetic-\U0001f680-\u00e9'" + lab_mod.WinRMLab(profile="winlap").run_ps(script) + argv, kw = calls[0] + units, payload = kw["input"].decode("utf-8").split("\n", 1) + assert payload == script + assert int(units) == len(script) + 1 + assert not any(script in arg for arg in argv) + assert "OAFlowLastOk" in base64.b64decode(argv[-1].rsplit(" ", 1)[1]).decode( + "utf-16-le" + ) + + +def test_stop_task_has_a_short_redacted_timeout_without_retry(monkeypatch): + monkeypatch.setenv("OAFLOW_WINLAP_HOST", "192.0.2.1") + monkeypatch.setenv("OAFLOW_WINLAP_USER", "operator") + monkeypatch.setenv("OAFLOW_WINLAP_SSH_KEY", "/synthetic/key") + calls = [] + + def fake_run(argv, **kw): + calls.append((argv, kw)) + raise subprocess.TimeoutExpired(argv, kw["timeout"], output=b"sensitive") + + monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) + with pytest.raises(lab_mod.WinRMLabError) as err: + lab_mod.WinRMLab(profile="winlap").stop_interactive("SyntheticAgent") + assert len(calls) == 1 + assert calls[0][1]["timeout"] == 30 + assert "stop interactive task" in str(err.value) + assert "30s" in str(err.value) + assert "sensitive" not in str(err.value) + assert "SyntheticAgent" not in str(err.value) + assert err.value.__cause__ is None and err.value.__suppress_context__ + + +def test_ssh_exit_status_and_check_false_are_preserved(monkeypatch): + monkeypatch.setenv("OAFLOW_WINLAP_HOST", "192.0.2.1") + monkeypatch.setenv("OAFLOW_WINLAP_USER", "operator") + monkeypatch.setenv("OAFLOW_WINLAP_SSH_KEY", "/synthetic/key") + monkeypatch.setattr( + lab_mod.subprocess, + "run", + lambda argv, **kw: subprocess.CompletedProcess(argv, 7, b"partial", b"failed"), + ) + lab = lab_mod.WinRMLab(profile="winlap") + result = lab.run_ps("Write-Output 'value'", check=False) + assert (result.status, result.stdout, result.stderr) == (7, "partial", "failed") + with pytest.raises(lab_mod.WinRMLabError, match="guest PowerShell exited 7"): + lab.run_ps("Write-Output 'value'") + + +@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")]) +def test_invalid_timeout_refuses_before_ssh(monkeypatch, timeout): + monkeypatch.setenv("OAFLOW_WINLAP_HOST", "192.0.2.1") + monkeypatch.setenv("OAFLOW_WINLAP_USER", "operator") + monkeypatch.setenv("OAFLOW_WINLAP_SSH_KEY", "/synthetic/key") + monkeypatch.setattr( + lab_mod.subprocess, "run", lambda *args, **kw: pytest.fail("SSH was called") + ) + with pytest.raises(ValueError, match="finite and positive"): + lab_mod.WinRMLab(profile="winlap").run_ps( + "Write-Output 'value'", timeout_s=timeout + ) + + def test_explicit_host_uses_the_winlab_password(monkeypatch): import types From 160079a528980b3e8efe0effdd8b3f5d64514a5e Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 23:00:27 -0800 Subject: [PATCH 15/21] No command falls back to the demo app on its own (#1764) * No command falls back to the demo app on its own replay with no --url (and no backend.url) served the bundled MockMed app; it now needs --demo-app for that, and refuses otherwise. --drift needs --demo-app. The launcher help, quickstart docs, the quickstart lifecycle script, e2e tests and Desktop's frozen smoke test pass it. The variant router's first-screen probe returns nothing without a URL. qualify accept's break-it gate ran every workflow's proposed oracle against MockMed (its banner text, /api/encounter, /api/db). It now builds the lie from the workflow: an in-process store that did not change, and a screen that shows the workflow's own success text, every typed value, and whatever a screen read looks for. The reference Execute worker ran the MockMed tutorial driver against any admission's target_url (resetting and reading it as MockMed). It now refuses such an admission before anything runs. * Review fixes: --demo-app tests, attached-browser refusal, synthetic targets Adversarial review of the fallback change: two tests replayed without a target (now --demo-app); --demo-app with an attached browser was silently ignored (now refused); a synthetic Execute admission that names a target_url was reported as verified without running (now refused before anything runs); the bug template shows --demo-app. * Frozen smoke test passes --demo-app only to a Flow that has it; format Desktop's frozen engine pins a released Flow, which serves the demo app whenever no URL is given and doesn't know --demo-app. * Re-run CI * Lower the engine generality baseline: no command names the demo app now --------- Signed-off-by: abrichr --- docs/cli.md | 2 +- docs/getting-started/quickstart.md | 2 +- openadapt/cli.py | 4 +- .../desktop/scripts/smoke_test_frozen_flow.py | 5 + .../.github/ISSUE_TEMPLATE/bug_report.yml | 2 +- packages/flow/CHANGELOG.md | 4 + packages/flow/docs/TUTORIAL.md | 4 +- packages/flow/openadapt_flow/__main__.py | 45 ++++-- packages/flow/openadapt_flow/api.py | 6 +- packages/flow/openadapt_flow/cli_undo.py | 7 +- packages/flow/openadapt_flow/cli_variants.py | 15 +- .../flow/openadapt_flow/execute/dispatch.py | 65 +++++---- .../qualification_oracle_gate.py | 136 ++++++++---------- .../flow/scripts/generality_baseline.json | 14 -- packages/flow/scripts/quickstart_lifecycle.py | 2 + packages/flow/tests/e2e/test_free_path_e2e.py | 11 +- .../tests/e2e/test_record_compile_replay.py | 5 +- packages/flow/tests/test_api.py | 2 +- .../test_execute_post_dispatch_faults.py | 39 +++++ packages/flow/tests/test_privacy.py | 2 +- .../tests/test_qualification_oracle_gate.py | 53 +++++++ 21 files changed, 262 insertions(+), 163 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 3bc94019e4..7505a57479 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -73,7 +73,7 @@ openadapt flow demo-record --out rec openadapt flow compile rec --out bundle --name my-task openadapt flow lint bundle --strict openadapt flow certify bundle --policy permissive -openadapt flow replay bundle --run-dir run +openadapt flow replay bundle --demo-app --run-dir run ``` The strict lint step refuses the unarmed demo write. The permissive policy is a diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 21e0da90da..7a88d0fd84 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -50,7 +50,7 @@ openadapt flow demo-record --out rec openadapt flow compile rec --out bundle --name my-task openadapt flow lint bundle --strict openadapt flow certify bundle --policy permissive -openadapt flow replay bundle --run-dir run +openadapt flow replay bundle --demo-app --run-dir run ``` The strict lint step returns a nonzero exit code. The bundled manual demo has diff --git a/openadapt/cli.py b/openadapt/cli.py index 28bb728ebb..d0a9b0b32e 100644 --- a/openadapt/cli.py +++ b/openadapt/cli.py @@ -3,7 +3,7 @@ Usage: openadapt flow record --url --out rec # record a workflow openadapt flow compile rec --out bundle # compile it - openadapt flow replay bundle # run it, local, $0 + openadapt flow replay bundle --url # run it, local, $0 openadapt flow lint bundle openadapt flow certify bundle --policy clinical-write @@ -117,7 +117,7 @@ def main(): openadapt flow compile rec --out bundle --name demo openadapt flow lint bundle openadapt flow certify bundle --policy permissive - openadapt flow replay bundle + openadapt flow replay bundle --demo-app Lint lists the demo's coverage gaps. The demo has warnings, so `lint --strict` exits 1 on it. diff --git a/packages/desktop/scripts/smoke_test_frozen_flow.py b/packages/desktop/scripts/smoke_test_frozen_flow.py index e836bc71aa..cb3eb1b3bb 100644 --- a/packages/desktop/scripts/smoke_test_frozen_flow.py +++ b/packages/desktop/scripts/smoke_test_frozen_flow.py @@ -663,11 +663,16 @@ def main() -> int: ], env=env, ) + # A Flow that serves the demo app only when asked takes --demo-app; + # an older pinned Flow serves it whenever no URL is given. + replay_help, _ = _run([*flow, "replay", "--help"], env=env) + demo_app = ["--demo-app"] if "--demo-app" in replay_help else [] _, replay_seconds = _run( [ *flow, "replay", str(root / "bundle"), + *demo_app, "--run-dir", str(root / "run"), ], diff --git a/packages/flow/.github/ISSUE_TEMPLATE/bug_report.yml b/packages/flow/.github/ISSUE_TEMPLATE/bug_report.yml index be6258e74a..3c4e088c8e 100644 --- a/packages/flow/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/packages/flow/.github/ISSUE_TEMPLATE/bug_report.yml @@ -22,7 +22,7 @@ body: placeholder: | 1. openadapt-flow demo-record --out rec 2. openadapt-flow compile rec --out bundle --name my-task - 3. openadapt-flow replay bundle + 3. openadapt-flow replay bundle --demo-app validations: required: true - type: textarea diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index ffcdd9ddb4..75a582a84b 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -17,6 +17,10 @@ ## Unreleased +### Changed + +- No command falls back to the demo app on its own. `replay` needs a target: `--url`, `backend.url` in `--config`, or `--demo-app` to serve the bundled MockMed (the tutorial and quickstart commands pass it; `--drift` needs it). The variant router's first-screen probe captures nothing without a URL. `qualify accept`'s break-it check no longer runs every workflow's oracle against MockMed: it builds the lie from the workflow itself (a store that did not change, and a screen showing the workflow's own success text and every value the run typed). The reference Execute worker refuses, before running anything, an admission whose `target_url` it cannot drive, instead of resetting and reading that app as if it were the demo. + ### Fixes - The OpenEMR contracts (`benchmark/lifecycle_proof/verify.yaml`, `benchmark/openemr_local/verify.yaml`) now declare how the New Patient form stores what was typed: street and city in capitals, names with each word capitalized. Without it the exact record check refuted 19 of 20 correct saves on the stand-in suites (#1811). A contract value's `stored` form gains `capitalize_words` (OpenEMR's "C" fields); the set stays closed and the check stays exact. diff --git a/packages/flow/docs/TUTORIAL.md b/packages/flow/docs/TUTORIAL.md index 32b8dbec2a..63a18b54a9 100644 --- a/packages/flow/docs/TUTORIAL.md +++ b/packages/flow/docs/TUTORIAL.md @@ -125,8 +125,8 @@ openadapt-flow compile rec --out bundle --name my-task # compile it openadapt-flow lint bundle # expected: finds demo gaps openadapt-flow certify bundle --policy permissive # smoke-policy pass openadapt-flow certify bundle --policy clinical-write # expected: strict refusal -openadapt-flow replay bundle # replay: local, $0 -openadapt-flow replay bundle --drift theme \ +openadapt-flow replay bundle --demo-app # replay: local, $0 +openadapt-flow replay bundle --demo-app --drift theme \ --save-healed-to healed # deterministic repair openadapt-flow visualize bundle -o graph.html # see what compiled ``` diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 8f1aae14be..bd65924ff0 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -2985,12 +2985,29 @@ def _cmd_replay(args: argparse.Namespace) -> int: headed = args.headed or cfg.backend.headed url = args.url or cfg.backend.url - refuse_unqualified_skill(bundle, target_url=url) - if url and args.drift: + demo_app = bool(getattr(args, "demo_app", False)) + if demo_app and ( + url or backend_cfg.browser_cdp_endpoint or backend_cfg.browser_page_url + ): + raise SystemExit( + "--demo-app serves its own app; drop --url, backend.url or the " + "attached browser." + ) + if args.drift and not demo_app: + raise SystemExit( + "--drift only applies to the bundled MockMed demo app (--demo-app); " + "drift your own app for real." + ) + if ( + url is None + and not demo_app + and not (backend_cfg.browser_cdp_endpoint or backend_cfg.browser_page_url) + ): raise SystemExit( - "--drift only applies to the bundled MockMed demo app; " - "omit --url to use it (drift your own app for real)." + "replay needs a target: pass --url, set backend.url in --config, or " + "pass --demo-app to replay against the bundled MockMed demo app." ) + refuse_unqualified_skill(bundle, target_url=url) held_backend: list = [] hold_s = min(max(float(getattr(args, "hold_on_halt", 0.0) or 0.0), 0.0), 1800.0) @@ -3076,13 +3093,13 @@ def run_on_page(page, *, entry_url: Optional[str]): ) stop = None - if url is None: + if url is None and demo_app: from openadapt_flow.mockmed.server import serve url, stop = serve(port=0) url = _with_drift(url, args.drift) drift_note = f" (drift: {args.drift})" if args.drift else "" - print(f"No --url given; replaying against bundled MockMed{drift_note}") + print(f"--demo-app: replaying against bundled MockMed{drift_note}") video_dir = getattr(args, "record_video", None) from playwright.sync_api import sync_playwright @@ -9795,22 +9812,30 @@ def build_parser() -> argparse.ArgumentParser: p = sub.add_parser( "replay", help=( - "Replay a bundle (serves the bundled MockMed demo app when " - "no --url is given)" + "Replay a bundle against --url, the config's backend.url, or " + "with --demo-app the bundled demo app" ), ) p.add_argument("bundle", help="Workflow bundle directory") p.add_argument( "--url", default=None, - help=("URL of the target app (default: serve the bundled MockMed demo app)"), + help="URL of the target app", + ) + p.add_argument( + "--demo-app", + action="store_true", + help=( + "Serve the bundled MockMed demo app and replay against it (for a " + "bundle recorded on it, such as the tutorial's)" + ), ) p.add_argument( "--drift", default=None, help=( "Comma-separated MockMed drift modes (theme,move,rename,modal) " - "to demonstrate bounded drift resolution; only valid without --url" + "to demonstrate bounded drift resolution; only with --demo-app" ), ) p.add_argument( diff --git a/packages/flow/openadapt_flow/api.py b/packages/flow/openadapt_flow/api.py index 94e6cae9d0..5ccc4f6abc 100644 --- a/packages/flow/openadapt_flow/api.py +++ b/packages/flow/openadapt_flow/api.py @@ -78,16 +78,20 @@ def replay( *, params: Optional[Mapping[str, str]] = None, run_dir: str | Path = "run", + demo_app: bool = False, ) -> RunReport: """Rehearse ``bundle`` against ``url`` and return the run report. - Without ``url`` it serves the bundled MockMed demo app, as the CLI does. + ``demo_app=True`` (and no ``url``) serves the bundled demo app instead, as + ``--demo-app`` does on the CLI. With neither, the replay is refused. A halted run is not an exception: check ``report.transaction_outcome``. """ argv = ["replay", str(bundle), "--run-dir", str(run_dir)] if url: argv += ["--url", url] + if demo_app: + argv.append("--demo-app") for key, value in (params or {}).items(): argv += ["--param", f"{key}={value}"] from openadapt_flow.__main__ import main diff --git a/packages/flow/openadapt_flow/cli_undo.py b/packages/flow/openadapt_flow/cli_undo.py index 3159281628..a2a518098e 100644 --- a/packages/flow/openadapt_flow/cli_undo.py +++ b/packages/flow/openadapt_flow/cli_undo.py @@ -178,7 +178,7 @@ def register( p.add_argument( "--url", default=None, - help="URL of the target app (default: the bundled MockMed demo app)", + help="URL of the target app (required unless the config names one)", ) p.add_argument("--headed", action="store_true", help="Run the browser headed") for add in add_run_flags: @@ -535,9 +535,8 @@ def _cmd_undo_accept(args: argparse.Namespace) -> int: def _cmd_undo_run(args: argparse.Namespace) -> int: run_dir = Path(args.run_dir) workflow = Workflow.load(args.bundle) - # Without a target, replay would start the bundled MockMed demo app: the - # undo would act on the demo, leave the real record alone and use up the - # approval. Refuse before claiming it. + # Without a target, replay has nowhere to act: the undo would use up the + # approval and change nothing. Refuse before claiming it. problem = args._undo_target_problem(args, workflow) if problem: print( diff --git a/packages/flow/openadapt_flow/cli_variants.py b/packages/flow/openadapt_flow/cli_variants.py index a846cc17ff..c61ae4eeb0 100644 --- a/packages/flow/openadapt_flow/cli_variants.py +++ b/packages/flow/openadapt_flow/cli_variants.py @@ -128,8 +128,8 @@ def first_screen_probe( ``--first-screen PNG`` wins. Otherwise the probe reads one frame through the same backend selection ``run`` uses: the attached signed-in browser tab, the native or remote-display window, or a freshly launched browser - at ``--url`` (the bundled MockMed app when no URL is given). It only - reads; it never clicks or types. + at ``--url``. With no target it captures nothing. It only reads; it + never clicks or types. """ def capture() -> Optional[bytes]: @@ -148,7 +148,6 @@ def _probe() -> Optional[bytes]: _attached_web_page, _deployment_sections, _resolve_backend_config, - _with_drift, ) from openadapt_flow.admitted_composition import resolve_pointer from openadapt_flow.backends.factory import _normalize_kind, build_backend @@ -172,15 +171,7 @@ def _probe() -> Optional[bytes]: with _attached_web_page(backend_cfg, url=url, workflow=workflow) as page: return build_backend(backend_cfg, page=page).screenshot() if url is None: - from openadapt_flow.mockmed.server import serve - - served, stop = serve(port=0) - try: - return _launched_web_screen( - _with_drift(served, getattr(args, "drift", None)) - ) - finally: - stop() + return None return _launched_web_screen(url) return capture diff --git a/packages/flow/openadapt_flow/execute/dispatch.py b/packages/flow/openadapt_flow/execute/dispatch.py index d0178f3d8f..b63d9793d3 100644 --- a/packages/flow/openadapt_flow/execute/dispatch.py +++ b/packages/flow/openadapt_flow/execute/dispatch.py @@ -64,6 +64,12 @@ def default_runner( ) -> DispatchResult: """Run synthetic MockMed or local governed replay.""" + if admission.synthetic and admission.target_url: + # A synthetic outcome describes the demo app, never another target. + raise DispatchError( + "a synthetic admission can't name a target_url; nothing was executed", + pre_effect=True, + ) if admission.synthetic or not admission.bundle_dir: return synthetic_mockmed(admission, request) return live_replay(admission, request, run_dir) @@ -114,8 +120,23 @@ def live_replay( request: ExecuteRequestV1, run_dir: Path, ) -> DispatchResult: - """Replay a compiled bundle in this process under the standard profile.""" + """Replay a compiled bundle in this process under the standard profile. + + The reference worker replays only on the demo app it serves itself: its + runner resets and reads that app's store. An admission naming another + target is refused before anything runs; such a bundle runs through + ``openadapt-flow run BUNDLE --config deploy.yaml``, which reads the app's + own deployment config. + """ + if admission.target_url: + raise DispatchError( + "the reference Execute worker replays only on its bundled demo app; " + f"it does not drive {admission.target_url!r}. Run this bundle with " + "`openadapt-flow run BUNDLE --config deploy.yaml` or `openadapt-flow " + "connector run`. Nothing was executed.", + pre_effect=True, + ) bundle = Path(admission.bundle_dir or "") if not bundle.is_dir(): raise DispatchError( @@ -158,36 +179,18 @@ def live_replay( ) stop: Optional[Callable[[], None]] = None try: - if admission.target_url: - base_url = admission.target_url - entry_query = ( - TUTORIAL_BREAK_ENTRY_QUERY - if admission.break_it - else TUTORIAL_ENTRY_QUERY - ) - report = run_tutorial_workflow( - base_url=base_url, - workflow=workflow, - bundle_dir=bundle, - run_dir=run_dir, - headed=False, - entry_query=entry_query, - ) - else: - base_url, _db, stop = serve_mockmed(port=0) - entry_query = ( - TUTORIAL_BREAK_ENTRY_QUERY - if admission.break_it - else TUTORIAL_ENTRY_QUERY - ) - report = run_tutorial_workflow( - base_url=base_url, - workflow=workflow, - bundle_dir=bundle, - run_dir=run_dir, - headed=False, - entry_query=entry_query, - ) + base_url, _db, stop = serve_mockmed(port=0) + entry_query = ( + TUTORIAL_BREAK_ENTRY_QUERY if admission.break_it else TUTORIAL_ENTRY_QUERY + ) + report = run_tutorial_workflow( + base_url=base_url, + workflow=workflow, + bundle_dir=bundle, + run_dir=run_dir, + headed=False, + entry_query=entry_query, + ) except Exception as exc: # The replay may have dispatched a write before it failed. raise DispatchError(f"local replay failed: {exc}", pre_effect=False) from exc diff --git a/packages/flow/openadapt_flow/qualification_oracle_gate.py b/packages/flow/openadapt_flow/qualification_oracle_gate.py index 934cd2cc74..cdaf222739 100644 --- a/packages/flow/openadapt_flow/qualification_oracle_gate.py +++ b/packages/flow/openadapt_flow/qualification_oracle_gate.py @@ -1,8 +1,9 @@ """Execute the qualify-proposal effect oracle. Do not only list the fault. Before ``qualify accept`` can succeed, the proposed oracle is run against a -``--break-it`` world: the success banner says saved, the store did not change. -MockMed is the default fixture because it has a real persistence boundary. +``--break-it`` world: the screen says saved and shows every value the run +typed, and the store did not change. The world is built from the workflow +itself (its own success text and parameters), so it is the same for every app. A model must not invent an endpoint. Actor bytes and oracle bytes must be disjoint. Re-reading the acting screen or the same-session banner HALTs. @@ -10,11 +11,7 @@ from __future__ import annotations -import json -import urllib.error -import urllib.request from typing import Any, Literal, Optional -from urllib.parse import urlsplit from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -33,7 +30,7 @@ ORACLE_GATE_SCHEMA: Literal["openadapt.qualification-oracle-gate/v1"] = ( "openadapt.qualification-oracle-gate/v1" ) -DEFAULT_BREAK_IT_FIXTURE = "mockmed" +DEFAULT_BREAK_IT_FIXTURE = "unchanged_store" MISSING_SECOND_READ = "do not automate until a second read exists" BROKEN_CASE_ACCEPTED = ( "proposed oracle would have accepted the broken case: " @@ -67,7 +64,8 @@ } ) _BANNER_TOKENS = ("saved", "success", "banner") -_MOCKMED_BANNER_PREFIX = "Encounter saved — " +#: The break-it store's base URL: an in-process read, never a network call. +_BREAK_IT_STORE = "http://break-it.invalid" class BoundEffect(BaseModel): @@ -221,8 +219,17 @@ def _step_effect(workflow: Workflow, item: BoundEffect) -> Effect: def _banner_text(workflow: Workflow) -> str: - note = str(workflow.params.get("note") or "example") - return _MOCKMED_BANNER_PREFIX + note[:40] + """What the lying screen shows: the workflow's own success text, and every + value the run typed, so a screen read of any of them is fooled.""" + + success = [ + pc.text + for step in _write_steps(workflow) + for pc in step.expect + if pc.kind is PostconditionKind.TEXT_PRESENT and pc.text + ] + values = [str(v) for v in workflow.params.values() if v not in (None, "")] + return " — ".join([*(success or ["Saved"]), *values]) def _png() -> bytes: @@ -256,30 +263,23 @@ def ocr(self, png: bytes, *, region: Any = None) -> list[_BannerLine]: return [_BannerLine(self._text)] -def _http_json( - url: str, *, method: str = "GET", body: Optional[dict[str, Any]] = None -) -> tuple[int, dict[str, Any]]: - data = None if body is None else json.dumps(body).encode("utf-8") - request = urllib.request.Request(url, data=data, method=method) - if data is not None: - request.add_header("Content-Type", "application/json") - try: - with urllib.request.urlopen(request, timeout=2) as response: - payload = json.loads(response.read().decode("utf-8")) - status = int(response.status) - except urllib.error.HTTPError as exc: - raw = exc.read().decode("utf-8") if exc.fp is not None else "" - try: - payload = json.loads(raw) if raw else {} - except json.JSONDecodeError: - payload = {} - return int(exc.code), payload if isinstance(payload, dict) else {} - return status, payload if isinstance(payload, dict) else {} +class _UnchangedStore: + """A records read that returns the same empty store before and after: + the write the screen claims never landed.""" + + class _Response: + status_code = 200 + headers: dict[str, str] = {} + def json(self) -> dict[str, Any]: + return {"records": []} -def _origin(url: str) -> str: - parts = urlsplit(url) - return f"{parts.scheme}://{parts.netloc}" + def raise_for_status(self) -> None: + return None + + def get(self, url: str, **kwargs: Any) -> "_UnchangedStore._Response": + del url, kwargs + return self._Response() def _resolved_effect(workflow: Workflow, effect: Effect) -> Effect: @@ -326,12 +326,12 @@ def evaluate_oracle_gate( *, fixture: str = DEFAULT_BREAK_IT_FIXTURE, ) -> OracleGateResult: - """Run channel disjointness and the MockMed --break-it fault. + """Run channel disjointness and the --break-it fault. Status stays draft/halted when the proposed oracle would have accepted the lie, when actor and oracle share a channel, or when no second read - exists. Nothing is guessed. The MockMed ``/api/db`` URL is the fixture's - persistence boundary, not a production endpoint invented for the pin. + exists. Nothing is guessed. The break-it store is read in process; it + names no app's endpoint. """ actor = actor_channel_ids(workflow) @@ -385,50 +385,28 @@ def evaluate_oracle_gate( ), ) - from openadapt_flow.mockmed.fault_server import serve - banner_text = _banner_text(workflow) - records: list[dict[str, Any]] = [] - store_unchanged = False - screen_claimed_success = False verdicts: list[EffectVerdict] = [] - status = 0 - base_url, db, stop = serve() - try: - verifier = RestRecordVerifier( - _origin(base_url), - records_path="/api/db", - records_key="records", - timeout_s=2.0, - poll_interval_s=0.05, - ) - before = verifier.capture_pre_state() - encounter_url = f"{base_url.rstrip('/')}/api/encounter?fault=optimistic" - status, _payload = _http_json( - encounter_url, - method="POST", - body={ - "patient_id": str(workflow.params.get("record_id") or "p1"), - "type": "Triage", - "note": str(workflow.params.get("note") or "example"), - }, - ) - snapshot = db.snapshot() - records = list(snapshot.get("records") or []) - store_unchanged = len(records) == 0 - screen_claimed_success = True - for item in effects: - effect = _step_effect(workflow, item) - if set(item.oracle_channels) & _SCREEN_CHANNELS: - verdicts.append(_execute_screen_oracle(workflow, effect, banner_text)) - else: - verdicts.append( - _execute_sor_oracle( - workflow, effect, verifier=verifier, before=before - ) - ) - finally: - stop() + verifier = RestRecordVerifier( + _BREAK_IT_STORE, + records_path="/records", + records_key="records", + session=_UnchangedStore(), + timeout_s=2.0, + poll_interval_s=0.05, + ) + before = verifier.capture_pre_state() + records = list(before.records or []) + store_unchanged = before.reachable and not records + screen_claimed_success = True + for item in effects: + effect = _step_effect(workflow, item) + if set(item.oracle_channels) & _SCREEN_CHANNELS: + verdicts.append(_execute_screen_oracle(workflow, effect, banner_text)) + else: + verdicts.append( + _execute_sor_oracle(workflow, effect, verifier=verifier, before=before) + ) dumped = [ { @@ -451,8 +429,8 @@ def evaluate_oracle_gate( "proposed oracle did not refute the lying backend; " "refusing to guess that it would catch --break-it" ) - if status not in {409, 200} and not store_unchanged: - reasons.append("MockMed optimistic write did not leave the store unchanged") + if not store_unchanged: + reasons.append("the break-it store could not be read as unchanged") halt_reason = "; ".join(reasons) if reasons else None return OracleGateResult( fixture=fixture, diff --git a/packages/flow/scripts/generality_baseline.json b/packages/flow/scripts/generality_baseline.json index 50ebd1a16c..269cbec315 100644 --- a/packages/flow/scripts/generality_baseline.json +++ b/packages/flow/scripts/generality_baseline.json @@ -3,9 +3,6 @@ "mockmed": 43, "textedit": 1 }, - "api.py": { - "mockmed": 1 - }, "backend.py": { "mockmed-records": 1 }, @@ -21,12 +18,6 @@ "backends/win_agent/server.py": { "notepad": 1 }, - "cli_undo.py": { - "mockmed": 2 - }, - "cli_variants.py": { - "mockmed": 2 - }, "compile_agent/code_action.py": { "openemr": 1, "token_main": 1 @@ -141,11 +132,6 @@ "qualification_dev_signer.py": { "mockmed": 1 }, - "qualification_oracle_gate.py": { - "mockmed": 8, - "mockmed-records": 2, - "mockmed-write": 1 - }, "qualification_proposal.py": { "mockmed": 1 }, diff --git a/packages/flow/scripts/quickstart_lifecycle.py b/packages/flow/scripts/quickstart_lifecycle.py index 7ef099ac27..07cc31d397 100644 --- a/packages/flow/scripts/quickstart_lifecycle.py +++ b/packages/flow/scripts/quickstart_lifecycle.py @@ -408,6 +408,7 @@ def run_lifecycle( *cli, "replay", str(bundle), + "--demo-app", "--run-dir", str(artifacts / "baseline-run"), ], @@ -420,6 +421,7 @@ def run_lifecycle( *cli, "replay", str(bundle), + "--demo-app", "--drift", "theme", "--save-healed-to", diff --git a/packages/flow/tests/e2e/test_free_path_e2e.py b/packages/flow/tests/e2e/test_free_path_e2e.py index 63f2242112..b988edc6f0 100644 --- a/packages/flow/tests/e2e/test_free_path_e2e.py +++ b/packages/flow/tests/e2e/test_free_path_e2e.py @@ -244,7 +244,16 @@ def test_demo_replay_still_cannot_reach_verified( run_dir = tmp_path / "demo-run" assert ( - cli_main(["replay", str(free_path.bundle_dir), "--run-dir", str(run_dir)]) == 1 + cli_main( + [ + "replay", + str(free_path.bundle_dir), + "--demo-app", + "--run-dir", + str(run_dir), + ] + ) + == 1 ) report = _report(run_dir) assert report.execution_profile == "demo" diff --git a/packages/flow/tests/e2e/test_record_compile_replay.py b/packages/flow/tests/e2e/test_record_compile_replay.py index 64c32896ed..46fd5ed44d 100644 --- a/packages/flow/tests/e2e/test_record_compile_replay.py +++ b/packages/flow/tests/e2e/test_record_compile_replay.py @@ -460,13 +460,14 @@ def cli(*args: str) -> subprocess.CompletedProcess: assert NOTE_TEXT not in content assert "mockmed-demo-pass" not in content - # README quickstart contract: `replay` with no --url self-serves - # MockMed, and --drift demonstrates healing in one command. + # README quickstart contract: `replay --demo-app` serves MockMed, + # and --drift demonstrates healing in one command. selfserve_run = tmp_path / "run-selfserve" healed = tmp_path / "healed" proc = cli( "replay", str(bundle), + "--demo-app", "--drift", "theme", "--run-dir", diff --git a/packages/flow/tests/test_api.py b/packages/flow/tests/test_api.py index b02820265e..fd8d3ed434 100644 --- a/packages/flow/tests/test_api.py +++ b/packages/flow/tests/test_api.py @@ -18,7 +18,7 @@ def test_api_record_compile_replay_on_mockmed( assert main(["demo-record", "--out", "rec"]) == 0 bundle = api.compile("rec", out="bundle", name="api-test") - report = api.replay(bundle, params={"note": "from the API"}) + report = api.replay(bundle, params={"note": "from the API"}, demo_app=True) # replay is a rehearsal: it never reads the record back. assert report.transaction_outcome == "COMPLETED_UNVERIFIED" diff --git a/packages/flow/tests/test_execute_post_dispatch_faults.py b/packages/flow/tests/test_execute_post_dispatch_faults.py index 069996a29d..ec3b4a05bc 100644 --- a/packages/flow/tests/test_execute_post_dispatch_faults.py +++ b/packages/flow/tests/test_execute_post_dispatch_faults.py @@ -243,3 +243,42 @@ def finalize_fails_once(self: Any, *args: Any, **kwargs: Any) -> None: receipt = store.get_receipt(accepted.execution_id) assert receipt.outcome is ExecuteTerminalOutcomeV1.RECONCILIATION_REQUIRED assert receipt.delivery_uncertain is True + + +def test_live_replay_refuses_a_target_it_cannot_drive(tmp_path: Path) -> None: + """The reference worker never resets or reads someone else's app.""" + from openadapt_flow.execute.dispatch import DispatchError, live_replay + from openadapt_flow.execute.models import AdmittedBundle + + admission = AdmittedBundle( + qualification_id="q", + workflow_version="v1", + workflow_digest="sha256:" + "0" * 64, + environment_id="env", + minimum_effect_strength="independent_system_of_record", + bundle_dir=str(tmp_path), + target_url="https://app.example/emr", + ) + with pytest.raises( + DispatchError, match="replays only on its bundled demo app" + ) as err: + live_replay(admission, None, tmp_path / "run") # type: ignore[arg-type] + assert err.value.pre_effect + + +def test_a_synthetic_admission_cannot_name_a_target(tmp_path: Path) -> None: + from openadapt_flow.execute.dispatch import DispatchError, default_runner + from openadapt_flow.execute.models import AdmittedBundle + + admission = AdmittedBundle( + qualification_id="q", + workflow_version="v1", + workflow_digest="sha256:" + "0" * 64, + environment_id="env", + minimum_effect_strength="independent_system_of_record", + synthetic=True, + target_url="https://app.example/emr", + ) + with pytest.raises(DispatchError, match="synthetic admission") as err: + default_runner(admission, None, tmp_path / "run") # type: ignore[arg-type] + assert err.value.pre_effect diff --git a/packages/flow/tests/test_privacy.py b/packages/flow/tests/test_privacy.py index 1f1cab5d89..20b3f444dd 100644 --- a/packages/flow/tests/test_privacy.py +++ b/packages/flow/tests/test_privacy.py @@ -386,7 +386,7 @@ def fake_sync_playwright(): monkeypatch.setattr(cli, "_build_and_run_replayer", fake_run) args = cli.build_parser().parse_args( - ["replay", str(bundle), "--run-dir", str(run_dir)] + ["replay", str(bundle), "--demo-app", "--run-dir", str(run_dir)] ) with pytest.warns(PlaintextPHIWarning): diff --git a/packages/flow/tests/test_qualification_oracle_gate.py b/packages/flow/tests/test_qualification_oracle_gate.py index 6a9cdae816..d54a82f64d 100644 --- a/packages/flow/tests/test_qualification_oracle_gate.py +++ b/packages/flow/tests/test_qualification_oracle_gate.py @@ -255,3 +255,56 @@ def propose(self, target: str, kind: str, context: dict[str, Any]) -> str: assert accepted.status == "accepted" # The sketch is not the pin. The structural SoR effect still is. assert accepted.pin("effect")["payload"]["effects"][0]["kind"] == "field_equals" + + +def _loan_workflow(effects: list[Effect]) -> Workflow: + """A Frappe-Lending-shaped write: nothing in it is MockMed's.""" + return Workflow( + name="loan-application", + params={"applicant": "Synthetic Applicant", "loan_amount": "125000"}, + steps=[_step(effects)], + ) + + +def test_break_it_on_another_apps_record_read_refutes() -> None: + """A record read for another app's write catches the unchanged store.""" + gate = evaluate_oracle_gate( + _loan_workflow( + [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "applicant": ValueExpr(param="applicant"), + "loan_amount": ValueExpr(param="loan_amount"), + }, + expected_count=1, + risk="irreversible", + ) + ] + ) + ) + assert gate.fixture == "unchanged_store" + assert gate.break_it_executed and gate.store_unchanged + assert gate.break_it_verdicts[0]["verdict"] == "refuted" + assert BROKEN_CASE_ACCEPTED not in (gate.halt_reason or "") + + +def test_break_it_fools_another_apps_screen_read() -> None: + """A screen read for another app's success text is accepted by the lie.""" + gate = evaluate_oracle_gate( + _loan_workflow( + [ + Effect( + kind=EffectKind.FIELD_EQUALS, + value="Loan Application saved", + readback=ReadbackSpec( + region=(10, 10, 40, 20), different_path=False + ), + risk="irreversible", + ) + ] + ) + ) + assert gate.break_it_verdicts[0]["verdict"] == "confirmed" + assert BROKEN_CASE_ACCEPTED in (gate.halt_reason or "") + assert not gate.passed From 81f3b05388ae77ab8e23af065f0fc7da1e717844 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 23:04:38 -0800 Subject: [PATCH 16/21] feat(policy): add record-write, a general write policy for non-clinical systems of record (#1937) * feat(policy): add record-write, a general write policy for non-clinical systems of record * release: register record-write.yaml in the public artifact inventory check_release_consistency.py refused the tree: the new policy file was not in public-artifacts.json. --- packages/flow/README.md | 4 +- packages/flow/docs/QUALIFY_A_WORKFLOW.md | 4 +- .../openadapt_flow/policies/record-write.yaml | 60 +++++++++++++++++++ packages/flow/public-artifacts.json | 1 + packages/flow/tests/test_policy.py | 31 +++++++++- 5 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 packages/flow/openadapt_flow/policies/record-write.yaml diff --git a/packages/flow/README.md b/packages/flow/README.md index 7e7fbdc5ca..600d4720e4 100644 --- a/packages/flow/README.md +++ b/packages/flow/README.md @@ -265,7 +265,9 @@ no implicit browser default in production; with a hand-written config, set `backend.kind` or pass `--backend`. `certify` checks the bundle against a policy. `clinical-write` is the strict -built-in policy, and `permissive` is a smoke check. A new recording usually +built-in policy, `record-write` asks for the same evidence for a system of +record that is not clinical (claims or back-office, say), and `permissive` is +a smoke check. A new recording usually fails `clinical-write` until its contracts are complete; `lint` lists the gaps. `run --profile standard` checks the policy again, replays, and reads the record back. diff --git a/packages/flow/docs/QUALIFY_A_WORKFLOW.md b/packages/flow/docs/QUALIFY_A_WORKFLOW.md index 280bd367ec..6ee81f4e79 100644 --- a/packages/flow/docs/QUALIFY_A_WORKFLOW.md +++ b/packages/flow/docs/QUALIFY_A_WORKFLOW.md @@ -167,7 +167,9 @@ What each part does: lists inputs to suffix per campaign, so reruns against a database that was not reset do not collide. - **campaign**: the id written into the report, trials per case, the cases - (all six by default) and the policy `--certify` uses when given no name. + (all six by default) and the policy `--certify` uses when given no name + (`clinical-write` by default; `record-write` is the same gate for a system + of record that is not clinical). - **business_rules** (optional): rules the application itself enforces, which the workflow must not click through. The six cases inject faults; a business-rule case injects nothing. After the reset, `seed` (one of diff --git a/packages/flow/openadapt_flow/policies/record-write.yaml b/packages/flow/openadapt_flow/policies/record-write.yaml new file mode 100644 index 0000000000..9fe1c440c5 --- /dev/null +++ b/packages/flow/openadapt_flow/policies/record-write.yaml @@ -0,0 +1,60 @@ +# Strict policy for write workflows against any system of record. +# +# The general counterpart of clinical-write, for systems that are not +# clinical: claims, billing, back-office and line-of-business apps. It asks for +# exactly the same evidence as clinical-write (a test pins the two rule sets +# together), so choosing it never lowers the bar; it only stops a non-clinical +# app from having to certify under a policy named for clinical data. Select it +# per app with `campaign.policy: record-write` in that app's verify.yaml, or +# `--policy record-write` on certify. +name: record-write +description: >- + Strict gate for irreversible writes against a system of record: no unarmed + clicks, no vacuous steps, identity required on every record-navigation and + write step, a real system-of-record effect (not just a screen assertion) plus + an idempotency key required on every write, no unconfirmed effect bindings, + and human sign-off for any thinly-evidenced step. + +# No click may act without a verified target identity. +prohibit_unarmed_clicks: true + +# Every effect-expecting step must carry outcome evidence. +prohibit_vacuous_postconditions: true + +# The wrong-record surface (navigational data clicks) and every write must be +# identity-armed. +require_identity_for: + - entity_navigation + - write + +# Every consequential write must verify its effect against the system of +# record (step.effects), not merely the screen. +require_system_effects_for: + - write + +# ...and carry an idempotency / at-most-once key, so a retried or +# double-delivered submission collapses to a single record. +require_idempotency_key_for: + - write + +# A placeholder effect whose binding was not derivable from the demonstration +# must never be certified. +prohibit_unconfirmed_effect_bindings: true + +# Screen postconditions are still required, in addition to the effect above. +require_screen_postconditions_for: + - write + - save + - submit + - create + - update + - delete + - confirm + - send + +# Zero tolerance for steps that assert nothing. +max_unverified_steps: 0 + +# Any step resolving on thin evidence needs human sign-off before it runs +# unattended. +require_human_approval_below_confidence: 0.8 diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 2b23af02aa..23dac01d5c 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -636,6 +636,7 @@ "openadapt_flow/mockmed/static/widgets.js", "openadapt_flow/policies/clinical-write.yaml", "openadapt_flow/policies/permissive.yaml", + "openadapt_flow/policies/record-write.yaml", "openadapt_flow/registry_trusted_root.json", "openadapt_flow/visualize/static/program_graph.css", "openadapt_flow/visualize/static/program_graph.js", diff --git a/packages/flow/tests/test_policy.py b/packages/flow/tests/test_policy.py index 23939b8f3b..06176a0b1a 100644 --- a/packages/flow/tests/test_policy.py +++ b/packages/flow/tests/test_policy.py @@ -5,6 +5,8 @@ from __future__ import annotations +import re + import pytest from openadapt_flow.ir import ( @@ -27,6 +29,7 @@ is_vacuous, lint_workflow, load_policy, + policy_contract_sha256, step_confidence, ) from openadapt_flow.runtime.effects import Effect, EffectKind, ValueExpr @@ -302,8 +305,9 @@ def test_builtins_present(self): names = builtin_policy_names() assert "permissive" in names assert "clinical-write" in names + assert "record-write" in names - @pytest.mark.parametrize("name", ["permissive", "clinical-write"]) + @pytest.mark.parametrize("name", ["permissive", "clinical-write", "record-write"]) def test_example_policies_parse(self, name): pol = load_policy(name) assert isinstance(pol, Policy) @@ -332,6 +336,31 @@ def test_clinical_write_fails_a_gappy_bundle(self): report = evaluate_policy(_gappy_workflow(), load_policy("clinical-write")) assert not report.passed + def test_record_write_asks_for_exactly_what_clinical_write_asks(self): + # The general write policy is for non-clinical systems of record. It + # must never be a weaker gate, so its rules are pinned to clinical-write. + def rules(policy: Policy) -> dict: + return policy.model_dump(exclude={"name", "description"}) + + record, clinical = load_policy("record-write"), load_policy("clinical-write") + assert rules(record) == rules(clinical) + assert policy_contract_sha256(record) != policy_contract_sha256(clinical) + assert not re.search(r"clinic|patient|emr\b", record.description, re.I) + + def test_record_write_refuses_what_clinical_write_refuses(self): + def refusals(name: str) -> list[tuple[str, str | None]]: + report = evaluate_policy(_gappy_workflow(), load_policy(name)) + return [(v.rule, v.step_id) for v in report.violations] + + assert refusals("record-write") + assert refusals("record-write") == refusals("clinical-write") + + def test_an_app_selects_record_write_in_its_verify_yaml(self): + from openadapt_flow.qualify.config import CampaignConfig + + campaign = CampaignConfig.model_validate({"policy": "record-write"}) + assert load_policy(campaign.policy).name == "record-write" + def test_string_marker_cannot_suppress_inferred_write(self): step = _click( "continue", From 62c48493277d13728fe56aabe56bd476427a1fb6 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 5 Nov 2026 02:44:10 -0500 Subject: [PATCH 17/21] on-prem image: copy only the named deploy files, and refuse a base image that is not loaded --- packages/flow/deploy/on-prem/Dockerfile | 3 +- packages/flow/deploy/on-prem/README.md | 2 +- .../flow/deploy/on-prem/bin/build-image.sh | 46 ++++++++++++++++--- .../flow/deploy/on-prem/bin/test-container.sh | 42 +++++++++++++++-- 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/packages/flow/deploy/on-prem/Dockerfile b/packages/flow/deploy/on-prem/Dockerfile index b71c2bdffa..6fff41072d 100644 --- a/packages/flow/deploy/on-prem/Dockerfile +++ b/packages/flow/deploy/on-prem/Dockerfile @@ -7,7 +7,8 @@ # the release's own wheels (the ones pinned by uv.lock): # # release/ the extracted release: VERSION, wheels/, requirements.lock.txt -# on-prem/ this directory (scripts, units, this Dockerfile) +# on-prem/ the runtime scripts, compose file, docs and this Dockerfile +# (build-image.sh names each file; nothing else is copied) # # The base image must already be in the local image store (`docker load` it # on an air-gapped host). Its default is pinned by digest; the release must be diff --git a/packages/flow/deploy/on-prem/README.md b/packages/flow/deploy/on-prem/README.md index a2f1f81afa..d0b2bc1607 100644 --- a/packages/flow/deploy/on-prem/README.md +++ b/packages/flow/deploy/on-prem/README.md @@ -25,7 +25,7 @@ install story) and **[COMPLIANCE.md](COMPLIANCE.md)** (posture + data-flow). | `bin/test-offline-install.sh` | Builds two signed releases, then as root with egress blocked installs the real engine from the first, refuses a tampered second, updates to the signed second, rolls back, and runs the air-gap attestation. CI runs it with `bin/test-update.sh` (job `flow-onprem-offline`). | REAL (Linux, root) | | `UPDATE.md` | Offline update + rollback runbook: build a signed release, apply it atomically, revert instantly. | REAL (doc) | | `systemd/openadapt-flow-runner.service` + `.path` | Event-driven runner unit with kernel-level egress denial (`IPAddressDeny=any`). | REAL (Linux/systemd) | -| `docker-compose.yml` + `Dockerfile` + `bin/build-image.sh` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. `bin/build-image.sh` verifies a signed release and builds the image from its wheels with `--network none`. `bin/test-container.sh` (CI, job `flow-onprem-offline`) builds it, refuses a tampered archive, checks a missing wheel fails the build, and runs the runner on the compose network with no route out. | REAL (Linux, Docker) | +| `docker-compose.yml` + `Dockerfile` + `bin/build-image.sh` | Container alternative on an `internal:true` (egress-blocked) network; optional LAN-only VLM appliance profile. `bin/build-image.sh` verifies a signed release and builds the image from its wheels with `--network none`, copying only the deploy files it names and refusing a base image that is not already loaded. `bin/test-container.sh` (CI, job `flow-onprem-offline`) builds it, refuses a tampered archive and an absent base image, checks the image holds only the named deploy files and a missing wheel fails the build, and runs the runner on the compose network with no route out. | REAL (Linux, Docker) | | `COMPLIANCE.md` | Regulatory posture, data-flow diagram, what's encrypted, boundaries. | REAL (doc; non-legal) | ## 60-second install (systemd host) diff --git a/packages/flow/deploy/on-prem/bin/build-image.sh b/packages/flow/deploy/on-prem/bin/build-image.sh index f75b2d04ef..97bfa330b2 100755 --- a/packages/flow/deploy/on-prem/bin/build-image.sh +++ b/packages/flow/deploy/on-prem/bin/build-image.sh @@ -7,7 +7,9 @@ # `install.sh --update`), extracts it with the bounded extractor, and runs # `docker build --network none` on ../Dockerfile, so the image holds exactly # the release's wheels. A missing wheel fails the build instead of reaching -# PyPI. The base image must already be loaded locally. +# PyPI. The base image must already be loaded locally; the script refuses +# before building when it is not. Only the deploy files it names are copied +# into the image. # # Usage: # build-image.sh --archive release-V.tar.gz --signature release-V.tar.gz.sig @@ -64,14 +66,44 @@ VERSION="$(rel_declared_version "$SRC" "$SRC/wheels")" || exit 1 mv "$SRC" "$CTX/release" rm -rf "$CTX/extract" -# The deploy scripts, without anything a site may have left beside them. -mkdir -p "$CTX/on-prem" -tar -C "$ONPREM" --exclude=./wheels --exclude=./storage --exclude=./release \ - --exclude='*.pem' --exclude='*.env' -cf - . | tar -C "$CTX/on-prem" -xf - +# Only the files the image runs or documents, named one by one, so nothing a +# site keeps beside the scripts (keys, configs, notes) reaches the image. +IMAGE_FILES=( + Dockerfile + docker-compose.yml + onprem.example.yaml + README.md + COMPLIANCE.md + UPDATE.md + bin/run-queue.sh + bin/audit-log.sh + bin/verify-airgap.sh + bin/run-egress-blocked.sh +) +for f in "${IMAGE_FILES[@]}"; do + if [[ ! -f "$ONPREM/$f" || -L "$ONPREM/$f" ]]; then + echo "build-image.sh: $ONPREM/$f is missing or not a regular file" >&2; exit 1 + fi + mkdir -p "$CTX/on-prem/$(dirname "$f")" + cp -p "$ONPREM/$f" "$CTX/on-prem/$f" +done + +# The base image must already be in the local image store. Otherwise docker +# would resolve FROM over the network even with --network none and +# --pull=false, so refuse before building. +if [[ -z "$BASE" ]]; then + BASE="$(awk -F= '/^ARG BASE_IMAGE=/ {print $2; exit}' "$CTX/on-prem/Dockerfile")" +fi +[[ -n "$BASE" ]] || { echo "build-image.sh: no base image named" >&2; exit 1; } +if ! docker image inspect "$BASE" >/dev/null 2>&1; then + echo "build-image.sh: base image $BASE is not in the local image store;" \ + "docker load it first (the build never pulls)" >&2 + exit 1 +fi TAG="${TAG:-openadapt-flow-onprem:$VERSION}" -args=(build --network none --pull=false -f "$CTX/on-prem/Dockerfile" -t "$TAG") -[[ -n "$BASE" ]] && args+=(--build-arg "BASE_IMAGE=$BASE") +args=(build --network none --pull=false -f "$CTX/on-prem/Dockerfile" -t "$TAG" + --build-arg "BASE_IMAGE=$BASE") echo "== building $TAG from release $VERSION with no network ==" docker "${args[@]}" "$CTX" diff --git a/packages/flow/deploy/on-prem/bin/test-container.sh b/packages/flow/deploy/on-prem/bin/test-container.sh index cc4a07c4a3..98771592a2 100755 --- a/packages/flow/deploy/on-prem/bin/test-container.sh +++ b/packages/flow/deploy/on-prem/bin/test-container.sh @@ -9,9 +9,13 @@ # release and the vendor public key). Needs Linux, Docker with compose, and a # connected host only to pull the pinned base image. Checks: # - build-image.sh refuses a tampered archive and builds no image; +# - build-image.sh refuses a base image that is not loaded locally, before +# building; # - the image build has no network: the same release with one dependency # wheel removed fails to build; -# - the signed release builds, and inside the image (no network) the engine +# - the signed release builds from a deploy directory with site files left +# beside the scripts, and the image holds only the named deploy files; +# inside the image (no network) the engine # and privacy extra import from the image's venv at the release version; # - the default command resolves, and run-queue.sh files a job and writes # the audit log on a bind-mounted storage root; it refuses to start with @@ -60,7 +64,7 @@ cleanup() { # The container writes the storage root as root; empty it from inside. docker run --rm --network none -v "$store:/s" "$tag" \ sh -c 'rm -rf /s/* /s/.[!.]*' >/dev/null 2>&1 || true - docker image rm -f "$tag" "$tag-tampered" "$tag-missing" >/dev/null 2>&1 || true + docker image rm -f "$tag" "$tag-tampered" "$tag-missing" "$tag-nobase" >/dev/null 2>&1 || true rm -rf "$work" } trap cleanup EXIT @@ -83,6 +87,24 @@ else ok "no image built from the tampered archive" fi +echo "== build-image.sh refuses a base image that is not loaded locally ==" +absent="openadapt-flow-onprem-test-absent-base:$$" +if bash "$HERE/build-image.sh" --archive "$archive" --signature "$sig" --pubkey "$pub" \ + --sig-tool openssl --base "$absent" --tag "$tag-nobase" >"$work/nobase.log" 2>&1; then + bad "absent base image refused" +elif grep -q "is not in the local image store" "$work/nobase.log" \ + && ! grep -q "== building" "$work/nobase.log"; then + ok "absent base image refused before building" +else + bad "absent base image refused before building (failed another way)" + tail -n 20 "$work/nobase.log" +fi +if docker image inspect "$tag-nobase" >/dev/null 2>&1; then + bad "no image built on an absent base" +else + ok "no image built on an absent base" +fi + echo "== the image build has no network: a release missing one wheel fails ==" mkdir -p "$work/missing/on-prem" tar xzf "$archive" -C "$work/missing" @@ -107,8 +129,16 @@ else fi fi -echo "== build the signed release ==" -if bash "$HERE/build-image.sh" --archive "$archive" --signature "$sig" --pubkey "$pub" \ +echo "== build the signed release from a deploy directory with site files beside it ==" +site="$work/site/on-prem" +mkdir -p "$site" +tar -C "$ONPREM" -cf - . | tar -C "$site" -xf - +printf 'site secret\n' > "$site/bin/site-notes.txt" +printf 'not a real key\n' > "$site/vendor.key" +printf 'site: local\n' > "$site/onprem.yaml" +mkdir -p "$site/storage" +printf 'job\n' > "$site/storage/left.job" +if bash "$site/bin/build-image.sh" --archive "$archive" --signature "$sig" --pubkey "$pub" \ --sig-tool openssl --tag "$tag" >"$work/build.log" 2>&1; then ok "signed release $v1 builds with no network" else @@ -133,6 +163,10 @@ if openadapt_flow.__version__ != importlib.metadata.version("openadapt-flow"): sys.exit("module and distribution versions differ") print(openadapt_flow.__version__) ' 2>&1)" +check_eq "image: holds only the named deploy files" \ + "COMPLIANCE.md Dockerfile README.md UPDATE.md bin/audit-log.sh bin/run-egress-blocked.sh bin/run-queue.sh bin/verify-airgap.sh docker-compose.yml onprem.example.yaml" \ + "$(run -w /opt/openadapt-flow/deploy/on-prem "$tag" find . ! -type d 2>&1 \ + | sed 's|^\./||' | LC_ALL=C sort | tr '\n' ' ' | sed 's/ $//')" check_eq "image: VERSION file" "$v1" "$(run "$tag" cat /opt/openadapt-flow/VERSION 2>&1)" check_eq "image: default command is the absolute run-queue path" \ '["/opt/openadapt-flow/deploy/on-prem/bin/run-queue.sh","watch"]' \ From 5f5894df16494e2f5aed933750bbd16717535f5f Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 4 Nov 2026 23:46:47 -0800 Subject: [PATCH 18/21] ensure_record: bind the read-back to the last version written and name version authors (#1854 a+b) (#1873) * ensure_record: bind the read-back to the last version written, and name each version's author (#1854) * ensure_record: keep the version_trail docstring app-neutral for the engine generality check * fix(flow): order ensure-record versions by instant, not timestamp text (#1925) * fix(flow): order ensure-record versions by instant, not timestamp text version_trail sorted versions by when.isoformat(), which keeps each timestamp's own offset, so versions written with mixed offsets came out in the wrong order and the audit trail's n-th write named the wrong version's author. bookkeeping_only sorted its chain by the raw lastUpdated text the same way; with a later version's time out of place, a bookkeeping stamp ten minutes from its own version's time was checked against the later version and excused. Both now sort by the parsed instant. In bookkeeping_only a version time that cannot be read (unparseable or with no time zone) now refuses the bookkeeping excuse instead of being sorted as text. * fix(flow): name version authors from the audit order only when it is unambiguous * fix(flow): refuse bookkeeping when two versions share an instant Versions with the same lastUpdated instant have no known order, and the server's return order is not a chronology. bookkeeping_only now refuses the excuse instead of keeping response order for ties, so a newest-first history can no longer hide a final stamp change. * fix(flow): narrow version instants to datetime for the sort key --- packages/flow/docs/ENSURE_RECORD.md | 17 ++ packages/flow/openadapt_flow/ensure_record.py | 127 ++++++++- packages/flow/tests/test_ensure_record.py | 255 +++++++++++++++++- .../tests/test_ensure_record_operations.py | 134 ++++++++- 4 files changed, 516 insertions(+), 17 deletions(-) diff --git a/packages/flow/docs/ENSURE_RECORD.md b/packages/flow/docs/ENSURE_RECORD.md index 908ce1f2fd..f24905ee01 100644 --- a/packages/flow/docs/ENSURE_RECORD.md +++ b/packages/flow/docs/ENSURE_RECORD.md @@ -24,6 +24,23 @@ The outcome is one of three: The agent's own report is not evidence. A save that landed while the agent reported a timeout is judged like any other save. +### The read-back is the last version written + +When `_history` is read, the record judged must be the newest version in the +window, matched by `meta.versionId`. It is `needs_review` when the read-back +is an older version, or a version saved after the window closed (by anyone), +or carries no `versionId`; when any version in the window has no +`versionId` or time; or when two versions share the newest time. Without +`_history` there is nothing to bind to, and the other checks decide as +before. + +The evidence lists each version in the window as `versions`: its id and its +author, oldest first, and never its body. The author is `meta.author` where +the server returns it. Where it doesn't (Medplum 5.2), and the audit trail +logged exactly one timed create or update per version, the n-th logged write +names the n-th version. Otherwise the author is `null`: unknown, not +guessed. + ### When the agent corrects its own record An agent that reads its create back may find a wrong value (a tool changed diff --git a/packages/flow/openadapt_flow/ensure_record.py b/packages/flow/openadapt_flow/ensure_record.py index db6560f428..e084b01dfe 100644 --- a/packages/flow/openadapt_flow/ensure_record.py +++ b/packages/flow/openadapt_flow/ensure_record.py @@ -772,10 +772,10 @@ def bookkeeping_only( return False if any(not (b.get("meta") or {}).get("lastUpdated") for b in change.bodies): return False # a version with no time can't be put in order - chain = [ - change.prior, - *sorted(change.bodies, key=lambda v: str(v["meta"]["lastUpdated"])), - ] + ordered = _by_instant(change.bodies) + if ordered is None: + return False # a version with no readable time can't be put in order + chain = [change.prior, *ordered] if not all(_same_outside(a, b, paths) for a, b in zip(chain, chain[1:])): return False timed = [ @@ -790,6 +790,26 @@ def bookkeeping_only( return True +def _by_instant(bodies: list[dict[str, Any]]) -> Optional[list[dict[str, Any]]]: + """``bodies`` oldest first by the instant of ``meta.lastUpdated``, or None + when one has no readable time (none, unparseable, or no time zone) or + two share an instant. Compared as instants, not text: with mixed offsets + the text order is not the time order. Tied versions have no known order, + and the server's return order is not a chronology, so a tie refuses.""" + + try: + read = [_when((b.get("meta") or {}).get("lastUpdated")) for b in bodies] + except Unreadable: + return None + times = [when for when in read if when is not None] + if len(times) != len(bodies): + return None + if len(set(times)) != len(times): + return None + order = sorted(range(len(bodies)), key=lambda i: times[i]) + return [bodies[i] for i in order] + + def _times_near_now( before: Mapping[str, Any], after: Mapping[str, Any], rule: BookkeepingRule ) -> bool: @@ -1342,9 +1362,16 @@ def judge( reasons.append(refused) if target in others or (changes[target].authors - selfs): reasons.append("someone else changed the created record") + if changes[target].versions is not None: + # Who wrote each version in the window, for the evidence: version + # ids and actor references only, never a value. + evidence["versions"] = version_trail(changes[target]) if created is None or str(created.get("id")) != tid: reasons.append("the created record can't be read back") else: + unbound = read_back_unbound(changes[target], created) + if unbound: + reasons.append(unbound) wrong = wrong_fields(contract, created, params) if wrong: reasons.append("wrong or missing: " + ", ".join(wrong)) @@ -1391,6 +1418,98 @@ def judge( return _review(reasons, evidence) +def _version_meta(version: Mapping[str, Any]) -> tuple[Optional[datetime], str]: + meta = version.get("meta") or {} + if not isinstance(meta, Mapping): + return None, "" + vid = meta.get("versionId") + return _when(meta.get("lastUpdated")), str(vid) if vid not in (None, "") else "" + + +def read_back_unbound(change: Change, created: Mapping[str, Any]) -> Optional[str]: + """Why the read-back can't be shown to be the last version written, or None. + + With ``_history`` read, the record judged must be the newest version in + the window, by ``meta.versionId``: a version written after the window + (by anyone) or a stale read is not what the agent's writes left. Without + history (``versions`` None) there is nothing to bind to, and the other + checks decide as before. + """ + + if change.versions is None: + return None + if not change.bodies: + return "the created record's _history shows no version in the window" + timed = [_version_meta(body) for body in change.bodies] + if any(when is None or not vid for when, vid in timed): + return "a version in the window has no versionId or time to bind to" + newest = max(when for when, _ in timed if when is not None) + last = {vid for when, vid in timed if when == newest} + if len(last) != 1: + return "two versions share the newest time, so the last write can't be told" + _, got = _version_meta(created) + if not got: + return "the read-back carries no versionId to bind to" + if got not in last: + return "the read-back is not the last version written in the window" + return None + + +_NO_TIME = datetime.min.replace(tzinfo=timezone.utc) + + +def version_trail(change: Change) -> list[dict[str, Optional[str]]]: + """Each version in the window, oldest first: its id and who wrote it. + + Only ids and actor references; the version bodies stay out of evidence. + The author is ``meta.author`` when the server returns it. When it + doesn't (some servers omit it), and the audit trail logged exactly one + timed create or update per version, the n-th logged write names the + n-th version, but only when that order is unambiguous: every version + has a time, and no two versions and no two logged writes share one (a + tie keeps whatever order the server returned). Otherwise ``author`` is + None: unknown, never guessed. + """ + + rows: list[tuple[Optional[datetime], str, Optional[str]]] = [] + for body in change.bodies: + when, vid = _version_meta(body) + meta = body.get("meta") or {} + who = ( + (meta.get("author") or {}).get("reference") + if isinstance(meta, Mapping) + else None + ) + rows.append((when, vid, _ref(who) if isinstance(who, str) and who else None)) + # By instant, not by text: with mixed offsets the text order is not the + # time order, and the n-th logged write would name the wrong version. + # A version with no time sorts first, as before. + rows.sort(key=lambda r: (r[0] is not None, r[0] or _NO_TIME)) + writes = [ + (when, who) + for when, action, who in change.events + if action in ("C", "U") and when is not None + ] + logged: list[Optional[str]] = [None] * len(rows) + version_times = [r[0] for r in rows] + write_times = [w[0] for w in writes] + if ( + len(writes) == len(rows) + and len(writes) == sum(1 for _, a, _ in change.events if a in ("C", "U")) + and None not in version_times + and len(set(version_times)) == len(version_times) + and len(set(write_times)) == len(write_times) + ): + logged = [ + ", ".join(sorted(who)) if who else None + for _, who in sorted(writes, key=lambda w: w[0]) + ] + return [ + {"version": vid or None, "author": author or audit} + for (_, vid, author), audit in zip(rows, logged) + ] + + def correction_refused( contract: EnsureContract, change: Change, diff --git a/packages/flow/tests/test_ensure_record.py b/packages/flow/tests/test_ensure_record.py index f07daa5ddb..41de9ebca4 100644 --- a/packages/flow/tests/test_ensure_record.py +++ b/packages/flow/tests/test_ensure_record.py @@ -975,6 +975,19 @@ def fixed(*extra_events, reads=(), versions=None, bodies=None, ref="Patient/x"): events = [(_t(1), "C", frozenset({BOT})), (_t(3), "U", frozenset({BOT}))] events += list(extra_events) n = len(events) if versions is None else versions + bodies = [patient("x")] * n if bodies is None else bodies + # As ``_history`` returns them: each version has its own id and time. + bodies = [ + { + **body, + "meta": { + **(body.get("meta") or {}), + "versionId": str(i + 1), + "lastUpdated": _iso(1 + 2 * i), + }, + } + for i, body in enumerate(bodies) + ] return { ref: Change( ref=ref, @@ -983,13 +996,13 @@ def fixed(*extra_events, reads=(), versions=None, bodies=None, ref="Patient/x"): events=events, reads=list(reads), versions=n, - bodies=[patient("x")] * n if bodies is None else bodies, + bodies=bodies, ) } def test_a_create_the_agent_corrected_is_verified_and_says_so(): - x = patient("x") + x = fixed()["Patient/x"].bodies[-1] r = judge(cc(), PARAMS, base(), fixed(), {"x": x}, x) assert (r.outcome, r.record_id) == ("created_verified", "x") assert r.evidence["corrected"] is True and r.evidence["writes_to_record"] == 2 @@ -1048,7 +1061,7 @@ def test_a_read_logged_just_after_the_last_fix_still_counts(): def test_the_agents_own_reads_dont_count(): - x = patient("x") + x = fixed()["Patient/x"].bodies[-1] seen = [(_t(2), frozenset({BOT})), (_t(4), frozenset({BOT}))] r = judge(cc(), PARAMS, base(), fixed(reads=seen), {"x": x}, x) assert r.outcome == "created_verified" @@ -1179,7 +1192,16 @@ class _Server: def __init__(self, *writes): self.events = [_stamped(a, "Patient/x", _iso(s)) for a, s in writes] - self.history = [versioned(patient("x"), BOT, _iso(s)) for _, s in writes] + self.history = [ + { + **versioned(patient("x"), BOT, _iso(s)), + "meta": { + "versionId": str(i + 1), + **versioned({}, BOT, _iso(s))["meta"], + }, + } + for i, (_, s) in enumerate(writes) + ] self.now = 12.0 # the verifier's reads land after the window def get(self, url, params): @@ -1211,7 +1233,7 @@ def inside(e): _stamped("R", "Patient/x", _iso(self.now), "ClientApplication/verifier") ) self.now += 0.1 - return 200, {}, patient("x") + return 200, {}, self.history[-1] return 200, {}, bundle(patient("x")) @@ -1253,8 +1275,8 @@ def test_a_correction_into_void_needs_review(): def test_a_correction_beside_a_void_lookalike_is_verified(): - x = vax("x") - changes = fixed(ref="Immunization/x", bodies=[x, x]) + changes = fixed(ref="Immunization/x", bodies=[vax("x"), vax("x")]) + x = changes["Immunization/x"].bodies[-1] r = judge(vax_contract(settle_s=1), VAX_PARAMS, base(), changes, {"x": x}, x) assert (r.outcome, r.record_id) == ("created_verified", "x") assert r.evidence["corrected"] is True @@ -1333,3 +1355,222 @@ def get(url, params): c, PARAMS, base(e), reader=FhirReader(c, get), sleep=lambda s: None ) assert r.outcome == "needs_review" and r.evidence.get("unreadable") + + +# -- the read-back is bound to the last version written (#1854) ------------------------- + + +def _bound(changes, created): + return judge(cc(), PARAMS, base(), changes, {"x": created}, created) + + +def test_the_last_version_written_is_bound_and_its_authors_are_named(): + changes = fixed() + r = _bound(changes, changes["Patient/x"].bodies[-1]) + assert r.outcome == "created_verified", r.reasons + # Only ids and actors: no version body reaches the evidence. + # No meta.author (as on Medplum): the audit trail's n-th write names it. + assert r.evidence["versions"] == [ + {"version": "1", "author": BOT}, + {"version": "2", "author": BOT}, + ] + + +def test_a_version_author_is_never_guessed(): + # meta.author, when the server gives it, names the version. + changes = fixed() + bodies = changes["Patient/x"].bodies + bodies[1] = versioned(bodies[1], OTHER, bodies[1]["meta"]["lastUpdated"]) + bodies[1]["meta"]["versionId"] = "2" + from openadapt_flow.ensure_record import version_trail + + assert [v["author"] for v in version_trail(changes["Patient/x"])] == [BOT, OTHER] + # One logged write for two versions: which wrote which is unknown. + changes = fixed() + changes["Patient/x"].events = changes["Patient/x"].events[:1] + assert [v["author"] for v in version_trail(changes["Patient/x"])] == [None, None] + + +def test_a_stale_read_back_is_refused(): + changes = fixed() + r = _bound(changes, changes["Patient/x"].bodies[0]) + assert r.outcome == "needs_review" + assert "the read-back is not the last version written in the window" in r.reasons + + +def test_a_version_written_after_the_window_is_refused(): + # Someone saved the record again after the window closed: the read-back + # is a version the trail never covered, whoever wrote it. + changes = fixed() + later = {**changes["Patient/x"].bodies[-1]} + later["meta"] = {**later["meta"], "versionId": "3", "lastUpdated": LATER} + r = _bound(changes, later) + assert r.outcome == "needs_review" + assert "the read-back is not the last version written in the window" in r.reasons + + +def test_a_read_back_without_a_version_id_is_refused(): + changes = fixed() + bare = {k: v for k, v in changes["Patient/x"].bodies[-1].items() if k != "meta"} + r = _bound(changes, bare) + assert r.outcome == "needs_review" + assert "the read-back carries no versionId to bind to" in r.reasons + + +def test_a_version_without_an_id_or_time_is_refused(): + changes = fixed() + body = changes["Patient/x"].bodies[0] + changes["Patient/x"].bodies[0] = {**body, "meta": {"versionId": "1"}} + r = _bound(changes, changes["Patient/x"].bodies[-1]) + assert r.outcome == "needs_review" + assert "a version in the window has no versionId or time to bind to" in r.reasons + + +def test_two_versions_at_the_newest_time_are_refused(): + changes = fixed() + first, second = changes["Patient/x"].bodies + tied = { + **first, + "meta": {**first["meta"], "lastUpdated": second["meta"]["lastUpdated"]}, + } + changes["Patient/x"].bodies[0] = tied + r = _bound(changes, second) + assert r.outcome == "needs_review" + assert ( + "two versions share the newest time, so the last write can't be told" + in r.reasons + ) + + +class _LateServer(_Server): + """Serves, on read-back, a version saved after the window by another user.""" + + def get(self, url, params): + status, headers, body = super().get(url, params) + if url.split("/fhir/R4/", 1)[-1] == "Patient/x": + meta = {"versionId": "9", "lastUpdated": _iso(11.5)} + body = {**body, "meta": {**meta, "author": {"reference": OTHER}}} + return status, headers, body + + +@pytest.mark.parametrize( + "server,outcome", [(_Server, "created_verified"), (_LateServer, "needs_review")] +) +def test_the_reader_binds_the_read_back_to_history(server, outcome): + from openadapt_flow.ensure_record import verify_ensure + + s = server(("C", 2), ("U", 3)) + c = cc(change_feed={"audit": {"heartbeat": True}, "history_author": False}) + r = verify_ensure( + c, PARAMS, base(), reader=FhirReader(c, s.get), sleep=lambda _: None + ) + assert r.outcome == outcome, r.reasons + if outcome == "needs_review": + assert r.reasons == [ + "the read-back is not the last version written in the window" + ] + authors = [v["author"] for v in r.evidence.get("versions", [])] + assert authors == [BOT, BOT] + + +def test_version_trail_orders_mixed_offsets_by_instant(): + # v1 at 12:00:01Z written as +01:00, v2 at 12:00:02Z: as text, v2 sorts + # first and the audit trail's n-th write would name the wrong version. + from openadapt_flow.ensure_record import version_trail + + v1 = { + **patient("x"), + "meta": {"versionId": "1", "lastUpdated": "2026-10-28T13:00:01+01:00"}, + } + v2 = { + **patient("x"), + "meta": {"versionId": "2", "lastUpdated": "2026-10-28T12:00:02Z"}, + } + change = Change( + ref="Patient/x", + actions={"C", "U"}, + authors={BOT, OTHER}, + events=[ + (T0 + timedelta(seconds=1), "C", frozenset({BOT})), + (T0 + timedelta(seconds=2), "U", frozenset({OTHER})), + ], + versions=2, + bodies=[v2, v1], + ) + assert version_trail(change) == [ + {"version": "1", "author": BOT}, + {"version": "2", "author": OTHER}, + ] + + +def _trail_change(v_times, events): + bodies = [ + {**patient("x"), "meta": {"versionId": str(i + 1), "lastUpdated": when}} + for i, when in enumerate(v_times) + ] + return Change( + ref="Patient/x", + actions={"C", "U"}, + authors={BOT, OTHER}, + events=events, + versions=len(bodies), + bodies=list(reversed(bodies)), + ) + + +def test_version_trail_leaves_authors_unknown_when_writes_share_a_time(): + # Two writes logged in the same second, update returned before create: + # their order is the server's, not the writes', so no one is named. + from openadapt_flow.ensure_record import version_trail + + change = _trail_change( + ["2026-10-28T12:00:01.100Z", "2026-10-28T12:00:01.200Z"], + [ + (T0 + timedelta(seconds=1), "U", frozenset({BOT})), + (T0 + timedelta(seconds=1), "C", frozenset({OTHER})), + ], + ) + assert version_trail(change) == [ + {"version": "1", "author": None}, + {"version": "2", "author": None}, + ] + + +def test_version_trail_leaves_authors_unknown_when_versions_share_a_time(): + from openadapt_flow.ensure_record import version_trail + + change = _trail_change( + ["2026-10-28T12:00:01Z", "2026-10-28T12:00:01Z"], + [ + (T0 + timedelta(seconds=1), "C", frozenset({BOT})), + (T0 + timedelta(seconds=2), "U", frozenset({OTHER})), + ], + ) + assert [v["author"] for v in version_trail(change)] == [None, None] + + +def test_version_trail_leaves_authors_unknown_when_a_version_has_no_time(): + from openadapt_flow.ensure_record import version_trail + + change = _trail_change( + ["2026-10-28T12:00:01Z", None], + [ + (T0 + timedelta(seconds=1), "C", frozenset({BOT})), + (T0 + timedelta(seconds=2), "U", frozenset({OTHER})), + ], + ) + assert [v["author"] for v in version_trail(change)] == [None, None] + + +def test_version_trail_names_authors_when_the_order_is_unambiguous(): + # The control: distinct times on both sides still name each version. + from openadapt_flow.ensure_record import version_trail + + change = _trail_change( + ["2026-10-28T12:00:01.100Z", "2026-10-28T12:00:01.200Z"], + [ + (T0 + timedelta(seconds=2), "U", frozenset({OTHER})), + (T0 + timedelta(seconds=1), "C", frozenset({BOT})), + ], + ) + assert [v["author"] for v in version_trail(change)] == [BOT, OTHER] diff --git a/packages/flow/tests/test_ensure_record_operations.py b/packages/flow/tests/test_ensure_record_operations.py index eff0a181fe..19578750c5 100644 --- a/packages/flow/tests/test_ensure_record_operations.py +++ b/packages/flow/tests/test_ensure_record_operations.py @@ -91,10 +91,33 @@ def base(*matches: dict) -> Baseline: return Baseline(started=T0, matches={m["id"]: m for m in matches}) +def _iso(seconds: float) -> str: + return _t(seconds).isoformat().replace("+00:00", "Z") + + +def last(changes: dict[str, Change], ref: str = "Patient/x") -> dict: + """The newest version ``_history`` shows: what a read-back returns.""" + + return changes[ref].bodies[-1] + + def written(ref: str, *steps: tuple, bodies: list, reads=(), prior=None) -> Change: """``steps`` are (seconds, action, actor); one version per create or update.""" events = [(_t(s), a, frozenset({who})) for s, a, who in steps] + # As ``_history`` returns them: each version has its own id and time. + times = [s for s, _, _ in steps] or [1] + bodies = [ + { + **body, + "meta": { + **(body.get("meta") or {}), + "versionId": str(i + 1), + "lastUpdated": _iso(times[min(i, len(times) - 1)] + 0.01 * i), + }, + } + for i, body in enumerate(bodies) + ] return Change( ref=ref, actions={a for _, a, _ in events}, @@ -122,8 +145,8 @@ def stepped(*first: dict) -> dict[str, Change]: def test_a_record_saved_in_steps_counts_as_one_write_when_consistent(): - x = patient("x") partial = [{"resourceType": "Patient", "id": "x"}, patient("x", birthDate=None)] + x = last(stepped(*partial)) r = judge(contract(**CONSISTENT), PARAMS, base(), stepped(*partial), {"x": x}, x) assert (r.outcome, r.record_id) == ("created_verified", "x") assert r.evidence["corrected"] is True @@ -153,8 +176,8 @@ def test_a_version_holding_any_other_value_still_refuses(first): def test_a_person_version_with_a_wrong_phone_passes_person_but_not_consistent(): # ``person`` checks identity only; ``consistent`` checks every value. - x = patient("x") wrong = patient("x", telecom=[{"system": "phone", "value": "617-555-0199"}]) + x = last(stepped(wrong)) assert judge(contract(), PARAMS, base(), stepped(wrong), {"x": x}, x).outcome == ( "created_verified" ) @@ -181,7 +204,7 @@ def test_a_person_version_with_a_wrong_phone_passes_person_but_not_consistent(): ], ) def test_either_is_never_stricter_than_person_and_accepts_steps(first, outcome): - x = patient("x") + x = last(stepped(first)) either = {"correction": {"between": "either"}} r = judge(contract(**either), PARAMS, base(), stepped(first), {"x": x}, x) assert r.outcome == outcome, r.reasons @@ -192,8 +215,8 @@ def test_either_is_never_stricter_than_person_and_accepts_steps(first, outcome): def test_consistent_still_needs_every_other_correction_condition(): - x = patient("x") partial = {"resourceType": "Patient", "id": "x"} + x = last(stepped(partial)) # Someone else read the record between the agent's writes. changes = stepped(partial) changes["Patient/x"].reads = [(_t(1.5), frozenset({OTHER}))] @@ -639,8 +662,8 @@ def _pair_cases() -> list[ ops, stepped(partial), base(), - {"x": x}, - x, + {"x": last(stepped(partial))}, + last(stepped(partial)), "created_verified", ), ( @@ -755,3 +778,102 @@ def test_the_frozen_inputs_are_the_leaderboards_rows(): r["_source"].startswith("04e76283a03c37a18e47887c1954728812d21533:") for r in rows ) + + +def test_bookkeeping_versions_are_ordered_by_instant_not_text(): + # The stamp first changes in a version written at 12:00:02, ten minutes + # before the time it names, so it is not that version's own time. Its + # lastUpdated is written as +01:00, which as text sorts after the next + # version's 12:10:00Z; checked in that order, the stamp would look like + # the later version's own time. + book = { + "bookkeeping": [ + {"resource_type": "Practitioner", "paths": BOOK_PATHS, "now_checks": True} + ] + } + seen = [{"url": "urn:app:last-seen", "valueInstant": "2026-10-28T12:10:00Z"}] + v2 = { + **DOC, + "meta": {"versionId": "2", "lastUpdated": "2026-10-28T13:00:02+01:00"}, + "extension": seen, + } + v3 = { + **DOC, + "meta": {"versionId": "3", "lastUpdated": "2026-10-28T12:10:00Z"}, + "extension": seen, + } + c = Change( + ref="Practitioner/d", + actions={"U"}, + authors={BOT}, + events=[(_t(2), "U", frozenset({BOT})), (_t(600), "U", frozenset({BOT}))], + versions=2, + bodies=[v2, v3], + prior=DOC, + ) + x = patient("x") + r = judge(contract(**book), PARAMS, base(), created_with(c), {"x": x}, x) + assert r.outcome == "needs_review" + # The same versions with one offset are refused as well. + v2z = {**v2, "meta": {"versionId": "2", "lastUpdated": "2026-10-28T12:00:02Z"}} + c.bodies = [v2z, v3] + r = judge(contract(**book), PARAMS, base(), created_with(c), {"x": x}, x) + assert r.outcome == "needs_review" + + +def test_bookkeeping_with_a_version_time_without_a_zone_is_refused(): + # Its place in the order can't be known, so it excuses nothing. + x = patient("x") + after = {**DOC, "meta": {"versionId": "2", "lastUpdated": "2026-10-28T12:00:02Z"}} + c = touched(after) + c.bodies = [ + {**after, "meta": {"versionId": "2", "lastUpdated": "2026-10-28T12:00:02"}} + ] + r = judge(contract(**BOOK), PARAMS, base(), created_with(c), {"x": x}, x) + assert r.outcome == "needs_review" + + +def test_bookkeeping_versions_at_the_same_instant_are_refused(): + # The prior stamp is 12:10:00. The first update moves it to its own time, + # 12:00:02; the second moves it back to 12:10:00, ten minutes past that + # version's time. Both versions name the same instant, so nothing says + # which came last, and the server's newest-first order read as oldest + # first would hide the final move. + book = { + "bookkeeping": [ + {"resource_type": "Practitioner", "paths": BOOK_PATHS, "now_checks": True} + ] + } + late = [{"url": "urn:app:last-seen", "valueInstant": "2026-10-28T12:10:00Z"}] + own = [{"url": "urn:app:last-seen", "valueInstant": "2026-10-28T12:00:02Z"}] + prior = {**DOC, "extension": late} + first = { + **DOC, + "meta": {"versionId": "2", "lastUpdated": "2026-10-28T12:00:02Z"}, + "extension": own, + } + second = { + **DOC, + "meta": {"versionId": "3", "lastUpdated": "2026-10-28T17:00:02+05:00"}, + "extension": late, + } + c = Change( + ref="Practitioner/d", + actions={"U"}, + authors={BOT}, + events=[(_t(2), "U", frozenset({BOT})), (_t(2), "U", frozenset({BOT}))], + versions=2, + bodies=[second, first], + prior=prior, + ) + x = patient("x") + r = judge(contract(**book), PARAMS, base(), created_with(c), {"x": x}, x) + assert r.outcome == "needs_review" + # Written with one offset the tie is just as unknown. + second_z = { + **second, + "meta": {"versionId": "3", "lastUpdated": "2026-10-28T12:00:02Z"}, + } + c.bodies = [second_z, first] + r = judge(contract(**book), PARAMS, base(), created_with(c), {"x": x}, x) + assert r.outcome == "needs_review" From 47ec7f4888cb06933b59955f5085f334e8700fd9 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Thu, 5 Nov 2026 00:05:15 -0800 Subject: [PATCH 19/21] fix: retain lab SSH operation budgets and unknown guest timeouts (#1949) Signed-off-by: abrichr --- packages/flow/scripts/desktop/winrm_lab.py | 56 ++++++- packages/flow/tests/test_winrm_lab.py | 165 ++++++++++++++++++++- 2 files changed, 213 insertions(+), 8 deletions(-) diff --git a/packages/flow/scripts/desktop/winrm_lab.py b/packages/flow/scripts/desktop/winrm_lab.py index 4264aad35d..0e337c874e 100644 --- a/packages/flow/scripts/desktop/winrm_lab.py +++ b/packages/flow/scripts/desktop/winrm_lab.py @@ -31,6 +31,11 @@ ``winlab``), so a harness written for one lab host runs on another by changing that variable. Nothing here prints a credential. ``pywinrm`` is a lab dependency only and is imported lazily, so nothing ships with it. + +SSH commands default to a finite 600-second budget. The three desktop task +preparation calls use 30 seconds. A local SSH timeout does not confirm guest +exit: ``SSHTimeoutError`` retains ``MAY_STILL_RUN`` and requires reconciliation. +It does not attempt remote cancellation. """ from __future__ import annotations @@ -85,6 +90,32 @@ class WinRMLabError(RuntimeError): """A lab host command failed; carries the exit status and stderr tail.""" +class SSHTimeoutError(WinRMLabError): + """The local SSH deadline expired; the guest outcome remains unknown.""" + + def __init__(self, host: str, operation: str, timeout_s: float) -> None: + self.host = host + self.operation = operation + self.timeout_s = timeout_s + super().__init__( + f"{operation} over SSH to {host} timed out after {timeout_s:g}s; " + "guest state unknown (MAY_STILL_RUN); ownership unverified; " + "no cancellation attempted; reconciliation required" + ) + + def as_dict(self) -> dict[str, str | float | bool]: + """Retain the unknown guest outcome without payload or partial output.""" + return { + "host": self.host, + "operation": self.operation, + "timeout_s": self.timeout_s, + "guest_state": "MAY_STILL_RUN", + "identity_verified": False, + "cancellation_attempted": False, + "reconciliation_required": True, + } + + @dataclass class Result: status: int @@ -221,7 +252,7 @@ def _ssh( remote_command: str, stdin: bytes = b"", *, - timeout_s: float = 60.0, + timeout_s: float = 600.0, operation: str = "PowerShell command", ) -> Result: if not math.isfinite(timeout_s) or timeout_s <= 0: @@ -235,9 +266,7 @@ def _ssh( ) except subprocess.TimeoutExpired: # The exception's own text repeats the command line. - raise WinRMLabError( - f"{operation} over SSH to {self.host} timed out after {timeout_s:g}s" - ) from None + raise SSHTimeoutError(self.host, operation, timeout_s) from None return Result( proc.returncode, proc.stdout.decode("utf-8", "replace"), @@ -249,7 +278,7 @@ def run_ps( script: str, *, check: bool = True, - timeout_s: float = 60.0, + timeout_s: float = 600.0, operation: str = "PowerShell command", ) -> Result: if self._session is not None: @@ -276,7 +305,15 @@ def run_ps( ) return result - def run_cmd(self, command: str, *args: str, check: bool = True) -> Result: + def run_cmd( + self, + command: str, + *args: str, + check: bool = True, + timeout_s: float = 600.0, + operation: str = "guest command", + ) -> Result: + """Run a command, forwarding its finite budget and phase on SSH.""" if self._session is not None: raw = self._session.run_cmd(command, list(args)) result = Result( @@ -289,7 +326,12 @@ def run_cmd(self, command: str, *args: str, check: bool = True) -> Result: quoted = " ".join( "'" + a.replace("'", "''") + "'" for a in (command, *args) ) - result = self.run_ps(f"& {quoted}; exit $LASTEXITCODE", check=False) + result = self.run_ps( + f"& {quoted}; exit $LASTEXITCODE", + check=False, + timeout_s=timeout_s, + operation=operation, + ) if check and result.status != 0: raise WinRMLabError( f"guest command exited {result.status}: {result.stderr[-600:]}" diff --git a/packages/flow/tests/test_winrm_lab.py b/packages/flow/tests/test_winrm_lab.py index ae2d0bd483..856fe7c056 100644 --- a/packages/flow/tests/test_winrm_lab.py +++ b/packages/flow/tests/test_winrm_lab.py @@ -4,6 +4,7 @@ import base64 import importlib.util +import json import subprocess import sys from pathlib import Path @@ -86,7 +87,7 @@ def test_ssh_timeout_names_only_the_host(monkeypatch): monkeypatch.setenv("OAFLOW_WINLAP_SSH_KEY", "/k") def fake_run(argv, **kw): - assert kw["timeout"] == 60 + assert kw["timeout"] == 600 raise subprocess.TimeoutExpired(argv, 600) monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) @@ -227,3 +228,165 @@ def test_lab_harnesses_name_no_interpreter_path(): desktop = _PATH.parent for script in desktop.glob("qualify_lab_*.py"): assert "Python312-arm64" not in script.read_text(), script.name + + +@pytest.fixture +def synthetic_ssh_lab(monkeypatch): + monkeypatch.setenv("OAFLOW_SYNTHETIC_HOST", "192.0.2.1") + monkeypatch.setenv("OAFLOW_SYNTHETIC_USER", "operator") + monkeypatch.setenv("OAFLOW_SYNTHETIC_SSH_KEY", "/synthetic/not-a-key") + return lab_mod.WinRMLab(profile="synthetic") + + +@pytest.mark.parametrize("method", ["_ssh", "run_ps", "run_cmd"]) +def test_generic_ssh_budget_preserves_ten_minutes( + monkeypatch, synthetic_ssh_lab, method +): + calls = [] + + def fake_run(argv, **kw): + calls.append((argv, kw)) + return subprocess.CompletedProcess(argv, 0, b"ok", b"") + + monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) + getattr(synthetic_ssh_lab, method)("synthetic-command") + assert len(calls) == 1 + assert calls[0][1]["timeout"] == 600 + + +@pytest.mark.parametrize( + "method,check", + [ + ("_ssh", None), + ("run_ps", False), + ("run_ps", True), + ("run_cmd", False), + ("run_cmd", True), + ], +) +def test_timeout_retains_unknown_guest_outcome_without_payload( + monkeypatch, synthetic_ssh_lab, method, check +): + calls = [] + payload = "synthetic-private-payload" + + def fake_run(argv, **kw): + calls.append((argv, kw)) + raise subprocess.TimeoutExpired( + argv, kw["timeout"], output=payload.encode(), stderr=b"partial-output" + ) + + monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) + kwargs = {"timeout_s": 90.0, "operation": "owned command phase"} + if check is not None: + kwargs["check"] = check + with pytest.raises(lab_mod.WinRMLabError) as err: + getattr(synthetic_ssh_lab, method)(payload, **kwargs) + assert isinstance(err.value, lab_mod.SSHTimeoutError) + assert len(calls) == 1 and calls[0][1]["timeout"] == 90 + assert err.value.as_dict() == { + "host": "192.0.2.1", + "operation": "owned command phase", + "timeout_s": 90.0, + "guest_state": "MAY_STILL_RUN", + "identity_verified": False, + "cancellation_attempted": False, + "reconciliation_required": True, + } + assert "owned command phase" in str(err.value) and "90s" in str(err.value) + assert "MAY_STILL_RUN" in str(err.value) + assert "reconciliation required" in str(err.value) + retained = str(err.value) + json.dumps(err.value.as_dict()) + assert payload not in retained and "partial-output" not in retained + assert "/synthetic/not-a-key" not in retained + assert err.value.__cause__ is None and err.value.__suppress_context__ + + +@pytest.mark.parametrize( + "method,phase", + [ + ("launch_interactive", "launch interactive task"), + ("stop_interactive", "stop interactive task"), + ("attach_console", "attach desktop console"), + ], +) +def test_task_preparation_retains_short_budget_and_unknown_timeout( + monkeypatch, synthetic_ssh_lab, method, phase +): + calls = [] + + def fake_run(argv, **kw): + calls.append((argv, kw)) + raise subprocess.TimeoutExpired(argv, kw["timeout"]) + + monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) + with pytest.raises(lab_mod.WinRMLabError) as err: + if method == "attach_console": + synthetic_ssh_lab.attach_console() + else: + getattr(synthetic_ssh_lab, method)("synthetic-owned-task") + assert len(calls) == 1 and calls[0][1]["timeout"] == 30 + assert phase in str(err.value) + assert err.value.as_dict()["guest_state"] == "MAY_STILL_RUN" + assert not err.value.as_dict()["cancellation_attempted"] + assert "synthetic-owned-task" not in str(err.value) + + +@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")]) +def test_command_budget_refuses_invalid_before_ssh( + monkeypatch, synthetic_ssh_lab, timeout +): + monkeypatch.setattr( + lab_mod.subprocess, "run", lambda *args, **kw: pytest.fail("SSH was called") + ) + with pytest.raises(ValueError, match="finite and positive"): + synthetic_ssh_lab.run_cmd("synthetic-command", timeout_s=timeout) + + +def test_file_hash_check_retains_generic_budget( + monkeypatch, synthetic_ssh_lab, tmp_path +): + import hashlib + + source = tmp_path / "source.py" + source.write_bytes(b"synthetic source") + digest = hashlib.sha256(source.read_bytes()).hexdigest().encode() + calls = [] + + def fake_run(argv, **kw): + calls.append((argv, kw)) + return subprocess.CompletedProcess(argv, 0, digest, b"") + + monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) + synthetic_ssh_lab.push_file(source, r"C:\synthetic\source.py") + assert len(calls) == 1 and calls[0][1]["timeout"] == 600 + assert "Get-FileHash" in calls[0][1]["input"].decode() + + +def test_interpreter_discovery_retains_generic_budget(monkeypatch, synthetic_ssh_lab): + calls = [] + + def fake_run(argv, **kw): + calls.append((argv, kw)) + return subprocess.CompletedProcess(argv, 0, b"C:\\synthetic\\pythonw.exe", b"") + + monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) + assert synthetic_ssh_lab.resolve_python() == r"C:\synthetic\pythonw.exe" + assert len(calls) == 1 and calls[0][1]["timeout"] == 600 + + +def test_command_explicit_budget_preserves_nonzero_status( + monkeypatch, synthetic_ssh_lab +): + calls = [] + + def fake_run(argv, **kw): + calls.append((argv, kw)) + return subprocess.CompletedProcess(argv, 7, b"partial", b"failed") + + monkeypatch.setattr(lab_mod.subprocess, "run", fake_run) + result = synthetic_ssh_lab.run_cmd("synthetic-command", check=False, timeout_s=90) + assert (result.status, result.stdout, result.stderr) == (7, "partial", "failed") + with pytest.raises(lab_mod.WinRMLabError, match="guest command exited 7"): + synthetic_ssh_lab.run_cmd("synthetic-command", timeout_s=90) + assert len(calls) == 2 and all(call[1]["timeout"] == 90 for call in calls) From e3765a4e55925f879019408c39dd5fa8a82ec1d0 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Thu, 5 Nov 2026 00:35:34 -0800 Subject: [PATCH 20/21] qualify: verifier.session for FHIR and GraphQL read-back, renewed each trial (#1936) * fix(qualify): let verifier.session renew FHIR and GraphQL read-back credentials * fix(qualify): refuse a session hook that opens no client --- packages/flow/docs/QUALIFY_A_WORKFLOW.md | 6 +- packages/flow/openadapt_flow/deployment.py | 31 ++- .../flow/openadapt_flow/qualify/config.py | 17 +- .../flow/openadapt_flow/qualify/session.py | 35 +++- .../tests/test_qualify_verifier_session.py | 176 ++++++++++++++++++ 5 files changed, 255 insertions(+), 10 deletions(-) create mode 100644 packages/flow/tests/test_qualify_verifier_session.py diff --git a/packages/flow/docs/QUALIFY_A_WORKFLOW.md b/packages/flow/docs/QUALIFY_A_WORKFLOW.md index 6ee81f4e79..5577f5d6c2 100644 --- a/packages/flow/docs/QUALIFY_A_WORKFLOW.md +++ b/packages/flow/docs/QUALIFY_A_WORKFLOW.md @@ -113,7 +113,11 @@ What each part does: `auth: {bearer_env: NAME}`, `{header: X-Api-Key, value_env: NAME}` or `{basic_env: NAME}`, with the secret in the named environment variable. `kind: sql`, `fhir` and `graphql` take the same keys as the effects section - of a deployment file ([EFFECT_KIT.md](EFFECT_KIT.md)). An on-screen check is + of a deployment file ([EFFECT_KIT.md](EFFECT_KIT.md)). For `kind: rest`, + `fhir` or `graphql`, `session: module:function` can name a hook that + returns an authenticated client instead. `qualify` calls it again for each + trial's read-back, so a token that expires during a long campaign is + renewed. An on-screen check is refused: it cannot tell a real write from a screen that only looks right. - **effects**: the save's checks as an effect contract list, the same entries an `effect_contract.yaml` holds: diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 3a256e3cb0..600a937c3c 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -1954,7 +1954,10 @@ def plugin_secrets( def build_effect_verifier( - cfg: EffectsConfig, params: Optional[Mapping[str, object]] = None + cfg: EffectsConfig, + params: Optional[Mapping[str, object]] = None, + *, + session: Any = None, ) -> Optional[Any]: """Construct the configured ``EffectVerifier`` (or None for ``kind: none``). @@ -1966,6 +1969,11 @@ def build_effect_verifier( ``graphql_variables``) are resolved against these AT CONSTRUCTION, so the verifier probes the record THIS run writes. A config with no references ignores ``params`` entirely (fully back-compatible). + session: An authenticated ``requests``-style client the read-back + uses instead of building its own, for ``kind: rest``, ``fhir`` + or ``graphql`` (for example one from a qualify + ``verifier.session`` hook that renews an expiring token). Any + other kind refuses it rather than read without it. Resolution order for ``kind``: built-in adapters, then plugin factories registered under the ``openadapt_flow.effect_verifiers`` entry-point group or via @@ -1980,6 +1988,14 @@ def build_effect_verifier( unresolved ``{param: ...}`` reference, or a missing secret env var (fail loud rather than wire a broken verifier). """ + if session is not None and ( + cfg.candidates or (cfg.kind or "").strip().lower() not in _SESSION_KINDS + ): + raise ValueError( + "an injected read-back session is supported for kind: " + + ", ".join(_SESSION_KINDS) + + f", not {cfg.kind!r}" + ) if cfg.candidates: built = [ build_effect_verifier(candidate, params) for candidate in cfg.candidates @@ -2026,7 +2042,7 @@ def build_effect_verifier( return RedactingVerifier(candidate_selector, policy) return candidate_selector - verifier = _build_effect_verifier_unredacted(cfg, params) + verifier = _build_effect_verifier_unredacted(cfg, params, session=session) if verifier is None: return None from openadapt_flow.runtime.effects.adapter import set_verifier_identity @@ -2048,6 +2064,9 @@ def build_effect_verifier( _PLUGIN_ONLY_FIELDS = ("options", "option_params", "secret_env") +#: The built-in read-backs that can read through an injected client. +_SESSION_KINDS: Final[tuple[str, ...]] = ("rest", "fhir", "graphql") + #: What these fields held when they still had defaults. Verifier identities #: and check digests issued before then hashed these values, so for hashing @@ -2172,7 +2191,10 @@ def sanitized(value: Any, *, key: str = "") -> Any: def _build_effect_verifier_unredacted( - cfg: EffectsConfig, params: Optional[Mapping[str, object]] = None + cfg: EffectsConfig, + params: Optional[Mapping[str, object]] = None, + *, + session: Any = None, ) -> Optional[Any]: """The per-kind construction behind :func:`build_effect_verifier`.""" kind = (cfg.kind or "none").strip().lower() @@ -2291,6 +2313,7 @@ def _build_effect_verifier_unredacted( cfg.base_url, records_path=records_path, records_key=cfg.records_key, + session=session, headers=headers, timeout_s=cfg.timeout_s, poll_interval_s=cfg.poll_interval_s, @@ -2338,6 +2361,7 @@ def _build_effect_verifier_unredacted( field_paths=cfg.field_paths, access_token=access_token, token_source=token_source, + session=session, verify_tls=cfg.verify_tls, timeout_s=cfg.timeout_s, poll_interval_s=cfg.poll_interval_s, @@ -2434,6 +2458,7 @@ def _build_effect_verifier_unredacted( query=cfg.graphql_query, variables=variables, records_path=cfg.graphql_records_path, + session=session, headers=headers, timeout_s=cfg.timeout_s, poll_interval_s=cfg.poll_interval_s, diff --git a/packages/flow/openadapt_flow/qualify/config.py b/packages/flow/openadapt_flow/qualify/config.py index 2bdb67d821..3e3e75379e 100644 --- a/packages/flow/openadapt_flow/qualify/config.py +++ b/packages/flow/openadapt_flow/qualify/config.py @@ -12,7 +12,9 @@ screen. Exactly the deployment ``effects`` section (``kind: rest``, ``sql``, ``fhir``, ``graphql`` or a plugin kind). ``none`` and ``onscreen`` are refused. ``session`` (``module:function``) may add an authenticated client - for ``kind: rest``; it is not part of the verifier's identity; + for ``kind: rest``, ``fhir`` or ``graphql``, built again for every trial's + read-back so an expiring token is renewed; it is not part of the + verifier's identity; - ``effects``: the effect contract list (``effect_contract.yaml`` ``effects:``), bound to the save step. Or ``record``, a shorthand that expands to that list (record fields from inputs, the selecting fields, the @@ -175,11 +177,16 @@ def origin(self) -> str: return self.url.rstrip("/") +#: The read-back kinds a ``verifier.session`` hook can authenticate. +SESSION_KINDS = ("rest", "fhir", "graphql") + + class SystemOfRecordConfig(BaseModel): """The independent read-back, as a deployment ``effects`` section. ``session`` (``module:function``) may supply an authenticated HTTP client - for ``kind: rest`` when ``auth`` cannot (for example an OAuth flow). + for ``kind: rest``, ``fhir`` or ``graphql`` when ``auth`` cannot (for + example an OAuth flow). It is called again for every trial's read-back. """ model_config = ConfigDict(extra="forbid") @@ -205,8 +212,10 @@ def _check(self) -> "SystemOfRecordConfig": "(rest, sql, fhir, graphql, ui_readback or a plugin), not " f"{self.verifier.kind!r}" ) - if self.session is not None and self.verifier.kind != "rest": - raise ValueError("verifier.session is supported for kind: rest") + if self.session is not None and self.verifier.kind not in SESSION_KINDS: + raise ValueError( + "verifier.session is supported for kind: " + ", ".join(SESSION_KINDS) + ) return self diff --git a/packages/flow/openadapt_flow/qualify/session.py b/packages/flow/openadapt_flow/qualify/session.py index 849ee5c562..60b3fd3268 100644 --- a/packages/flow/openadapt_flow/qualify/session.py +++ b/packages/flow/openadapt_flow/qualify/session.py @@ -148,13 +148,44 @@ def run_http() -> None: return run_http +def _hook_client(hook: Callable[..., Any], reference: str, config: Any) -> Any: + """The client a ``verifier.session`` hook opened. A hook that opens none + is refused: the verifier would fall back to an anonymous client and read + whatever that client can see instead of the protected view.""" + + client = hook(config) + if client is None: + raise QualifyConfigError( + f"verifier.session {reference!r} returned no client; " + "refusing to read back without the configured authentication" + ) + return client + + def verifier_factory(config: VerifyConfig) -> Callable[[Mapping[str, object]], Any]: """Build the read-back verifier for one run's inputs.""" sor = config.system_of_record cfg = sor.verifier + if sor.session and cfg.kind != "rest": + reference = sor.session + session_hook = load_hook(reference) + from openadapt_flow.deployment import build_effect_verifier + + def build_with_session(params: Mapping[str, object]) -> Any: + # A fresh client per read-back, so a token that expired during a + # long campaign is renewed rather than replayed. + verifier = build_effect_verifier( + cfg, params, session=_hook_client(session_hook, reference, config) + ) + if verifier is None: + raise QualifyConfigError("verifier built no verifier") + return verifier + + return build_with_session if sor.session: - hook = load_hook(sor.session) + rest_reference = sor.session + hook = load_hook(rest_reference) from openadapt_flow.runtime.effects import RestRecordVerifier if not cfg.base_url: @@ -168,7 +199,7 @@ def build_rest(params: Mapping[str, object]) -> Any: cfg.base_url.rstrip("/"), # type: ignore[union-attr] records_path=records_path, records_key=cfg.records_key, - session=hook(config), + session=_hook_client(hook, rest_reference, config), timeout_s=10.0, poll_interval_s=0.2, ) diff --git a/packages/flow/tests/test_qualify_verifier_session.py b/packages/flow/tests/test_qualify_verifier_session.py new file mode 100644 index 0000000000..d545adfd5a --- /dev/null +++ b/packages/flow/tests/test_qualify_verifier_session.py @@ -0,0 +1,176 @@ +"""``verifier.session`` renews read-back credentials for FHIR and GraphQL too. + +A static token can expire during a long qualification campaign; a session hook +is called again for every trial's read-back, so the next read uses a fresh +token instead of replaying the expired one. +""" + +from __future__ import annotations + +import sys +import types +from types import SimpleNamespace +from typing import Any, Optional + +import pytest + +from openadapt_flow.deployment import EffectsConfig, build_effect_verifier +from openadapt_flow.qualify.config import SystemOfRecordConfig +from openadapt_flow.qualify.session import verifier_factory + + +class _Response: + def __init__(self, status_code: int, body: Any = None) -> None: + self.status_code = status_code + self._body = body + + def json(self) -> Any: + return self._body + + +class _Authority: + """Issues one bearer token at a time; expiring it refuses every read.""" + + def __init__(self) -> None: + self.issued = 0 + self.current: Optional[str] = None + + def mint(self) -> str: + self.issued += 1 + self.current = f"token-{self.issued}" + return self.current + + def expire(self) -> None: + self.current = None + + +class _Session: + """A ``requests``-style client bound to the token it was opened with.""" + + def __init__(self, authority: _Authority, token: str) -> None: + self.authority = authority + self.token = token + + def _authorized(self) -> bool: + return self.token == self.authority.current + + def get(self, url: str, **_kwargs: Any) -> _Response: + if not self._authorized(): + return _Response(401) + return _Response( + 200, + { + "resourceType": "Bundle", + "entry": [{"resource": {"id": "p1", "name": "Ada"}}], + }, + ) + + def post(self, url: str, **_kwargs: Any) -> _Response: + if not self._authorized(): + return _Response(401) + return _Response(200, {"data": {"patients": [{"id": "p1", "name": "Ada"}]}}) + + +@pytest.fixture +def authority(monkeypatch) -> _Authority: + authority = _Authority() + module = types.ModuleType("fake_sor_auth") + + def open_session(_config: Any) -> _Session: + return _Session(authority, authority.mint()) + + module.open_session = open_session # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "fake_sor_auth", module) + return authority + + +_VERIFIERS = { + "fhir": { + "kind": "fhir", + "base_url": "https://sor.invalid/fhir", + "resource_type": "Patient", + "field_paths": {"name": "name"}, + }, + "graphql": { + "kind": "graphql", + "base_url": "https://sor.invalid/graphql", + "graphql_query": "{ patients { id name } }", + "graphql_records_path": "data.patients", + }, +} + + +@pytest.mark.parametrize("kind", sorted(_VERIFIERS)) +def test_session_hook_renews_an_expired_token_for_each_read(authority, kind): + sor = SystemOfRecordConfig( + session="fake_sor_auth:open_session", + verifier=EffectsConfig(**_VERIFIERS[kind]), + ) + make_verifier = verifier_factory(SimpleNamespace(system_of_record=sor)) + + first = make_verifier({}) + before = first.capture_pre_state() + assert before.reachable is True + assert [record["id"] for record in before.records] == ["p1"] + + # The token expires mid-run: the client it was issued to can't read any + # more, which is what a static token would leave every later trial with. + authority.expire() + assert first.capture_pre_state().reachable is False + + # The next trial's read-back gets a freshly authenticated client. + second = make_verifier({}) + after = second.capture_pre_state() + assert after.reachable is True + assert [record["id"] for record in after.records] == ["p1"] + assert authority.issued == 2 + + +def test_session_hook_is_still_refused_for_a_kind_that_cannot_use_it(): + with pytest.raises(ValueError, match="verifier.session is supported for kind"): + SystemOfRecordConfig( + session="fake_sor_auth:open_session", + verifier=EffectsConfig(kind="sql", sqlite_database="/nonexistent.db"), + ) + + +def test_build_effect_verifier_refuses_a_session_it_would_ignore(): + with pytest.raises(ValueError, match="injected read-back session"): + build_effect_verifier( + EffectsConfig(kind="sql", sqlite_database="/nonexistent.db"), + session=object(), + ) + + +_REST = { + "kind": "rest", + "base_url": "https://sor.invalid/api", + "records_path": "/patients", +} + + +@pytest.mark.parametrize("kind", [*sorted(_VERIFIERS), "rest"]) +def test_a_session_hook_that_opens_no_client_is_refused_before_any_read( + monkeypatch, kind +): + import httpx + + from openadapt_flow.qualify.config import QualifyConfigError + + module = types.ModuleType("fake_sor_no_client") + module.open_session = lambda _config: None # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "fake_sor_no_client", module) + + def no_anonymous_client(*_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("an anonymous client was opened") + + monkeypatch.setattr(httpx, "Client", no_anonymous_client) + spec = _REST if kind == "rest" else _VERIFIERS[kind] + sor = SystemOfRecordConfig( + session="fake_sor_no_client:open_session", + verifier=EffectsConfig(**spec), + ) + make_verifier = verifier_factory(SimpleNamespace(system_of_record=sor)) + + with pytest.raises(QualifyConfigError, match="returned no client"): + make_verifier({}) From 72b743e41efeb96c2c249719425ffcd342869a95 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Thu, 5 Nov 2026 00:57:44 -0800 Subject: [PATCH 21/21] fix(flow): read bundle manifest and API proposal sidecars as UTF-8 (#1944) * fix(flow): read bundle manifest and API proposal sidecars as UTF-8 (#1845) * fix(flow): read and write the deploy config and its workflow inputs as UTF-8 --- packages/flow/openadapt_flow/cli_api.py | 5 +- packages/flow/openadapt_flow/cli_config.py | 8 +- packages/flow/openadapt_flow/deployment.py | 6 +- packages/flow/openadapt_flow/ir.py | 4 +- .../flow/tests/test_bundle_utf8_sidecars.py | 137 ++++++++++++++++++ 5 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 packages/flow/tests/test_bundle_utf8_sidecars.py diff --git a/packages/flow/openadapt_flow/cli_api.py b/packages/flow/openadapt_flow/cli_api.py index 51abfc878a..df7396e610 100644 --- a/packages/flow/openadapt_flow/cli_api.py +++ b/packages/flow/openadapt_flow/cli_api.py @@ -84,7 +84,7 @@ def cmd_api_show(args: argparse.Namespace) -> int: reasons: list[str] = [] sidecar = bundle / "api_proposal.json" if sidecar.is_file(): - reasons = json.loads(sidecar.read_text()).get("reasons") or [] + reasons = json.loads(sidecar.read_text(encoding="utf-8")).get("reasons") or [] if args.as_json: rows = [] for s in found: @@ -144,7 +144,8 @@ def cmd_api_propose(args: argparse.Namespace) -> int: }, indent=2, ) - + "\n" + + "\n", + encoding="utf-8", ) if step is None: print(f"No API step proposed: {'; '.join(reasons)}.") diff --git a/packages/flow/openadapt_flow/cli_config.py b/packages/flow/openadapt_flow/cli_config.py index 7d06618085..b5521372e7 100644 --- a/packages/flow/openadapt_flow/cli_config.py +++ b/packages/flow/openadapt_flow/cli_config.py @@ -49,7 +49,7 @@ def bundle_digest(bundle: Path) -> str: manifest_path = bundle / "manifest.json" if manifest_path.is_file(): try: - manifest = json.loads(manifest_path.read_text()) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise SystemExit(f"{manifest_path} is not valid JSON: {exc}") from exc digest = manifest.get("content_digest") @@ -63,7 +63,7 @@ def bundle_surface(bundle: Path) -> Optional[str]: payload = _require_bundle(bundle) try: - raw = json.loads(payload.read_text()) + raw = json.loads(payload.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise SystemExit(f"{payload} is not valid JSON: {exc}") from exc surface = raw.get("surface") @@ -75,7 +75,7 @@ def bundle_surface(bundle: Path) -> Optional[str]: def bundle_name(bundle: Path) -> str: payload = _require_bundle(bundle) try: - raw = json.loads(payload.read_text()) + raw = json.loads(payload.read_text(encoding="utf-8")) except json.JSONDecodeError: return bundle.name name = raw.get("name") @@ -192,7 +192,7 @@ def init_deploy_config( draft = build_draft(bundle, backend=backend) text = render_draft(draft) out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(text) + out.write_text(text, encoding="utf-8") # Draft must parse, and must be refused by the governed loader. parsed = load_deployment(out, allow_unresolved=True) if parsed.bundle_digest != draft["bundle_digest"]: diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 600a937c3c..9484a13305 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -1457,7 +1457,11 @@ def load_deployment( if pinned is None and not path.is_file(): raise FileNotFoundError(f"deployment config {source!r} is not an existing file") try: - text = pinned.decode("utf-8") if pinned is not None else path.read_text() + text = ( + pinned.decode("utf-8") + if pinned is not None + else path.read_text(encoding="utf-8") + ) data = yaml.safe_load(text) or {} except yaml.YAMLError as e: # pragma: no cover - passthrough raise ValueError(f"could not parse deployment YAML {path}: {e}") from e diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 88341fc0b8..dd946abc4b 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -3693,7 +3693,9 @@ def load( if wf.manifest is None: sidecar = bundle / "manifest.json" if sidecar.is_file(): - wf.manifest = BundleManifest.model_validate_json(sidecar.read_text()) + wf.manifest = BundleManifest.model_validate_json( + sidecar.read_text(encoding="utf-8") + ) persisted = wf.manifest if sealed_assets is not None: diff --git a/packages/flow/tests/test_bundle_utf8_sidecars.py b/packages/flow/tests/test_bundle_utf8_sidecars.py new file mode 100644 index 0000000000..9fce3fa35a --- /dev/null +++ b/packages/flow/tests/test_bundle_utf8_sidecars.py @@ -0,0 +1,137 @@ +"""Bundle sidecars are read as UTF-8 whatever the platform's default codec. + +Windows reads text in its ANSI code page (cp1252) unless told otherwise, so a +sidecar with non-ASCII text would load wrong or fail there. Each reader runs +in a child Python whose default codec is not UTF-8 (``PYTHONUTF8=0`` and the C +locale with no coercion), where an unencoded read of the same bytes fails. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from openadapt_flow.ir import ActionKind, Step, Workflow + +NAME = "Zoë Müller – 東京" + + +def _bundle(tmp_path: Path, *, manifest_name: str = NAME) -> Path: + bundle = tmp_path / "bundle" + bundle.mkdir() + wf = Workflow( + name="demo", + steps=[Step(id="s1", intent="submit", action=ActionKind.KEY, key="Enter")], + ) + wf.save(bundle) + # Move the manifest out of workflow.json into the sidecar, as a sealed + # bundle keeps it, with non-ASCII text in it. + raw = json.loads((bundle / "workflow.json").read_bytes()) + manifest = raw.pop("manifest", None) or json.loads( + (bundle / "manifest.json").read_bytes() + ) + manifest["provenance"]["compiler_version"] = manifest_name + (bundle / "workflow.json").write_bytes(json.dumps(raw).encode("utf-8")) + (bundle / "manifest.json").write_bytes( + json.dumps(manifest, ensure_ascii=False).encode("utf-8") + ) + (bundle / "api_proposal.json").write_bytes( + json.dumps({"proposed": False, "reasons": [NAME]}, ensure_ascii=False).encode( + "utf-8" + ) + ) + return bundle + + +def _run(code: str, bundle: Path) -> str: + env = { + **os.environ, + "PYTHONUTF8": "0", + "PYTHONCOERCECLOCALE": "0", + "LC_ALL": "C", + "LANG": "C", + "PYTHONIOENCODING": "utf-8", + } + probe = subprocess.run( + [ + sys.executable, + "-c", + "import locale; print(locale.getpreferredencoding(False))", + ], + env=env, + capture_output=True, + text=True, + check=True, + ) + if probe.stdout.strip().lower().replace("-", "") in ("utf8", "utf_8"): + pytest.skip("this platform's default codec can't be set away from UTF-8") + done = subprocess.run( + [sys.executable, "-c", code, str(bundle)], + env=env, + capture_output=True, + encoding="utf-8", + ) + assert done.returncode == 0, done.stderr + return done.stdout + + +def test_the_manifest_sidecar_loads_as_utf8(tmp_path: Path) -> None: + out = _run( + "import sys\n" + "from openadapt_flow.ir import Workflow\n" + "wf = Workflow.load(sys.argv[1])\n" + "print(wf.manifest.provenance.compiler_version)\n", + _bundle(tmp_path), + ) + assert out.strip() == NAME + + +def test_the_bundle_digest_reads_the_sidecar_as_utf8(tmp_path: Path) -> None: + bundle = _bundle(tmp_path) + digest = json.loads((bundle / "manifest.json").read_bytes())["content_digest"] + out = _run( + "import sys\n" + "from pathlib import Path\n" + "from openadapt_flow.cli_config import bundle_digest\n" + "print(bundle_digest(Path(sys.argv[1])))\n", + bundle, + ) + assert out.strip() == digest + + +def test_api_show_reads_the_proposal_sidecar_as_utf8(tmp_path: Path) -> None: + out = _run( + "import argparse, sys\n" + "from openadapt_flow.cli_api import cmd_api_show\n" + "cmd_api_show(argparse.Namespace(bundle=sys.argv[1], as_json=True))\n", + # An ASCII manifest, so only the proposal sidecar has non-ASCII text. + _bundle(tmp_path, manifest_name="demo"), + ) + assert json.loads(out)["not_proposed_because"] == [NAME] + + +@pytest.mark.parametrize("backend", [[], ["--backend", "web"]]) +def test_config_init_reads_a_non_ascii_workflow_as_utf8( + tmp_path: Path, backend: list[str] +) -> None: + # The workflow name itself is non-ASCII; ``init`` reads workflow.json for + # the surface (no --backend) and for the run name (either way). + bundle = tmp_path / "bundle" + Workflow( + name=NAME, + steps=[Step(id="s1", intent="submit", action=ActionKind.KEY, key="Enter")], + ).save(bundle) + out = tmp_path / "flow.toml" + _run( + "import sys\n" + "from openadapt_flow.cli_config import main\n" + f"code = main(['init', sys.argv[1], '--out', {str(out)!r}, *{backend!r}])\n" + "sys.exit(code)\n", + bundle, + ) + assert NAME in out.read_text(encoding="utf-8")