Restore complete claims JUnit checks in monorepo CI

.github/ci-tools/pyproject.toml

@@ -16,6 +16,7 @@ test = ["pytest>=9.1.1", "packaging>=25.0"]
 build = ["build>=1.6.1", "pip>=25.3"]
 pip = ["pip>=25.3"]
 lint = ["ruff==0.17.0"]
+claims = ["pyyaml>=6.0"]
 release = ["build>=1.6.1", "twine>=7.0.0"]
 # The in-guest Windows agent's screen and input modules, for the hosted
 # Windows replay (desktop-replay.yml), which runs Python 3.12. CI only: Flow

.github/workflows/packages.yml

@@ -114,6 +114,7 @@ jobs:
     timeout-minutes: 5
     outputs:
       flow: ${{ steps.classify.outputs.flow }}
+      flow_unit_shards: ${{ steps.classify.outputs.flow_unit_shards }}
       types: ${{ steps.classify.outputs.types }}
       agent: ${{ steps.classify.outputs.agent }}
       capture: ${{ steps.classify.outputs.capture }}
@@ -136,6 +137,9 @@ jobs:
           GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }}
         run: |
           set -euo pipefail
+          # Both the matrix and the claims consumer use this exact count.
+          if [ "$EVENT_NAME" = pull_request ]; then unit_shards=3; else unit_shards=2; fi
+          echo "flow_unit_shards=$unit_shards" >> "$GITHUB_OUTPUT"
           all() {
             for pkg in flow types agent capture privacy desktop tray program_graph weight; do echo "$pkg=true" >> "$GITHUB_OUTPUT"; done
             exit 0
@@ -439,7 +443,7 @@ jobs:
         # together: a pull request's fast lane takes three shards, about five
         # minutes each, to fit `ci` in ten minutes. The full suite on main and
         # nightly takes two.
-        include: ${{ fromJSON(github.event_name == 'pull_request' && '[{"shard":1,"of":3},{"shard":2,"of":3},{"shard":3,"of":3}]' || '[{"shard":1,"of":2},{"shard":2,"of":2}]') }}
+        include: ${{ fromJSON(needs.scope.outputs.flow_unit_shards == '3' && '[{"shard":1,"of":3},{"shard":2,"of":3},{"shard":3,"of":3}]' || '[{"shard":1,"of":2},{"shard":2,"of":2}]') }}
     defaults:
       run:
         working-directory: packages/flow
@@ -528,7 +532,16 @@ jobs:
           set -euo pipefail
           mkdir -p runs
           mapfile -t files < "$RUNNER_TEMP/shard.txt"
-          pytest -n logical --dist loadfile -v --durations=20 --basetemp=runs/ci "${files[@]}"
+          pytest -n logical --dist loadfile -v --durations=20 --basetemp=runs/ci \
+            --junitxml=runs/flow-unit-shard-${{ matrix.shard }}-of-${{ matrix.of }}.xml "${files[@]}"
+      - name: Retain this shard's claims evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: flow-unit-junit-${{ matrix.shard }}-of-${{ matrix.of }}
+          path: packages/flow/runs/flow-unit-shard-${{ matrix.shard }}-of-${{ matrix.of }}.xml
+          if-no-files-found: error
+          retention-days: 7
 
   flow-critical:
     name: flow core critical (golden VERIFIED, zero model calls)
@@ -621,6 +634,11 @@ jobs:
           sys.exit(1 if bad else 0)
           PY
 
+      - name: Validate passing browser claim evidence
+        run: >-
+          python scripts/validate_claims.py --check --ci-job e2e-browser
+          --junit runs/critical-e2e-junit.xml
+
   # The agent-framework adapters (openadapt_flow.integrations.agents): each
   # framework's own agent loop with a scripted model, writing through a real
   # HTTP app, from the lock with every framework extra (CrewAI pinned beside
@@ -1528,3 +1546,37 @@ jobs:
               print(f"failed: {bad}")
           sys.exit(1 if bad else 0)
           PY
+      # Unit evidence is complete only after every current-run shard passed.
+      # The merge queue deliberately rechecks critical tests, not unit shards.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group'
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group'
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+        if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group'
+        with:
+          enable-cache: true
+          cache-dependency-glob: .github/ci-tools/uv.lock
+          cache-suffix: flow-claims
+      - name: Install the locked claims validator dependency
+        if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group'
+        run: python scripts/ci/locked_install.py --lock .github/ci-tools:claims
+      - name: Download this run's unit claim evidence
+        if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group'
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          pattern: flow-unit-junit-*
+          path: runs/flow-unit-junit
+          merge-multiple: false
+      - name: Require every unit shard and validate all supported unit evidence
+        if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group'
+        run: |
+          set -euo pipefail
+          python scripts/ci/merge_flow_junit.py --artifacts-dir runs/flow-unit-junit \
+            --shards '${{ needs.scope.outputs.flow_unit_shards }}' --out runs/flow-unit-junit.xml
+          cd packages/flow
+          python scripts/validate_claims.py --check --ci-job test --junit ../../runs/flow-unit-junit.xml

packages/flow/docs/VERIFICATION.md

@@ -3,9 +3,9 @@
 > GENERATED by `scripts/validate_claims.py --report` from `claims/` (one file per claim). Do not edit by hand — edit the registry and regenerate.
 
 - Generated at: **committed registry state (regenerate: scripts/validate_claims.py --report)**
-- JUnit pass check: **not embedded in this generated registry view** (required CI jobs enforce pass evidence)
+- JUnit pass check: **not embedded in this generated registry view** (see the current CI run for file-level pass evidence)
 - Structure gate: `python scripts/validate_claims.py --check --structure-only` (a claim whose tier outranks its strongest backing evidence fails CI).
-- Pass gates: required `test` and `e2e-browser` jobs supply their own JUnit files; an absent, all-skipped, or failed supported evidence file fails that required job.
+- File-level pass check: the validator requires each supported backing file to appear in its job's JUnit result, with at least one passing case and no failed or errored cases. This generated report does not itself prove a CI run passed.
 - Scoped validating refreshes name each selected evidence file; each selected file must pass on its declared substrate. Unselected validating evidence is not represented as checked.
 
 **What this validator does and does not do.** It makes each public capability claim a *function* of automated evidence: a `supported` claim must be backed by a test file that has a real passing case in its required default CI job; a `validating` claim must be grounded in a REAL opt-in / infra-gated proof or a field test, and is never presented as supported. These are evidence tiers, not product lifecycle states. The validator does not replace workflow- and deployment-specific acceptance: application controls, identity rules, effect oracles, and live transport conditions remain bound to their counted evidence.

packages/flow/scripts/validate_claims.py

@@ -635,17 +635,18 @@ def render_markdown(
     else:
         lines.append(
             "- JUnit pass check: **not embedded in this generated registry view** "
-            "(required CI jobs enforce pass evidence)"
+            "(see the current CI run for file-level pass evidence)"
         )
     lines.append(
         "- Structure gate: `python scripts/validate_claims.py --check "
         "--structure-only` (a claim whose tier outranks its strongest backing "
         "evidence fails CI)."
     )
     lines.append(
-        "- Pass gates: required `test` and `e2e-browser` jobs supply their own "
-        "JUnit files; an absent, all-skipped, or failed supported evidence file "
-        "fails that required job."
+        "- File-level pass check: the validator requires each supported backing "
+        "file to appear in its job's JUnit result, with at least one passing "
+        "case and no failed or errored cases. This generated report does not "
+        "itself prove a CI run passed."
     )
     lines.append(
         "- Scoped validating refreshes name each selected evidence file; each "

packages/flow/tests/e2e/test_tutorial_break_it_e2e.py

@@ -32,9 +32,9 @@
 from openadapt_flow.tutorial import TutorialResult, run_tutorial
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def break_it_path(tmp_path_factory: pytest.TempPathFactory) -> TutorialResult:
-    """Run the tutorial once with ``break_it=True`` for the whole module."""
+    """Run after the function-scoped synthetic at-rest isolation is active."""
 
     return run_tutorial(tmp_path_factory.mktemp("break-it"), break_it=True)
 

packages/flow/tests/nightly_only.txt

@@ -18,8 +18,6 @@ tests/test_desktop_check_it_cli_mockmed.py  # 126s
 tests/test_effectbench_task_pack.py  # 34s
 tests/test_ap_invoice_benchmark.py  # 12s
 tests/test_o2c_recon_benchmark.py  # 9s
-tests/test_ui_readback_benchmark.py  # 24s
-tests/test_silent_wrong_action_benchmark.py  # 31s
 
 tests/test_agent_teach_branch.py::test_two_demonstrations_become_one_approved_branch  # 5s
 tests/test_agent_teach_repair.py::test_dismiss_without_a_dialog_box_stays_unarmed  # 8s

packages/flow/tests/test_native_app_startup.py

@@ -255,14 +255,31 @@ def test_wpf_fixture_uses_signalled_handle_and_owned_window(
     from tests import test_native_census_real_apps as native
 
     calls = []
-    items = [SimpleNamespace(ControlTypeName="ListItemControl") for _ in range(2)]
+    items = []
+    points = {}
+    for index in range(2):
+        rect = SimpleNamespace(
+            left=10, top=20 * index, right=100, bottom=20 * index + 10
+        )
+        cell = SimpleNamespace(BoundingRectangle=rect)
+        item = SimpleNamespace(
+            ControlTypeName="ListItemControl",
+            ProcessId=1234,
+            GetChildren=lambda cell=cell: [cell],
+        )
+        items.append(item)
+        points[(55, 20 * index + 5)] = item
     listing = SimpleNamespace(Exists=lambda *args: True, GetChildren=lambda: items)
     window = SimpleNamespace(
         ProcessId=1234 if identity != "other_pid" else 999,
         Name=native.TITLE if identity != "other_title" else "other window",
         ListControl=lambda **kwargs: listing,
     )
-    auto = SimpleNamespace(ControlFromHandle=lambda hwnd: calls.append(hwnd) or window)
+    auto = SimpleNamespace(
+        ControlFromHandle=lambda hwnd: calls.append(hwnd) or window,
+        ControlFromPoint=lambda x, y: points[(x, y)],
+        ControlsAreSame=lambda left, right: left is right,
+    )
 
     class Owned(Process):
         def terminate(self):
@@ -296,3 +313,79 @@ def spawn(argv, stdout, stderr):
     finally:
         fixture.close()
     assert proc.poll() == -15
+
+
+@pytest.mark.parametrize("fault", ["empty", "missing", "other_pid", "other_row"])
+def test_wpf_points_are_not_ready_when_the_physical_target_is_unproven(fault):
+    from tests import test_native_census_real_apps as native
+
+    rect = SimpleNamespace(
+        left=10, top=20, right=10 if fault == "empty" else 100, bottom=30
+    )
+    item = SimpleNamespace(
+        GetChildren=lambda: [SimpleNamespace(BoundingRectangle=rect)]
+    )
+    hit = SimpleNamespace(
+        ProcessId=999 if fault == "other_pid" else 1234,
+        ControlTypeName="TextControl",
+        GetParentControl=lambda: None,
+    )
+    auto = SimpleNamespace(
+        ControlFromPoint=lambda *point: None if fault == "missing" else hit,
+        ControlsAreSame=lambda left, right: left is right,
+    )
+    ready, detail = native._wpf_points_ready(auto, [item], 1234)
+    assert ready is False
+    assert "row=0" in detail
+
+
+def test_wpf_points_accept_the_owned_row_ancestor_only():
+    from tests import test_native_census_real_apps as native
+
+    rect = SimpleNamespace(left=10, top=20, right=100, bottom=30)
+    item = SimpleNamespace(
+        GetChildren=lambda: [SimpleNamespace(BoundingRectangle=rect)]
+    )
+    hit = SimpleNamespace(
+        ProcessId=1234,
+        ControlTypeName="TextControl",
+        GetParentControl=lambda: item,
+    )
+    auto = SimpleNamespace(
+        ControlFromPoint=lambda *point: hit,
+        ControlsAreSame=lambda left, right: left is right,
+    )
+    assert native._wpf_points_ready(auto, [item], 1234)[0] is True
+
+
+def test_wpf_readiness_waits_for_the_point_before_returning_items(fixture_state):
+    from tests import test_native_census_real_apps as native
+
+    clock, _, ready_path, _, _ = fixture_state
+    _signal(ready_path)
+    rect = SimpleNamespace(left=10, top=20, right=100, bottom=30)
+    item = SimpleNamespace(
+        GetChildren=lambda: [SimpleNamespace(BoundingRectangle=rect)]
+    )
+    outside = SimpleNamespace(
+        ProcessId=999, ControlTypeName="PaneControl", GetParentControl=lambda: None
+    )
+    inside = SimpleNamespace(
+        ProcessId=1234, ControlTypeName="TextControl", GetParentControl=lambda: item
+    )
+    hits = iter([outside, inside, inside])
+    auto = SimpleNamespace(
+        ControlFromPoint=lambda *point: next(hits),
+        ControlsAreSame=lambda left, right: left is right,
+    )
+
+    def find_items(hwnd):
+        assert hwnd == 5678
+        return (
+            [item, item]
+            if native._wpf_points_ready(auto, [item, item], 1234)[0]
+            else None
+        )
+
+    assert _wait(fixture_state, find_items) == [item, item]
+    assert 0.1 <= clock.now < 0.2

packages/flow/tests/test_native_census_real_apps.py

@@ -28,7 +28,11 @@
 
 from openadapt_flow.runtime import identity_template as itmpl
 from openadapt_flow.runtime.identity_census import CensusDecision, decide
-from tests._native_app_startup import stop_owned_process, wait_for_rendered_items
+from tests._native_app_startup import (
+    NativeAppStartupError,
+    stop_owned_process,
+    wait_for_rendered_items,
+)
 
 pytestmark = pytest.mark.timeout(180)
 
@@ -112,6 +116,42 @@ def _wait(predicate, timeout_s: float = 30.0) -> Any:
 """.replace("__TITLE__", TITLE)
 
 
+def _wpf_points_ready(auto, items: list[Any], pid: int) -> tuple[bool, str]:
+    """Require the points used by the tests to hit their owned rendered rows."""
+    for index, item in enumerate(items):
+        try:
+            children = item.GetChildren()
+            if not children:
+                return False, f"row={index} has no first cell"
+            rect = children[0].BoundingRectangle
+            bounds = (rect.left, rect.top, rect.right, rect.bottom)
+            if rect.right <= rect.left or rect.bottom <= rect.top:
+                return False, f"row={index} has empty cell bounds={bounds}"
+            point = ((rect.left + rect.right) // 2, (rect.top + rect.bottom) // 2)
+            hit = auto.ControlFromPoint(*point)
+            if hit is None:
+                return False, f"row={index} bounds={bounds} point={point} has no hit"
+            hit_pid, hit_type = hit.ProcessId, hit.ControlTypeName
+            detail = (
+                f"row={index} bounds={bounds} point={point} "
+                f"hit_pid={hit_pid} hit_type={hit_type}"
+            )
+            if hit_pid != pid:
+                return False, detail + " is outside the owned process"
+            current = hit
+            for _ in range(16):
+                if auto.ControlsAreSame(current, item):
+                    break
+                current = current.GetParentControl()
+                if current is None:
+                    return False, detail + " is outside the expected row"
+            else:
+                return False, detail + " ancestry did not reach the expected row"
+        except Exception as exc:
+            return False, f"row={index} point discovery failed ({type(exc).__name__})"
+    return True, f"all {len(items)} first-cell points hit their owned rows"
+
+
 @pytest.fixture
 def wpf_worklist(tmp_path: Path):
     if sys.platform != "win32" or os.environ.get("OPENADAPT_TEST_WINDOWS_UIA") != "1":
@@ -153,8 +193,10 @@ def start(rows: list[dict[str, Any]]) -> list[Any]:
             )
             procs.append(proc)
             owned.callback(stop_owned_process, proc)
+            last_point_state = "point discovery not reached"
 
             def items_from_owned_window(hwnd: int) -> Optional[list[Any]]:
+                nonlocal last_point_state
                 window = auto.ControlFromHandle(hwnd)
                 if window is None:
                     return None
@@ -165,23 +207,33 @@ def items_from_owned_window(hwnd: int) -> Optional[list[Any]]:
                 listing = window.ListControl(searchDepth=3)
                 if not listing.Exists(0, 0):
                     return None
-                return [
+                items = [
                     child
                     for child in listing.GetChildren()
                     if child.ControlTypeName == "ListItemControl"
                 ]
+                if len(items) != len(rows):
+                    last_point_state = f"UIA exposes {len(items)} of {len(rows)} rows"
+                    return None
+                ready, last_point_state = _wpf_points_ready(auto, items, proc.pid)
+                return items if ready else None
 
             # One startup budget inside the existing 180s test limit leaves
             # 60s for the unchanged census assertions and owned cleanup.
-            return wait_for_rendered_items(
-                proc,
-                ready,
-                len(rows),
-                items_from_owned_window,
-                stdout_path,
-                stderr_path,
-                timeout_s=120.0,
-            )
+            try:
+                return wait_for_rendered_items(
+                    proc,
+                    ready,
+                    len(rows),
+                    items_from_owned_window,
+                    stdout_path,
+                    stderr_path,
+                    timeout_s=120.0,
+                )
+            except NativeAppStartupError as exc:
+                raise NativeAppStartupError(
+                    f"{exc}; point_readiness={last_point_state}"
+                ) from exc
 
         yield auto, start