diff --git a/packages/capture/docs/WINDOW_CAPTURE.md b/packages/capture/docs/WINDOW_CAPTURE.md index 7390414c57..9230077172 100644 --- a/packages/capture/docs/WINDOW_CAPTURE.md +++ b/packages/capture/docs/WINDOW_CAPTURE.md @@ -75,6 +75,22 @@ In this mode: viewport, and geometry generation. Recorder shutdown retains one final frame after input has stopped, so a consumer can select an exact after-action frame by ordinal instead of by nearest timestamp. +- **Dropped input is recorded, not silently missing.** Each native input + reserves a source ordinal when it arrives. Input that did not go to the + target window (a click on another window, typing in another app, the + release of a dropped press) is dropped, and its ordinal holds no event. + Capture writes each dropped ordinal and its reason code to + `source-ordinal-skips.json`, which the artifact manifest seals. The verifier + still requires every ordinal from 1 to the last one. It accepts an empty + ordinal only when that record names it with a known reason, in a + window-scoped capture. Any other hole fails verification. The record holds + ordinals and reason codes only. +- **macOS system chrome is hit-tested.** The Quartz window list can include + full-screen windows at the Dock level and above (the Dock, Notification + Center) that take clicks only where they draw. Such a window counts as the + window under the pointer only when the system's Accessibility hit test + says its process takes the click there. When the hit test can't answer, + the click is dropped (fail closed). Other apps' windows block as before. - **Window movement and resize are supported.** The first frame fixes the encoded video size. Later source frames scale to fit and letterbox into that viewport. Input uses the exact current bounds and content rectangle. No frame diff --git a/packages/capture/openadapt_capture/capture.py b/packages/capture/openadapt_capture/capture.py index 30189f8f5c..9f2c230f9d 100644 --- a/packages/capture/openadapt_capture/capture.py +++ b/packages/capture/openadapt_capture/capture.py @@ -47,6 +47,10 @@ WindowCaptureStateV2, ) from openadapt_capture.processing import process_events +from openadapt_capture.source_skips import ( + SourceOrdinalSkipError, + load_source_ordinal_skips, +) from openadapt_capture.structural import StructuralObservation if TYPE_CHECKING: @@ -631,7 +635,14 @@ def _validate_database_contract( ordinals_by_kind[kind] = set(typed_ordinals) all_ordinals.extend(typed_ordinals) - calculated_last = max(all_ordinals, default=None) + # Ordinals that window-scoped recording dropped by policy hold no event. + # Only the sealed skip record can account for them. + try: + skipped_ordinals = load_source_ordinal_skips(capture.capture_dir) + except SourceOrdinalSkipError as exc: + raise InvalidCaptureEvent(str(exc)) from exc + + calculated_last = max([*all_ordinals, *skipped_ordinals], default=None) if calculated_last != last_source_ordinal: raise InvalidCaptureEvent("sealed last source ordinal differs from the immutable database") @@ -716,6 +727,10 @@ def _validate_database_contract( and desktop_metadata.get("coordinate_space") == "virtual_desktop_pixels" ) is_native_scoped = is_v2 or is_desktop_v1 + if skipped_ordinals and not is_v2: + raise InvalidCaptureEvent( + "sealed source-ordinal skips are only valid for a window-scoped capture" + ) if is_native_scoped and not rows_by_kind["screen"]: raise InvalidCaptureEvent("sealed native capture has no retained frames") if is_desktop_v1: @@ -846,7 +861,9 @@ def _validate_database_contract( ordinal in allowed_pair_ordinals and kinds == {"screen", "window"} ): raise InvalidCaptureEvent("sealed journal reuses a source ordinal across events") - if is_native_scoped and sorted(owners) != list( + if not owners.keys().isdisjoint(skipped_ordinals): + raise InvalidCaptureEvent("sealed journal has an event at a skipped source ordinal") + if is_native_scoped and sorted(set(owners) | set(skipped_ordinals)) != list( range(1, (last_source_ordinal or 0) + 1) ): raise InvalidCaptureEvent("sealed native source journal has a missing ordinal") diff --git a/packages/capture/openadapt_capture/recorder.py b/packages/capture/openadapt_capture/recorder.py index 131ef71748..06740598e4 100644 --- a/packages/capture/openadapt_capture/recorder.py +++ b/packages/capture/openadapt_capture/recorder.py @@ -80,6 +80,11 @@ create_input_observer, ) from openadapt_capture.live import LiveEventTap, publish_live +from openadapt_capture.source_skips import ( + INPUT_CONTINUES_DROPPED_INPUT, + SOURCE_ORDINAL_SKIP_REASONS, + write_source_ordinal_skips, +) from openadapt_capture.structural import ( StructuralObservationRequest, StructuralObserver, @@ -290,6 +295,7 @@ class _JournalEntry: error: BaseException | None = None ready: bool = False discarded: bool = False + discard_reason: str | None = None class EventReservation: @@ -335,12 +341,20 @@ def fail(self, error: BaseException) -> None: self._finished = True self._journal._condition.notify_all() - def discard(self) -> None: - """Release this position without an event; the consumer skips it.""" + def discard(self, reason: str) -> None: + """Release this position without an event because policy dropped it. + + The consumer skips the position and the journal records the ordinal + with ``reason`` (one of ``SOURCE_ORDINAL_SKIP_REASONS``), so the + sealed journal names every intentional hole. + """ if self._finished: raise RuntimeError("the event journal reservation is already complete") + if reason not in SOURCE_ORDINAL_SKIP_REASONS: + raise EventJournalOrderingError(f"unknown source-ordinal skip reason: {reason!r}") with self._journal._condition: self._entry.discarded = True + self._entry.discard_reason = reason self._entry.ready = True self._finished = True self._journal._condition.notify_all() @@ -387,9 +401,9 @@ def admit_continuation(self) -> None: """Accept the release or drag of a press the target already received.""" self._admitted = True - def discard(self) -> None: + def discard(self, reason: str) -> None: """Drop this input: its journal position completes without an event.""" - self._reservation.discard() + self._reservation.discard(reason) @property def source_ordinal(self) -> int: @@ -579,6 +593,7 @@ def __init__(self) -> None: self._condition = threading.Condition() self._entries: deque[_JournalEntry] = deque() self._next_sequence = 1 + self._skipped: list[tuple[int, str]] = [] def reserve(self, timestamp: float) -> EventReservation: timestamp = float(timestamp) @@ -594,6 +609,24 @@ def last_source_ordinal(self) -> int | None: with self._condition: return self._next_sequence - 1 or None + @property + def skipped_source_ordinals(self) -> tuple[tuple[int, str], ...]: + """Return ``(ordinal, reason)`` for each dropped position consumed so far.""" + with self._condition: + return tuple(self._skipped) + + def _pop_discarded_head_locked(self) -> bool: + """Consume one ready discarded head entry and record its skip.""" + if not (self._entries and self._entries[0].ready and self._entries[0].discarded): + return False + entry = self._entries.popleft() + if entry.discard_reason not in SOURCE_ORDINAL_SKIP_REASONS: + raise EventJournalReservationError( + "an ordered event reservation was dropped without a policy reason" + ) + self._skipped.append((entry.sequence, entry.discard_reason)) + return True + def _reserve_locked(self, timestamp: float) -> _JournalEntry: entry = _JournalEntry(timestamp, self._next_sequence) self._next_sequence += 1 @@ -623,8 +656,9 @@ def reserve_window_action( ) else: binding = window_scope.generation_for_action() - except WindowInputNotOnTarget: + except WindowInputNotOnTarget as exc: entry.discarded = True + entry.discard_reason = exc.reason entry.ready = True reservation._finished = True self._condition.notify_all() @@ -702,10 +736,10 @@ def get(self, block: bool = True, timeout: float | None = None) -> Event: deadline = None if timeout is None else time.monotonic() + timeout with self._condition: while True: + if self._pop_discarded_head_locked(): + continue if self._entries and self._entries[0].ready: entry = self._entries.popleft() - if entry.discarded: - continue if entry.error is not None: raise EventJournalReservationError( "an ordered event producer failed after reservation" @@ -727,8 +761,8 @@ def get_nowait(self) -> Event: def empty(self) -> bool: with self._condition: - while self._entries and self._entries[0].ready and self._entries[0].discarded: - self._entries.popleft() + while self._pop_discarded_head_locked(): + pass return not self._entries def qsize(self) -> int: @@ -2648,9 +2682,9 @@ def __init__(self, scope: WindowCaptureScope, gap_log: WindowInputGapLog) -> Non self._held_buttons: set[str] = set() @staticmethod - def _discard(reservation: object | None) -> None: + def _discard(reservation: object | None, reason: str) -> None: if isinstance(reservation, WindowActionReservation): - reservation.discard() + reservation.discard(reason) def dropped(self, event: ObservedInput, reason: str) -> None: """Record a marker for dropped input, never its content.""" @@ -2667,13 +2701,13 @@ def admit(self, event: ObservedInput, reservation: object | None) -> bool: if isinstance(event, ObservedKey) and not event.pressed: identity = _key_identity(event) if identity not in self._held_keys: - self._discard(reservation) + self._discard(reservation, INPUT_CONTINUES_DROPPED_INPUT) return False self._held_keys.discard(identity) continuation = True elif isinstance(event, ObservedMouseButton) and not event.pressed: if event.button not in self._held_buttons: - self._discard(reservation) + self._discard(reservation, INPUT_CONTINUES_DROPPED_INPUT) return False self._held_buttons.discard(event.button) continuation = True @@ -2696,7 +2730,7 @@ def admit(self, event: ObservedInput, reservation: object | None) -> bool: else: evidence = self._scope.assert_input_on_target(window, point=point) except WindowInputNotOnTarget as exc: - self._discard(reservation) + self._discard(reservation, exc.reason) self.dropped(event, exc.reason) return False @@ -3833,6 +3867,9 @@ def record( _raise_for_failed_processes(task_by_name, process_errors) if window_scope is not None: window_scope.assert_current() + # Name every ordinal policy dropped, so the sealed journal's holes + # are explicit. Unrecorded holes still fail verification. + write_source_ordinal_skips(capture_dir, event_q.skipped_source_ordinals) elif desktop_scope is not None: # Close the interval between the last captured frame and operator stop. # A topology change in that interval still invalidates the session. diff --git a/packages/capture/openadapt_capture/source_skips.py b/packages/capture/openadapt_capture/source_skips.py new file mode 100644 index 0000000000..816957ae09 --- /dev/null +++ b/packages/capture/openadapt_capture/source_skips.py @@ -0,0 +1,179 @@ +"""Sealed record of source ordinals that window-scoped recording dropped. + +Every native input reserves a source ordinal when it arrives, before Capture +knows whether the target window received it. Input that went elsewhere is +dropped by policy, so its ordinal holds no event. Capture writes each such +ordinal and its reason code here. The artifact manifest seals the file, and +the verifier accepts a source journal hole only when this file names it. A +hole with no record (a crash, a lost write, or a bug) still fails. + +The file holds ordinals and reason codes only: never key identity, text, +points, or window names. +""" + +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path +from typing import Iterable, Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator + +SOURCE_ORDINAL_SKIPS_FILENAME = "source-ordinal-skips.json" +SOURCE_ORDINAL_SKIPS_SCHEMA_VERSION = "openadapt.capture.source-ordinal-skips/v1" + +# The release (or drag) of a press that was itself dropped. It continues +# input the target never received, so it is dropped too. +INPUT_CONTINUES_DROPPED_INPUT = "continues_dropped_input" + +# Every reason a window-scoped recorder may drop input by policy. These match +# the ``INPUT_*`` reason codes in ``openadapt_capture.window_capture``. +SourceOrdinalSkipReason = Literal[ + "keyboard_focus_elsewhere", + "keyboard_focus_in_dialog", + "keyboard_focus_unknown", + "pointer_over_other_window", + "pointer_target_unknown", + "target_not_on_screen", + "continues_dropped_input", +] +SOURCE_ORDINAL_SKIP_REASONS: frozenset[str] = frozenset(SourceOrdinalSkipReason.__args__) + + +class SourceOrdinalSkipError(ValueError): + """The sealed source-ordinal skip record is malformed.""" + + +class SourceOrdinalSkipRun(BaseModel): + """Consecutive ordinals ``first..last`` dropped for one reason.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + first: int = Field(ge=1) + last: int = Field(ge=1) + reason: SourceOrdinalSkipReason + + @model_validator(mode="after") + def _ordered(self) -> "SourceOrdinalSkipRun": + if self.last < self.first: + raise ValueError("a skip run must not end before it starts") + return self + + +class SourceOrdinalSkips(BaseModel): + """Canonical, ordered, non-overlapping runs of dropped ordinals.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["openadapt.capture.source-ordinal-skips/v1"] + runs: tuple[SourceOrdinalSkipRun, ...] = Field(min_length=1) + + @model_validator(mode="after") + def _canonical_runs(self) -> "SourceOrdinalSkips": + previous: SourceOrdinalSkipRun | None = None + for run in self.runs: + if previous is not None: + if run.first <= previous.last: + raise ValueError("skip runs must be ordered and must not overlap") + if run.first == previous.last + 1 and run.reason == previous.reason: + raise ValueError("adjacent skip runs with one reason must be merged") + previous = run + return self + + def ordinals(self) -> dict[int, str]: + """Return every skipped ordinal and its reason.""" + return { + ordinal: run.reason + for run in self.runs + for ordinal in range(run.first, run.last + 1) + } + + +def _canonical_bytes(payload: object) -> bytes: + return ( + json.dumps(payload, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n" + ).encode("utf-8") + + +def build_source_ordinal_skips( + skips: Iterable[tuple[int, str]], +) -> SourceOrdinalSkips | None: + """Merge ``(ordinal, reason)`` pairs into canonical runs; ``None`` if empty.""" + ordered = sorted(skips) + runs: list[dict[str, object]] = [] + for ordinal, reason in ordered: + if reason not in SOURCE_ORDINAL_SKIP_REASONS: + raise SourceOrdinalSkipError(f"unknown source-ordinal skip reason: {reason!r}") + if runs and ordinal <= runs[-1]["last"]: + raise SourceOrdinalSkipError("a source ordinal was skipped twice") + if runs and ordinal == runs[-1]["last"] + 1 and reason == runs[-1]["reason"]: + runs[-1]["last"] = ordinal + else: + runs.append({"first": ordinal, "last": ordinal, "reason": reason}) + if not runs: + return None + try: + return SourceOrdinalSkips( + schema_version=SOURCE_ORDINAL_SKIPS_SCHEMA_VERSION, + runs=tuple(SourceOrdinalSkipRun(**run) for run in runs), + ) + except ValidationError as exc: + raise SourceOrdinalSkipError("invalid source-ordinal skips") from exc + + +def write_source_ordinal_skips( + capture_dir: str | os.PathLike[str], + skips: Iterable[tuple[int, str]], +) -> Path | None: + """Write the skip record once, durably; write nothing when there are none.""" + record = build_source_ordinal_skips(skips) + if record is None: + return None + directory = Path(capture_dir) + path = directory / SOURCE_ORDINAL_SKIPS_FILENAME + if os.path.lexists(path): + raise SourceOrdinalSkipError("the source-ordinal skip record already exists") + data = _canonical_bytes(record.model_dump(mode="json")) + fd, temporary = tempfile.mkstemp(prefix=".source-ordinal-skips-", dir=directory) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.link(temporary, path) # create-only: never replaces an existing record + finally: + os.unlink(temporary) + return path + + +def load_source_ordinal_skips(capture_dir: str | os.PathLike[str]) -> dict[int, str]: + """Return the sealed skipped ordinals and reasons; empty when there is no record.""" + path = Path(capture_dir) / SOURCE_ORDINAL_SKIPS_FILENAME + if not os.path.lexists(path): + return {} + if path.is_symlink() or not path.is_file(): + raise SourceOrdinalSkipError("the source-ordinal skip record is not a regular file") + raw = path.read_bytes() + try: + record = SourceOrdinalSkips.model_validate_json(raw) + except ValidationError as exc: + raise SourceOrdinalSkipError("the source-ordinal skip record is malformed") from exc + if raw != _canonical_bytes(record.model_dump(mode="json")): + raise SourceOrdinalSkipError("the source-ordinal skip record is not canonical") + return record.ordinals() + + +__all__ = [ + "INPUT_CONTINUES_DROPPED_INPUT", + "SOURCE_ORDINAL_SKIPS_FILENAME", + "SOURCE_ORDINAL_SKIPS_SCHEMA_VERSION", + "SOURCE_ORDINAL_SKIP_REASONS", + "SourceOrdinalSkipError", + "SourceOrdinalSkipRun", + "SourceOrdinalSkips", + "build_source_ordinal_skips", + "load_source_ordinal_skips", + "write_source_ordinal_skips", +] diff --git a/packages/capture/openadapt_capture/window_input_focus.py b/packages/capture/openadapt_capture/window_input_focus.py index d296ec4408..e07342869f 100644 --- a/packages/capture/openadapt_capture/window_input_focus.py +++ b/packages/capture/openadapt_capture/window_input_focus.py @@ -69,6 +69,12 @@ def _point_in_bounds(x: float, y: float, bounds: tuple[float, float, float, floa _MACOS_STATUS_LAYER = 25 # kCGModalPanelWindowLevel: an NSAlert or a password prompt. _MACOS_MODAL_PANEL_LAYER = 8 +# kCGDockWindowLevel. From here up to the status level are system chrome +# layers: the Dock (20) and system overlays such as Notification Center (23). +# Their windows can span a whole display while taking clicks only where they +# draw (the Dock's tiles, a notification banner), so their bounds alone do not +# say whether they are under the pointer. +_MACOS_DOCK_LAYER = 20 # Accessibility subroles of a focused window that is a dialog, not a # document window. Typing there never counts as typing into the target. _MACOS_DIALOG_SUBROLES = frozenset({"AXDialog", "AXSystemDialog"}) @@ -286,15 +292,62 @@ def keyboard_focus(self, *, at_receipt: bool = False) -> FocusOwner | None: return FocusOwner(window_id=raised["window_id"] or None, pid=pid, dialog=True) return FocusOwner(window_id=window_id, pid=pid) + @staticmethod + def _ax_pid_at(x: float, y: float) -> int | None: + """Return the pid that owns the Accessibility element at a point. + + This is the system's own hit test: it passes through the transparent + parts of a window. ``None`` when Accessibility cannot answer. + """ + try: + import ApplicationServices as ax + + success = ax.kAXErrorSuccess + system = ax.AXUIElementCreateSystemWide() + ax.AXUIElementSetMessagingTimeout(system, _MACOS_AX_FOCUS_TIMEOUT_SECONDS) + error, element = ax.AXUIElementCopyElementAtPosition(system, float(x), float(y), None) + if error != success or element is None: + return None + error, pid = ax.AXUIElementGetPid(element, None) + return int(pid) if error == success and pid else None + except Exception: + return None + def window_at(self, x: float, y: float) -> FocusOwner | None: - """Return the first front-to-back app window containing the point. + """Return the first front-to-back app window that takes a click at the point. A raised panel over the target (a modal password prompt) is returned as itself, so a click on it never counts as a click on the target. + + A system chrome window (Dock level and above, below the status level) + counts only where its process takes the click: the Dock's tiles count, + the transparent rest of a full-screen Dock or Notification Center + window does not. The Accessibility hit test decides. When it cannot + answer, the point's owner is unknown (fail closed). """ + hit_pid: int | None = None + hit_tested = False for info in self._onscreen_app_windows(): - if _point_in_bounds(x, y, info["bounds"]): + if not _point_in_bounds(x, y, info["bounds"]): + continue + if info["layer"] >= _MACOS_DOCK_LAYER: + if not hit_tested: + hit_pid = self._ax_pid_at(x, y) + hit_tested = True + if hit_pid is None: + return None + if hit_pid != info["pid"]: + continue # click-through system chrome return FocusOwner(window_id=info["window_id"], pid=info["pid"]) + # Normal window. Accessibility already named an owner while we + # skipped chrome: only accept this candidate when that owner + # agrees. An unresolved or contradictory owner (AX names a + # process absent from the Quartz candidates, or one present only + # at the filtered status-window level) must not attribute the + # click to the target behind the chrome. + if hit_tested and hit_pid != info["pid"]: + return None + return FocusOwner(window_id=info["window_id"], pid=info["pid"]) return None diff --git a/packages/capture/tests/test_capture_terminal.py b/packages/capture/tests/test_capture_terminal.py index 331d3aeb72..b6b21fc07c 100644 --- a/packages/capture/tests/test_capture_terminal.py +++ b/packages/capture/tests/test_capture_terminal.py @@ -15,6 +15,10 @@ from openadapt_capture.db import create_db, crud from openadapt_capture.db.models import ActionEvent, Recording, Screenshot, WindowEvent from openadapt_capture.events import window_geometry_epoch_sha256 +from openadapt_capture.source_skips import ( + SOURCE_ORDINAL_SKIPS_FILENAME, + write_source_ordinal_skips, +) from openadapt_capture.terminal import ( ARTIFACT_MANIFEST_FILENAME, CAPTURE_TERMINAL_FILENAME, @@ -74,6 +78,7 @@ def _v2_capture_directory( before_binding_ordinal: int | None = None, after_binding_ordinal: int | None = None, wrong_png_digest: bool = False, + skipped: tuple[tuple[int, str], ...] = (), video: bool = False, ) -> Path: """Build one sealed v2 capture for artifact-contract regressions.""" @@ -199,6 +204,7 @@ def _v2_capture_directory( if video: (capture_dir / "video.mp4").write_bytes(b"fixture") + write_source_ordinal_skips(capture_dir, skipped) seal_capture( capture_dir, session_id="v2-session", @@ -212,7 +218,9 @@ def _v2_capture_directory( "browser": 0, "video": len(frame_ordinals) if video else 0, }, - last_source_ordinal=max((*frame_ordinals, action_ordinal or 0)), + last_source_ordinal=max( + (*frame_ordinals, action_ordinal or 0, *(ordinal for ordinal, _ in skipped)) + ), ) return capture_dir @@ -226,6 +234,7 @@ def _desktop_capture_directory( before_binding_ordinal: int | None = None, after_binding_ordinal: int | None = None, wrong_png_digest: bool = False, + skipped: tuple[tuple[int, str], ...] = (), retain_png: bool = True, ) -> Path: """Build one sealed virtual-desktop capture for contract regressions.""" @@ -326,6 +335,7 @@ def _desktop_capture_directory( session.close() engine.dispose() + write_source_ordinal_skips(capture_dir, skipped) seal_capture( capture_dir, session_id="desktop-session", @@ -340,7 +350,12 @@ def _desktop_capture_directory( "video": 0, }, last_source_ordinal=max( - (*frame_ordinals, *window_ordinals, action_ordinal or 0) + ( + *frame_ordinals, + *window_ordinals, + action_ordinal or 0, + *(ordinal for ordinal, _ in skipped), + ) ) or None, ) @@ -632,6 +647,107 @@ def test_verified_loader_rejects_a_gap_in_the_v2_source_journal(tmp_path) -> Non CaptureSession.load_verified(capture_dir) +def test_verified_loader_accepts_a_sealed_policy_skip_in_the_v2_journal(tmp_path) -> None: + """A dropped click's ordinal holds no event; the sealed skip record names it.""" + capture_dir = _v2_capture_directory( + tmp_path, + frame_ordinals=(1, 3), + action_ordinal=None, + skipped=((2, "pointer_over_other_window"),), + ) + + CaptureSession.validate_sealed(capture_dir) + with CaptureSession.load_verified(capture_dir) as capture: + assert [frame.source_ordinal for frame in capture.frames()] == [1, 3] + + +def test_verified_loader_accepts_policy_skips_after_the_last_event(tmp_path) -> None: + capture_dir = _v2_capture_directory( + tmp_path, + frame_ordinals=(1, 3), + action_ordinal=2, + skipped=((4, "pointer_over_other_window"), (5, "continues_dropped_input")), + ) + + CaptureSession.validate_sealed(capture_dir) + + +def test_verified_loader_rejects_an_unrecorded_hole_beside_a_recorded_skip(tmp_path) -> None: + capture_dir = _v2_capture_directory( + tmp_path, + frame_ordinals=(1, 5), + action_ordinal=None, + skipped=((2, "pointer_over_other_window"), (4, "keyboard_focus_elsewhere")), + ) + + with pytest.raises(ValueError, match="source journal has a missing ordinal"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_a_skip_that_hides_a_retained_event(tmp_path) -> None: + capture_dir = _v2_capture_directory( + tmp_path, + skipped=((2, "pointer_over_other_window"),), + ) + + with pytest.raises(ValueError, match="event at a skipped source ordinal"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_policy_skips_outside_window_scope(tmp_path) -> None: + capture_dir = _desktop_capture_directory( + tmp_path, + frame_ordinals=(1, 3), + action_ordinal=None, + skipped=((2, "pointer_over_other_window"),), + ) + + with pytest.raises(ValueError, match="only valid for a window-scoped capture"): + CaptureSession.load_verified(capture_dir) + + +@pytest.mark.parametrize( + "payload", + [ + # An unknown reason is not a policy drop. + b'{"runs":[{"first":2,"last":2,"reason":"crashed"}],' + b'"schema_version":"openadapt.capture.source-ordinal-skips/v1"}\n', + # Not canonical: whitespace. + b'{"runs": [{"first": 2, "last": 2, "reason": "pointer_over_other_window"}],' + b' "schema_version": "openadapt.capture.source-ordinal-skips/v1"}\n', + # An empty record never stands in for a hole. + b'{"runs":[],"schema_version":"openadapt.capture.source-ordinal-skips/v1"}\n', + ], + ids=["unknown-reason", "non-canonical", "empty"], +) +def test_verified_loader_rejects_a_malformed_skip_record(tmp_path, payload) -> None: + capture_dir = _v2_capture_directory(tmp_path, frame_ordinals=(1, 3), action_ordinal=None) + # Re-seal over a hand-written record, as a writer bug would. + (capture_dir / ARTIFACT_MANIFEST_FILENAME).unlink() + (capture_dir / CAPTURE_TERMINAL_FILENAME).unlink() + (capture_dir / SOURCE_ORDINAL_SKIPS_FILENAME).write_bytes(payload) + seal_capture( + capture_dir, + session_id="v2-session", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=14.0, + event_counts={"action": 0, "screen": 2, "window": 2, "browser": 0, "video": 0}, + last_source_ordinal=3, + ) + + with pytest.raises(ValueError, match="source-ordinal skip record"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_a_skip_record_added_after_sealing(tmp_path) -> None: + capture_dir = _v2_capture_directory(tmp_path, frame_ordinals=(1, 3), action_ordinal=None) + write_source_ordinal_skips(capture_dir, [(2, "pointer_over_other_window")]) + + with pytest.raises(CaptureSealError): + CaptureSession.load_verified(capture_dir) + + def test_verified_loader_requires_a_desktop_after_frame_for_every_action( tmp_path, ) -> None: diff --git a/packages/capture/tests/test_source_skips.py b/packages/capture/tests/test_source_skips.py new file mode 100644 index 0000000000..458fa92a4e --- /dev/null +++ b/packages/capture/tests/test_source_skips.py @@ -0,0 +1,128 @@ +"""Policy-dropped source ordinals are recorded, never silently missing.""" + +from __future__ import annotations + +import queue + +import pytest + +from openadapt_capture import window_capture +from openadapt_capture.recorder import ( + Event, + EventJournalOrderingError, + OrderedEventJournal, +) +from openadapt_capture.source_skips import ( + INPUT_CONTINUES_DROPPED_INPUT, + SOURCE_ORDINAL_SKIP_REASONS, + SOURCE_ORDINAL_SKIPS_FILENAME, + SourceOrdinalSkipError, + build_source_ordinal_skips, + load_source_ordinal_skips, + write_source_ordinal_skips, +) + + +def test_skip_reasons_are_exactly_the_window_input_drop_reasons(): + drop_reasons = { + window_capture.INPUT_KEYBOARD_FOCUS_ELSEWHERE, + window_capture.INPUT_KEYBOARD_FOCUS_IN_DIALOG, + window_capture.INPUT_KEYBOARD_FOCUS_UNKNOWN, + window_capture.INPUT_POINTER_OVER_OTHER_WINDOW, + window_capture.INPUT_POINTER_TARGET_UNKNOWN, + window_capture.INPUT_TARGET_NOT_ON_SCREEN, + INPUT_CONTINUES_DROPPED_INPUT, + } + assert SOURCE_ORDINAL_SKIP_REASONS == drop_reasons + + +def test_skips_merge_into_canonical_runs_and_round_trip(tmp_path): + skips = [ + (7, "pointer_over_other_window"), + (2, "pointer_over_other_window"), + (3, "pointer_over_other_window"), + (4, "continues_dropped_input"), + ] + record = build_source_ordinal_skips(skips) + assert [(run.first, run.last, run.reason) for run in record.runs] == [ + (2, 3, "pointer_over_other_window"), + (4, 4, "continues_dropped_input"), + (7, 7, "pointer_over_other_window"), + ] + path = write_source_ordinal_skips(tmp_path, skips) + assert path == tmp_path / SOURCE_ORDINAL_SKIPS_FILENAME + assert load_source_ordinal_skips(tmp_path) == dict(skips) + # Only reason codes and ordinals are stored. + assert set(path.read_text()) <= set('{}[]":,_/.\n0123456789abcdefghijklmnopqrstuvwxyz-') + assert [p.name for p in tmp_path.iterdir()] == [SOURCE_ORDINAL_SKIPS_FILENAME] + + +def test_no_skips_writes_nothing(tmp_path): + assert write_source_ordinal_skips(tmp_path, []) is None + assert list(tmp_path.iterdir()) == [] + assert load_source_ordinal_skips(tmp_path) == {} + + +def test_skip_record_is_create_only(tmp_path): + write_source_ordinal_skips(tmp_path, [(2, "pointer_over_other_window")]) + with pytest.raises(SourceOrdinalSkipError, match="already exists"): + write_source_ordinal_skips(tmp_path, [(3, "pointer_over_other_window")]) + assert load_source_ordinal_skips(tmp_path) == {2: "pointer_over_other_window"} + + +@pytest.mark.parametrize( + "skips", + [ + [(2, "crashed")], + [(2, "pointer_over_other_window"), (2, "keyboard_focus_elsewhere")], + [(0, "pointer_over_other_window")], + ], + ids=["unknown-reason", "duplicate", "zero"], +) +def test_invalid_skips_are_refused(tmp_path, skips): + with pytest.raises(SourceOrdinalSkipError): + write_source_ordinal_skips(tmp_path, skips) + assert list(tmp_path.iterdir()) == [] + + +def test_journal_records_each_policy_drop_it_consumes(): + journal = OrderedEventJournal() + first = journal.reserve(1.0) + dropped = journal.reserve(2.0) + third = journal.reserve(3.0) + trailing = journal.reserve(4.0) + first.complete(Event(1.0, "action", {})) + dropped.discard("pointer_over_other_window") + third.complete(Event(3.0, "action", {})) + trailing.discard(INPUT_CONTINUES_DROPPED_INPUT) + + assert journal.get(timeout=1).source_ordinal == 1 + assert journal.get(timeout=1).source_ordinal == 3 + assert journal.empty() + with pytest.raises(queue.Empty): + journal.get(block=False) + assert journal.skipped_source_ordinals == ( + (2, "pointer_over_other_window"), + (4, INPUT_CONTINUES_DROPPED_INPUT), + ) + assert journal.last_source_ordinal == 4 + + +def test_journal_refuses_a_drop_without_a_policy_reason(): + journal = OrderedEventJournal() + reservation = journal.reserve(1.0) + with pytest.raises(EventJournalOrderingError, match="skip reason"): + reservation.discard("crashed") + assert not reservation.finished + assert journal.skipped_source_ordinals == () + + +def test_journal_never_records_a_failed_producer_as_a_skip(): + from openadapt_capture.recorder import EventJournalReservationError + + journal = OrderedEventJournal() + reservation = journal.reserve(1.0) + reservation.fail(RuntimeError("producer crashed")) + with pytest.raises(EventJournalReservationError): + journal.get(timeout=1) + assert journal.skipped_source_ordinals == () diff --git a/packages/capture/tests/test_window_input_focus.py b/packages/capture/tests/test_window_input_focus.py index 1d6d5cf26d..93ba72e01f 100644 --- a/packages/capture/tests/test_window_input_focus.py +++ b/packages/capture/tests/test_window_input_focus.py @@ -142,6 +142,7 @@ def __init__(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fake: FakePl self.capture_dir = tmp_path / "capture" self.observers: list[FedObserver] = [] self.error: BaseException | None = None + self.last_source_ordinal: int | None = None self.terminate_processing = multiprocessing.Event() self.terminate_recording = multiprocessing.Event() self._status_recv, self._status_send = multiprocessing.Pipe(duplex=False) @@ -200,7 +201,7 @@ def no_display(*_args, **_kwargs): def __enter__(self) -> FedObserver: def run(): try: - recorder_module.record( + self.last_source_ordinal = recorder_module.record( "window input focus", capture_dir=str(self.capture_dir), terminate_processing=self.terminate_processing, @@ -246,6 +247,36 @@ def actions(self) -> list[dict]: connection.close() return [dict(row) for row in rows] + def seal_and_validate(self) -> None: + """Finish the capture as ``Recorder`` does: verify, seal, re-validate.""" + from openadapt_capture.capture import CaptureSession + from openadapt_capture.db import finalize_capture_database + + db_path = self.capture_dir / "recording.db" + finalize_capture_database(str(db_path)) + connection = sqlite3.connect(db_path) + try: + counts = { + table: connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + for table in ("action_event", "screenshot", "window_event", "browser_event") + } + finally: + connection.close() + recorder = recorder_module.Recorder( + str(self.capture_dir), + capture_video=False, + capture_images=True, + ) + recorder._num_action_events.value = counts["action_event"] + recorder._num_screen_events.value = counts["screenshot"] + recorder._num_window_events.value = counts["window_event"] + recorder._num_browser_events.value = counts["browser_event"] + recorder._last_source_ordinal = self.last_source_ordinal + recorder._verify_completed_capture() + recorder._stage_completed_control_state() + recorder._seal_completed_capture() + CaptureSession.validate_sealed(str(self.capture_dir)) + def gaps(self) -> list[dict]: path = self.capture_dir / WINDOW_INPUT_GAPS_FILENAME if not path.exists(): @@ -325,6 +356,38 @@ def test_click_on_overlapping_window_inside_target_rectangle_is_dropped( assert left["coarse_point"] == [400.0, 300.0] +def test_dropped_input_leaves_a_sealed_contiguous_source_journal(tmp_path, monkeypatch, fake): + """A click in another window must not make the whole recording unverifiable. + + Its ordinals are named in the sealed skip record, and the capture still + passes the strict no-missing-ordinal check. + """ + from openadapt_capture.source_skips import load_source_ordinal_skips + + canary = f"pw-canary-{uuid.uuid4().hex}" + recording = WindowRecording(tmp_path, monkeypatch, fake) + with recording as observer: + click(observer) + fake.pointer_owner = SLACK # the user clicks another app's window + click(observer) + fake.keyboard_owner = SLACK + type_text(observer, "x", canary=canary) + fake.pointer_owner = TARGET + fake.keyboard_owner = TARGET + click(observer) + + assert len(_click_rows(recording.actions())) == 4 + skips = load_source_ordinal_skips(recording.capture_dir) + assert sorted(set(skips.values())) == [ + "continues_dropped_input", + INPUT_KEYBOARD_FOCUS_ELSEWHERE, + INPUT_POINTER_OVER_OTHER_WINDOW, + ] + assert len(skips) == 4 # press and release of the click and of the key + recording.seal_and_validate() + recording.assert_nowhere(canary) + + def test_password_dialog_over_target_drops_keys_and_clicks(tmp_path, monkeypatch, fake): canary = f"pw-canary-{uuid.uuid4().hex}" recording = WindowRecording(tmp_path, monkeypatch, fake) @@ -625,6 +688,8 @@ def fake_macos(monkeypatch): ) # Accessibility is unavailable unless a test names a focused window. state.ax_window = None # (window number, subrole, focused element role) + # The pid owning the Accessibility element at any point; None: unknown. + state.hit_pid = None success = 0 def copy_attribute(element, name, _out): @@ -650,7 +715,13 @@ def copy_attribute(element, name, _out): kAXErrorSuccess=success, AXUIElementCreateApplication=lambda _pid: "app", AXUIElementCreateSystemWide=lambda: "system", - AXUIElementGetPid=lambda _element, _out: (success, state.focused_app_pid), + AXUIElementGetPid=lambda element, _out: ( + success, + state.hit_pid if element == "hit" else state.focused_app_pid, + ), + AXUIElementCopyElementAtPosition=lambda _system, _x, _y, _out: ( + (success, "hit") if state.hit_pid is not None else (-25211, None) + ), AXUIElementSetMessagingTimeout=lambda _element, _seconds: success, AXUIElementCopyAttributeValue=copy_attribute, ) @@ -756,6 +827,113 @@ def test_macos_probe_window_at_is_front_to_back_layer_zero(fake_macos): assert probe.window_at(5000.0, 5000.0) is None +def test_macos_probe_window_at_passes_through_transparent_system_chrome(fake_macos): + """Full-screen Dock and Notification Center windows take clicks only where they draw.""" + from openadapt_capture.window_input_focus import MacOSInputFocusProbe + + probe = MacOSInputFocusProbe() + dock = _quartz_window(5, 50, (0, 0, 2000, 1200), layer=20) + notification_center = _quartz_window(6, 51, (0, 0, 2000, 1200), layer=23) + fake_macos.windows = [ + _quartz_window(1, 99, (0, 0, 2000, 25), layer=25), # menu bar extra + notification_center, + dock, + _quartz_window(7, 555, (350, 250, 200, 200)), # Slack over the target + _quartz_window(42, 1234, (300, 150, 800, 600)), + ] + # The system hit test lands on the target: the chrome is transparent there. + fake_macos.hit_pid = 1234 + assert probe.window_at(310.0, 170.0) == FocusOwner(window_id=42, pid=1234) + # Another app's window still blocks, whatever the chrome does. + fake_macos.hit_pid = 555 + assert probe.window_at(400.0, 300.0) == FocusOwner(window_id=7, pid=555) + # A Dock tile or a notification banner under the pointer takes the click. + fake_macos.hit_pid = 50 + assert probe.window_at(310.0, 170.0) == FocusOwner(window_id=5, pid=50) + fake_macos.hit_pid = 51 + assert probe.window_at(310.0, 170.0) == FocusOwner(window_id=6, pid=51) + # Without a system hit test the owner is unknown: fail closed. + fake_macos.hit_pid = None + assert probe.window_at(310.0, 170.0) is None + # Outside the chrome no hit test is needed. + fake_macos.windows = [dock, _quartz_window(42, 1234, (2100, 150, 800, 600))] + assert probe.window_at(2200.0, 170.0) == FocusOwner(window_id=42, pid=1234) + + +def test_macos_probe_window_at_refuses_ax_owner_absent_from_quartz(fake_macos): + """AX naming a process Quartz never listed must not pick the target. + + Codex BLOCK on #1984: after full-screen chrome is skipped, returning the + first normal window without reconciling its PID with the hit test can + attribute another process's click to the target. + """ + from openadapt_capture.window_input_focus import MacOSInputFocusProbe + + probe = MacOSInputFocusProbe() + fake_macos.windows = [ + _quartz_window(5, 50, (0, 0, 2000, 1200), layer=20), # Dock chrome + _quartz_window(42, 1234, (300, 150, 800, 600)), + ] + fake_macos.hit_pid = 9999 # process absent from the Quartz candidates + assert probe.window_at(310.0, 170.0) is None + + +def test_macos_probe_window_at_refuses_ax_owner_only_at_status_level(fake_macos): + """AX naming a status-bar PID (filtered from candidates) must refuse. + + The menu-bar extra is layer 25 and never enters ``_onscreen_app_windows``. + Skipping chrome then accepting the target behind it would admit the click. + """ + from openadapt_capture.window_input_focus import MacOSInputFocusProbe + + probe = MacOSInputFocusProbe() + fake_macos.windows = [ + _quartz_window(1, 99, (0, 0, 2000, 25), layer=25), # status; filtered + _quartz_window(5, 50, (0, 0, 2000, 1200), layer=20), # Dock chrome + _quartz_window(42, 1234, (300, 150, 800, 600)), + ] + fake_macos.hit_pid = 99 # status-window owner, not a normal candidate + assert probe.window_at(310.0, 170.0) is None + + +def test_macos_ownership_disagreement_refuses_assert_input_on_target(fake_macos): + """Admission must refuse when window_at reports unknown after disagreement.""" + from openadapt_capture.window_capture import TargetWindow + from openadapt_capture.window_input_focus import MacOSInputFocusProbe + + probe = MacOSInputFocusProbe() + fake_macos.windows = [ + _quartz_window(5, 50, (0, 0, 2000, 1200), layer=20), + _quartz_window(42, 1234, (300, 150, 800, 600)), + ] + fake_macos.hit_pid = 9999 + target = TargetWindow( + window_id=42, + owner="FakeApp", + title="Target", + pid=1234, + bounds=(300, 150, 800, 600), + on_screen=True, + ) + scope = WindowCaptureScope.__new__(WindowCaptureScope) + scope._focus_probe = probe + with pytest.raises(WindowInputNotOnTarget) as excinfo: + scope.assert_input_on_target(target, point=(310.0, 170.0)) + assert excinfo.value.reason == INPUT_POINTER_TARGET_UNKNOWN + + +def test_macos_chrome_does_not_change_keyboard_focus(fake_macos): + from openadapt_capture.window_input_focus import MacOSInputFocusProbe + + probe = MacOSInputFocusProbe() + fake_macos.windows = [ + _quartz_window(6, 51, (0, 0, 2000, 1200), layer=23), + _quartz_window(5, 50, (0, 0, 2000, 1200), layer=20), + _quartz_window(42, 1234, (300, 150, 800, 600)), + ] + assert probe.keyboard_focus() == FocusOwner(window_id=42, pid=1234) + + def test_windows_probe_uses_root_ancestor_of_foreground_and_point(monkeypatch): import ctypes