Native backends expose focused_field_label from the a11y tree

packages/flow/openadapt_flow/backends/linux_backend.py

@@ -699,6 +699,36 @@ def text_value_at(self, x: int, y: int) -> Optional[str]:
             return None
         return value if isinstance(value, str) else None
 
+    def focused_field_label(self) -> Optional[str]:
+        """Return the focused field's AT-SPI label, or None (never raises).
+
+        RECORD-TIME seam (:class:`openadapt_flow.backend.FieldLabelBackend`):
+        reads the focused element's ``LABELLED_BY`` relation, then its
+        accessible name. Never by position; None when there is no label so
+        compile-time OCR fallback still applies.
+        """
+
+        try:
+            window = (
+                self._require_same_window(self._captured_window)
+                if self._captured_window is not None
+                else self._resolve_window()
+            )
+            element = self._client.focused_element(window)
+            if element is None or not self._in_scope(element, window):
+                return None
+            reader = getattr(self._client, "field_label", None)
+            if callable(reader):
+                value = reader(element)
+            else:
+                value = element.name
+        except Exception:
+            return None
+        if isinstance(value, str):
+            text = " ".join(value.split())
+            return text or None
+        return None
+
     def focused_text_value(self) -> Optional[str]:
         """The exact text of the one focused editable field in this window."""
         reader = getattr(self._client, "editable_text", None)
@@ -1696,6 +1726,54 @@ def perform_native(self, element: LinuxElement, operation: str) -> str:
     #: Raw AT-SPI roles whose text is an editable value (not a password).
     _EDITABLE_ROLES = frozenset({"text", "entry", "editbar"})
 
+    def field_label(self, element: LinuxElement) -> Optional[str]:
+        """LABELLED_BY relation name, else the accessible name; None if empty."""
+
+        node = element.native
+        if node is not None:
+            labelled = self._labelled_by_name(node)
+            if labelled:
+                return labelled
+            name = _clean_text(self._call(node, "get_name", default=None))
+            if name:
+                return name
+        return _clean_text(element.name)
+
+    def _labelled_by_name(self, node: Any) -> Optional[str]:
+        """Name of the first LABELLED_BY relation target, if any."""
+
+        relation_set = self._call(node, "get_relation_set", default=None)
+        if not relation_set:
+            return None
+        labelled_by = getattr(self._atspi.RelationType, "LABELLED_BY", None)
+        if labelled_by is None:
+            return None
+        relations: list[Any] = []
+        try:
+            relations = list(relation_set)
+        except TypeError:
+            if isinstance(relation_set, list):
+                relations = relation_set
+            elif isinstance(relation_set, tuple):
+                relations = list(relation_set)
+        for relation in relations:
+            rel_type = self._call(relation, "get_relation_type", default=None)
+            if rel_type != labelled_by:
+                continue
+            target = None
+            get_target = getattr(relation, "get_target", None)
+            if callable(get_target):
+                try:
+                    target = get_target(0)
+                except Exception:
+                    target = None
+            if target is None:
+                continue
+            name = _clean_text(self._call(target, "get_name", default=None))
+            if name:
+                return name
+        return None
+
     def editable_text(self, element: LinuxElement) -> Optional[str]:
         """The full text of an editable text field; None for anything else."""
         node = element.native

packages/flow/openadapt_flow/backends/macos_backend.py

@@ -345,6 +345,28 @@ def _fingerprint(element: MacElement) -> str:
     return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
 
 
+def _ax_field_label(element: Any, copy_attr) -> Optional[str]:
+    """Best human label for one focused AX element, or None.
+
+    Prefers ``AXTitleUIElement`` (the associated label control), then the
+    element's own ``AXTitle`` / ``AXDescription``. Used by
+    :meth:`MacOSBackend.focused_field_label`; ``copy_attr`` is injectable so
+    unit tests can exercise the priority without ApplicationServices.
+    """
+
+    title_ui = copy_attr(element, "AXTitleUIElement")
+    if title_ui is not None:
+        for attribute in ("AXTitle", "AXDescription"):
+            text = _clean_text(copy_attr(title_ui, attribute))
+            if text:
+                return text
+    for attribute in ("AXTitle", "AXDescription"):
+        text = _clean_text(copy_attr(element, attribute))
+        if text:
+            return text
+    return None
+
+
 class QuartzMacAXClient:
     """Live :class:`MacAXClient` over the macOS Accessibility (AX) API.
 
@@ -1887,6 +1909,38 @@ def identity_candidates_at(self, x: int, y: int) -> Optional[dict[str, Any]]:
                 candidate["dx"] = candidate["dy"] = None
         return census
 
+    def focused_field_label(self) -> Optional[str]:
+        """Return the focused field's AX label, or None (never raises).
+
+        RECORD-TIME seam (:class:`openadapt_flow.backend.FieldLabelBackend`):
+        reads ``AXTitleUIElement`` then ``AXTitle`` / ``AXDescription`` on the
+        one focused AX element in this window. Never by position; None when
+        there is no label so compile-time OCR fallback still applies.
+        """
+
+        try:
+            window = (
+                self._captured_window
+                if self._captured_window is not None
+                else self._resolve_window()
+            )
+            if window is None:
+                return None
+            token = self._mac_client.focused_element_token(window)
+            if token is None:
+                return None
+            from ApplicationServices import AXUIElementCopyAttributeValue
+
+            def copy_attr(element: Any, attribute: str) -> Any:
+                error, value = AXUIElementCopyAttributeValue(element, attribute, None)
+                if error != 0:
+                    return None
+                return value
+
+            return _ax_field_label(token, copy_attr)
+        except Exception:  # noqa: BLE001 - a momentary AX failure is not fatal
+            return None
+
     def displayed_text_at(self, x: int, y: int) -> Optional[str]:
         """The text of the one AX element under (x, y), for a read-back.
 

packages/flow/openadapt_flow/backends/win_agent/server.py

@@ -756,6 +756,52 @@ def _target_fingerprint(control: object, candidate: dict[str, Any]) -> str:
     return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
 
 
+def _uia_labeled_by(control: object) -> Optional[object]:
+    """The UIA LabeledBy target for ``control``, or None."""
+
+    # uiautomation exposes LabeledBy as property id 30092; prefer a named
+    # constant when the package provides one.
+    try:
+        import uiautomation as auto  # noqa: PLC0415
+    except Exception:
+        auto = None
+    prop_id = 30092
+    if auto is not None:
+        prop_id = int(getattr(auto, "LabeledByProperty", prop_id) or prop_id)
+    getter = getattr(control, "GetPropertyValue", None)
+    if not callable(getter):
+        return None
+    try:
+        labeled = getter(prop_id)
+    except Exception:  # noqa: BLE001
+        return None
+    return labeled if labeled is not None else None
+
+
+def _uia_field_label(control: object) -> Optional[str]:
+    """Best human label for one focused UIA control: LabeledBy then Name."""
+
+    labeled = _uia_labeled_by(control)
+    if labeled is not None:
+        name = " ".join(str(_control_value(labeled, "Name", "") or "").split())
+        if name:
+            return name
+    name = " ".join(str(_control_value(control, "Name", "") or "").split())
+    return name or None
+
+
+def _focused_field_label(auto: Any) -> Optional[str]:
+    """Label of the one focused UIA control, or None."""
+
+    try:
+        focused = auto.GetFocusedControl()
+    except Exception:  # noqa: BLE001
+        return None
+    if focused is None:
+        return None
+    return _uia_field_label(focused)
+
+
 def _candidate(control: object) -> Optional[dict[str, Any]]:
     locator = _locator_from_control(control)
     bounds = _bounds(control)
@@ -1333,6 +1379,10 @@ def _perform_uia_initialized(
             locator = None
         return {"status": "ok", "locator": locator}
 
+    if operation == "focused-field-label":
+        _exact_object(payload, required=frozenset(), label=operation)
+        return {"status": "ok", "label": _focused_field_label(auto)}
+
     if operation == "tree":
         data = _exact_object(
             payload,
@@ -2634,6 +2684,7 @@ def do_POST(self) -> None:  # noqa: N802 - stdlib naming
                 "/window/root-at": ("uia", "window-root-at"),
                 "/window/owned-over": ("uia", "owned-window-over"),
                 "/uia/text-at-point": ("uia", "text-at-point"),
+                "/uia/focused-field-label": ("uia", "focused-field-label"),
                 "/uia/identity-candidates-at": ("uia", "identity-candidates-at"),
                 "/uia/find": ("uia", "find"),
                 "/uia/act": ("uia", "act"),

packages/flow/openadapt_flow/backends/windows_backend.py

@@ -681,6 +681,24 @@ def workflow_state_identity(self) -> Optional[str]:
 
     # -- structured-text identity (openadapt_flow.backend.IdentityBackend) --
 
+    def focused_field_label(self) -> Optional[str]:
+        """Return the focused field's UIA label, or None (never raises).
+
+        RECORD-TIME seam (:class:`openadapt_flow.backend.FieldLabelBackend`):
+        reads UIA ``LabeledBy`` then ``Name`` on the focused control via the
+        in-guest agent. Never by position; None when there is no label so
+        compile-time OCR fallback still applies.
+        """
+
+        typed = self._post_typed_read("/uia/focused-field-label", {})
+        if not typed.available or typed.payload is None:
+            return None
+        value = typed.payload.get("label")
+        if isinstance(value, str):
+            text = " ".join(value.split())
+            return text or None
+        return None
+
     def structured_text_at(self, x: int, y: int) -> Optional[str]:
         """Return the UI Automation text of the element/row under (x, y).