diff --git a/CHANGELOG.md b/CHANGELOG.md index 58ebd56b..e77fafec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Notable changes in each release, newest first. Releases before 0.9.7 are describ ## Unreleased -- Pause everything (Beacon iOS): `PUT /api/pause` stops the whole agent and keeps it stopped. It writes the state to disk first, withdraws every pending approval, stops desktop tasks, tasks and workers, and pauses every coding run through its control file, so the work is kept. Turns started through the WebSocket, the REST chat routes and `/v1` are cancelled at once. Every other turn (Telegram, Slack, Discord, the scheduler) stops at its next round, so a tool call already running finishes first, until a follow-up gives those turns tasks of their own; nothing new that needs an approval can start, since every pending approval is withdrawn. While paused, no new turn starts: the chat and OpenAI-compatible routes answer `423 paused`, and the agent loop itself refuses to start. A restart comes back paused, and an unreadable pause file counts as paused. `DELETE /api/pause` resumes, and only for a person: a device carrying the new person mark, which the owner sets in their own Telegram with `/person allow ` (`/person disallow` removes it, `/person` lists). The mark does nothing but allow resume: it is not the computer-use mark, and no REST route sets it. The owner can also pause and resume from their own Telegram with `/pause [reason]` and `/resume`, which nobody else can use, so a pause can always be lifted, even with no device marked. The API token cannot resume, and neither can a device without the mark, an owner device included, since the token can mint one. Resuming starts nothing. A pause holds even when its file cannot be written, and the answer then says `persisted: false`. The pause file is in the trusted-state floor (the gate, Seatbelt and the AppArmor profile), so the agent's shell cannot write or remove it; on Linux, reload the installed AppArmor profile to get the rule. `/api/status` gains `paused`, `GET /api/pause` answers `{paused, since}` to any device (a chat-only phone included, never who or why), `server_capabilities` lists `pause`, and a coding run's control file is now written atomically. The push alert and a doctor row follow. Every worker and coding run the daemon starts, and every model-written shell it starts as a task, a watcher or a cron job, is told the exact pause file (`PROMETHEUS_PAUSE_FILE`) and passes it on; one that cannot reach its folder treats the agent as paused. A turn from Telegram, Slack or Discord now runs as a task of its own, so a pause cancels it at once, like a turn from the web, instead of letting it finish its round. While paused, a cron job that falls due is skipped and recorded as `skipped` (its next run moves on, nothing is saved up), side model calls (titles, conversation summaries, memory extraction, the curator's model review) are not made and do not count as failures, a prompt hook that cannot be asked blocks its event, `POST /api/code`, `POST /api/computer/tasks` and `POST /api/cron/{name}/run` answer `423`, and Telegram's `/computer yes` starts no desktop task. +- Pause everything (Beacon iOS): `PUT /api/pause` stops the whole agent and keeps it stopped. It writes the state to disk first, withdraws every pending approval, stops desktop tasks, tasks and workers, and pauses every coding run through its control file, so the work is kept. Turns started through the WebSocket, the REST chat routes and `/v1` are cancelled at once. Every other turn (Telegram, Slack, Discord, the scheduler) stops at its next round, so a tool call already running finishes first, until a follow-up gives those turns tasks of their own; nothing new that needs an approval can start, since every pending approval is withdrawn. While paused, no new turn starts: the chat and OpenAI-compatible routes answer `423 paused`, and the agent loop itself refuses to start. A restart comes back paused, and an unreadable pause file counts as paused. `DELETE /api/pause` resumes, and only for a person: a device carrying the new person mark, which the owner sets in their own Telegram with `/person allow ` (`/person disallow` removes it, `/person` lists). The mark does nothing but allow resume: it is not the computer-use mark, and no REST route sets it. The owner can also pause and resume from their own Telegram with `/pause [reason]` and `/resume`, which nobody else can use, so a pause can always be lifted, even with no device marked. The API token cannot resume, and neither can a device without the mark, an owner device included, since the token can mint one. Resuming starts nothing. A pause holds even when its file cannot be written, and the answer then says `persisted: false`. The pause file is in the trusted-state floor (the gate, Seatbelt and the AppArmor profile), so the agent's shell cannot write or remove it; on Linux, reload the installed AppArmor profile to get the rule. `/api/status` gains `paused`, `GET /api/pause` answers `{paused, since}` to any device (a chat-only phone included, never who or why), `server_capabilities` lists `pause`, and a coding run's control file is now written atomically. A new pause pushes one "Prometheus paused" alert, saying who paused (never why), to owner devices with no live connection; `oara doctor` has a Pause row (an unreadable pause file is an error that counts as paused); the curator archives nothing while paused (`/curator run` refuses); and a pause file with a nonsense time counts as paused instead of stopping the daemon from starting. Every worker and coding run the daemon starts, and every model-written shell it starts as a task, a watcher or a cron job, is told the exact pause file (`PROMETHEUS_PAUSE_FILE`) and passes it on; one that cannot reach its folder treats the agent as paused. A turn from Telegram, Slack or Discord now runs as a task of its own, so a pause cancels it at once, like a turn from the web, instead of letting it finish its round. While paused, a cron job that falls due is skipped and recorded as `skipped` (its next run moves on, nothing is saved up), side model calls (titles, conversation summaries, memory extraction, the curator's model review) are not made and do not count as failures, a prompt hook that cannot be asked blocks its event, `POST /api/code`, `POST /api/computer/tasks` and `POST /api/cron/{name}/run` answer `423`, and Telegram's `/computer yes` starts no desktop task. - Docs: `docs/reference/pairing-join-requests.md` is the reference for how a new device asks to join and how the owner lets it in: the routes and who may call them, every request and response field, the statuses, the expiry and rate limits, the sealed token, the operator's WebSocket frames and the owner code (the explicit, single-use grant that makes an owner device). It is written from the code and pinned to it by `tests/test_pairing_reference.py`. No behaviour changes. - Approvals: Prometheus now says when nothing can answer an approval prompt. `security.approval_queue.enabled` ships `false`, and the queue is only built together with the Telegram gateway; without it every call that needs an approval (a third-party MCP tool, a write or a shell command in `cautious` or `strict` mode, a network or install command in a background turn) is refused within seconds as "Permission denied" and none is ever listed to approve, so `cautious` read as "asks the owner" and behaved as "refuses everything". The daemon logs a warning at start and `oara doctor` has an "Approval prompts" row, both naming the keys to set. Nothing is switched on for you. - Approvals: the grants in `security.grants` are now loaded when the daemon starts. A remembered approval (`/approve always`) was written to the file and then ignored at every restart, so the prompt it silenced came back each time, and a `kind: tool` row written by hand (for one third-party MCP tool) never took effect. After this, a grant lasts until you `/revoke` it. **On upgrade, grants saved by earlier versions take effect again at the first start:** the startup log lists each one by id, kind and tool, `/grants` shows them in full, and `/revoke` removes one. The daemon and `SecurityGate.from_config` now read the key in one place. diff --git a/docs/guide/api.md b/docs/guide/api.md index b8656ec2..d44b2156 100644 --- a/docs/guide/api.md +++ b/docs/guide/api.md @@ -43,7 +43,7 @@ app and kept current by CI, see | POST | `/api/identity/challenge` | Unauthenticated, so a paired client can check which Prometheus it reached **before** it sends its token. Body `{"v": 1, "nonce": }`; answer `{v, instance_id, node_pub, tls_spki_sha256, signed_at, sig}`, where `sig` is the node key's Ed25519 signature over six lines, each ending in a newline: `prometheus-identity-challenge/v1`, `nonce=`, `instance_id=`, `node_pub=`, `tls_spki_sha256=`, `signed_at=`. `tls_spki_sha256` is the served certificate's SPKI digest on the TLS port and `null` on the plain one. Over TLS the signature binds the node key to the channel; over a plain listener it proves the real instance is reachable, not that nothing sits in between. `400 bad_nonce`, `503 no_node_key` before first run (the route never makes the key), and hello's limits: no `Origin`, 60 a minute per peer shared with hello, never cached, no cookie. Stateless. | | POST | `/api/restart` | Operator only. Answers `202 {"restarting": true}`, then the daemon shuts down gracefully and starts again in the same process (it re-execs its own command line), so a saved setting such as the network mode takes effect without anyone touching the machine. Every socket drops; wait for `GET /api/hello` to answer with the same `fp`. `503 restart_unavailable` from a process that was not started as the daemon. The restart runs with the environment the daemon first started with, so the env file is read again (a rotated token takes effect). Refused to a web page: an `Origin` header is `400 browser_not_allowed`. | | GET | `/api/status` | Model, uptime, tools, memory, subsystem states — plus `node_pub` (the node's Ed25519 public key) and `instance_id` (the vault's UUID), both `null` until identity exists, and `name` and `fp`, the same values `GET /api/hello` answers (`fp` is `""` until the instance key exists). Bearer-gated deliberately; `/health` never carries identity. `paused`: `null`, or `{since, by, reason}` while everything is paused (`PUT /api/pause`) | -| PUT | `/api/pause` | Pause everything. Operator (the API token or an owner device); a scoped device gets `403 operator_only`. Body `{reason?}`. Writes the state to disk first, withdraws every pending approval, cancels every turn running in the daemon (the WebSocket, the REST chat routes, `/v1`, Telegram, Slack and Discord), stops desktop tasks, tasks and workers, and pauses (keeps) every coding run. A worker or coding run, a process of its own, also stops at its next round, and one that cannot reach the pause file's folder counts as paused. Answers `{paused: true, since, by, reason, persisted, stopped: {turns, desktop_tasks, tasks, coding_runs, approvals_withdrawn}}` and broadcasts `paused_changed` `{paused, since}` (who and why are on `/api/status`). The pause holds even if the file cannot be written; `persisted: false` then says it will not survive a restart. Idempotent: a second call is the same pause (same `since`). While paused, `POST /api/chat/send`, `POST /api/chat`, story dispatch, the Paperclip wake `/v1/chat/completions`, `POST /api/code`, `POST /api/computer/tasks` and `POST /api/cron/{name}/run` answer `423 {"error": "paused", since, detail}` (plus `by` and `reason` for an operator) and keep nothing; a cron job that falls due is skipped (status `skipped`, its next run moves on); no desktop task starts from any door (Telegram's `/computer yes` included); side model calls (titles, summaries, memory extraction, the curator) are not made; the WebSocket send gets an `error` frame of kind `paused`; reads and every stop route still work, and no slash command runs over the WebSocket. Survives a restart; an unreadable pause file means paused. `pause.json` is in the trusted-state floor, so the agent's shell cannot write or remove it (the gate, Seatbelt on macOS, the AppArmor profile on Linux; an installed AppArmor profile must be reloaded to get the rule) | +| PUT | `/api/pause` | Pause everything. Operator (the API token or an owner device); a scoped device gets `403 operator_only`. Body `{reason?}`. Writes the state to disk first, withdraws every pending approval, cancels every turn running in the daemon (the WebSocket, the REST chat routes, `/v1`, Telegram, Slack and Discord), stops desktop tasks, tasks and workers, and pauses (keeps) every coding run. A worker or coding run, a process of its own, also stops at its next round, and one that cannot reach the pause file's folder counts as paused. A pause stops what the daemon runs: a process the model has already detached from its shell (`nohup`, `setsid`) is outside it, and a process the model starts itself can be pointed at another pause file. Answers `{paused: true, since, by, reason, persisted, stopped: {turns, desktop_tasks, tasks, coding_runs, approvals_withdrawn}}` and broadcasts `paused_changed` `{paused, since}` (who and why are on `/api/status`). A new pause also pushes one alert, "Prometheus paused" with who paused (never why, and no category), to each owner device with no live connection; a second call pushes nothing. `oara doctor` has a Pause row. The pause holds even if the file cannot be written; `persisted: false` then says it will not survive a restart. Idempotent: a second call is the same pause (same `since`). While paused, `POST /api/chat/send`, `POST /api/chat`, story dispatch, the Paperclip wake `/v1/chat/completions`, `POST /api/code`, `POST /api/computer/tasks` and `POST /api/cron/{name}/run` answer `423 {"error": "paused", since, detail}` (plus `by` and `reason` for an operator) and keep nothing; a cron job that falls due is skipped (status `skipped`, its next run moves on); no desktop task starts from any door (Telegram's `/computer yes` included); side model calls (titles, summaries, memory extraction, the curator) are not made; the WebSocket send gets an `error` frame of kind `paused`; reads and every stop route still work, and no slash command runs over the WebSocket. Survives a restart; an unreadable pause file means paused. `pause.json` is in the trusted-state floor, so the agent's shell cannot write or remove it (the gate, Seatbelt on macOS, the AppArmor profile on Linux; an installed AppArmor profile must be reloaded to get the rule) | | GET | `/api/pause` | Whether everything is paused: `{paused, since}`. Any valid credential, scoped devices included; never who or why (those are on `/api/status`, operator-only) | | DELETE | `/api/pause` | Resume. The owner can also pause and resume from their own Telegram (`/pause [reason]`, `/resume`); nobody else can use those commands. **A person only**: a device carrying the person mark, which the owner sets in their own Telegram (`/person allow `, removed with `/person disallow `, listed with `/person`). The mark does nothing but allow resume: it is not the computer-use mark, no REST route sets it, and a held device cannot carry it. Anything else gets `403 person_required`: the API token, an open daemon, and any device without the mark, an owner device included (the token can mint one on this machine). Mark a device that is NOT on the daemon's own machine (your phone, say): a client on that machine keeps its token where the agent's shell could reach it if a floor were down, and the mark would then let the agent resume. Starts nothing: interrupted turns stay interrupted, coding runs stay paused until resumed one by one (`/api/code/{id}/resume`) | | GET | `/api/packs` | Discovered packs with load/refuse state, refusal reasons, quarantined-draft ids, and panel declarations. `wired: false` means the pack loader didn't run this boot (the bare `web` entrypoint), distinct from "no packs installed" | diff --git a/src/prometheus/cli/doctor.py b/src/prometheus/cli/doctor.py index a3aaec2e..ff65b058 100644 --- a/src/prometheus/cli/doctor.py +++ b/src/prometheus/cli/doctor.py @@ -47,6 +47,7 @@ class existed (daemon startup check + /doctor Telegram command) but had ADVERTISED_TOOLS_ROW, DiagnosticCheck, always_loaded_case, + check_pause, check_router, yaml_error_summary, ) @@ -1600,6 +1601,7 @@ def run_extended_checks( check_memory_extraction(config), check_whisper(config), *check_voice_remotes(config, timeout=timeout), + check_pause(), ] @@ -1629,9 +1631,11 @@ async def _scan_and_diagnose(): # per-gateway checks (SPRINT G3). "Router" is the same # check_router row the extended checks already carry, and # "Advertised tools" is check_advertised_tools' row, which adds - # the count of tools actually offered to the same case. + # the count of tools actually offered to the same case. "Pause" is + # check_pause, which the extended checks carry so it shows without + # the scan. if check.name in ("Config", "Inference", "Model", "Whisper STT", - "Telegram", "Router", ADVERTISED_TOOLS_ROW): + "Telegram", "Router", "Pause", ADVERTISED_TOOLS_ROW): continue # Missing SOUL.md/AGENTS.md doesn't break the loop — the fast # setup path deliberately skips identity. Warn, don't fail. diff --git a/src/prometheus/engine/pause.py b/src/prometheus/engine/pause.py index 295d32de..9f29f200 100644 --- a/src/prometheus/engine/pause.py +++ b/src/prometheus/engine/pause.py @@ -24,6 +24,7 @@ import asyncio import json import logging +import math import os import time from dataclasses import dataclass @@ -106,7 +107,17 @@ def read_file(path: Path) -> PauseInfo | None: since, by, reason = data.get("since"), data.get("by"), data.get("reason", "") if not isinstance(since, (int, float)) or isinstance(since, bool) or not isinstance(by, str): return _unreadable("bad fields") - return PauseInfo(since=float(since), by=by, reason=reason if isinstance(reason, str) else "") + try: + when = float(since) + except (OverflowError, ValueError): # an integer too large for a float + return _unreadable("bad time") + if not math.isfinite(when) or not 0 <= when < SINCE_MAX: + return _unreadable("bad time") # Infinity, NaN, negative, or past what a clock shows + return PauseInfo(since=when, by=by, reason=reason if isinstance(reason, str) else "") + + +#: Seconds since the epoch past which a ``since`` is nonsense (the year 5138): every clock function can show it. +SINCE_MAX = 1e11 def _write_file(path: Path, data: dict[str, Any]) -> None: diff --git a/src/prometheus/gateway/commands.py b/src/prometheus/gateway/commands.py index 6a19e4d4..c9e13d28 100644 --- a/src/prometheus/gateway/commands.py +++ b/src/prometheus/gateway/commands.py @@ -1227,6 +1227,11 @@ async def cmd_curator_run(*, dry_run: bool = False) -> str: instance = get_curator() if instance is None: return "Curator not wired (check learning.curator_enabled in config)." + from prometheus.engine import pause as _pause + + if _pause.current() is not None: + # Its non-model pass archives skill files: nothing runs while everything is paused. + return "Prometheus is paused. The curator runs again once it is resumed." try: run = await instance.run_once(dry_run=dry_run) except Exception as exc: diff --git a/src/prometheus/infra/doctor.py b/src/prometheus/infra/doctor.py index b6352015..6f301007 100644 --- a/src/prometheus/infra/doctor.py +++ b/src/prometheus/infra/doctor.py @@ -7,6 +7,7 @@ from __future__ import annotations import logging +import time from dataclasses import dataclass, field from pathlib import Path @@ -244,6 +245,34 @@ def check_always_loaded(config: dict | None) -> DiagnosticCheck: # Doctor # --------------------------------------------------------------------------- +def check_pause() -> DiagnosticCheck: + """Is everything paused (engine/pause.py)? Since when, never who or why (those are on ``/api/status``). An + unreadable pause file counts as paused, so it is an error with what to do.""" + from prometheus.engine import pause + + resume = ("Resume from the owner's Telegram (/resume), or DELETE /api/pause from a device carrying the " + "person mark (/person allow ).") + # The file this process reads: the daemon's own, or (a CLI run) the one it was told, else this machine's default. + where = pause.child_env_entry()[pause.CHILD_ENV] + try: + info = pause.current() + except Exception as exc: # noqa: BLE001 - a doctor row must not take the report down + return DiagnosticCheck("Pause", "platform", "error", + f"the pause state could not be read ({type(exc).__name__}) from {where}", fix=resume) + if info is None: + return DiagnosticCheck("Pause", "platform", "ok", f"not paused ({where})") + if info.by == "unreadable" and info.reason.startswith("pause state unreadable"): + return DiagnosticCheck( + "Pause", "platform", "error", + f"the pause file is unreadable ({info.reason.split(': ', 1)[-1]}), so everything counts as paused", + fix=f"{resume} Resuming rewrites the file; check its owner and permissions if it stays unreadable.") + try: + since = time.strftime("%Y-%m-%d %H:%M %Z", time.localtime(info.since)) + except (OverflowError, ValueError, OSError): + since = f"{info.since:.0f}" + return DiagnosticCheck("Pause", "platform", "warning", f"everything is PAUSED (since {since})", fix=resume) + + class Doctor: """Diagnoses Prometheus configuration against model capabilities.""" @@ -296,6 +325,7 @@ async def diagnose(self, state: AnatomyState) -> DiagnosticReport: checks.append(self._check_config_valid()) checks.append(check_router(self.config)) checks.append(check_always_loaded(self.config)) + checks.append(check_pause()) checks.append(self._check_data_dir()) checks.append(self._check_bootstrap_files()) checks.append(self._check_dependencies()) diff --git a/src/prometheus/learning/curator.py b/src/prometheus/learning/curator.py index b1aea102..8bdf8fdd 100644 --- a/src/prometheus/learning/curator.py +++ b/src/prometheus/learning/curator.py @@ -545,6 +545,13 @@ async def run_once(self, *, dry_run: bool = False) -> CuratorRun: log.exception("Curator: LLM review failed") run.errors.append(f"LLM review failed: {exc!s}") + # Paused while the model reviewed (engine/pause.py): nothing is moved. The suggestions come again next run. + from prometheus.engine import pause as _pause + + if run.prunings and _pause.current() is not None: + log.info("Curator: Prometheus was paused during the review; archiving nothing") + run.prunings = [] + # Apply prunings (move to .archive/, never delete). if run.prunings and not dry_run: never_loaded = {s["name"] for s in skills if s["days_ago"] is None} diff --git a/src/prometheus/push/dispatcher.py b/src/prometheus/push/dispatcher.py index fca6cd84..46073bf6 100644 --- a/src/prometheus/push/dispatcher.py +++ b/src/prometheus/push/dispatcher.py @@ -8,6 +8,9 @@ turn_completed → only to devices with NO live WS client. TURN. task_completed / task_failed → only when no live WS client. TASK. coding_complete → only when no live WS client. TASK. + a NEW pause (engine/pause.py) → owner devices with no live WS client, one + alert "Prometheus paused", no category + (``push_paused``, called by the pause itself). "Has a live WS client" is PER DEVICE (the bridge's websocket→identity map, which is why Piece 1 gates this piece): a desktop being open must not @@ -27,7 +30,9 @@ from __future__ import annotations +import asyncio import logging +import re import time from typing import Any @@ -45,6 +50,10 @@ } +#: "owner in Telegram (telegram:)": the chat account's id stays on the daemon (``/api/status``), out of a push. +_CHAT_ACCOUNT = re.compile(r"\s*\((?:telegram|slack|discord):[^)]*\)\s*$") + + def first_line(text: str, cap: int = 120) -> str: line = (text or "").strip().splitlines()[0] if (text or "").strip() else "" return line[:cap] @@ -137,6 +146,38 @@ async def _push_approval(self, payload: dict) -> None: for target in self._store.push_targets(): await self._deliver(target, body) + async def push_paused(self, info: Any) -> None: + """One alert for a NEW pause (Piece 17): "Prometheus paused", who paused, to every live OWNER device with + no live socket. A pause nobody knows about looks like a broken agent. + + No category: a lock screen offers nothing to act on (resuming is a person's, in the app). Never the reason, + which stays on ``/api/status``, and never a chat account's id. A store that cannot name its owners sends + nothing (fail closed). + + OWNER devices on purpose, not the approval or quiet-push target rules: the alert is the owner's, whatever + else a device may see. Sent to every target at once, so one device that never answers does not keep it + from the others (the pause caps the whole push).""" + try: + owners = self._store.owner_device_ids() + except Exception: + logger.warning("pause alert not sent: the device store could not name its owner devices", exc_info=True) + return + live = self._live_device_ids() + who = _CHAT_ACCOUNT.sub("", first_line(str(getattr(info, "by", "") or ""), cap=80)) or "someone" + body = { + "aps": { + "alert": {"title": "Prometheus paused", + "body": f"Paused by {who}. Nothing runs until a person resumes it."}, + "sound": "default", + }, + "paused_since": getattr(info, "since", None), + } + targets = [t for t in self._store.push_targets() if t.id in owners and t.id not in live] + results = await asyncio.gather(*(self._deliver(t, body) for t in targets), return_exceptions=True) + for target, result in zip(targets, results): + if isinstance(result, Exception): + logger.warning("pause alert to device %s failed: %s", target.id, type(result).__name__) + async def _push_quiet(self, kind: str, payload: dict) -> None: live = self._live_device_ids() category = _QUIET_KINDS[kind] diff --git a/src/prometheus/web/pause_routes.py b/src/prometheus/web/pause_routes.py index 59a1f082..a94402f3 100644 --- a/src/prometheus/web/pause_routes.py +++ b/src/prometheus/web/pause_routes.py @@ -216,9 +216,27 @@ async def pause_everything(app: Any, *, by: str, reason: str) -> dict: stopped, "" if persisted else "; NOT written to disk, so it will not survive a restart") # Every client hears THAT it is paused; who paused and why are the operator's (GET /api/status). await _broadcast(app, {"paused": True, "since": info.since}) + if _new: + await _push_alert(bridge, info) return {"paused": True, **info.to_dict(), "persisted": persisted, "stopped": stopped} +#: The pause has taken effect before the alert goes; a slow APNs must not hold the answer long. +PUSH_ALERT_TIMEOUT_S = 5.0 + + +async def _push_alert(bridge: Any, info: pause.PauseInfo) -> None: + """The "Prometheus paused" push (PR 3), once per pause. A failure is logged and never fails the pause.""" + dispatcher = getattr(bridge, "push_dispatcher", None) + if dispatcher is None or not hasattr(dispatcher, "push_paused"): + return + try: + async with asyncio.timeout(PUSH_ALERT_TIMEOUT_S): + await dispatcher.push_paused(info) + except Exception: # noqa: BLE001 - the pause stands regardless + logger.warning("pause: the push alert failed", exc_info=True) + + def register_pause_routes(app: Any, *, devices: Any = None) -> None: """*devices*: a callable returning the daemon's device store (the one the marks live in).""" global _app diff --git a/tests/test_pause_alert_and_doctor.py b/tests/test_pause_alert_and_doctor.py new file mode 100644 index 00000000..dd0a44dc --- /dev/null +++ b/tests/test_pause_alert_and_doctor.py @@ -0,0 +1,311 @@ +"""Pause everything (Beacon iOS Piece 17, PR 3): the push alert and the doctor row. + +Held here: + + * a NEW pause pushes one alert, "Prometheus paused", saying who paused, to every live owner device with no live + socket (the spec: "A pause nobody knows about looks like a broken agent"); no category, so a lock screen offers + nothing to act on; never the reason (it stays on ``/api/status``); never to a scoped or revoked device; a second + ``PUT /api/pause`` (the same pause) pushes nothing; a store that cannot name its owners sends nothing (fail + closed); a push that fails does not fail the pause; + * ``oara doctor`` says whether everything is paused: ok when not, a warning with since when (never who or why) + when paused, and an error with what to do when the pause file is unreadable (which counts as paused); + * ``/curator run`` refuses while paused instead of running its non-model pass. +""" + +from __future__ import annotations + +import asyncio +import types + +import pytest + +from prometheus.engine import pause + + +@pytest.fixture +def state(tmp_path): + s = pause.PauseState(tmp_path / "pause.json") + pause.install(s) + yield s + pause.install(None) + + +class _Sender: + def __init__(self) -> None: + self.sent: list[dict] = [] + + async def send(self, **kwargs): + from prometheus.push.apns import SendResult + + self.sent.append(kwargs) + return SendResult(outcome="ok", status=200) + + +def _store(tmp_path): + from prometheus.config.device_store import DeviceStore + + store = DeviceStore(tmp_path / "devices.db") + away = store.mint_owner("owner phone", "ios", by="test") + watching = store.mint_owner("owner tablet", "ios", by="test") + scoped = store.mint("scoped phone", "ios") + gone = store.mint_owner("old owner phone", "ios", by="test") + for d, tok in ((away, "tok-away"), (watching, "tok-watching"), (scoped, "tok-scoped"), (gone, "tok-gone")): + assert store.set_push(d["id"], tok, "sandbox", "ai.test.beacon") + store.revoke(gone["id"]) + return store, watching["id"] + + +def _dispatcher(store, sender, live_ids=()): + from prometheus.push.dispatcher import PushDispatcher + + bridge = types.SimpleNamespace(_ws_identity={object(): types.SimpleNamespace(id=i) for i in live_ids}) + return PushDispatcher(store, sender, bridge) + + +# ── the push ───────────────────────────────────────────────────────────────── + +def test_a_pause_alerts_owner_devices_with_no_live_socket(tmp_path): + store, watching = _store(tmp_path) + sender = _Sender() + info = pause.PauseInfo(since=1_700_000_000.0, by="owner phone", reason="private words") + asyncio.run(_dispatcher(store, sender, live_ids=[watching]).push_paused(info)) + assert [s["apns_token"] for s in sender.sent] == ["tok-away"] + body = sender.sent[0]["payload"] + assert body["aps"]["alert"]["title"] == "Prometheus paused" + assert "owner phone" in body["aps"]["alert"]["body"] + assert "category" not in body["aps"] + assert body["paused_since"] == 1_700_000_000.0 + assert "private words" not in str(body) + + +def test_a_store_that_cannot_name_its_owners_sends_no_pause_alert(tmp_path): + class _Store: + def push_targets(self): + return [types.SimpleNamespace(id="d1", apns_token="tok", environment="sandbox", bundle_id="", + push_failures=0)] + + def owner_device_ids(self): + raise RuntimeError("database is locked") + + sender = _Sender() + asyncio.run(_dispatcher(_Store(), sender).push_paused(pause.PauseInfo(since=1.0, by="owner"))) + assert sender.sent == [] + + +@pytest.fixture +def w(tmp_path, state): + pytest.importorskip("fastapi") + from tests.support.device_world import World + + return World(tmp_path) + + +class _Recorder: + def __init__(self, fail: bool = False) -> None: + self.calls: list[pause.PauseInfo] = [] + self.fail = fail + + async def push_paused(self, info): + self.calls.append(info) + if self.fail: + raise RuntimeError("apns down") + + +def test_pausing_pushes_once_and_pausing_again_pushes_nothing(w): + rec = _Recorder() + w.bridge.push_dispatcher = rec + assert w.call("op", "PUT", "/api/pause", json={"reason": "away"}).status_code == 200 + assert w.call("op", "PUT", "/api/pause", json={}).status_code == 200 + assert len(rec.calls) == 1 and rec.calls[0].reason == "away" + + +def test_a_push_that_fails_does_not_fail_the_pause(w): + w.bridge.push_dispatcher = _Recorder(fail=True) + r = w.call("op", "PUT", "/api/pause", json={}) + assert r.status_code == 200 and r.json()["paused"] is True + assert pause.current() is not None + + +def test_pausing_from_the_owners_telegram_pushes_too(w): + from prometheus.web.pause_routes import pause_from_chat + + rec = _Recorder() + w.bridge.push_dispatcher = rec + asyncio.run(pause_from_chat(by="telegram owner", reason="")) + assert len(rec.calls) == 1 + + +# ── the doctor row ─────────────────────────────────────────────────────────── + +def test_the_doctor_says_not_paused(state): + from prometheus.infra.doctor import check_pause + + row = check_pause() + assert row.status == "ok" and row.name == "Pause" + + +def test_the_doctor_says_paused_and_since_when_but_not_who_or_why(state): + from prometheus.infra.doctor import check_pause + + state.pause(by="owner phone", reason="private words") + row = check_pause() + assert row.status == "warning" and "paused" in row.message.lower() + assert "owner phone" not in row.message and "private words" not in row.message + assert row.fix and "/resume" in row.fix + + +def test_the_doctor_says_an_unreadable_pause_file_is_an_error_that_counts_as_paused(tmp_path, monkeypatch): + from prometheus.infra.doctor import check_pause + + pause.install(None) + path = tmp_path / "pause.json" + path.write_text("{not json", encoding="utf-8") + monkeypatch.setenv(pause.CHILD_ENV, str(path)) + row = check_pause() + assert row.status == "error" + assert "unreadable" in row.message.lower() and "paused" in row.message.lower() + assert row.fix and "/resume" in row.fix + + +def test_the_doctor_runs_the_pause_row(): + import inspect + + from prometheus.infra.doctor import Doctor + + assert "check_pause()" in inspect.getsource(Doctor.diagnose) + + +# ── /curator run ───────────────────────────────────────────────────────────── + +def test_curator_run_refuses_while_paused(state, monkeypatch): + from prometheus.gateway import commands + from prometheus.learning import curator + + ran: list[bool] = [] + + class _Curator: + async def run_once(self, *, dry_run=False): + ran.append(dry_run) + raise RuntimeError("stop here") + + monkeypatch.setattr(curator, "get_curator", lambda: _Curator()) + state.pause(by="owner", reason="") + reply = asyncio.run(commands.cmd_curator_run()) + assert ran == [] and "paused" in reply.lower() + + +# ── review fixes (#920): the doctor row everywhere, odd times, the archive step, Apple, the cap ───────────────── + +def test_the_pause_row_is_in_the_checks_that_always_run(tmp_path, monkeypatch): + """Review of #920: the row came only from the deep scan, so `oara doctor --no-scan`, a missing config or a failed + scan showed no Pause row and exited 0 with an unreadable pause file.""" + import inspect + + from prometheus.cli import doctor + + ok = doctor.DiagnosticCheck + monkeypatch.setattr(doctor, "check_inference", lambda *a, **k: (ok("Inference", "connectivity", "ok", "x"), + ok("Model", "model", "ok", "x"))) + monkeypatch.setattr(doctor, "check_web_port", lambda *a, **k: ok("Web", "connectivity", "ok", "x")) + pause.install(None) + path = tmp_path / "pause.json" + path.write_text("{not json", encoding="utf-8") + monkeypatch.setenv(pause.CHILD_ENV, str(path)) + rows = doctor.run_extended_checks({}, config_check=ok("Config", "platform", "ok", "x")) + [row] = [r for r in rows if r.name == "Pause"] + assert row.status == "error" + assert '"Pause"' in inspect.getsource(doctor.run_anatomy_checks) # not reported twice + + +def test_the_not_paused_row_names_the_file_it_read(state): + from prometheus.infra.doctor import check_pause + + assert str(state.path) in check_pause().message + + +@pytest.mark.parametrize("since", ["Infinity", "NaN", "-Infinity", "1e300", "-5", "1" + "0" * 400]) +def test_an_out_of_range_since_is_unreadable_not_a_crash(tmp_path, since): + from prometheus.infra.doctor import check_pause + + path = tmp_path / "pause.json" + path.write_text('{"paused": true, "since": ' + since + ', "by": "owner"}', encoding="utf-8") + info = pause.read_file(path) + assert info is not None and info.by == "unreadable" + s = pause.PauseState(path) # a daemon booting on it: paused, no raise + pause.install(s) + try: + assert s.current is not None and check_pause().status == "error" + finally: + pause.install(None) + + +def test_a_pause_landing_during_the_curators_review_archives_nothing(state, tmp_path, monkeypatch): + from prometheus.learning.curator import Curator + from prometheus.learning.skill_state import SkillStateStore + + auto = tmp_path / "skills" / "auto" + auto.mkdir(parents=True) + (auto / "a.md").write_text("---\nname: a\n---\n# A\n") + c = Curator(object(), auto_dir=auto, reports_dir=tmp_path / "curator", + state_store=SkillStateStore(tmp_path / "_state.json"), interval_seconds=60, telemetry=None) + + async def _review(skills): + state.pause(by="owner", reason="") # the pause lands while the model reviews + return [], [{"name": "a", "reason": "stale"}], "{}" + + archived: list = [] + monkeypatch.setattr(c, "_review_via_llm", _review) + monkeypatch.setattr(c, "_apply_prunings", lambda *a, **k: archived.append(a) or ([], [], [])) + asyncio.run(c.run_once()) + assert archived == [] and (auto / "a.md").exists() + + +def test_the_alert_does_not_carry_a_chat_id_to_apple(tmp_path): + store, _ = _store(tmp_path) + sender = _Sender() + info = pause.PauseInfo(since=1.0, by="owner in Telegram (telegram:123456789)") + asyncio.run(_dispatcher(store, sender).push_paused(info)) + assert sender.sent + for s in sender.sent: + assert "123456789" not in str(s["payload"]) and "owner in Telegram" in s["payload"]["aps"]["alert"]["body"] + + +def test_one_slow_device_does_not_keep_the_alert_from_the_others(tmp_path): + from prometheus.config.device_store import DeviceStore + + store = DeviceStore(tmp_path / "devices.db") + for name, tok in (("phone", "tok-1"), ("tablet", "tok-2")): + d = store.mint_owner(name, "ios", by="test") + assert store.set_push(d["id"], tok, "sandbox", "ai.test.beacon") + tried: list[str] = [] + + class _Slow(_Sender): + async def send(self, **kwargs): + tried.append(kwargs["apns_token"]) + if len(tried) == 1: + await asyncio.Event().wait() # the first device never answers + return await super().send(**kwargs) + + async def go(): + with pytest.raises(TimeoutError): + async with asyncio.timeout(0.5): + await _dispatcher(store, _Slow()).push_paused(pause.PauseInfo(since=1.0, by="owner")) + + asyncio.run(go()) + assert sorted(tried) == ["tok-1", "tok-2"] + + +def test_a_hanging_push_is_cut_off_and_the_pause_answers(w, monkeypatch): + import time as _time + + from prometheus.web import pause_routes + + class _Hang: + async def push_paused(self, info): + await asyncio.Event().wait() + + monkeypatch.setattr(pause_routes, "PUSH_ALERT_TIMEOUT_S", 0.2) + w.bridge.push_dispatcher = _Hang() + started = _time.monotonic() + r = w.call("op", "PUT", "/api/pause", json={}) + assert r.status_code == 200 and _time.monotonic() - started < 3