diff --git a/README.md b/README.md index 11ec23d..ce80d68 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ docs/METRICS.md -- the metrics seed: format, codex≈0, regeneration (it' docs/PROVIDERS.md -- per-provider auth + the AntSeed node docs/OPENAI-CODEX.md -- the ChatGPT-subscription provider (unofficial / ToS-risky) docs/USAGE_ENDPOINTS.md -- per-key usage stats API endpoints and auth behavior +model_lines.py / model_lines.json -- model lines: `line_` fields that follow the qualified head of a model series (docs/MODEL-LINES.md) +scripts/qualify_model_lines.py -- the job: probes successors with prompt suites and promotes or proposes them live_smoke.py -- drive real providers end-to-end tests/ -- the full host unit-test suite features/ -- BDD user-flow suite (behave); user_flows.json is the spec @@ -137,6 +139,11 @@ benchmarks/modalities/capabilities (`bench_intelligence`, `in_image`, `cap_tools`, …). See the core's `core/docs/SIGMA-POL.md` for the algebra; `config.live.lua` declares the host fields and the `default` policy. +To keep a policy from going stale when a model's successor ships, name a +**model line** instead of an exact family: `["is", "line_openai_sol"]` follows +whichever family is currently qualified as that series' head. See +[model lines](docs/MODEL-LINES.md). + ### Blessed policy templates Most callers should not write raw weights. `GET /x/policy/templates` lists the diff --git a/auth_proxy.py b/auth_proxy.py index d83aa61..e006724 100644 --- a/auth_proxy.py +++ b/auth_proxy.py @@ -2314,6 +2314,79 @@ async def _router_post_json(path: str, payload: dict) -> Response: "code": "upstream"}}) +async def _router_lines_call(method: str, path: str, + payload: dict | None = None) -> Response: + """Admin-dashboard passthrough to the router's model-line endpoints, which + sit behind the internal secret. Forwards status and body verbatim so a lost + compare-and-set (409) reaches the operator as such.""" + if _client is None: + return JSONResponse(status_code=503, content={"error": { + "message": "router client not ready", "type": "router_error", + "code": "upstream"}}) + try: + r = await _client.request(method, f"{UPSTREAM}{path}", json=payload, + headers=_router_admin_headers(), timeout=10.0) + return Response(content=r.content, status_code=r.status_code, + media_type="application/json") + except (httpx.ConnectError, httpx.ConnectTimeout): + # Never reached the router: nothing happened, safe to retry. + return JSONResponse(status_code=502, content={"error": { + "message": f"router {path} unavailable", "type": "router_error", + "code": "upstream"}}) + except Exception: + # Sent, but no answer came back. A head move may have committed: say so + # rather than inviting a blind retry that would undo it. + if method == "GET": + return JSONResponse(status_code=502, content={"error": { + "message": f"router {path} unavailable", "type": "router_error", + "code": "upstream"}}) + return JSONResponse(status_code=504, content={"error": { + "message": "no response from the router: the move may or may not " + "have been applied. Read /dashboard/api/lines before retrying.", + "type": "router_error", "code": "model_line_outcome_unknown"}}) + + +@app.get("/dashboard/api/lines") +async def dashboard_model_lines(request: Request) -> Response: + """Every model line with its current head, seed and recent audit events.""" + ctx, error = _require_admin_dashboard_auth(request) + if error: + return error + return await _router_lines_call("GET", "/x/lines") + + +async def _dashboard_model_line_move(line: str, action: str, + request: Request) -> Response: + ctx, error = _require_admin_dashboard_auth(request) + if error: + return error + if not re.fullmatch(r"[a-z][a-z0-9_]{0,47}", line): + return JSONResponse(status_code=400, content={"error": { + "message": "invalid line name", "type": "invalid_request_error", + "code": "model_line"}}) + try: + body = await request.json() + except Exception: + body = None + if not isinstance(body, dict): + return JSONResponse(status_code=400, content={"error": { + "message": "JSON body must be an object", + "type": "invalid_request_error", "code": "invalid_json"}}) + # The audit actor is the authenticated dashboard viewer, never a body field. + body = {**body, "actor": str(ctx.get("viewer"))} + return await _router_lines_call("POST", f"/x/lines/{line}/{action}", body) + + +@app.post("/dashboard/api/lines/{line}/promote") +async def dashboard_model_line_promote(line: str, request: Request) -> Response: + return await _dashboard_model_line_move(line, "promote", request) + + +@app.post("/dashboard/api/lines/{line}/rollback") +async def dashboard_model_line_rollback(line: str, request: Request) -> Response: + return await _dashboard_model_line_move(line, "rollback", request) + + def _backtest_timeframe_options(value: Any) -> dict[str, Any]: timeframe = _dashboard_timeframe(value or "7d") if timeframe == "runtime": diff --git a/docs/MODEL-LINES.md b/docs/MODEL-LINES.md new file mode 100644 index 0000000..2b7b004 --- /dev/null +++ b/docs/MODEL-LINES.md @@ -0,0 +1,218 @@ +# Model lines + +A policy that names an exact family goes stale the day its successor ships: + +```json +["family_eq", "gpt-6.1-sol"] +``` + +Every caller that copied the term has to be edited and redeployed, and until +someone does, the fleet quietly runs last quarter's model. A **model line** +moves that one fact into the router. The policy names the line; the router +knows which family is its head today: + +```json +["is", "line_openai_sol"] +``` + +The policy bytes, and so the policy id, never change when a line moves. + +## What a line is + +A line is a named pointer to the family currently **qualified** as the head of +a model series. It is declared in [`model_lines.json`](../model_lines.json): + +```json +"openai_sol": { + "description": "OpenAI general flagship (the -sol tier).", + "head": "gpt-6.1-sol", + "match": "^gpt-(\\d(?:\\.\\d+)?)-sol$" +} +``` + +- `head` is the **seed**: the family the line points at until someone moves it. +- `match` says which family names belong to the series. Its capture groups are + the version: groups compare in order, each split on `.`/`-` into integers, so + `gpt-6.2-sol` sorts after `gpt-6.1-sol` and a dated `…-flash-1201` after + `…-flash-1017`. Tier siblings (`-pro`, `:batch`, vendor-prefixed aliases) do + not match and are never candidates. Keep the pattern as tight as the real + naming: marketplace catalogs carry look-alike aliases (`glm-52` for 5.2), and + a loose `\d+` would rank one above `glm-5.3`. `null` means the line has no + series to follow and only an operator moves it. + +Every declared line becomes a model-group Bool observation field, +`line_`, true for exactly the head family. It composes like any other +field: + +```json +["policy", + ["and", ["meets_req"], ["not", ["is", "disabled"]], + ["or", ["is", "line_anthropic_sonnet"], ["is", "line_openai_sol"]]], + ["add", ["scale", 100, ["gate", ["is", "line_anthropic_sonnet"], ["lit", 1]]], + ["scale", 10, ["gate", ["is", "line_openai_sol"], ["lit", 1]]], + ["scale", 0.45, ["neg", ["normalize", ["field", "price_out"]]]]], + ["argmax"], ["id"], ["always", {"action": "next_candidate"}]] +``` + +The set of lines is static on purpose. A policy naming an undeclared line fails +admission (`undeclared field 'line_x'`) instead of resolving to "no candidate". + +This uses the `fields.lua` extension seam, like `cache_hot`: the engine observes +a Bool and learns nothing about lines. No engine change, no signature change. +Each engine holds the heads as a native table, so the field costs one lookup per +candidate — the same order as `family_eq`. + +## Where the head lives + +Which family a line points at **now** is operator state in the host store +(`model_line_heads`), with an append-only audit trail (`model_line_events`). A +line with no row serves its seed. Every router replica re-reads the heads on a +short timer (`MODEL_LINES_REFRESH_S`, default 30; `0` pins the boot-time heads), +so a move reaches the whole pool without a reload fan-out or a restart. + +Fail-safe directions: + +- Store unreachable: replicas keep their last-known heads. They never snap back + to the seeds mid-outage. +- Definitions file missing: no lines are declared. Invalid: no lines are + declared and the reason is logged. Either way a policy naming a line is + refused loudly rather than served from a half-read set. +- A stored head for an undeclared line, or one that is not a plausible family + name, is ignored. + +`MODEL_LINES_FILE` points the router at a different definitions file. + +## Moving a head + +``` +GET /x/lines every line, head, seed, source, recent events +POST /x/lines/{line}/promote {"family", "expected_current"?, "force"?, "detail"?} +POST /x/lines/{line}/rollback {"expected_current"?, "expected_previous"?, "detail"?} +``` + +These are operator surfaces behind the router's internal secret. The dashboard +exposes the same three to an authenticated admin as `/dashboard/api/lines…` and +stamps the audit actor from the session, never from the request body. + +Both moves are compare-and-set when you say what you saw. `expected_current` +refuses the move with 409 if the line no longer points there. Send it on a +rollback too: the store picks the rollback target inside its own transaction, +and a rollback retried after a lost response then conflicts instead of undoing +itself. If the dashboard answers 504 `model_line_outcome_unknown`, the request +reached the router and may have been applied: read `/dashboard/api/lines` +before retrying. A family outside the line's own pattern is refused unless +`force` is set. + +## Qualification + +"Newer" is not "better for this workload". A successor can refuse a prompt the +head answers, blow the caller's deadline, or cost several times more through the +routes that actually serve it. So a line never moves on a version number alone. +[`scripts/qualify_model_lines.py`](../scripts/qualify_model_lines.py) is meant to +run on a schedule, outside the request-serving pods: + +1. Probe every line's current head with the prompt suites. This is the baseline + and a health check of what is live. +2. Find successors by the line's pattern and probe the newest few + (`max_candidates`) the same way: one family, no fallback, the caller's own + deadlines. Prompts within a family run sequentially, so the qualifier's own + burst is never what trips a provider's rate limit. +3. Gate each candidate against the head, on the same prompts in the same run: + + | Gate | Passes when | Knob | + |---|---|---| + | answers | share of valid answers inside the deadlines ≥ the overall bar, **and** ≥ the per-suite bar in every suite | `min_pass_rate` (0.9), `min_suite_pass_rate` (0.5) | + | latency | p90 of successful probes ≤ the absolute bound, **or** ≤ head's p90 × ratio | `max_latency_ms` (15000), `max_latency_ratio` (1.25) | + | cost | measured cost ≤ head's × ratio, summed over the probes both reported | `max_cost_ratio` (1.1) | + + - Cost is what the router **measured** for those calls, not a list price: + marketplace routes make list prices a poor predictor of what a call costs. + It is compared over the same probes on both sides, and a probe that + returned an unusable answer still counts what it charged. + - Every gate fails closed. An unknown cost, or too few probes with a cost on + both sides, qualifies nobody. + - A head that is merely degraded still serves as the baseline: it is the one + most worth replacing. + - `min_pass_rate` is also the bar a head must clear to count as healthy. It is + below 1 because one transient provider error in a run is noise; + `consecutive_passes` is what guards against a lucky run, and the per-suite + bar is what stops a model that fails one whole workload from hiding behind + the others. Give a one-prompt suite `repeats` if a single miss should not + decide it. +4. A candidate that qualified on `consecutive_passes` runs in a row (2), all of + them against the head that is current now, is promoted or proposed, newest + first. A pass measured against an earlier head does not count. +5. A head that stops passing is always recorded. It is rolled back only in + `auto`, only if the move that made it current is this qualifier's own + promotion, and only if the head it replaced still passes. A head an operator + chose is never moved automatically, and the qualifier never undoes its own + rollback. +6. A probe stopped by the ingress or the network — the qualifier's own rate + limit or budget, an overloaded or unreachable router — is **inconclusive**, + not failed. A family with any such probe is not judged on that run: a + candidate keeps its streak, and a line whose head was affected is left + alone. + +The run stops with a non-zero exit if the store is unreachable or an audit +event cannot be written: the streak a promotion rests on is read from those +events. + +The qualifier reads the same definitions file as the router and writes to the +same store. Run it from the router's image with the same `MODEL_LINES_FILE` (or +none on both). A job with different knobs would promote into the shared store +by its own rules. + +Modes (`--mode` / `LINE_QUALIFIER_MODE`): + +| Mode | Probes | Audit events | Moves heads | +|---|---|---|---| +| `report` | yes | no | no | +| `propose` (default) | yes | yes, plus one standing `proposed` event | no | +| `auto` | yes | yes | yes | + +Start in `propose`: read the proposals in `/x/lines`, approve with a promote, +and switch to `auto` once the proposals match what you would have done by hand. + +Probes go through the public ingress with a consumer key +(`LINE_QUALIFIER_API_KEY`), so their spend lands in the call ledger under that +key like any other caller's. Give the key its own budget. + +### Prompt suites + +A suite is the workload a line has to keep serving. It is a JSON file; pass +files or directories with `--suites`: + +```json +{ + "name": "support-triage", + "request": {"temperature": 0.7, "max_tokens": 4000, + "response_format": {"type": "json_object"}, + "first_token_timeout_ms": 7000, "timeout_ms": 22000}, + "expect": {"type": "json", "required": { + "label": {"enum": ["bug", "billing", "other"]}, + "confidence": {"type": "integer", "min": 0, "max": 100}}}, + "repeats": 1, + "prompts": ["…", "…"] +} +``` + +- `request` carries what the real caller sends, deadlines included. A candidate + that only answers given more time than production allows has not qualified. +- `expect` is `text` (non-empty), `regex`, or `json` with `required` keys + (`enum`, or `type` with optional `min`/`max`). JSON is read from the first `{` + to the last `}`, the way lenient callers accept a fenced answer. +- A candidate must pass every suite. Use the caller's real prompt templates; + keep user-generated content out of the files. + +[`evaluations/line_suites/example.json`](../evaluations/line_suites/example.json) +is a runnable sample. + +## Limits + +- The series pattern is a heuristic for *what to try*, never for what to serve. + An odd naming scheme costs a wasted probe, not a bad promotion. +- Suites are a sample of the workload. A regression outside them is caught by + the live reliability fields a policy already gates on (`success_rate`, + breakers) and by the next run's head check, not prevented. +- One head per line, for every caller. Two workloads that need different heads + of the same series are two lines. diff --git a/evaluations/line_suites/example.json b/evaluations/line_suites/example.json new file mode 100644 index 0000000..eae84ac --- /dev/null +++ b/evaluations/line_suites/example.json @@ -0,0 +1,25 @@ +{ + "name": "example-ticket-triage", + "description": "Sample suite: classify a support ticket. Replace with your own workload's prompt templates.", + "request": { + "temperature": 0.7, + "max_tokens": 1000, + "response_format": {"type": "json_object"}, + "first_token_timeout_ms": 7000, + "timeout_ms": 22000 + }, + "expect": { + "type": "json", + "required": { + "label": {"enum": ["bug", "billing", "other"]}, + "confidence": {"type": "integer", "min": 0, "max": 100}, + "summary": {"type": "string"} + } + }, + "repeats": 1, + "prompts": [ + "Classify this support ticket.\n\nTicket: The export button returns a blank CSV since yesterday's update. Other pages work.\n\nReturn a JSON object with exactly: {\"label\": \"bug\" or \"billing\" or \"other\", \"confidence\": integer 0-100, \"summary\": \"one sentence\"}", + "Classify this support ticket.\n\nTicket: I was charged twice for the March invoice and need one of the charges refunded.\n\nReturn a JSON object with exactly: {\"label\": \"bug\" or \"billing\" or \"other\", \"confidence\": integer 0-100, \"summary\": \"one sentence\"}", + "Classify this support ticket.\n\nTicket: Do you have an office in Lisbon? I would like to visit next month.\n\nReturn a JSON object with exactly: {\"label\": \"bug\" or \"billing\" or \"other\", \"confidence\": integer 0-100, \"summary\": \"one sentence\"}" + ] +} diff --git a/host_store.py b/host_store.py index 7f7bd69..9b405c3 100644 --- a/host_store.py +++ b/host_store.py @@ -346,6 +346,30 @@ def _retention_days() -> int: )""", "CREATE INDEX IF NOT EXISTS idx_subject_budget_reservations_subject" " ON subject_budget_reservations(tenant_id, subject, period, expires_at)", + # Model lines (model_lines.py): the operator-visible pointer from a line name + # to the family currently qualified as its head. One row per line that has + # been moved off its seed; a line with no row reads its seed from + # model_lines.json. Operator state: never pruned by the retention sweep. + """CREATE TABLE IF NOT EXISTS model_line_heads ( + line TEXT PRIMARY KEY, + family TEXT NOT NULL, + previous_family TEXT, + updated_at BIGINT NOT NULL, + updated_by TEXT + )""", + # Append-only audit of every line decision: promotions, rollbacks, proposals + # awaiting approval and qualification runs, each with its evidence. + """CREATE TABLE IF NOT EXISTS model_line_events ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + ts BIGINT NOT NULL, + line TEXT NOT NULL, + kind TEXT NOT NULL, + family TEXT, + previous_family TEXT, + actor TEXT, + detail JSONB + )""", + "CREATE INDEX IF NOT EXISTS idx_model_line_events_line ON model_line_events(line, id)", ] _pool_lock = threading.Lock() @@ -746,6 +770,153 @@ def set_provider_overlays(providers: dict[str, Any]) -> bool: return False +# ---- model lines (durable operator state) --------------------------------------- + +def get_model_line_heads() -> "dict[str, dict[str, Any]] | None": + """Every line moved off its seed: line -> {family, previous_family, + updated_at, updated_by}. None (NOT {}) when the store is unreachable, so the + caller keeps its last-known heads instead of snapping back to the seeds.""" + try: + with _get_pool().connection() as conn: + cur = conn.execute( + "SELECT line, family, previous_family, updated_at, updated_by" + " FROM model_line_heads") + return {r[0]: {"family": r[1], "previous_family": r[2], + "updated_at": r[3], "updated_by": r[4]} + for r in cur.fetchall()} + except Exception as exc: # noqa: BLE001 + _log.warning("host_store get_model_line_heads failed: %s", exc) + return None + + +def set_model_line_head(line: str, family: str, *, seed_family: "str | None", + actor: str, kind: str = "promoted", + detail: "dict[str, Any] | None" = None, + expected_current: "str | None" = None) -> dict[str, Any]: + """Move a line's head and append the audit event, atomically. + + `seed_family` is the head a line with no row is currently serving, so the + first move records the right `previous_family`. `expected_current` makes the + move a compare-and-set: two qualifier runs (or a run and an operator) cannot + both promote from the same starting head. Returns {ok, family, + previous_family} or {ok: False, reason}.""" + try: + now = int(time.time()) + with _get_pool().connection() as conn: # one transaction + # Serialize movers of this line even when it has no row yet. + conn.execute("SELECT pg_advisory_xact_lock(hashtext(%s))", + ("model_line:" + line,)) + row = conn.execute( + "SELECT family FROM model_line_heads WHERE line=%s", + (line,)).fetchone() + current = row[0] if row else seed_family + if expected_current is not None and current != expected_current: + return {"ok": False, "reason": "conflict", "family": current} + if current == family: + return {"ok": False, "reason": "unchanged", "family": current} + conn.execute( + "INSERT INTO model_line_heads(line, family, previous_family," + " updated_at, updated_by) VALUES (%s,%s,%s,%s,%s)" + " ON CONFLICT (line) DO UPDATE SET family=EXCLUDED.family," + " previous_family=EXCLUDED.previous_family," + " updated_at=EXCLUDED.updated_at, updated_by=EXCLUDED.updated_by", + (line, family, current, now, actor)) + conn.execute( + "INSERT INTO model_line_events(ts, line, kind, family," + " previous_family, actor, detail) VALUES (%s,%s,%s,%s,%s,%s,%s)", + (now, line, kind, family, current, actor, + json.dumps(detail) if detail is not None else None)) + return {"ok": True, "family": family, "previous_family": current} + except Exception as exc: # noqa: BLE001 + _log.warning("host_store set_model_line_head failed: %s", exc) + return {"ok": False, "reason": "store_unavailable"} + + +def rollback_model_line_head(line: str, *, seed_family: "str | None", actor: str, + detail: "dict[str, Any] | None" = None, + expected_current: "str | None" = None, + expected_previous: "str | None" = None) -> dict[str, Any]: + """Swap a line back to the head it had before its last move, atomically. + + The current and previous heads are read INSIDE the locked transaction, so + the target is the predecessor of the move that is actually current — not of + one the caller read earlier. `expected_current` / `expected_previous` turn + it into a compare-and-set on that move. Returns {ok, family, + previous_family} or {ok: False, reason, family}.""" + try: + now = int(time.time()) + with _get_pool().connection() as conn: + conn.execute("SELECT pg_advisory_xact_lock(hashtext(%s))", + ("model_line:" + line,)) + row = conn.execute( + "SELECT family, previous_family FROM model_line_heads WHERE line=%s", + (line,)).fetchone() + current = row[0] if row else seed_family + previous = row[1] if row else None + if expected_current is not None and current != expected_current: + return {"ok": False, "reason": "conflict", "family": current} + if not previous or previous == current: + return {"ok": False, "reason": "no_previous", "family": current} + if expected_previous is not None and previous != expected_previous: + return {"ok": False, "reason": "conflict", "family": current} + conn.execute( + "UPDATE model_line_heads SET family=%s, previous_family=%s," + " updated_at=%s, updated_by=%s WHERE line=%s", + (previous, current, now, actor, line)) + conn.execute( + "INSERT INTO model_line_events(ts, line, kind, family," + " previous_family, actor, detail) VALUES (%s,%s,%s,%s,%s,%s,%s)", + (now, line, "rolled_back", previous, current, actor, + json.dumps(detail) if detail is not None else None)) + return {"ok": True, "family": previous, "previous_family": current} + except Exception as exc: # noqa: BLE001 + _log.warning("host_store rollback_model_line_head failed: %s", exc) + return {"ok": False, "reason": "store_unavailable"} + + +def add_model_line_event(line: str, kind: str, *, family: "str | None" = None, + previous_family: "str | None" = None, actor: str = "", + detail: "dict[str, Any] | None" = None) -> bool: + """Append one audit event that does not move a head (a proposal, a failed + candidate, a qualification run).""" + try: + with _get_pool().connection() as conn: + conn.execute( + "INSERT INTO model_line_events(ts, line, kind, family," + " previous_family, actor, detail) VALUES (%s,%s,%s,%s,%s,%s,%s)", + (int(time.time()), line, kind, family, previous_family, actor, + json.dumps(detail) if detail is not None else None)) + return True + except Exception as exc: # noqa: BLE001 + _log.warning("host_store add_model_line_event failed: %s", exc) + return False + + +def model_line_events(line: "str | None" = None, limit: int = 50, + kinds: "tuple[str, ...] | None" = None) -> list[dict[str, Any]]: + """Newest-first audit events, optionally for one line and/or some kinds.""" + try: + where, params = [], [] + if line is not None: + where.append("line=%s") + params.append(line) + if kinds: + where.append("kind = ANY(%s)") + params.append(list(kinds)) + sql = ("SELECT id, ts, line, kind, family, previous_family, actor, detail" + " FROM model_line_events" + + (" WHERE " + " AND ".join(where) if where else "") + + " ORDER BY id DESC LIMIT %s") + with _get_pool().connection() as conn: + cur = conn.execute(sql, params + [max(1, min(int(limit), 500))]) + return [{"id": r[0], "ts": r[1], "line": r[2], "kind": r[3], + "family": r[4], "previous_family": r[5], "actor": r[6], + "detail": r[7]} for r in cur.fetchall()] + except Exception as exc: # noqa: BLE001 + _log.warning("host_store model_line_events failed: %s", exc) + return [] + + # ---- consumer_keys (durable operator state, credentials) ----------------------- def get_consumer_keys() -> "tuple[dict[str, Any], bool]": @@ -2656,7 +2827,8 @@ def truncate_all_for_tests() -> None: " peer_concurrency_leases," " consumer_budget_usage, analytics_hourly, analytics_rollup_state, peer_offers, buyer_status, route_observations," " login_history, provider_prices, wallet_ops," - " subject_budget_usage, subject_budget_reservations") + " subject_budget_usage, subject_budget_reservations," + " model_line_heads, model_line_events") def key_usage_rows(consumer_sha: str, since_ts: "int | None" = None, diff --git a/llm_router_host.py b/llm_router_host.py index f75a446..95bb01d 100644 --- a/llm_router_host.py +++ b/llm_router_host.py @@ -24,6 +24,7 @@ import host_store import route_reliability as _route_reliability +import model_lines as _model_lines from byo_http import is_byo_buyer from provider_adapters.anthropic import make_anthropic_async_call_provider from provider_adapters.bedrock import make_bedrock_async_call_provider @@ -216,6 +217,51 @@ def _inject_host_fields(self) -> None: } end """)(self.config) + self._inject_model_line_fields() + + def _inject_model_line_fields(self) -> None: + """Declare one model-group Bool per model line: `line_` is true for + the family currently qualified as that line's head (see model_lines.py). + + Same extension seam as `cache_hot`. The set of fields is fixed by the + definitions file, so a policy naming an undeclared line fails admission. + Which family a line points at lives in this runtime's `host_line_heads` + table, so the getter is one native table lookup per candidate — it runs + for every candidate of every request and must not cross into Python. + model_lines republishes that table to every live engine when a head + moves (tenant children and flow nodes included), so a promotion needs no + engine rebuild. A name the catalog already declares is left alone.""" + lines = _model_lines.definitions() + if not lines: + return + self.lua.execute("host_line_heads = {}") + declare = self.lua.eval(""" +function(cfg, line, field) + cfg.fields = cfg.fields or {} + if cfg.fields[field] ~= nil then return false end + cfg.fields[field] = { + sort = "Bool", default = false, group = "model", + get = function(c) + local fam = c.model_family + if fam ~= nil and host_line_heads[line] == fam then return true end + -- nil, not false: the engine applies the field default (false), and + -- model_meta()/the Market view list a line only on its head instead + -- of one false per line on every family in the catalog. + return nil + end, + } + return true +end +""") + for name in lines: + declare(self.config, name, _model_lines.field_name(name)) + _model_lines.subscribe(self) + + def sync_model_line_heads(self, heads: dict[str, str]) -> None: + """Replace this runtime's line -> head table. Called by model_lines on + subscribe and on every head move; rebinding one global is atomic to any + Lua code running between attempts.""" + self.lua.globals().host_line_heads = self.lua.table_from(heads) # ---- public API ----------------------------------------------------- diff --git a/model_line_qualifier.py b/model_line_qualifier.py new file mode 100644 index 0000000..b736b66 --- /dev/null +++ b/model_line_qualifier.py @@ -0,0 +1,590 @@ +"""Qualify successors for model lines by probing them with real workloads. + +"Newer" is not "better for this workload": a successor can refuse a prompt the +head answers, blow the caller's deadline, or cost several times more through the +routes that actually serve it. So a line never moves on a version number alone. +Each run: + +1. Probes every line's current head with the prompt suites — the baseline, and + a health check of what is live. +2. Finds successors by the line's name pattern (model_lines.successors) and + probes the newest few exactly the same way: one family, no fallback, the + caller's own deadlines. +3. A candidate QUALIFIES when it answers validly on enough probes, is no slower + than an absolute bound or than the head by more than a ratio, and its MEASURED + cost per probe is within the allowed ratio of the head's measured cost on the + same prompts. List prices are not + consulted: marketplace routes make them a poor predictor of what a call costs. +4. A candidate that qualified on enough consecutive runs is promoted (`auto`) or + recorded as a proposal for an operator (`propose`). `report` writes nothing. +5. A head that stops passing is flagged; in `auto`, a head this qualifier itself + promoted is rolled back when its predecessor still passes. +6. A probe the ingress or the network stopped (our own rate limit or budget, an + overloaded router) is inconclusive, not failed: a family with any such probe + is not judged on that run. + +Probes go through the public ingress with a consumer key, so their spend lands +in the call ledger like any other caller's. + +The decision logic is pure and takes its transport (`post`, `get`) and store as +arguments; `scripts/qualify_model_lines.py` is the thin CLI around it. +""" +from __future__ import annotations + +import json +import math +import re +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any, Callable + +import model_lines + +ACTOR = "qualifier" +MODES = ("report", "propose", "auto") + +# Body keys a suite may set on its probe requests. Everything else in the body +# (model, messages, the single-family policy) is owned by the qualifier. +_REQUEST_KEYS = {"temperature", "max_tokens", "response_format", + "first_token_timeout_ms", "timeout_ms", "top_p", "seed"} + + +class SuiteError(ValueError): + """A prompt suite file is not usable.""" + + +# ---- suites --------------------------------------------------------------------- + +def parse_suite(data: Any, source: str = "suite") -> dict[str, Any]: + if not isinstance(data, dict): + raise SuiteError(f"{source}: suite must be an object") + name = data.get("name") + if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_.\-]{0,63}", name): + raise SuiteError(f"{source}: `name` must be a short slug") + prompts = data.get("prompts") + if (not isinstance(prompts, list) or not prompts + or not all(isinstance(p, str) and p.strip() for p in prompts)): + raise SuiteError(f"{source}: `prompts` must be a non-empty list of strings") + request = data.get("request") or {} + if not isinstance(request, dict) or set(request) - _REQUEST_KEYS: + raise SuiteError( + f"{source}: `request` may only set {sorted(_REQUEST_KEYS)}") + expect = data.get("expect") or {"type": "text"} + if not isinstance(expect, dict) or expect.get("type") not in {"text", "json", "regex"}: + raise SuiteError(f"{source}: `expect.type` must be text, json or regex") + if expect["type"] == "regex": + pattern = expect.get("pattern") + # An absent pattern would compile to the empty expression, which matches + # every answer — a refusal included. + if not isinstance(pattern, str) or not pattern: + raise SuiteError(f"{source}: expect.pattern must be a non-empty string") + try: + re.compile(pattern) + except re.error as exc: + raise SuiteError(f"{source}: bad expect.pattern: {exc}") from exc + repeats = data.get("repeats", 1) + if isinstance(repeats, bool) or not isinstance(repeats, int) or not 1 <= repeats <= 5: + raise SuiteError(f"{source}: `repeats` must be an integer in [1, 5]") + return {"name": name, "prompts": prompts, "request": request, + "expect": expect, "repeats": repeats} + + +def load_suites(paths: "list[str | Path]") -> list[dict[str, Any]]: + """Load suites from files and/or directories (every *.json inside).""" + files: list[Path] = [] + for raw in paths: + p = Path(raw) + files.extend(sorted(p.glob("*.json")) if p.is_dir() else [p]) + suites, seen = [], set() + for f in files: + suite = parse_suite(json.loads(f.read_text(encoding="utf-8")), str(f)) + if suite["name"] in seen: + raise SuiteError(f"duplicate suite name {suite['name']!r}") + seen.add(suite["name"]) + suites.append(suite) + if not suites: + raise SuiteError("no prompt suites found") + return suites + + +def _json_object(text: str) -> Any: + """The JSON object in a model answer, tolerating a code fence or prose + around it the way lenient callers do: first `{` to last `}`.""" + start, end = text.find("{"), text.rfind("}") + if start < 0 or end <= start: + raise ValueError("no JSON object") + return json.loads(text[start:end + 1]) + + +def _check_value(value: Any, spec: dict[str, Any]) -> bool: + kind = spec.get("type") + if "enum" in spec: + return value in spec["enum"] + if kind == "integer": + ok = isinstance(value, int) and not isinstance(value, bool) + elif kind == "number": + ok = isinstance(value, (int, float)) and not isinstance(value, bool) + elif kind == "string": + ok = isinstance(value, str) and (bool(value.strip()) or spec.get("allow_empty", False)) + elif kind == "boolean": + ok = isinstance(value, bool) + else: + ok = value is not None + if ok and kind in {"integer", "number"}: + if "min" in spec and value < spec["min"]: + return False + if "max" in spec and value > spec["max"]: + return False + return ok + + +def answer_is_valid(text: str, expect: dict[str, Any]) -> bool: + """Whether a model answer satisfies a suite's `expect` block.""" + if not isinstance(text, str) or not text.strip(): + return False + kind = expect.get("type", "text") + if kind == "text": + return True + if kind == "regex": + return re.search(expect.get("pattern") or "", text) is not None + try: + obj = _json_object(text) + except (ValueError, TypeError): + return False + if not isinstance(obj, dict): + return False + for key, spec in (expect.get("required") or {}).items(): + if key not in obj or not _check_value(obj[key], spec if isinstance(spec, dict) else {}): + return False + return True + + +# ---- probing -------------------------------------------------------------------- + +# Errors the INGRESS returns before any provider sees the request, or that mean +# the probe never reached the router: our own key's rate limit or budget, the +# router being overloaded or unreachable. They say nothing about the model, so a +# probe that ends this way is inconclusive, not failed. +_INGRESS_CODES = frozenset({ + "consumer_auth", "caller_rate_limit", "caller_rate_limit_unavailable", + "key_rate_limit", "key_rate_limit_unavailable", "key_budget_exhausted", + "key_budget_unavailable", "consumer_budget_exhausted", + "consumer_budget_unavailable", "consumer_store_unavailable", + "router_overloaded", "route_unavailable", "upstream", "upstream_error", +}) + + +def probe_policy(family: str) -> list: + """One family, no fallback: a failure must surface as a failure, not as + another model's answer. Ranks that family's routes the way the validator + policies do (price, reliability, latency).""" + return ["policy", + ["and", ["meets_req"], ["not", ["is", "disabled"]], ["family_eq", family]], + ["add", + ["scale", 0.25, ["neg", ["normalize", ["field", "price_in"]]]], + ["scale", 0.45, ["neg", ["normalize", ["field", "price_out"]]]], + ["scale", 0.2, ["field", "success_rate"]], + ["scale", 0.1, ["neg", ["normalize", ["field", "latency_ms"]]]]], + ["argmax"], ["id"], ["always", {"action": "next_candidate"}]] + + +def probe_once(post: Callable[[dict], "tuple[int, Any]"], family: str, + suite: dict[str, Any], prompt: str, label: str) -> dict[str, Any]: + """One probe. `post(body) -> (http_status, parsed_json_or_None)`. Never + raises: a transport error is an inconclusive probe.""" + body = {**suite["request"], + "model": f"policy:{label}", + "messages": [{"role": "user", "content": prompt}], + "policy_ir": probe_policy(family)} + started = time.monotonic() + try: + status, data = post(body) + except Exception as exc: # noqa: BLE001 — a probe never takes the run down + status, data = 0, {"error": {"code": type(exc).__name__, "message": str(exc)}} + out: dict[str, Any] = {"suite": suite["name"], "status": status, + "latency_ms": int((time.monotonic() - started) * 1000), + "valid": False, "inconclusive": False, "cost_usd": None} + data = data if isinstance(data, dict) else {} + if status != 200: + err = data.get("error") if isinstance(data.get("error"), dict) else {} + code = str(err.get("code") or err.get("type") or "error")[:60] + out["error"] = code + out["inconclusive"] = status in (0, 401, 403) or code in _INGRESS_CODES + return out + x_router = data.get("x_router") if isinstance(data.get("x_router"), dict) else {} + served = x_router.get("model_family") + out["provider"] = x_router.get("provider") + cost = x_router.get("cost_usd") + if isinstance(cost, (int, float)) and not isinstance(cost, bool) and cost >= 0: + out["cost_usd"] = float(cost) + choice = (data.get("choices") or [{}])[0] + text = (choice.get("message") or {}).get("content") or "" + if served is not None and served != family: + # The single-family policy makes this impossible; treat a mismatch as a + # failed probe rather than crediting the candidate with another model. + out["error"] = "served_other_family" + return out + out["valid"] = answer_is_valid(text, suite["expect"]) + if not out["valid"]: + # A reasoning model that spends the whole output budget thinking returns + # an empty or cut-off answer. Name it: the fix is a budget, not a model. + out["error"] = ("truncated" if choice.get("finish_reason") == "length" + else "invalid_answer") + return out + + +def probe_family(post: Callable[[dict], "tuple[int, Any]"], family: str, + suites: list[dict[str, Any]], label: str) -> dict[str, Any]: + """Run every suite prompt against one family, sequentially — a burst of our + own probes must not be what trips the provider's rate limit. The order is + fixed, so probe i of one family is the same prompt as probe i of another.""" + results = [] + for suite in suites: + for _ in range(suite["repeats"]): + for prompt in suite["prompts"]: + results.append(probe_once(post, family, suite, prompt, label)) + return summarize(family, results) + + +def summarize(family: str, results: list[dict[str, Any]]) -> dict[str, Any]: + total = len(results) + passed = [r for r in results if r["valid"]] + latencies = sorted(r["latency_ms"] for r in passed) + errors: dict[str, int] = {} + by_suite: dict[str, dict[str, int]] = {} + for r in results: + stats = by_suite.setdefault(r["suite"], {"probes": 0, "passed": 0}) + stats["probes"] += 1 + stats["passed"] += 1 if r["valid"] else 0 + if not r["valid"]: + key = f"{r['suite']}:{r.get('error') or r['status']}" + errors[key] = errors.get(key, 0) + 1 + providers: dict[str, int] = {} + for r in passed: + providers[str(r.get("provider"))] = providers.get(str(r.get("provider")), 0) + 1 + # What each probe cost, valid answer or not: a model that charges for an + # unusable answer has still charged. None where the router reported no cost + # (a probe that never produced a response has none to report). + costs = [r["cost_usd"] for r in results] + answered = [r for r in results if r["status"] == 200] + known = [c for c in costs if c is not None] + + def pct(q: float) -> "int | None": + if not latencies: + return None + return latencies[min(len(latencies) - 1, math.ceil(q * len(latencies)) - 1)] + + return { + "family": family, "probes": total, "passed": len(passed), + "inconclusive": sum(1 for r in results if r.get("inconclusive")), + "pass_rate": round(len(passed) / total, 4) if total else 0.0, + "suites": by_suite, + "latency_p50_ms": pct(0.5), "latency_p90_ms": pct(0.9), + "latency_p95_ms": pct(0.95), + # Known only when every response that came back carried a cost. + "mean_cost_usd": (round(sum(known) / len(known), 8) + if known and len(known) == len(answered) else None), + "costs": costs, + "providers": providers, "errors": errors, + } + + +# ---- decisions (pure) ----------------------------------------------------------- + +def conclusive(summary: dict[str, Any]) -> bool: + """False when any probe was stopped by the ingress or the transport rather + than answered or failed by the model: such a run decides nothing about it.""" + return summary["probes"] > 0 and not summary.get("inconclusive") + + +def healthy(summary: dict[str, Any], definition: dict[str, Any]) -> bool: + """Enough valid answers overall AND in every suite. The overall bar leaves + room for one transient error; the per-suite bar stops a model that fails a + whole workload from hiding behind the others.""" + if summary["probes"] <= 0 or summary["pass_rate"] < definition["min_pass_rate"]: + return False + for stats in (summary.get("suites") or {}).values(): + if stats["probes"] and stats["passed"] / stats["probes"] < definition["min_suite_pass_rate"]: + return False + return True + + +def _paired_cost_ratio(candidate: dict[str, Any], head: dict[str, Any]) -> dict[str, Any]: + """Candidate vs head cost over the probes where BOTH reported one, so the + two sides are summed over the same prompts. Fails closed when the candidate + has an answered probe without a cost, or when too few probes pair up to say + anything.""" + c_costs, h_costs = candidate.get("costs") or [], head.get("costs") or [] + if candidate["mean_cost_usd"] is None: + return {"ok": False, "reason": "candidate_cost_unknown"} + pairs = [(c, h) for c, h in zip(c_costs, h_costs) if c is not None and h is not None] + if len(pairs) * 2 < max(len(c_costs), 1): + return {"ok": False, "reason": "no_head_baseline", "paired": len(pairs)} + c_sum, h_sum = sum(c for c, _ in pairs), sum(h for _, h in pairs) + out: dict[str, Any] = {"paired": len(pairs), + "candidate_usd": round(c_sum / len(pairs), 8), + "head_usd": round(h_sum / len(pairs), 8)} + if h_sum == 0: + return {**out, "ok": c_sum == 0, "reason": "head_is_free"} + return {**out, "ok": None, "ratio": round(c_sum / h_sum, 4)} + + +def judge(candidate: dict[str, Any], head: dict[str, Any], + definition: dict[str, Any]) -> dict[str, Any]: + """Gate a candidate's probe summary against the head's. Every gate fails + closed: an unknown cost, or a head with no successful probe to compare + against, does not qualify anyone. + + The head does NOT have to be healthy to serve as the baseline. A head that + still answers some probes is exactly the one worth replacing, and its + successful probes still say what it costs and how fast it is.""" + gates: dict[str, dict[str, Any]] = {} + gates["answers"] = { + "ok": healthy(candidate, definition), + "pass_rate": candidate["pass_rate"], "min": definition["min_pass_rate"], + "suites": {name: round(s["passed"] / s["probes"], 4) if s["probes"] else 0.0 + for name, s in (candidate.get("suites") or {}).items()}, + "min_suite": definition["min_suite_pass_rate"]} + # p90, not the maximum: with a few dozen probes p95 IS the slowest call, and + # one slow provider hop would decide the gate. + p90, head_p90 = candidate["latency_p90_ms"], head["latency_p90_ms"] + relative = (round(head_p90 * definition["max_latency_ratio"]) + if head_p90 is not None else None) + gates["latency"] = { + "ok": p90 is not None and (p90 <= definition["max_latency_ms"] + or (relative is not None and p90 <= relative)), + "p90_ms": p90, "head_p90_ms": head_p90, + "max_ms": definition["max_latency_ms"], "max_vs_head_ms": relative} + cost = _paired_cost_ratio(candidate, head) + if cost["ok"] is None: + cost["ok"] = cost["ratio"] <= definition["max_cost_ratio"] + cost["max_ratio"] = definition["max_cost_ratio"] + gates["cost"] = cost + return {"qualified": all(g["ok"] for g in gates.values()), "gates": gates} + + +def consecutive_passes(events: list[dict[str, Any]], family: str, head: str) -> int: + """How many of this family's most recent qualification results (newest + first) were passes AGAINST THE CURRENT HEAD, stopping at the first failure + or at the first result measured against a different head: a pass against a + baseline that is no longer live says nothing about replacing this one.""" + n = 0 + for event in events: + if event.get("family") != family: + continue + if event.get("previous_family") != head: + break + if event.get("kind") == "candidate_passed": + n += 1 + elif event.get("kind") == "candidate_failed": + break + return n + + +# ---- the run -------------------------------------------------------------------- + +def list_families(get: Callable[[str], "tuple[int, Any]"]) -> list[str]: + """Family names the router can currently route, from GET /v1/models.""" + status, data = get("/v1/models") + if status != 200 or not isinstance(data, dict): + raise RuntimeError(f"/v1/models returned {status}") + out = [] + for row in data.get("data") or []: + ident = row.get("id") if isinstance(row, dict) else None + if isinstance(ident, str) and ident.startswith("family:"): + out.append(ident[len("family:"):]) + if not out: + raise RuntimeError("/v1/models listed no families") + return out + + +def _stored(summary: dict[str, Any]) -> dict[str, Any]: + """A summary as kept in the audit trail: without the per-probe cost list.""" + return {k: v for k, v in summary.items() if k != "costs"} + + +def run(*, post: Callable[[dict], "tuple[int, Any]"], + get: Callable[[str], "tuple[int, Any]"], + suites: list[dict[str, Any]], mode: str = "propose", + only_lines: "list[str] | None" = None, workers: int = 4, + store: Any = None) -> dict[str, Any]: + """One qualification run over every (or the selected) line. Returns the + report; in `propose`/`auto` it also writes audit events and, in `auto`, + moves heads. `store` defaults to host_store (injected in tests). + + Raises when the run cannot be trusted to continue: the store is unreachable, + or an audit event could not be written. The streak a promotion rests on is + read from those events, so a run that cannot record a result must stop + before it decides anything on top of it.""" + if mode not in MODES: + raise ValueError(f"mode must be one of {MODES}") + if store is None: + import host_store as store # noqa: PLC0415 + writes = mode != "report" + definitions = model_lines.definitions() + store_ok = model_lines.reload() + if writes and not store_ok: + raise RuntimeError("host store unavailable: refusing to decide on stale heads") + names = [n for n in definitions if not only_lines or n in only_lines] + unknown = sorted(set(only_lines or []) - set(definitions)) + if unknown: + raise ValueError(f"unknown line(s): {unknown}") + families = list_families(get) + available = set(families) + snapshot = {row["line"]: row for row in model_lines.snapshot(events=0)["lines"]} + + def record(line: str, kind: str, **kw: Any) -> None: + if writes and not store.add_model_line_event(line, kind, actor=ACTOR, **kw): + raise RuntimeError(f"audit write failed ({line}: {kind}): stopping " + "before any decision rests on a missing result") + + plan: dict[str, dict[str, Any]] = {} + to_probe: dict[str, None] = {} + for name in names: + d = definitions[name] + head = snapshot[name]["head"] + cands = model_lines.successors(d, head, families, int(d["max_candidates"])) + plan[name] = {"head": head, "candidates": cands, + "previous": snapshot[name].get("previous")} + for family in [head, *cands]: + to_probe[family] = None + + def probe(family: str) -> dict[str, Any]: + return probe_family(post, family, suites, f"qualify-{family}"[:60]) + + with ThreadPoolExecutor(max_workers=max(1, int(workers))) as pool: + summaries = dict(zip(to_probe, pool.map(probe, list(to_probe)))) + + report: dict[str, Any] = {"mode": mode, "ts": int(time.time()), + "suites": [s["name"] for s in suites], + "lines": {}, "moves": [], "proposals": [], + "unhealthy_heads": [], "inconclusive_lines": []} + for name in names: + d = definitions[name] + head = plan[name]["head"] + head_summary = summaries[head] + line_report: dict[str, Any] = { + "head": head, "head_available": head in available, + "head_summary": _stored(head_summary), "candidates": [], "action": None} + report["lines"][name] = line_report + if not conclusive(head_summary): + # The ingress or the network stopped some of the head's probes. With + # no trustworthy baseline nothing on this line is judged this run: + # not its candidates, and not the head's own health. + line_report["head_healthy"] = None + report["inconclusive_lines"].append(name) + continue + head_ok = healthy(head_summary, d) + line_report["head_healthy"] = head_ok + events = (store.model_line_events( + name, limit=200, kinds=("candidate_passed", "candidate_failed")) + if writes else []) + + eligible: "tuple[str, dict[str, Any]] | None" = None + needed = int(d["consecutive_passes"]) + for family in plan[name]["candidates"]: + summary = summaries[family] + if not conclusive(summary): + line_report["candidates"].append({ + "family": family, "summary": _stored(summary), + "qualified": None, "inconclusive": True}) + continue # neither a pass nor a failure: the streak stands + verdict = judge(summary, head_summary, d) + detail = {"summary": _stored(summary), "head": head, + "head_summary": _stored(head_summary), **verdict, + "suites": report["suites"]} + streak = (consecutive_passes(events, family, head) + 1 + if verdict["qualified"] else 0) + line_report["candidates"].append({ + "family": family, "summary": _stored(summary), **verdict, + "streak": streak, "needed": needed}) + record(name, "candidate_passed" if verdict["qualified"] else "candidate_failed", + family=family, previous_family=head, detail=detail) + # Candidates are newest-first: the first eligible one is the pick. + if eligible is None and verdict["qualified"] and streak >= needed: + eligible = (family, {**detail, "streak": streak}) + + promoted = False + if eligible is not None: + family, detail = eligible + if mode == "auto": + result = model_lines.promote( + name, family, actor=ACTOR, detail=detail, expected_current=head) + line_report["action"] = {"promote": family, **result} + if result.get("ok"): + promoted = True + report["moves"].append({"line": name, "from": head, "to": family}) + else: + line_report["action"] = {"propose": family} + report["proposals"].append({"line": name, "from": head, "to": family}) + if writes and not _already_proposed(store, name, family, head): + record(name, "proposed", family=family, previous_family=head, + detail=detail) + + if not head_ok: + report["unhealthy_heads"].append(name) + if promoted: + continue # the line just moved to a qualified successor + action = _handle_unhealthy_head( + name, d, plan[name], head_summary, summaries, post, suites, + mode, store, record) + line_report["head_action"] = action + if action.get("rolled_back"): + report["moves"].append( + {"line": name, "from": head, "to": action["rolled_back"]}) + return report + + +def _already_proposed(store: Any, line: str, family: str, head: str) -> bool: + """A standing proposal is recorded once, not once per run.""" + latest = store.model_line_events( + line, limit=1, kinds=("proposed", "promoted", "rolled_back")) + return bool(latest and latest[0].get("kind") == "proposed" + and latest[0].get("family") == family + and latest[0].get("previous_family") == head) + + +def _qualifier_promoted(store: Any, line: str, head: str, previous: str) -> bool: + """True when the move that made `head` current is this qualifier's own + PROMOTION from `previous`. Not an operator's choice, and not one of the + qualifier's rollbacks: undoing a rollback would restore a head that was + removed for failing, on nothing more than the old head failing too.""" + latest = store.model_line_events(line, limit=1, kinds=("promoted", "rolled_back")) + return bool(latest and latest[0].get("kind") == "promoted" + and latest[0].get("actor") == ACTOR + and latest[0].get("family") == head + and latest[0].get("previous_family") == previous) + + +def _handle_unhealthy_head(name: str, d: dict[str, Any], plan: dict[str, Any], + head_summary: dict[str, Any], + summaries: dict[str, dict[str, Any]], + post: Callable, suites: list[dict[str, Any]], + mode: str, store: Any, + record: Callable[..., None]) -> dict[str, Any]: + """A head that no longer passes its probes. Always recorded. Rolled back only + in `auto`, only when the current move is this qualifier's own promotion, and + only when the head it replaced still passes — otherwise there is nothing + better to fall back to and an operator has to look.""" + head, previous = plan["head"], plan["previous"] + detail: dict[str, Any] = {"head": head, "head_summary": _stored(head_summary)} + action: dict[str, Any] = {"unhealthy": head} + if (mode == "auto" and previous and previous != head + and _qualifier_promoted(store, name, head, previous)): + prev_summary = summaries.get(previous) or probe_family( + post, previous, suites, f"qualify-{previous}"[:60]) + detail["previous"] = previous + detail["previous_summary"] = _stored(prev_summary) + if conclusive(prev_summary) and healthy(prev_summary, d): + # Compare-and-set on the exact move that was probed: if the line + # moved while the probes ran, this is no longer the move to undo. + result = model_lines.rollback( + name, actor=ACTOR, detail=detail, + expected_current=head, expected_previous=previous) + if result.get("ok"): + action["rolled_back"] = previous + return action + action["rollback_failed"] = result.get("reason") + record(name, "head_unhealthy", family=head, detail=detail) + return action diff --git a/model_lines.json b/model_lines.json new file mode 100644 index 0000000..db36b8a --- /dev/null +++ b/model_lines.json @@ -0,0 +1,83 @@ +{ + "version": 1, + "defaults": { + "max_cost_ratio": 1.1, + "max_latency_ms": 15000, + "max_latency_ratio": 1.25, + "min_pass_rate": 0.9, + "consecutive_passes": 2, + "max_candidates": 2 + }, + "lines": { + "openai_sol": { + "description": "OpenAI general flagship (the -sol tier).", + "head": "gpt-6.1-sol", + "match": "^gpt-(\\d(?:\\.\\d+)?)-sol$" + }, + "anthropic_sonnet": { + "description": "Anthropic Claude Sonnet.", + "head": "claude-sonnet-5.5", + "match": "^claude-sonnet-(\\d(?:\\.\\d+)?)$" + }, + "gemini_flash_lite": { + "description": "Google Gemini Flash-Lite.", + "head": "gemini-3.5-flash-lite", + "match": "^gemini-(\\d(?:\\.\\d+)?)-flash-lite$" + }, + "deepseek_flash": { + "description": "DeepSeek Flash, dated snapshots.", + "head": "deepseek-v4.1-flash-1017", + "match": "^deepseek-v(\\d(?:\\.\\d+)?)-flash-(\\d{4})$" + }, + "gpt_oss": { + "description": "OpenAI open-weights gpt-oss. No version series to follow: moved by an operator only.", + "head": "gpt-oss-120b", + "match": null + }, + "grok": { + "description": "xAI Grok.", + "head": "grok-4.3", + "match": "^grok-(\\d\\.\\d+)$" + }, + "mistral_large": { + "description": "Mistral Large, date-coded snapshots (YYMM).", + "head": "mistral-large-2512", + "match": "^mistral-large-(\\d{4})$" + }, + "kimi_code": { + "description": "Moonshot Kimi, the -code tier.", + "head": "kimi-k2.7-code", + "match": "^kimi-k(\\d(?:\\.\\d+)?)-code$" + }, + "qwen_27b": { + "description": "Alibaba Qwen, the 27b open-weights size.", + "head": "qwen3.8-27b", + "match": "^qwen(\\d(?:\\.\\d+)?)-27b$" + }, + "glm": { + "description": "Z.ai GLM base model.", + "head": "glm-5.3", + "match": "^glm-(\\d(?:\\.\\d+)?)$" + }, + "minimax": { + "description": "MiniMax M series base model.", + "head": "minimax-m3", + "match": "^minimax-m(\\d(?:\\.\\d+)?)$" + }, + "minimax_flash": { + "description": "MiniMax M series, the -flash tier.", + "head": "minimax-m3.1-flash", + "match": "^minimax-m(\\d(?:\\.\\d+)?)-flash$" + }, + "gemma": { + "description": "Google Gemma instruction-tuned, dense.", + "head": "gemma-4-31b-it", + "match": "^gemma-(\\d)-(\\d+)b-it$" + }, + "gemma_efficient": { + "description": "Google Gemma instruction-tuned, the active-parameter (aNb) variant.", + "head": "gemma-4-26b-a4b-it", + "match": "^gemma-(\\d)-(\\d+)b-a\\d+b-it$" + } + } +} diff --git a/model_lines.py b/model_lines.py new file mode 100644 index 0000000..94037f7 --- /dev/null +++ b/model_lines.py @@ -0,0 +1,406 @@ +"""Model lines: a named pointer to the family currently qualified as the head of +a model series. + +A policy that names an exact family (`family_eq "gpt-6.1-sol"`) goes stale the +day its successor ships, and every caller that copied the term has to be edited +and redeployed. A line moves that one fact into the host: the policy asks for +`is("line_openai_sol")` and the host answers which family that is today. The +policy bytes — and so the policy id — never change when a line moves. + +Two halves, kept apart on purpose: + +- DEFINITIONS (`model_lines.json`, reviewed in git): which lines exist, the seed + head of each, and the family-name pattern a successor must match. The set of + lines is static because a line is an observation field, and a policy naming an + undeclared field must fail admission, not resolve to "no candidate". +- HEADS (the `model_line_heads` table): which family each line points at now. + Moved by `promote`/`rollback`, written by the qualifier + (`scripts/qualify_model_lines.py`) or by an operator, and re-read by every + router replica on a short timer. A line with no row serves its seed. + +Fail-safe direction: an unreachable store keeps the last-known heads; a missing +or invalid definitions file declares no lines at all (the feature is off) rather +than a half-read set. + +This is the `fields.lua` extension seam, like `cache_hot`: the engine observes a +Bool per candidate and learns nothing about lines. Each engine keeps its own +copy of the heads as a Lua table (the getter runs per candidate per request and +must stay native); `subscribe` hands it over and every move republishes it. +""" +from __future__ import annotations + +import itertools +import json +import logging +import os +import re +import threading +import weakref +from pathlib import Path +from typing import Any + +import host_store + +_log = logging.getLogger("llm-router.model-lines") + +ROOT = Path(__file__).resolve().parent +DEFAULT_PATH = ROOT / "model_lines.json" +FIELD_PREFIX = "line_" + +_NAME_RE = re.compile(r"^[a-z][a-z0-9_]{0,47}$") +_FAMILY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/\-]{0,127}$") + +# The knobs a line may override; the values are the built-in defaults, used when +# neither the line nor the file's `defaults` block sets one. +QUALIFY_DEFAULTS: dict[str, float] = { + # A candidate's measured mean cost per probe may be at most this multiple of + # the head's, measured on the same prompts in the same run. + "max_cost_ratio": 1.1, + # Latency passes when the candidate's p90 over successful probes is under + # this absolute bound ... + "max_latency_ms": 15000, + # ... or within this multiple of the head's p90 on the same prompts: a line + # whose head is already slow is not held to a bar its own head misses. + "max_latency_ratio": 1.25, + # Share of probes that must return a valid answer inside the deadlines. Also + # the bar a head must clear to count as healthy. Below 1 on purpose: one + # transient provider error in a run is noise, not a regression. + "min_pass_rate": 0.9, + # ... and the share that must pass inside EACH suite, so a model that fails + # one whole workload cannot hide behind the others. Lower than the overall + # bar because a suite can be a handful of prompts. + "min_suite_pass_rate": 0.5, + # Qualifying runs in a row (on separate runs) before a promotion. + "consecutive_passes": 2, + # Newest candidates probed per line per run. + "max_candidates": 2, +} + + +class ModelLineError(ValueError): + """A definitions file or a head move is not acceptable.""" + + +# _lock guards the module's own state and is never held across I/O or a call +# into an engine. _install_lock orders "install a set of heads + deliver it to +# the engines"; it is held across engine deliveries (a table assignment each), +# never across a store read. +_lock = threading.Lock() +_install_lock = threading.RLock() +_tickets = itertools.count(1) +_installed_ticket = 0 +_definitions: "dict[str, dict[str, Any]] | None" = None +_heads: dict[str, str] = {} +_rows: dict[str, dict[str, Any]] = {} +_store_ok: "bool | None" = None +# Engines holding a copy of the heads (LLMRouterHost.sync_model_line_heads). +# Weak: a request-scoped tenant engine drops out when the request ends. +_subscribers: "weakref.WeakSet[Any]" = weakref.WeakSet() + + +# ---- definitions --------------------------------------------------------------- + +def _definitions_path() -> Path: + raw = os.getenv("MODEL_LINES_FILE", "").strip() + return Path(raw) if raw else DEFAULT_PATH + + +def parse_definitions(data: Any) -> dict[str, dict[str, Any]]: + """Validate a definitions document and return {line: definition} with the + qualification knobs resolved (line > file defaults > built-in defaults) and + the pattern compiled. Raises ModelLineError naming the first problem.""" + if not isinstance(data, dict) or data.get("version") != 1: + raise ModelLineError("model lines file must be an object with version 1") + file_defaults = data.get("defaults") or {} + lines = data.get("lines") + if not isinstance(file_defaults, dict) or not isinstance(lines, dict): + raise ModelLineError("`defaults` and `lines` must be objects") + unknown = set(file_defaults) - set(QUALIFY_DEFAULTS) + if unknown: + raise ModelLineError(f"unknown default knob(s): {sorted(unknown)}") + out: dict[str, dict[str, Any]] = {} + for name, raw in lines.items(): + if not isinstance(name, str) or not _NAME_RE.match(name): + raise ModelLineError(f"invalid line name {name!r}") + if not isinstance(raw, dict): + raise ModelLineError(f"line {name}: definition must be an object") + head = raw.get("head") + if not isinstance(head, str) or not _FAMILY_RE.match(head): + raise ModelLineError(f"line {name}: `head` must be a family name") + pattern = raw.get("match") + compiled = None + if pattern is not None: + if not isinstance(pattern, str): + raise ModelLineError(f"line {name}: `match` must be a string or null") + try: + compiled = re.compile(pattern) + except re.error as exc: + raise ModelLineError(f"line {name}: bad `match`: {exc}") from exc + if compiled.groups < 1: + raise ModelLineError( + f"line {name}: `match` needs a capture group for the version") + if version_key(compiled, head) is None: + raise ModelLineError( + f"line {name}: `head` {head!r} does not match its own `match`") + knobs: dict[str, float] = {} + for knob, builtin in QUALIFY_DEFAULTS.items(): + value = raw.get(knob, file_defaults.get(knob, builtin)) + if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0: + raise ModelLineError(f"line {name}: `{knob}` must be a positive number") + knobs[knob] = value + for rate in ("min_pass_rate", "min_suite_pass_rate"): + if knobs[rate] > 1: + raise ModelLineError(f"line {name}: `{rate}` must be <= 1") + extra = set(raw) - {"head", "match", "description"} - set(QUALIFY_DEFAULTS) + if extra: + raise ModelLineError(f"line {name}: unknown key(s) {sorted(extra)}") + out[name] = {"name": name, "head": head, "match": pattern, + "pattern": compiled, + "description": str(raw.get("description") or ""), **knobs} + return out + + +def load_definitions(path: "str | Path | None" = None) -> dict[str, dict[str, Any]]: + """Read and validate a definitions file. A missing file is an empty set.""" + p = Path(path) if path is not None else _definitions_path() + if not p.exists(): + return {} + return parse_definitions(json.loads(p.read_text(encoding="utf-8"))) + + +def configure(path: "str | Path | None" = None) -> dict[str, dict[str, Any]]: + """(Re)load the definitions and reset every head to its seed. An invalid file + declares NO lines and logs why — a policy naming one then fails admission + loudly instead of the host serving a partial set.""" + global _definitions, _heads, _rows, _store_ok, _installed_ticket + try: + defs = load_definitions(path) + except (OSError, ValueError) as exc: + _log.error("model lines disabled: %s", exc) + defs = {} + with _install_lock: + _installed_ticket = next(_tickets) # outdates any store read in flight + with _lock: + _definitions = defs + _heads = {name: d["head"] for name, d in defs.items()} + _rows = {} + _store_ok = None + _publish_locked() + return defs + + +def definitions() -> dict[str, dict[str, Any]]: + """The declared lines, loading them on first use.""" + with _lock: + defs = _definitions + return defs if defs is not None else configure() + + +def field_name(line: str) -> str: + return FIELD_PREFIX + line + + +# ---- heads --------------------------------------------------------------------- + +def heads() -> dict[str, str]: + definitions() + with _lock: + return dict(_heads) + + +def head_of(line: str) -> "str | None": + definitions() + with _lock: + return _heads.get(line) + + +def subscribe(engine: Any) -> None: + """Give `engine` the current heads now and on every later move. The engine + exposes `sync_model_line_heads(dict)`; it is held weakly. Registered and + served under the install lock, so the first delivery can never land after a + newer one.""" + with _install_lock: + with _lock: + _subscribers.add(engine) + current = dict(_heads) + engine.sync_model_line_heads(current) + + +def _publish_locked() -> None: + """Push the current heads to every live engine. Caller holds _install_lock, + which is what orders deliveries: two publishers cannot interleave, so an + engine never ends on an older table than the module's. Never raises: one + engine that cannot be updated must not stop the others; it keeps its last + table and is retried on the next successful store read.""" + with _lock: + engines = list(_subscribers) + current = dict(_heads) + for engine in engines: + try: + engine.sync_model_line_heads(current) + except Exception as exc: # noqa: BLE001 + _log.warning("model lines: engine sync failed: %s", exc) + + +def reload() -> bool: + """Re-read the heads from the store. Returns False — and changes nothing — + when the store is unreachable. A stored head for an undeclared line is + ignored, and one that is not a plausible family name falls back to the seed. + + Concurrent reloads are ordered by a ticket taken BEFORE the store read: a + read that finishes after a later-started one has installed is discarded, + because the later one necessarily saw every commit the earlier could have. + The store read itself runs outside every lock, so a slow or unreachable + store never stalls an engine being built for a request.""" + global _heads, _rows, _store_ok, _installed_ticket + defs = definitions() + ticket = next(_tickets) + stored = host_store.get_model_line_heads() + if stored is None: + with _lock: + _store_ok = False + return False + new_heads: dict[str, str] = {} + rows: dict[str, dict[str, Any]] = {} + for name, d in defs.items(): + row = stored.get(name) + family = (row or {}).get("family") + if isinstance(family, str) and _FAMILY_RE.match(family): + new_heads[name] = family + rows[name] = row + else: + new_heads[name] = d["head"] + with _install_lock: + if ticket < _installed_ticket: + return True # a newer read is already installed + _installed_ticket = ticket + with _lock: + if _definitions is not defs: + return True # reconfigured while reading: not our lines + moved = {n: (old, new_heads[n]) for n, old in _heads.items() + if n in new_heads and new_heads[n] != old} + _heads = new_heads + _rows = rows + _store_ok = True + # Publish on every successful read, moved or not: it is 14 table slots + # per engine, and it is what heals an engine whose last sync failed. + _publish_locked() + for name, (old, new) in moved.items(): + _log.info("model line %s head: %s -> %s", name, old, new) + return True + + +def _check_move(line: str, family: str, *, force: bool) -> dict[str, Any]: + d = definitions().get(line) + if d is None: + raise ModelLineError(f"unknown line {line!r}") + if not isinstance(family, str) or not _FAMILY_RE.match(family): + raise ModelLineError("family must be a family name") + if not force and d["pattern"] is not None and version_key(d["pattern"], family) is None: + raise ModelLineError( + f"{family!r} does not match line {line}'s pattern {d['match']!r}") + return d + + +def promote(line: str, family: str, *, actor: str, + detail: "dict[str, Any] | None" = None, + expected_current: "str | None" = None, + force: bool = False, kind: str = "promoted") -> dict[str, Any]: + """Point `line` at `family`. The store resolves the current head inside its + own transaction, so the recorded previous head is the real one even on a + replica that has not refreshed yet.""" + d = _check_move(line, family, force=force) + result = host_store.set_model_line_head( + line, family, seed_family=d["head"], actor=actor, kind=kind, + detail=detail, expected_current=expected_current) + if result.get("ok"): + reload() + return result + + +def rollback(line: str, *, actor: str, + detail: "dict[str, Any] | None" = None, + expected_current: "str | None" = None, + expected_previous: "str | None" = None) -> dict[str, Any]: + """Point `line` back at the head it had before its last move. The store + picks the target inside its transaction; `expected_current` / + `expected_previous` make it a compare-and-set on the move the caller saw, so + a retried or late rollback cannot undo a different move.""" + d = definitions().get(line) + if d is None: + raise ModelLineError(f"unknown line {line!r}") + result = host_store.rollback_model_line_head( + line, seed_family=d["head"], actor=actor, detail=detail, + expected_current=expected_current, expected_previous=expected_previous) + if result.get("ok"): + reload() + return result + + +def snapshot(events: int = 20) -> dict[str, Any]: + """Operator view: every line, its head, where that head came from, and the + latest audit events.""" + defs = definitions() + with _lock: + current = dict(_heads) + rows = dict(_rows) + store_ok = _store_ok + lines = [] + for name, d in defs.items(): + row = rows.get(name) or {} + lines.append({ + "line": name, "field": field_name(name), + "description": d["description"], "match": d["match"], + "head": current.get(name), "seed": d["head"], + "source": "store" if row else "seed", + "previous": row.get("previous_family"), + "updated_at": row.get("updated_at"), + "updated_by": row.get("updated_by"), + "qualify": {k: d[k] for k in QUALIFY_DEFAULTS}, + }) + return {"lines": lines, "store_ok": store_ok, + "events": host_store.model_line_events(limit=events) if events else []} + + +# ---- successor discovery (pure) ------------------------------------------------- + +def version_key(pattern: "re.Pattern[str]", family: str) -> "tuple[tuple[int, ...], ...] | None": + """The ordering key of `family` inside a line, or None when the family is + not in the line: one integer tuple per capture group, each group split on + `.`/`-`. Groups compare in order, so `v4.1 / 1017` is ((4, 1), (1017,)) and + sorts after `v4 / 0731` ((4,), (731,)) — the version decides before the + snapshot date does, however many parts either has.""" + m = pattern.fullmatch(family) + if m is None: + return None + key: list[tuple[int, ...]] = [] + for group in m.groups(): + if group is None: + key.append(()) + continue + parts = re.split(r"[.\-]", group) + if not all(part.isdigit() for part in parts): + return None + key.append(tuple(int(part) for part in parts)) + return tuple(key) if any(key) else None + + +def successors(definition: dict[str, Any], head: str, families: "list[str]", + limit: "int | None" = None) -> list[str]: + """Families of this line that sort strictly after `head`, newest first. + A line without a pattern, or a head that is not in its own line (an operator + forced it), has no automatic successors.""" + pattern = definition.get("pattern") + if pattern is None: + return [] + head_key = version_key(pattern, head) + if head_key is None: + return [] + keyed = [] + for family in set(families): + key = version_key(pattern, family) + if key is not None and key > head_key: + keyed.append((key, family)) + keyed.sort(reverse=True) + out = [family for _, family in keyed] + return out[:int(limit)] if limit else out diff --git a/scripts/qualify_model_lines.py b/scripts/qualify_model_lines.py new file mode 100644 index 0000000..6da2ee2 --- /dev/null +++ b/scripts/qualify_model_lines.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Qualify successors for model lines (see model_line_qualifier.py). + +Runs outside the request-serving pods, like scripts/rollup_analytics.py. Probes +go through the router ingress with a consumer key so their spend is ledgered; +heads and audit events are read and written in the host store (DATABASE_URL). + + LINE_QUALIFIER_ROUTER_URL=http://llm-router \\ + LINE_QUALIFIER_API_KEY=... \\ + python scripts/qualify_model_lines.py --suites /etc/line-suites --mode propose + +Modes: `report` probes and prints only; `propose` (default) also records audit +events and standing proposals; `auto` moves heads. Exit status is 0 when the run +completed (whatever it decided), 1 when it could not run at all. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import httpx # noqa: E402 + +import model_line_qualifier as qualifier # noqa: E402 + + +def make_transport(base_url: str, api_key: str, timeout_s: float): + """`post(body)` / `get(path)` against the router's OpenAI-compatible surface, + each returning (http_status, parsed_json_or_None).""" + headers = {"content-type": "application/json"} + if api_key: + headers["authorization"] = f"Bearer {api_key}" + client = httpx.Client(base_url=base_url.rstrip("/"), headers=headers, + timeout=httpx.Timeout(timeout_s, connect=10.0)) + + def parsed(response: httpx.Response): + try: + return response.status_code, response.json() + except ValueError: + return response.status_code, None + + def post(body: dict): + return parsed(client.post("/v1/chat/completions", json=body)) + + def get(path: str): + return parsed(client.get(path)) + + return post, get, client + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--router-url", + default=os.getenv("LINE_QUALIFIER_ROUTER_URL", ""), + help="router base URL (env LINE_QUALIFIER_ROUTER_URL)") + parser.add_argument("--suites", action="append", default=[], + help="suite file or directory of *.json (repeatable; " + "env LINE_QUALIFIER_SUITES, comma-separated)") + parser.add_argument("--mode", choices=qualifier.MODES, + default=os.getenv("LINE_QUALIFIER_MODE", "propose")) + parser.add_argument("--lines", default="", + help="comma-separated subset of lines (default: all)") + parser.add_argument("--workers", type=int, default=4, + help="families probed in parallel (prompts within a " + "family are always sequential)") + parser.add_argument("--timeout-s", type=float, default=60.0, + help="HTTP timeout per probe; keep it above the suite's " + "own timeout_ms so the router's verdict wins") + args = parser.parse_args() + + if args.mode not in qualifier.MODES: + parser.error(f"LINE_QUALIFIER_MODE must be one of {qualifier.MODES}") + if not args.router_url: + parser.error("--router-url / LINE_QUALIFIER_ROUTER_URL is required") + if not 1 <= args.workers <= 16: + parser.error("--workers must be between 1 and 16") + suite_paths = args.suites or [ + p.strip() for p in os.getenv("LINE_QUALIFIER_SUITES", "").split(",") if p.strip()] + if not suite_paths: + parser.error("--suites / LINE_QUALIFIER_SUITES is required") + + try: + suites = qualifier.load_suites(suite_paths) + except (OSError, ValueError) as exc: + print(json.dumps({"ok": False, "error": f"suites: {exc}"})) + return 1 + + # The key is read from the environment only: never accepted on the command + # line, where it would land in the process list and the job spec. + post, get, client = make_transport( + args.router_url, os.getenv("LINE_QUALIFIER_API_KEY", ""), args.timeout_s) + only = [n.strip() for n in args.lines.split(",") if n.strip()] or None + try: + report = qualifier.run(post=post, get=get, suites=suites, mode=args.mode, + only_lines=only, workers=args.workers) + except Exception as exc: # noqa: BLE001 — one JSON line either way + print(json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"})) + return 1 + finally: + client.close() + print(json.dumps({"ok": True, **report}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/serve.py b/serve.py index 7f92aee..64ce9cb 100644 --- a/serve.py +++ b/serve.py @@ -61,6 +61,15 @@ def main() -> None: import settings settings.reload() + # Model lines: load the definitions and pull the current heads before the + # engine is built, so the first request already sees promoted heads rather + # than the seeds. Best-effort: with the store down the seeds serve. + import model_lines + lines = model_lines.configure() + if lines: + print(f"model lines: {len(lines)} declared, heads from " + f"{'store' if model_lines.reload() else 'seeds (store unavailable)'}") + # Operator-managed provider keys persisted by the dashboard live on the PVC # (.env.secrets) and are the source of truth. Load them over the container # env BEFORE LLMRouterHost snapshots os.environ, so dashboard edits to @@ -255,6 +264,35 @@ def hook(discovery_id): return hook +def start_model_lines_refresh(): + """Background task re-reading the model line heads from the store. Returns + None when no lines are declared or the refresh is disabled. Call from an + async context. A failed read keeps the last-known heads (model_lines.reload + is fail-soft), so the loop only has to keep going.""" + import asyncio + import os + import random + + import model_lines + + try: + interval = float(os.getenv("MODEL_LINES_REFRESH_S", "30")) + except ValueError: + interval = 30.0 + if interval <= 0 or not model_lines.definitions(): + return None + + async def loop(): + while True: + await asyncio.sleep(interval * (0.9 + 0.2 * random.random())) + try: + await asyncio.to_thread(model_lines.reload) + except Exception as exc: # noqa: BLE001 — never kill the refresher + print(f"model lines refresh failed: {type(exc).__name__}: {exc}") + + return asyncio.create_task(loop()) + + def _enabled(name: str, default: bool = True) -> bool: import os fallback = "1" if default else "0" @@ -302,6 +340,12 @@ async def lifespan(app_): if _enabled("RUN_WALLET_KEEPER") else None) if keeper is not None: tasks.append(keeper) + # Every replica follows line promotions on its own short timer: the + # heads live in the shared store, so there is no reload fan-out to + # coordinate. MODEL_LINES_REFRESH_S=0 pins the boot-time heads. + lines_task = start_model_lines_refresh() + if lines_task is not None: + tasks.append(lines_task) try: yield finally: diff --git a/shim.py b/shim.py index d1defa5..05186c4 100644 --- a/shim.py +++ b/shim.py @@ -187,6 +187,27 @@ class FlowNormalizeRequest(BaseModel): flow_ir: list +class ModelLineMoveRequest(BaseModel): + """Body of POST /x/lines/{line}/promote.""" + family: str = Field(min_length=1, max_length=128) + actor: str = Field(default="", max_length=120) + # Compare-and-set: refuse the move unless the line still points here. + expected_current: str | None = Field(default=None, max_length=128) + # Accept a family outside the line's own name pattern (operator judgement). + force: bool = False + detail: dict | None = None + + +class ModelLineRollbackRequest(BaseModel): + """Body of POST /x/lines/{line}/rollback.""" + actor: str = Field(default="", max_length=120) + # Compare-and-set on the move being undone: send the head you saw. A retry + # after a lost response then conflicts instead of undoing the rollback. + expected_current: str | None = Field(default=None, max_length=128) + expected_previous: str | None = Field(default=None, max_length=128) + detail: dict | None = None + + class PolicyBuildRequest(BaseModel): """Body of POST /x/policy/build — the declarative surface the dashboard builder collects; lowered by the core's elaborate, never host-side.""" @@ -414,6 +435,45 @@ def recent_calls(limit: int = 100): return {"calls": host_store.recent_calls(min(max(int(limit), 1), 1000)), "total": host_store.count()} + # ---- model lines (operator view + head moves) ------------------------------ + # A line is a named pointer to the family currently qualified as the head of + # a model series (model_lines.py); policies observe it as `line_`. + # Admin-gated like the other operator surfaces: moving a head re-routes every + # caller whose policy names the line. + @app.get("/x/lines") + def model_lines_state(events: int = 20): + import model_lines + model_lines.reload() + return model_lines.snapshot(events=min(max(int(events), 0), 200)) + + @app.post("/x/lines/{line}/promote") + def model_line_promote(line: str, body: ModelLineMoveRequest): + import model_lines + try: + result = model_lines.promote( + line, body.family, actor=body.actor or "operator", + detail=body.detail, expected_current=body.expected_current, + force=body.force) + except model_lines.ModelLineError as exc: + return JSONResponse(status_code=400, content={"error": { + "message": str(exc), "type": "invalid_request_error", + "code": "model_line"}}) + return _model_line_move_response(result) + + @app.post("/x/lines/{line}/rollback") + def model_line_rollback(line: str, body: ModelLineRollbackRequest): + import model_lines + try: + result = model_lines.rollback( + line, actor=body.actor or "operator", detail=body.detail, + expected_current=body.expected_current, + expected_previous=body.expected_previous) + except model_lines.ModelLineError as exc: + return JSONResponse(status_code=400, content={"error": { + "message": str(exc), "type": "invalid_request_error", + "code": "model_line"}}) + return _model_line_move_response(result) + # ---- AntSeed buyer hot-wallet control (dashboard self-service) ----------- # Proxy deposit/withdraw/refresh to the sidecar control server, then refresh # SOURCE_STATE so /x/market reflects the new escrow at once. Internal — /x/* @@ -1702,7 +1762,21 @@ def _request_to_contract( _ROUTER_ADMIN_PATHS = ("/x/wallet", "/x/providers", "/x/provider-key", "/x/config/", - "/x/codex/", "/x/calls", "/x/sessions", "/x/session/") + "/x/codex/", "/x/calls", "/x/sessions", "/x/session/", + "/x/lines") + + +def _model_line_move_response(result: dict): + """A head move's outcome as HTTP: a lost compare-and-set or a no-op is the + caller's conflict (409), an unreachable store is ours (503).""" + if result.get("ok"): + return result + reason = result.get("reason") + status = 503 if reason == "store_unavailable" else 409 + return JSONResponse(status_code=status, content={"error": { + "message": f"model line not moved: {reason}", + "type": "router_error" if status == 503 else "invalid_request_error", + "code": f"model_line_{reason}", "family": result.get("family")}}) def _router_admin_ok(request: Request) -> bool: diff --git a/tests/test_model_line_qualifier.py b/tests/test_model_line_qualifier.py new file mode 100644 index 0000000..f6cb897 --- /dev/null +++ b/tests/test_model_line_qualifier.py @@ -0,0 +1,575 @@ +"""The model-line qualifier: answer validation, the three gates, the consecutive +pass requirement, and what each mode is allowed to write. + +Transport is faked at the `post`/`get` boundary (the router's OpenAI-compatible +surface); the store is the real one, so promotions, proposals and rollbacks are +asserted on what an operator would actually read back. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import model_line_qualifier as q # noqa: E402 +import model_lines # noqa: E402 + +GOOD = '{"decision": "authentic", "confidence": 80, "summary": "consistent"}' +SUITE = q.parse_suite({ + "name": "memories", + "request": {"temperature": 0.7, "max_tokens": 4000, + "first_token_timeout_ms": 7000, "timeout_ms": 22000}, + "expect": {"type": "json", "required": { + "decision": {"enum": ["authentic", "suspicious", "inconclusive"]}, + "confidence": {"type": "integer", "min": 0, "max": 100}, + "summary": {"type": "string"}}}, + "prompts": ["p1", "p2", "p3"], +}) + + +@pytest.fixture +def lines(tmp_path, monkeypatch, host_store_clean): + def install(**lines_): + path = tmp_path / "model_lines.json" + path.write_text(json.dumps({"version": 1, "lines": lines_}), encoding="utf-8") + monkeypatch.setenv("MODEL_LINES_FILE", str(path)) + model_lines.configure() + return host_store_clean + yield install + monkeypatch.delenv("MODEL_LINES_FILE", raising=False) + model_lines.configure() + + +class FakeRouter: + """`behaviour[family]` is a dict: text, cost, status. Records every body.""" + + def __init__(self, families, behaviour): + self.families = families + self.behaviour = behaviour + self.bodies: list[dict] = [] + + def get(self, path): + assert path == "/v1/models" + return 200, {"data": [{"id": "profile:default"}] + + [{"id": f"family:{f}"} for f in self.families]} + + def post(self, body): + self.bodies.append(body) + family = body["policy_ir"][1][-1][1] + b = self.behaviour.get(family, {}) + status = b.get("status", 200) + if body["messages"][0]["content"] in b.get("fail_on", ()): + status = 504 + if status != 200: + return status, {"error": {"code": b.get("code", "timeout")}} + return 200, { + "choices": [{"message": {"content": b.get("text", GOOD)}, + "finish_reason": b.get("finish", "stop")}], + "x_router": {"model_family": b.get("served", family), + "provider": "openrouter", "cost_usd": b.get("cost", 0.001)}} + + def run(self, mode, **kw): + return q.run(post=self.post, get=self.get, suites=[SUITE], mode=mode, + workers=2, **kw) + + +def _line(head, **kw): + return {"head": head, "match": r"^m-(\d+)$", **kw} + + +# ---- answers ------------------------------------------------------------------- + +@pytest.mark.parametrize("text, ok", [ + (GOOD, True), + ("```json\n" + GOOD + "\n```", True), # fenced, as lenient callers accept + ("Sure! " + GOOD + " Hope that helps.", True), + ('{"decision": "maybe", "confidence": 80, "summary": "x"}', False), + ('{"decision": "authentic", "confidence": "80", "summary": "x"}', False), + ('{"decision": "authentic", "confidence": 180, "summary": "x"}', False), + ('{"decision": "authentic", "confidence": true, "summary": "x"}', False), + ('{"decision": "authentic", "confidence": 80}', False), + ('{"decision": "authentic", "confidence": 80, "summary": " "}', False), + ("I cannot help with that.", False), + ("[1, 2]", False), + ("", False), +]) +def test_json_answers_are_checked_against_the_suite(text, ok): + assert q.answer_is_valid(text, SUITE["expect"]) is ok + + +def test_text_and_regex_expectations(): + assert q.answer_is_valid("anything", {"type": "text"}) + assert not q.answer_is_valid(" ", {"type": "text"}) + assert q.answer_is_valid("result: true", {"type": "regex", "pattern": r"\btrue\b"}) + assert not q.answer_is_valid("result: maybe", {"type": "regex", "pattern": r"\btrue\b"}) + + +@pytest.mark.parametrize("doc, message", [ + ({"name": "Bad Name", "prompts": ["p"]}, "short slug"), + ({"name": "s", "prompts": []}, "non-empty list"), + ({"name": "s", "prompts": ["p"], "request": {"model": "x"}}, "may only set"), + ({"name": "s", "prompts": ["p"], "request": {"policy_ir": []}}, "may only set"), + ({"name": "s", "prompts": ["p"], "expect": {"type": "xml"}}, "expect.type"), + ({"name": "s", "prompts": ["p"], "expect": {"type": "regex", "pattern": "("}}, "bad expect.pattern"), + # no pattern would compile to "match anything", a refusal included + ({"name": "s", "prompts": ["p"], "expect": {"type": "regex"}}, "non-empty string"), + ({"name": "s", "prompts": ["p"], "expect": {"type": "regex", "pattern": ""}}, "non-empty string"), + ({"name": "s", "prompts": ["p"], "repeats": 9}, "repeats"), +]) +def test_invalid_suites_are_refused(doc, message): + with pytest.raises(q.SuiteError, match=message): + q.parse_suite(doc) + + +def test_load_suites_reads_directories_and_rejects_duplicates(tmp_path): + for i in (1, 2): + (tmp_path / f"s{i}.json").write_text( + json.dumps({"name": f"s{i}", "prompts": ["p"]}), encoding="utf-8") + assert [s["name"] for s in q.load_suites([tmp_path])] == ["s1", "s2"] + (tmp_path / "dup.json").write_text( + json.dumps({"name": "s1", "prompts": ["p"]}), encoding="utf-8") + with pytest.raises(q.SuiteError, match="duplicate"): + q.load_suites([tmp_path]) + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(q.SuiteError, match="no prompt suites"): + q.load_suites([empty]) + + +# ---- probes -------------------------------------------------------------------- + +def test_probe_isolates_one_family_and_carries_the_suite_request(): + router = FakeRouter(["m-1"], {}) + summary = q.probe_family(router.post, "m-1", [SUITE], "qualify-m-1") + assert summary["probes"] == 3 and summary["pass_rate"] == 1.0 + body = router.bodies[0] + assert body["policy_ir"][1] == ["and", ["meets_req"], ["not", ["is", "disabled"]], + ["family_eq", "m-1"]] + assert body["policy_ir"][-1] == ["always", {"action": "next_candidate"}] + assert (body["first_token_timeout_ms"], body["timeout_ms"]) == (7000, 22000) + assert body["messages"] == [{"role": "user", "content": "p1"}] + assert body["model"] == "policy:qualify-m-1" + + +def test_probe_failures_are_counted_not_raised(): + def boom(body): + raise ConnectionError("down") + summary = q.probe_family(boom, "m-1", [SUITE], "x") + assert summary["passed"] == 0 and summary["pass_rate"] == 0.0 + assert summary["errors"] == {"memories:ConnectionError": 3} + assert summary["mean_cost_usd"] is None and summary["latency_p95_ms"] is None + # the network failed, not the model: nothing may be concluded from it + assert summary["inconclusive"] == 3 and not q.conclusive(summary) + + +@pytest.mark.parametrize("status, code, inconclusive", [ + (429, "caller_rate_limit", True), # our own key, before any provider + (402, "key_budget_exhausted", True), + (503, "router_overloaded", True), + (401, "consumer_auth", True), + (502, "upstream", True), + (429, "exhausted", False), # the provider's limit: the route's problem + (504, "timeout", False), + (400, "invalid_request_error", False), +]) +def test_ingress_rejections_are_inconclusive_model_failures_are_not(status, code, inconclusive): + router = FakeRouter(["m-1"], {"m-1": {"status": status, "code": code}}) + summary = q.probe_family(router.post, "m-1", [SUITE], "x") + assert summary["passed"] == 0 + assert (summary["inconclusive"] == 3) is inconclusive + assert q.conclusive(summary) is (not inconclusive) + + +def test_exhausted_output_budget_is_reported_as_truncated(): + router = FakeRouter(["m-1"], {"m-1": {"text": "", "finish": "length"}}) + summary = q.probe_family(router.post, "m-1", [SUITE], "x") + assert summary["passed"] == 0 + assert summary["errors"] == {"memories:truncated": 3} + + +def test_answer_from_another_family_does_not_credit_the_candidate(): + router = FakeRouter(["m-2"], {"m-2": {"served": "m-1"}}) + summary = q.probe_family(router.post, "m-2", [SUITE], "x") + assert summary["passed"] == 0 + assert summary["errors"] == {"memories:served_other_family": 3} + + +def test_cost_is_unknown_unless_every_answered_probe_reported_one(): + results = [{"suite": "s", "status": 200, "latency_ms": 10, "valid": True, "cost_usd": 0.002}, + {"suite": "s", "status": 200, "latency_ms": 30, "valid": True, "cost_usd": None}] + assert q.summarize("m", results)["mean_cost_usd"] is None + results[1]["cost_usd"] = 0.004 + s = q.summarize("m", results) + assert s["mean_cost_usd"] == 0.003 + assert (s["latency_p50_ms"], s["latency_p90_ms"], s["latency_p95_ms"]) == (10, 30, 30) + + +def test_p90_ignores_the_single_slowest_probe_of_a_run(): + results = [{"suite": "s", "status": 200, "latency_ms": ms, "valid": True, "cost_usd": 0.001} + for ms in [*range(1000, 16000, 1000), 40000]] # 16 probes + s = q.summarize("m", results) + assert (s["latency_p90_ms"], s["latency_p95_ms"]) == (15000, 40000) + + +# ---- gates --------------------------------------------------------------------- + +DEF = {"min_pass_rate": 0.9, "min_suite_pass_rate": 0.5, "max_latency_ms": 15000, + "max_latency_ratio": 1.25, "max_cost_ratio": 1.1} + + +def _summary(pass_rate=1.0, p90=5000, cost=0.001, probes=16, suites=None): + passed = round(pass_rate * probes) + return {"probes": probes, "pass_rate": pass_rate, "latency_p90_ms": p90, + "mean_cost_usd": cost, "costs": [cost] * probes, "inconclusive": 0, + "suites": suites or {"s": {"probes": probes, "passed": passed}}} + + +@pytest.mark.parametrize("candidate, head, qualified, failing", [ + (_summary(), _summary(), True, None), + (_summary(cost=0.0011), _summary(cost=0.001), True, None), # at the ratio + (_summary(cost=0.0012), _summary(cost=0.001), False, "cost"), + (_summary(pass_rate=0.9375), _summary(), True, None), # 15/16: noise + (_summary(pass_rate=0.875), _summary(), False, "answers"), # 14/16 + # over the absolute bound, and slower than the head by more than the ratio + (_summary(p90=15001), _summary(p90=5000), False, "latency"), + # over the absolute bound but no slower than a head that already is + (_summary(p90=24000), _summary(p90=20000), True, None), + (_summary(p90=25001), _summary(p90=20000), False, "latency"), + # the relative allowance needs a head latency to be relative to + (_summary(p90=24000), _summary(p90=None, pass_rate=0.0, cost=None), False, "latency"), + (_summary(p90=None, pass_rate=0.0), _summary(), False, "answers"), + (_summary(cost=None), _summary(), False, "cost"), + (_summary(), _summary(cost=None), False, "cost"), # no baseline + # a degraded head is still a baseline: it is the one worth replacing + (_summary(), _summary(pass_rate=0.5), True, None), + (_summary(cost=0.0012), _summary(pass_rate=0.5, cost=0.001), False, "cost"), + (_summary(cost=0.0), _summary(cost=0.0), True, None), + (_summary(cost=0.001), _summary(cost=0.0), False, "cost"), +]) +def test_gates_fail_closed(candidate, head, qualified, failing): + verdict = q.judge(candidate, head, DEF) + assert verdict["qualified"] is qualified + if failing: + assert verdict["gates"][failing]["ok"] is False + + +def test_whole_suite_failure_cannot_hide_behind_the_aggregate(): + # 9 general prompts pass, the single specialist prompt fails: 90% overall + candidate = _summary(pass_rate=0.9, probes=10, suites={ + "general": {"probes": 9, "passed": 9}, "specialist": {"probes": 1, "passed": 0}}) + verdict = q.judge(candidate, _summary(probes=10), DEF) + assert verdict["qualified"] is False + assert verdict["gates"]["answers"]["suites"] == {"general": 1.0, "specialist": 0.0} + # the same overall rate spread across suites is just noise + spread = _summary(pass_rate=0.9, probes=10, suites={ + "general": {"probes": 6, "passed": 5}, "specialist": {"probes": 4, "passed": 4}}) + assert q.judge(spread, _summary(probes=10), DEF)["qualified"] is True + + +def test_cost_is_compared_over_the_same_probes_and_counts_unusable_answers(): + # Nine prompts cost both sides $0.001. On the tenth the head answers for + # $0.01 and the candidate charges $0.02 for an answer that is not valid. + head = _summary(probes=10) + head["costs"] = [0.001] * 9 + [0.01] + candidate = _summary(pass_rate=0.9, probes=10) + candidate["costs"] = [0.001] * 9 + [0.02] + cost = q.judge(candidate, head, DEF)["gates"]["cost"] + assert cost["ok"] is False and cost["paired"] == 10 + assert cost["ratio"] == round(0.029 / 0.019, 4) + + +def test_cost_pairs_only_probes_both_sides_reported(): + head = _summary(probes=4) + head["costs"] = [0.004, None, 0.004, 0.004] # one head probe had no response + candidate = _summary(probes=4) + candidate["costs"] = [0.004, 0.5, 0.004, 0.004] # its cost there has no counterpart + cost = q.judge(candidate, head, DEF)["gates"]["cost"] + assert cost["ok"] is True and cost["paired"] == 3 and cost["ratio"] == 1.0 + # too few pairs to say anything: fail closed + head["costs"] = [0.004, None, None, None] + assert q.judge(candidate, head, DEF)["gates"]["cost"]["reason"] == "no_head_baseline" + + +def test_consecutive_passes_stop_at_the_first_failure(): + def ev(family, kind, head="m-1"): + return {"family": family, "kind": kind, "previous_family": head} + events = [ev("m-2", "candidate_passed"), + ev("m-3", "candidate_failed"), # another family + ev("m-2", "candidate_passed"), + ev("m-2", "candidate_failed"), + ev("m-2", "candidate_passed")] + assert q.consecutive_passes(events, "m-2", "m-1") == 2 + assert q.consecutive_passes(events, "m-3", "m-1") == 0 + assert q.consecutive_passes(events, "m-9", "m-1") == 0 + + +def test_passes_against_an_earlier_head_do_not_count(): + def ev(head): + return {"family": "m-3", "kind": "candidate_passed", "previous_family": head} + events = [ev("m-2"), ev("m-1"), ev("m-1")] # newest first + assert q.consecutive_passes(events, "m-3", "m-2") == 1 + assert q.consecutive_passes(events, "m-3", "m-1") == 0 # newest is vs another head + + +# ---- runs ---------------------------------------------------------------------- + +def test_auto_promotes_only_after_consecutive_qualifying_runs(lines): + store = lines(alpha=_line("m-1")) + router = FakeRouter(["m-1", "m-2"], {}) + + first = router.run("auto") + assert first["moves"] == [] and first["proposals"] == [] + assert first["lines"]["alpha"]["candidates"][0]["streak"] == 1 + assert model_lines.head_of("alpha") == "m-1" + + second = router.run("auto") + assert second["moves"] == [{"line": "alpha", "from": "m-1", "to": "m-2"}] + assert model_lines.head_of("alpha") == "m-2" + promoted = store.model_line_events("alpha", kinds=("promoted",))[0] + assert (promoted["actor"], promoted["family"], promoted["previous_family"]) == ( + "qualifier", "m-2", "m-1") + assert promoted["detail"]["gates"]["cost"]["ok"] is True + assert promoted["detail"]["streak"] == 2 + + third = router.run("auto") # nothing newer than the new head + assert third["moves"] == [] and third["lines"]["alpha"]["candidates"] == [] + + +def test_a_failed_run_resets_the_streak(lines): + lines(alpha=_line("m-1")) + router = FakeRouter(["m-1", "m-2"], {}) + router.run("auto") + router.behaviour["m-2"] = {"text": "I cannot help with that."} + assert router.run("auto")["moves"] == [] + router.behaviour["m-2"] = {} + assert router.run("auto")["moves"] == [] # streak restarted at 1 + assert router.run("auto")["moves"] == [{"line": "alpha", "from": "m-1", "to": "m-2"}] + + +def test_propose_records_one_standing_proposal_and_moves_nothing(lines): + store = lines(alpha=_line("m-1")) + router = FakeRouter(["m-1", "m-2"], {}) + router.run("propose") + for _ in range(3): + report = router.run("propose") + assert report["proposals"] == [{"line": "alpha", "from": "m-1", "to": "m-2"}] + assert report["moves"] == [] + assert model_lines.head_of("alpha") == "m-1" + assert len(store.model_line_events("alpha", kinds=("proposed",))) == 1 + + # an operator approves it; the next run has nothing left to propose + assert model_lines.promote("alpha", "m-2", actor="op", expected_current="m-1")["ok"] + assert router.run("propose")["proposals"] == [] + + +def test_report_mode_writes_nothing(lines): + store = lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2"], {}) + report = router.run("report") + assert report["proposals"] == [{"line": "alpha", "from": "m-1", "to": "m-2"}] + assert store.model_line_events() == [] + assert model_lines.head_of("alpha") == "m-1" + + +@pytest.mark.parametrize("behaviour, gate", [ + ({"cost": 0.005}, "cost"), # 5x the head's measured cost + ({"status": 504, "code": "timeout"}, "answers"), + ({"text": '{"decision": "authentic"}'}, "answers"), +]) +def test_candidate_failing_a_gate_is_recorded_and_never_promoted(lines, behaviour, gate): + store = lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2"], {"m-2": behaviour}) + report = router.run("auto") + assert report["moves"] == [] + candidate = report["lines"]["alpha"]["candidates"][0] + assert candidate["qualified"] is False and candidate["gates"][gate]["ok"] is False + assert store.model_line_events("alpha")[0]["kind"] == "candidate_failed" + assert model_lines.head_of("alpha") == "m-1" + + +def test_newest_qualifying_candidate_wins_and_older_ones_are_capped(lines): + lines(alpha=_line("m-1", consecutive_passes=1, max_candidates=2)) + router = FakeRouter(["m-1", "m-2", "m-3", "m-4"], {"m-4": {"cost": 0.01}}) + report = router.run("auto") + probed = [c["family"] for c in report["lines"]["alpha"]["candidates"]] + assert probed == ["m-4", "m-3"] # newest two only + assert report["moves"] == [{"line": "alpha", "from": "m-1", "to": "m-3"}] + + +def test_unhealthy_head_blocks_promotion_and_is_flagged(lines): + store = lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2"], {"m-1": {"status": 504}}) + report = router.run("auto") + assert report["unhealthy_heads"] == ["alpha"] + assert report["moves"] == [] # no baseline, no promotion + assert report["lines"]["alpha"]["candidates"][0]["gates"]["cost"]["reason"] == "no_head_baseline" + assert store.model_line_events("alpha", kinds=("head_unhealthy",))[0]["family"] == "m-1" + + +def test_degraded_head_is_flagged_and_can_still_be_replaced(lines): + store = lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2"], {"m-1": {"fail_on": ["p3"]}}) # 2/3 pass + report = router.run("auto") + assert report["unhealthy_heads"] == ["alpha"] + assert report["lines"]["alpha"]["head_summary"]["pass_rate"] < 0.9 + assert report["moves"] == [{"line": "alpha", "from": "m-1", "to": "m-2"}] + # replaced by a qualified successor: no separate unhealthy alarm for it + assert store.model_line_events("alpha", kinds=("head_unhealthy",)) == [] + + +def test_auto_rolls_back_its_own_promotion_when_the_predecessor_still_passes(lines): + store = lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2"], {}) + assert router.run("auto")["moves"] == [{"line": "alpha", "from": "m-1", "to": "m-2"}] + + router.behaviour["m-2"] = {"text": "refused"} + report = router.run("auto") + assert report["unhealthy_heads"] == ["alpha"] + assert report["moves"] == [{"line": "alpha", "from": "m-2", "to": "m-1"}] + assert model_lines.head_of("alpha") == "m-1" + assert store.model_line_events("alpha")[0]["kind"] == "rolled_back" + + +def test_auto_never_rolls_back_a_head_an_operator_chose(lines): + store = lines(alpha=_line("m-1")) + assert model_lines.promote("alpha", "m-2", actor="dashboard:alice")["ok"] + router = FakeRouter(["m-1", "m-2"], {"m-2": {"status": 504}}) + report = router.run("auto") + assert report["moves"] == [] and model_lines.head_of("alpha") == "m-2" + assert store.model_line_events("alpha")[0]["kind"] == "head_unhealthy" + + +def test_propose_mode_never_rolls_back(lines): + lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2"], {}) + router.run("auto") + router.behaviour["m-2"] = {"status": 504} + assert router.run("propose")["moves"] == [] + assert model_lines.head_of("alpha") == "m-2" + + +def test_streak_restarts_when_the_head_changes_under_the_candidate(lines): + lines(alpha=_line("m-1")) + router = FakeRouter(["m-1", "m-2", "m-3"], {"m-2": {"cost": 0.01}}) # m-2 never qualifies + assert router.run("auto")["moves"] == [] # m-3: one pass, against m-1 + # an operator moves the line to m-2: m-3's pass was measured against m-1 + assert model_lines.promote("alpha", "m-2", actor="op")["ok"] + router.behaviour["m-2"] = {} + report = router.run("auto") + assert report["moves"] == [] + assert report["lines"]["alpha"]["candidates"][0]["streak"] == 1 + assert router.run("auto")["moves"] == [{"line": "alpha", "from": "m-2", "to": "m-3"}] + + +def test_inconclusive_head_freezes_the_line(lines): + store = lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2"], {"m-1": {"status": 429, "code": "caller_rate_limit"}}) + report = router.run("auto") + assert report["inconclusive_lines"] == ["alpha"] + assert report["unhealthy_heads"] == [] and report["moves"] == [] + assert report["lines"]["alpha"]["head_healthy"] is None + assert store.model_line_events("alpha") == [] # nothing judged, nothing recorded + + +def test_inconclusive_head_is_never_rolled_back(lines): + lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2"], {}) + assert router.run("auto")["moves"] == [{"line": "alpha", "from": "m-1", "to": "m-2"}] + # our own key is rate limited while probing the new head; the old one answers + router.behaviour["m-2"] = {"status": 429, "code": "key_rate_limit"} + assert router.run("auto")["moves"] == [] + assert model_lines.head_of("alpha") == "m-2" + + +def test_inconclusive_candidate_neither_extends_nor_breaks_its_streak(lines): + store = lines(alpha=_line("m-1")) + router = FakeRouter(["m-1", "m-2"], {}) + router.run("auto") # streak 1 + router.behaviour["m-2"] = {"status": 503, "code": "router_overloaded"} + report = router.run("auto") + assert report["moves"] == [] + assert report["lines"]["alpha"]["candidates"][0]["inconclusive"] is True + assert len(store.model_line_events("alpha")) == 1 # no event for it + router.behaviour["m-2"] = {} + assert router.run("auto")["moves"] == [{"line": "alpha", "from": "m-1", "to": "m-2"}] + + +def test_run_stops_when_a_result_cannot_be_recorded(lines, monkeypatch): + store = lines(alpha=_line("m-1")) + router = FakeRouter(["m-1", "m-2"], {}) + router.run("auto") # streak 1 + router.behaviour["m-2"] = {"text": "refused"} # this run fails... + monkeypatch.setattr(store, "add_model_line_event", lambda *a, **k: False) + with pytest.raises(RuntimeError, match="audit write failed"): # ...and cannot be recorded + router.run("auto") + assert model_lines.head_of("alpha") == "m-1" + + +def test_auto_does_not_undo_its_own_rollback(lines): + store = lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2"], {}) + router.run("auto") # m-1 -> m-2 + router.behaviour["m-2"] = {"text": "refused"} + assert router.run("auto")["moves"] == [{"line": "alpha", "from": "m-2", "to": "m-1"}] + # now m-1 fails too, and m-2 answers again but at 5x the cost: it must not + # come back on answer rate alone just because it is the "previous" head + router.behaviour = {"m-1": {"status": 504}, "m-2": {"cost": 0.005}} + report = router.run("auto") + assert report["moves"] == [] and model_lines.head_of("alpha") == "m-1" + assert store.model_line_events("alpha")[0]["kind"] == "head_unhealthy" + + +def test_rollback_is_skipped_when_the_line_moved_during_the_probes(lines, monkeypatch): + store = lines(alpha=_line("m-1", consecutive_passes=1)) + router = FakeRouter(["m-1", "m-2", "m-3"], {"m-3": {"cost": 0.01}}) + router.run("auto") # m-1 -> m-2 + router.behaviour["m-2"] = {"text": "refused"} + real_post = router.post + moved = [] + + def post(body): + # an operator promotes m-3 while the qualifier is still probing + if not moved: + moved.append(model_lines.promote("alpha", "m-3", actor="op")["ok"]) + return real_post(body) + + report = q.run(post=post, get=router.get, suites=[SUITE], mode="auto", workers=1) + assert moved == [True] + # m-2 did fail its probes, but it is no longer the move to undo: the latest + # move is the operator's, so the qualifier only records what it saw + assert report["moves"] == [] + assert "rolled_back" not in report["lines"]["alpha"]["head_action"] + assert store.get_model_line_heads()["alpha"]["family"] == "m-3" # the operator's move stands + assert store.model_line_events("alpha", kinds=("rolled_back",)) == [] + + +def test_run_refuses_to_decide_without_the_store(lines, monkeypatch): + store = lines(alpha=_line("m-1")) + monkeypatch.setattr(store, "get_model_line_heads", lambda: None) + router = FakeRouter(["m-1", "m-2"], {}) + with pytest.raises(RuntimeError, match="host store unavailable"): + router.run("auto") + assert router.run("report")["lines"]["alpha"]["head"] == "m-1" # read-only is fine + + +def test_run_fails_loudly_when_the_router_lists_no_families(lines): + lines(alpha=_line("m-1")) + router = FakeRouter([], {}) + with pytest.raises(RuntimeError, match="no families"): + router.run("report") + with pytest.raises(ValueError, match="unknown line"): + FakeRouter(["m-1"], {}).run("report", only_lines=["ghost"]) + + +def test_only_lines_restricts_probing(lines): + lines(alpha=_line("m-1"), beta={"head": "n-1", "match": r"^n-(\d+)$"}) + router = FakeRouter(["m-1", "m-2", "n-1", "n-2"], {}) + report = router.run("report", only_lines=["beta"]) + assert list(report["lines"]) == ["beta"] + probed = {b["policy_ir"][1][-1][1] for b in router.bodies} + assert probed == {"n-1", "n-2"} diff --git a/tests/test_model_lines.py b/tests/test_model_lines.py new file mode 100644 index 0000000..f48d994 --- /dev/null +++ b/tests/test_model_lines.py @@ -0,0 +1,434 @@ +"""Model lines: definitions, the `line_` observation fields, head moves and +the operator endpoints. + +The engine-backed tests boot a real host so they exercise the actual Python/Lua +seam: a policy naming a line must rank the head family, follow a head move +without rebuilding the engine, and be refused when the line is not declared. +""" +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import model_lines # noqa: E402 +from llm_router_host import LLMRouterHost # noqa: E402 +from shim import create_app # noqa: E402 + + +def _doc(**lines): + return {"version": 1, "lines": lines} + + +def _line(head, match=None, **kw): + return {"head": head, "match": match, **kw} + + +@pytest.fixture +def lines_file(tmp_path, monkeypatch): + """Install a small definitions file for the test and restore the shipped one + afterwards, so no other test sees these lines or their moved heads.""" + def install(doc): + path = tmp_path / "model_lines.json" + path.write_text(json.dumps(doc), encoding="utf-8") + monkeypatch.setenv("MODEL_LINES_FILE", str(path)) + return model_lines.configure() + yield install + monkeypatch.delenv("MODEL_LINES_FILE", raising=False) + model_lines.configure() + + +def _host(): + h = LLMRouterHost(router_path=ROOT / "core" / "router.lua", + config_path=ROOT / "core" / "config.example.lua", + metrics_path=ROOT / "core" / "metrics.example.lua", + now_ms=lambda: 1_000_000) + h.init() + return h + + +def _policy(*lines): + preds = [["is", model_lines.field_name(n)] for n in lines] + pred = preds[0] if len(preds) == 1 else ["or", *preds] + return ["policy", ["and", ["meets_req"], pred], ["field", "success_rate"], + ["argmax"], ["id"], ["always", {"action": "next_candidate"}]] + + +def _replica_sees(monkeypatch, **heads): + """What a replica's refresh timer does after another process moved a head: + the next store read returns the new rows.""" + import host_store + rows = {line: {"family": family, "previous_family": None, + "updated_at": 1, "updated_by": "t"} for line, family in heads.items()} + monkeypatch.setattr(host_store, "get_model_line_heads", lambda: rows) + assert model_lines.reload() is True + + +def _ranked_families(host, policy): + ranked, _ = host.rank({"prompt": "hi", "policy_ir": policy}) + return {r["candidate"]["model_family"] for r in ranked} + + +# ---- definitions --------------------------------------------------------------- + +def test_shipped_definitions_are_valid_and_self_consistent(): + defs = model_lines.load_definitions(model_lines.DEFAULT_PATH) + assert defs, "the shipped model_lines.json declares lines" + for name, d in defs.items(): + assert re.fullmatch(r"[a-z][a-z0-9_]*", name) + if d["pattern"] is not None: + assert model_lines.version_key(d["pattern"], d["head"]) is not None, name + assert d["max_cost_ratio"] >= 1 + assert 0 < d["min_pass_rate"] <= 1 + + +@pytest.mark.parametrize("doc, message", [ + ({"version": 2, "lines": {}}, "version 1"), + (_doc(**{"Bad-Name": _line("m-1")}), "invalid line name"), + (_doc(a=_line("")), "`head` must be a family name"), + (_doc(a=_line("m-1", "m-(")), "bad `match`"), + (_doc(a=_line("m-1", r"^m-\d+$")), "capture group"), + (_doc(a=_line("other-1", r"^m-(\d+)$")), "does not match its own"), + (_doc(a=_line("m-1", None, max_cost_ratio=0)), "positive number"), + (_doc(a=_line("m-1", None, min_pass_rate=1.5)), "must be <= 1"), + (_doc(a=_line("m-1", None, min_suite_pass_rate=2)), "must be <= 1"), + (_doc(a=_line("m-1", None, surprise=1)), "unknown key"), + ({"version": 1, "defaults": {"nope": 1}, "lines": {}}, "unknown default knob"), +]) +def test_invalid_definitions_are_refused(doc, message): + with pytest.raises(model_lines.ModelLineError, match=re.escape(message)): + model_lines.parse_definitions(doc) + + +def test_knobs_resolve_line_over_file_defaults_over_builtins(): + defs = model_lines.parse_definitions({ + "version": 1, "defaults": {"max_latency_ms": 9000}, + "lines": {"a": _line("m-1"), "b": _line("m-2", None, max_latency_ms=4000)}}) + assert defs["a"]["max_latency_ms"] == 9000 + assert defs["b"]["max_latency_ms"] == 4000 + assert defs["a"]["max_cost_ratio"] == model_lines.QUALIFY_DEFAULTS["max_cost_ratio"] + + +def test_invalid_file_declares_no_lines(lines_file, tmp_path, monkeypatch): + bad = tmp_path / "bad.json" + bad.write_text("{not json", encoding="utf-8") + monkeypatch.setenv("MODEL_LINES_FILE", str(bad)) + assert model_lines.configure() == {} + assert model_lines.heads() == {} + + +# ---- successor discovery ------------------------------------------------------- + +CATALOG = [ + "gpt-6-sol", "gpt-6.1-sol", "gpt-6.2-sol", "gpt-6.2-sol-pro", "gpt-6.2-sol:batch", + "gpt-6.2-astra", "openai/gpt-6.2-sol", "gpt-oss-120b", + "deepseek-v4.1-flash-1017", "deepseek-v4.1-flash", "deepseek-v4.1-flash-1201", + "deepseek-v4.2-flash-0105", "deepseek-v4-flash-0731", + "grok-4.3", "grok-4.7", "grok-4.20", "grok-4.20-multi-agent", "grok-4.7-build-fast", + "mistral-large-2512", "mistral-large-4", "mistral-large-4-0", "mistral-large-2606", +] + + +def _def(match, head="x"): + return {"pattern": re.compile(match) if match else None, "head": head} + + +@pytest.mark.parametrize("match, head, expected", [ + # tier siblings (-pro, :batch, vendor-prefixed aliases) are not successors + (r"^gpt-(\d+(?:\.\d+)?)-sol$", "gpt-6.1-sol", ["gpt-6.2-sol"]), + (r"^gpt-(\d+(?:\.\d+)?)-sol$", "gpt-6-sol", ["gpt-6.2-sol", "gpt-6.1-sol"]), + (r"^gpt-(\d+(?:\.\d+)?)-sol$", "gpt-6.2-sol", []), + # version first, snapshot date second; the undated alias is not in the line + (r"^deepseek-v(\d+(?:\.\d+)?)-flash-(\d{4})$", "deepseek-v4.1-flash-1017", + ["deepseek-v4.2-flash-0105", "deepseek-v4.1-flash-1201"]), + (r"^grok-(\d+\.\d+)$", "grok-4.3", ["grok-4.20", "grok-4.7"]), + # a date-coded line ignores the differently-versioned `-4` naming + (r"^mistral-large-(\d{4})$", "mistral-large-2512", ["mistral-large-2606"]), +]) +def test_successors_are_strictly_newer_members_of_the_line(match, head, expected): + assert model_lines.successors(_def(match), head, CATALOG) == expected + + +@pytest.mark.parametrize("line, catalog, expected", [ + # marketplace look-alikes without the dot must not outrank the real series + ("glm", ["glm-5.3", "glm-52", "glm-5.4", "glm-6", "glm-5.3-flash"], ["glm-6", "glm-5.4"]), + ("minimax", ["minimax-m3", "minimax-m27", "minimax-m25", "minimax-m3.5", + "minimax-m3.1-flash"], ["minimax-m3.5"]), + ("openai_sol", ["gpt-6.1-sol", "gpt-62-sol", "gpt-6.2-sol", "gpt-6.2-sol-pro", + "gpt-6.2-sol:batch"], ["gpt-6.2-sol"]), + ("anthropic_sonnet", ["claude-sonnet-5.5", "claude-sonnet-5-5", "claude-sonnet-6", + "claude-sonnet-4-6"], ["claude-sonnet-6"]), + ("deepseek_flash", ["deepseek-v4.1-flash-1017", "deepseek-v4.1-flash", + "deepseek-v41-flash", "deepseek-v4.1-flash-1201"], + ["deepseek-v4.1-flash-1201"]), + ("gpt_oss", ["gpt-oss-120b", "gpt-oss-20b", "gpt-oss-240b"], []), +]) +def test_shipped_patterns_follow_the_real_series_only(line, catalog, expected): + d = model_lines.load_definitions(model_lines.DEFAULT_PATH)[line] + assert model_lines.successors(d, d["head"], catalog) == expected + + +def test_successors_respect_limit_and_patternless_lines(): + d = _def(r"^gpt-(\d+(?:\.\d+)?)-sol$") + assert model_lines.successors(d, "gpt-6-sol", CATALOG, limit=1) == ["gpt-6.2-sol"] + assert model_lines.successors(_def(None), "gpt-oss-120b", CATALOG) == [] + # a forced head outside the pattern has no automatic successors + assert model_lines.successors(d, "gpt-oss-120b", CATALOG) == [] + + +# ---- the observation field ----------------------------------------------------- + +def test_policy_naming_a_line_ranks_only_its_head(lines_file): + lines_file(_doc(alpha=_line("deepseek-v3"), beta=_line("llama-3.3-70b"))) + host = _host() + names = {f["name"]: f for f in host.field_schema()} + assert names["line_alpha"] == {"name": "line_alpha", "sort": "Bool", + "group": "model", "core": False} + assert _ranked_families(host, _policy("alpha")) == {"deepseek-v3"} + assert _ranked_families(host, _policy("alpha", "beta")) == { + "deepseek-v3", "llama-3.3-70b"} + + +def test_head_move_is_visible_to_a_running_engine(lines_file, monkeypatch): + lines_file(_doc(alpha=_line("deepseek-v3"))) + host = _host() + policy = _policy("alpha") + before = host.normalize_policy(policy, admit=True)["policy_id"] + assert _ranked_families(host, policy) == {"deepseek-v3"} + + _replica_sees(monkeypatch, alpha="hermes-3-405b") + assert _ranked_families(host, policy) == {"hermes-3-405b"} + # the policy term, and so its identity, did not change + assert host.normalize_policy(policy, admit=True)["policy_id"] == before + + +def test_every_live_engine_follows_a_move_and_dead_ones_are_dropped(lines_file, monkeypatch): + import gc + lines_file(_doc(alpha=_line("deepseek-v3"))) + first, second = _host(), _host() + policy = _policy("alpha") + _replica_sees(monkeypatch, alpha="hermes-3-405b") + assert _ranked_families(first, policy) == {"hermes-3-405b"} + assert _ranked_families(second, policy) == {"hermes-3-405b"} + + # an engine built after the move starts on the moved head, not the seed + assert _ranked_families(_host(), policy) == {"hermes-3-405b"} + + del second + gc.collect() + _replica_sees(monkeypatch, alpha="llama-3.3-70b") # must not touch a dead engine + assert _ranked_families(first, policy) == {"llama-3.3-70b"} + + +def test_engine_that_fails_to_sync_does_not_block_the_others(lines_file, monkeypatch): + lines_file(_doc(alpha=_line("deepseek-v3"))) + broken, healthy = _host(), _host() + monkeypatch.setattr(broken, "sync_model_line_heads", + lambda heads: (_ for _ in ()).throw(RuntimeError("lua busy"))) + _replica_sees(monkeypatch, alpha="hermes-3-405b") + policy = _policy("alpha") + assert _ranked_families(healthy, policy) == {"hermes-3-405b"} + assert _ranked_families(broken, policy) == {"deepseek-v3"} # kept its last table + + # once it accepts updates again, the next store read heals it even though + # nothing moved in between + monkeypatch.undo() + _replica_sees(monkeypatch, alpha="hermes-3-405b") + assert _ranked_families(broken, policy) == {"hermes-3-405b"} + + +def test_older_store_read_cannot_replace_a_newer_one(lines_file, monkeypatch): + import host_store + lines_file(_doc(alpha=_line("deepseek-v3"))) + host = _host() + row = lambda fam: {"alpha": {"family": fam, "previous_family": None, # noqa: E731 + "updated_at": 1, "updated_by": "t"}} + calls = [] + + def slow_read(): + # The timer's read returns the OLD head, but only after a promotion's + # reload — started later — has read the new one and installed it. + calls.append(1) + if len(calls) == 1: + assert model_lines.reload() is True + return row("hermes-3-405b") # stale by the time it lands + return row("llama-3.3-70b") # what the later read saw + + monkeypatch.setattr(host_store, "get_model_line_heads", slow_read) + assert model_lines.reload() is True + assert model_lines.head_of("alpha") == "llama-3.3-70b" + assert _ranked_families(host, _policy("alpha")) == {"llama-3.3-70b"} + + +def test_head_absent_from_catalog_ranks_nothing(lines_file): + lines_file(_doc(alpha=_line("not-a-real-family"))) + assert _ranked_families(_host(), _policy("alpha")) == set() + + +def test_undeclared_line_fails_admission(lines_file): + lines_file(_doc(alpha=_line("deepseek-v3"))) + with pytest.raises(Exception, match="undeclared field 'line_missing'"): + _host().normalize_policy(_policy("missing"), admit=True) + + +def test_model_meta_exposes_line_membership(lines_file): + lines_file(_doc(alpha=_line("deepseek-v3"))) + meta = _host().model_meta() + assert meta["deepseek-v3"]["line_alpha"] is True + # only the head carries the mark; other families are not padded with false + assert "line_alpha" not in meta["hermes-3-405b"] + + +def test_no_lines_declared_adds_no_fields(lines_file): + lines_file(_doc()) + assert not [f for f in _host().field_schema() if f["name"].startswith("line_")] + + +# ---- head moves (store-backed) ------------------------------------------------- + +def test_promote_rollback_and_audit_trail(host_store_clean, lines_file): + lines_file(_doc(alpha=_line("m-1", r"^m-(\d+)$"))) + assert model_lines.reload() is True + assert model_lines.head_of("alpha") == "m-1" + + moved = model_lines.promote("alpha", "m-2", actor="tester", detail={"why": "t"}) + assert moved == {"ok": True, "family": "m-2", "previous_family": "m-1"} + assert model_lines.head_of("alpha") == "m-2" + + snap = model_lines.snapshot() + row = snap["lines"][0] + assert (row["head"], row["seed"], row["source"], row["previous"], + row["updated_by"]) == ("m-2", "m-1", "store", "m-1", "tester") + assert snap["store_ok"] is True + assert snap["events"][0]["kind"] == "promoted" + assert snap["events"][0]["detail"] == {"why": "t"} + + back = model_lines.rollback("alpha", actor="tester") + assert back == {"ok": True, "family": "m-1", "previous_family": "m-2"} + assert model_lines.head_of("alpha") == "m-1" + assert [e["kind"] for e in host_store_clean.model_line_events("alpha")] == [ + "rolled_back", "promoted"] + + +def test_promote_is_a_compare_and_set(host_store_clean, lines_file): + lines_file(_doc(alpha=_line("m-1", r"^m-(\d+)$"))) + assert model_lines.promote("alpha", "m-2", actor="a", expected_current="m-1")["ok"] + stale = model_lines.promote("alpha", "m-3", actor="b", expected_current="m-1") + assert stale == {"ok": False, "reason": "conflict", "family": "m-2"} + same = model_lines.promote("alpha", "m-2", actor="b") + assert same == {"ok": False, "reason": "unchanged", "family": "m-2"} + assert model_lines.head_of("alpha") == "m-2" + + +def test_promote_refuses_families_outside_the_line_unless_forced( + host_store_clean, lines_file): + lines_file(_doc(alpha=_line("m-1", r"^m-(\d+)$"))) + with pytest.raises(model_lines.ModelLineError, match="does not match"): + model_lines.promote("alpha", "other-9", actor="t") + with pytest.raises(model_lines.ModelLineError, match="unknown line"): + model_lines.promote("nope", "m-2", actor="t") + assert model_lines.promote("alpha", "other-9", actor="t", force=True)["ok"] + + +def test_rollback_target_is_chosen_inside_the_store(host_store_clean, lines_file): + lines_file(_doc(alpha=_line("m-1", r"^m-(\d+)$"))) + assert model_lines.promote("alpha", "m-2", actor="t")["ok"] # m-1 -> m-2 + # A caller read "current m-2, previous m-1". Then the line went m-2 -> m-3 -> + # m-2: the head is m-2 again, but undoing its LAST move leads to m-3. + assert model_lines.promote("alpha", "m-3", actor="t")["ok"] + assert model_lines.promote("alpha", "m-2", actor="t", force=True)["ok"] + stale = model_lines.rollback("alpha", actor="t", expected_current="m-2", + expected_previous="m-1") + assert stale == {"ok": False, "reason": "conflict", "family": "m-2"} + assert model_lines.head_of("alpha") == "m-2" + # without the stale expectation the store undoes the move that is current + assert model_lines.rollback("alpha", actor="t") == { + "ok": True, "family": "m-3", "previous_family": "m-2"} + + +def test_retried_rollback_conflicts_instead_of_undoing_itself(host_store_clean, lines_file): + lines_file(_doc(alpha=_line("m-1", r"^m-(\d+)$"))) + model_lines.promote("alpha", "m-2", actor="t") + first = model_lines.rollback("alpha", actor="t", expected_current="m-2") + assert first["ok"] and model_lines.head_of("alpha") == "m-1" + # the response was lost and the operator retries the same request + retry = model_lines.rollback("alpha", actor="t", expected_current="m-2") + assert retry == {"ok": False, "reason": "conflict", "family": "m-1"} + assert model_lines.head_of("alpha") == "m-1" + + +def test_rollback_without_a_previous_head(host_store_clean, lines_file): + lines_file(_doc(alpha=_line("m-1"))) + assert model_lines.rollback("alpha", actor="t") == { + "ok": False, "reason": "no_previous", "family": "m-1"} + + +def test_store_outage_keeps_last_known_heads(host_store_clean, lines_file, monkeypatch): + lines_file(_doc(alpha=_line("m-1", r"^m-(\d+)$"))) + model_lines.promote("alpha", "m-2", actor="t") + monkeypatch.setattr(host_store_clean, "get_model_line_heads", lambda: None) + assert model_lines.reload() is False + assert model_lines.head_of("alpha") == "m-2" # not snapped back to the seed + assert model_lines.snapshot(events=0)["store_ok"] is False + + +def test_stored_heads_for_undeclared_or_malformed_lines_are_ignored( + host_store_clean, lines_file): + lines_file(_doc(alpha=_line("m-1"), beta=_line("n-1"))) + host_store_clean.set_model_line_head("ghost", "m-9", seed_family=None, actor="t") + host_store_clean.set_model_line_head("beta", "bad family!", seed_family="n-1", actor="t") + assert model_lines.reload() is True + assert model_lines.heads() == {"alpha": "m-1", "beta": "n-1"} + + +# ---- operator endpoints -------------------------------------------------------- + +def test_lines_endpoints_move_heads_and_report_conflicts(host_store_clean, lines_file): + lines_file(_doc(alpha=_line("deepseek-v3", r"^deepseek-v(\d+)$"))) + client = TestClient(create_app(_host())) + + state = client.get("/x/lines").json() + assert state["lines"][0]["head"] == "deepseek-v3" + + r = client.post("/x/lines/alpha/promote", + json={"family": "deepseek-v4", "actor": "op", + "expected_current": "deepseek-v3"}) + assert r.status_code == 200 and r.json()["previous_family"] == "deepseek-v3" + assert client.get("/x/lines").json()["lines"][0]["head"] == "deepseek-v4" + + r = client.post("/x/lines/alpha/promote", + json={"family": "deepseek-v5", "expected_current": "deepseek-v3"}) + assert r.status_code == 409 + assert r.json()["error"]["code"] == "model_line_conflict" + + r = client.post("/x/lines/alpha/promote", json={"family": "llama-3.3-70b"}) + assert r.status_code == 400 and r.json()["error"]["code"] == "model_line" + assert client.post("/x/lines/nope/rollback", json={}).status_code == 400 + + r = client.post("/x/lines/alpha/rollback", + json={"actor": "op", "expected_current": "deepseek-v9"}) + assert r.status_code == 409 # not the move the caller saw + r = client.post("/x/lines/alpha/rollback", + json={"actor": "op", "expected_current": "deepseek-v4"}) + assert r.status_code == 200 and r.json()["family"] == "deepseek-v3" + + +def test_lines_endpoints_are_admin_gated(tmp_path, monkeypatch, lines_file): + import control_plane_client as cp + lines_file(_doc(alpha=_line("deepseek-v3"))) + monkeypatch.delenv("ROUTER_ADMIN_ALLOW_UNAUTHENTICATED", raising=False) + monkeypatch.setattr(cp, "CONTROL_PLANE_INTERNAL_SECRET", "s3cret") + client = TestClient(create_app(_host())) + assert client.get("/x/lines").status_code == 403 + assert client.post("/x/lines/alpha/promote", + json={"family": "x"}).status_code == 403 + assert client.post("/x/lines/alpha/rollback", json={}).status_code == 403 + ok = client.get("/x/lines", headers={"x-internal-secret": "s3cret"}) + assert ok.status_code == 200 diff --git a/tests/test_model_lines_surfaces.py b/tests/test_model_lines_surfaces.py new file mode 100644 index 0000000..a724f23 --- /dev/null +++ b/tests/test_model_lines_surfaces.py @@ -0,0 +1,203 @@ +"""The surfaces around model lines: the admin-dashboard passthrough, the replica +refresh loop, the shipped example suite and the qualifier CLI.""" +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import sys +from pathlib import Path + +import httpx +import pytest +from fastapi.testclient import TestClient + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["CALLER_KEYS_JSON"] = '{"internal":"default"}' +os.environ["CALLER_KEYS_SHA256_JSON"] = "{}" +os.environ["CALLER_KEYS_BOOTSTRAP_JSON"] = "{}" +os.environ["DASHBOARD_TRUSTED_USER_HEADER"] = "" + +import auth_proxy # noqa: E402 +import model_line_qualifier as qualifier # noqa: E402 +import model_lines # noqa: E402 +import serve # noqa: E402 + + +# ---- dashboard passthrough ----------------------------------------------------- + +class _RecordingRouter: + """Stands in for the ingress's upstream client and records what it was sent.""" + + def __init__(self, status=200, body=None, raises=None): + self.calls: list[dict] = [] + self.status = status + self.body = body if body is not None else {"ok": True} + self.raises = raises + + async def request(self, method, url, json=None, headers=None, timeout=None): + self.calls.append({"method": method, "url": url, "json": json, + "headers": dict(headers or {})}) + if self.raises is not None: + raise self.raises + return httpx.Response(self.status, json=self.body) + + +def _admin(monkeypatch, user="alice"): + monkeypatch.setattr(auth_proxy, "DASHBOARD_SESSION_SECRET", "test-dashboard-session-secret") + client = TestClient(auth_proxy.app) + client.cookies.set(auth_proxy.DASHBOARD_COOKIE_NAME, + auth_proxy._make_dashboard_session(user)) + return client + + +def test_dashboard_lines_require_an_admin_session(): + client = TestClient(auth_proxy.app) + assert client.get("/dashboard/api/lines").status_code == 401 + for action in ("promote", "rollback"): + r = client.post(f"/dashboard/api/lines/alpha/{action}", json={"family": "m-2"}) + assert r.status_code == 401, action + + +def test_dashboard_promote_forwards_with_secret_and_session_actor(monkeypatch): + import control_plane_client as cp + router = _RecordingRouter(body={"ok": True, "family": "m-2", "previous_family": "m-1"}) + monkeypatch.setattr(auth_proxy, "_client", router) + monkeypatch.setattr(cp, "CONTROL_PLANE_INTERNAL_SECRET", "s3cret") + client = _admin(monkeypatch) + + r = client.post("/dashboard/api/lines/alpha/promote", + json={"family": "m-2", "expected_current": "m-1", + "actor": "someone-else"}) + assert r.status_code == 200 and r.json()["family"] == "m-2" + call = router.calls[0] + assert call["method"] == "POST" and call["url"].endswith("/x/lines/alpha/promote") + assert call["headers"]["x-internal-secret"] == "s3cret" + # the audit actor is the authenticated viewer, not what the body claimed + assert call["json"]["actor"] == "dashboard:alice" + assert call["json"]["expected_current"] == "m-1" + + assert client.get("/dashboard/api/lines").status_code == 200 + assert router.calls[1]["method"] == "GET" + assert router.calls[1]["url"].endswith("/x/lines") + + +def test_dashboard_relays_router_conflicts_and_rejects_bad_input(monkeypatch): + router = _RecordingRouter(status=409, body={"error": {"code": "model_line_conflict"}}) + monkeypatch.setattr(auth_proxy, "_client", router) + client = _admin(monkeypatch) + + r = client.post("/dashboard/api/lines/alpha/rollback", json={}) + assert r.status_code == 409 + assert r.json()["error"]["code"] == "model_line_conflict" + + assert client.post("/dashboard/api/lines/Bad-Name/promote", + json={"family": "m-2"}).status_code == 400 + assert client.post("/dashboard/api/lines/alpha/promote", + content=b"[1]", headers={"content-type": "application/json"} + ).status_code == 400 + assert len(router.calls) == 1 # nothing malformed reached the router + + +def test_dashboard_says_when_a_move_may_have_been_applied(monkeypatch): + client = _admin(monkeypatch) + # sent, no answer: the router may have committed the move + monkeypatch.setattr(auth_proxy, "_client", + _RecordingRouter(raises=httpx.ReadTimeout("slow"))) + r = client.post("/dashboard/api/lines/alpha/rollback", json={}) + assert r.status_code == 504 + assert r.json()["error"]["code"] == "model_line_outcome_unknown" + # never connected: nothing happened, a plain retry is safe + monkeypatch.setattr(auth_proxy, "_client", + _RecordingRouter(raises=httpx.ConnectError("refused"))) + r = client.post("/dashboard/api/lines/alpha/rollback", json={}) + assert r.status_code == 502 and r.json()["error"]["code"] == "upstream" + # a read has no outcome to be unsure about + monkeypatch.setattr(auth_proxy, "_client", + _RecordingRouter(raises=httpx.ReadTimeout("slow"))) + assert client.get("/dashboard/api/lines").status_code == 502 + + +# ---- replica refresh ----------------------------------------------------------- + +def test_refresh_task_rereads_heads_and_survives_failures(monkeypatch): + monkeypatch.setenv("MODEL_LINES_REFRESH_S", "0.01") + monkeypatch.setattr(model_lines, "definitions", lambda: {"alpha": {}}) + seen = [] + + def reload(): + seen.append(1) + if len(seen) == 1: + raise RuntimeError("store hiccup") + return True + + monkeypatch.setattr(model_lines, "reload", reload) + + async def scenario(): + task = serve.start_model_lines_refresh() + assert task is not None + for _ in range(200): + if len(seen) >= 3: + break + await asyncio.sleep(0.01) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + asyncio.run(scenario()) + assert len(seen) >= 3 # kept going after the failed read + + +@pytest.mark.parametrize("interval, lines", [("0", {"alpha": {}}), ("30", {}), + ("-5", {"alpha": {}})]) +def test_refresh_task_is_off_without_lines_or_interval(monkeypatch, interval, lines): + monkeypatch.setenv("MODEL_LINES_REFRESH_S", interval) + monkeypatch.setattr(model_lines, "definitions", lambda: lines) + + async def scenario(): + return serve.start_model_lines_refresh() + + assert asyncio.run(scenario()) is None + + +# ---- shipped suite and CLI ----------------------------------------------------- + +def test_shipped_example_suite_loads(): + suites = qualifier.load_suites([ROOT / "evaluations" / "line_suites"]) + assert [s["name"] for s in suites] == ["example-ticket-triage"] + assert suites[0]["request"]["timeout_ms"] == 22000 + good = '{"label": "bug", "confidence": 90, "summary": "export is broken"}' + assert qualifier.answer_is_valid(good, suites[0]["expect"]) + + +def _cli(*args, env=None): + return subprocess.run( + [sys.executable, str(ROOT / "scripts" / "qualify_model_lines.py"), *args], + capture_output=True, text=True, timeout=60, + env={**os.environ, "LINE_QUALIFIER_API_KEY": "", **(env or {})}) + + +def test_cli_requires_router_url_and_suites(): + assert _cli("--suites", str(ROOT / "evaluations" / "line_suites"), + env={"LINE_QUALIFIER_ROUTER_URL": ""}).returncode == 2 + assert _cli("--router-url", "http://127.0.0.1:9", + env={"LINE_QUALIFIER_SUITES": ""}).returncode == 2 + + +def test_cli_reports_an_unreachable_router_as_one_json_line(): + # port 9 (discard) refuses the connection: the run cannot start at all + done = _cli("--router-url", "http://127.0.0.1:9", "--mode", "report", + "--suites", str(ROOT / "evaluations" / "line_suites")) + assert done.returncode == 1 + out = json.loads(done.stdout.strip().splitlines()[-1]) + assert out["ok"] is False and out["error"] + + +def test_cli_reports_a_bad_suite_file(tmp_path): + bad = tmp_path / "bad.json" + bad.write_text('{"name": "x", "prompts": []}', encoding="utf-8") + done = _cli("--router-url", "http://127.0.0.1:9", "--suites", str(bad)) + assert done.returncode == 1 + assert json.loads(done.stdout.strip())["error"].startswith("suites:")