From f91cc7537b910993494b0ab3d7287da758f9c728 Mon Sep 17 00:00:00 2001 From: Erik LaBianca Date: Fri, 13 Nov 2026 16:44:03 -0500 Subject: [PATCH] Avoid idle Databricks SQL rate catalog refreshes --- README.md | 6 ++++++ erg/cloud_dashboard.py | 5 ++++- erg/rate_catalog.py | 36 ++++++++++++++++++++++++++++++------ scripts/rate-catalog.py | 4 ++-- tests/test_rate_refresh.py | 19 +++++++++++++++++++ 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index fd69418..e57823b 100644 --- a/README.md +++ b/README.md @@ -1087,6 +1087,12 @@ catalog with the existing reporting identity (replace profile/warehouse as neede python3 scripts/rate-catalog.py pull --profile erg-prod --warehouse WAREHOUSE_ID --namespace erg.telemetry --automatic ``` +With `--automatic`, the dashboard uses the saved snapshot and checks weekly after +fetching cloud records. A newly seen model missing from the catalog can trigger +an earlier check, at least 24 hours after the last attempt. Local reporting uses +SQLite and saved rates by default. An idle dashboard does not start SQL to check +prices. Run `pull` for an immediate refresh after a publication. + `publish --file complete-catalog.json` requires an explicitly supplied full JSON catalog and an identity permitted to write the application tables. Publication creates a new immutable revision; readers verify every row before atomically diff --git a/erg/cloud_dashboard.py b/erg/cloud_dashboard.py index bc0ce20..7b5b417 100644 --- a/erg/cloud_dashboard.py +++ b/erg/cloud_dashboard.py @@ -335,7 +335,6 @@ def worker(database, stop): sync_policy(db) from . import rate_catalog - rate_catalog.refresh(client, cfg["cloud"]["namespace"], cfg["cloud"]["host"]) known = receipt_cursor() cursor = db.execute("SELECT cursor FROM cloud_config WHERE id=1").fetchone()[0] metadata_count = fetch_metadata(db, client, ns, cfg["identity"], known) @@ -358,6 +357,10 @@ def worker(database, stop): break if not any(x.get("processed") for x in (usage, edits, work, winners)): break + if count: + rate_catalog.refresh( + client, cfg["cloud"]["namespace"], cfg["cloud"]["host"], db=db + ) project_rollups.capture_progress(db, usage, work) from . import quota_projection diff --git a/erg/rate_catalog.py b/erg/rate_catalog.py index de17bfc..8fe3311 100644 --- a/erg/rate_catalog.py +++ b/erg/rate_catalog.py @@ -9,7 +9,7 @@ import urllib.parse from pathlib import Path -from . import collect, databricks +from . import collect, databricks, rates def directory(): @@ -197,17 +197,36 @@ def _marker(value): tmp.unlink() -def enable(profile, target, host): +def enable(profile, target, host, snapshot=None): """Bind automatic pulls to the configured workspace and native profile.""" databricks.namespace(target) host = _host(host) if not host or profile is not None and (not isinstance(profile, str) or not profile): raise ValueError("rate source requires a workspace and native profile") - _marker(dict(version=1, namespace=target, host=host, profile=profile)) + cfg: dict = dict(version=1, namespace=target, host=host, profile=profile) + if snapshot is not None: + now = time.time() + cfg.update(snapshot, last_attempt=now, last_success=now) + _marker(cfg) + +DAY = 24 * 60 * 60 +WEEK = 7 * DAY -def refresh(client, target, host=None): - """Opt-in source binding; at most one attempt per 15 minutes per worker.""" + +def _unknown_models(db, now): + if db is None: + return set() + card = rates.load() + models = db.execute( + "SELECT DISTINCT model FROM llm WHERE ts>=? AND model IS NOT NULL AND model<>''", + (now - DAY,), + ) + return {model for (model,) in models if rates.value({"model": model}, card) is None} + + +def refresh(client, target, host=None, *, db=None): + """Refresh weekly, or sooner for a new model, at most once per day.""" marker = directory() / "enabled.json" try: if marker.stat().st_size > 65536: @@ -234,9 +253,14 @@ def refresh(client, target, host=None): or not math.isfinite(previous) ): return - if now - previous < 900: + if previous and now - previous < DAY: + return + checked = set(cfg.get("checked_unknown_models", [])) + unknown = _unknown_models(db, now) + if previous and now - previous < WEEK and not unknown - checked: return cfg["last_attempt"] = now + cfg["checked_unknown_models"] = sorted(checked | unknown)[:128] _marker(cfg) try: result = pull(client, target) diff --git a/scripts/rate-catalog.py b/scripts/rate-catalog.py index 52562e4..f4695a3 100644 --- a/scripts/rate-catalog.py +++ b/scripts/rate-catalog.py @@ -20,7 +20,7 @@ def main(): p.add_argument( "--automatic", action="store_true", - help="Enable a 15-minute pull in the existing cloud reporting worker", + help="Check weekly during cloud reporting, or sooner for a new model", ) a = p.parse_args() client = databricks.Client(a.profile, a.warehouse) @@ -44,7 +44,7 @@ def main(): _, _, cfg = cloud_history.connection() if cfg["cloud"]["profile"] != a.profile or cfg["cloud"]["namespace"] != a.namespace: p.error("--automatic requires the configured reporting profile and namespace") - rate_catalog.enable(a.profile, a.namespace, cfg["cloud"]["host"]) + rate_catalog.enable(a.profile, a.namespace, cfg["cloud"]["host"], snapshot=result) print(json.dumps(result)) diff --git a/tests/test_rate_refresh.py b/tests/test_rate_refresh.py index fea5511..fbbea84 100644 --- a/tests/test_rate_refresh.py +++ b/tests/test_rate_refresh.py @@ -66,3 +66,22 @@ def test_success_and_invalid_configuration_have_explicit_boundaries(self): self.assertEqual( json.loads((self.root / "enabled.json").read_text())["revision"], "verified" ) + + def test_new_model_refresh_has_daily_floor_and_weekly_retry(self): + host = "https://fixture.invalid" + target = "erg.telemetry" + now = 10 * r.WEEK + with patch.object(r.time, "time", return_value=now): + r.enable("fixture", target, host, snapshot=dict(revision="initial", models=1)) + with patch.object(r, "_unknown_models", return_value={"new-model"}), patch.object( + r, "pull", return_value=dict(revision="latest", models=1) + ) as pull: + for elapsed, expected in ( + (r.DAY - 1, 0), + (r.DAY, 1), + (2 * r.DAY, 1), + (r.DAY + r.WEEK, 2), + ): + with patch.object(r.time, "time", return_value=now + elapsed): + r.refresh(self.client, target, host) + self.assertEqual(pull.call_count, expected)