From 901f616606dd873164b3aaf100c9f782e43f8d85 Mon Sep 17 00:00:00 2001
From: Jared Lewis <jared@jared.kiwi.nz>
Date: Fri, 23 Oct 2026 13:38:54 +1100
Subject: [PATCH 1/5] fix(diagnostics): return a diagnostic whose provider is
 not installed

Fetching one diagnostic raised a 500 when the provider registry did not know it,
while the list endpoints already fell back to an empty description.
The three lookups now share one helper.
---
 backend/src/ref_backend/models/diagnostics.py | 39 ++++++++-----------
 .../test_api/test_routes/test_diagnostics.py  | 14 +++++++
 2 files changed, 31 insertions(+), 22 deletions(-)

diff --git a/backend/src/ref_backend/models/diagnostics.py b/backend/src/ref_backend/models/diagnostics.py
index 04f473f9..75fc411f 100644
--- a/backend/src/ref_backend/models/diagnostics.py
+++ b/backend/src/ref_backend/models/diagnostics.py
@@ -20,9 +20,19 @@
 from ref_backend.models.common import GroupBy, ProviderSummary
 
 if TYPE_CHECKING:
+    from climate_ref_core.diagnostics import Diagnostic
     from ref_backend.api.deps import AppContext
 
 
+def _concrete_diagnostic(diagnostic: models.Diagnostic, app_context: "AppContext") -> "Diagnostic | None":
+    """Find the registered diagnostic for a stored one, or None when its provider is not installed."""
+    try:
+        return app_context.provider_registry.get_metric(diagnostic.provider.slug, diagnostic.slug)
+    except KeyError:
+        logger.warning(f"Could not find concrete diagnostic for {diagnostic.provider.slug}/{diagnostic.slug}")
+        return None
+
+
 class DiagnosticCatalogEntry(BaseModel):
     """
     Summary information about a diagnostic, without whether it has metric values.
@@ -113,14 +123,8 @@ def _ensure_metadata_cache(app_context: "AppContext") -> Mapping[str, Diagnostic
     @staticmethod
     def _build_group_by_summary(diagnostic: models.Diagnostic, app_context: "AppContext") -> list[GroupBy]:
         """Extract and build group_by information from diagnostic data requirements."""
-        try:
-            concrete_diagnostic = app_context.provider_registry.get_metric(
-                diagnostic.provider.slug, diagnostic.slug
-            )
-        except KeyError:
-            logger.warning(
-                f"Could not find concrete diagnostic for {diagnostic.provider.slug}/{diagnostic.slug}"
-            )
+        concrete_diagnostic = _concrete_diagnostic(diagnostic, app_context)
+        if concrete_diagnostic is None:
             return []
         data_requirements = sorted(
             concrete_diagnostic.data_requirements,
@@ -196,16 +200,8 @@ def build_with_stats(  # noqa: PLR0913
         group_by_summary = DiagnosticCatalogEntry._build_group_by_summary(diagnostic, app_context)
         aft = DiagnosticCatalogEntry._get_aft_link(diagnostic)
 
-        try:
-            concrete_diagnostic = app_context.provider_registry.get_metric(
-                diagnostic.provider.slug, diagnostic.slug
-            )
-            description = concrete_diagnostic.__doc__ or ""
-        except KeyError:
-            logger.warning(
-                f"Could not find concrete diagnostic for {diagnostic.provider.slug}/{diagnostic.slug}"
-            )
-            description = ""
+        concrete_diagnostic = _concrete_diagnostic(diagnostic, app_context)
+        description = (concrete_diagnostic.__doc__ or "") if concrete_diagnostic else ""
 
         # Build the base diagnostic summary
         summary = DiagnosticCatalogEntry(
@@ -328,9 +324,8 @@ def has_values(value_type: StoredMetricValueType) -> bool:
             .count()
         )
 
-        concrete_diagnostic = app_context.provider_registry.get_metric(
-            diagnostic.provider.slug, diagnostic.slug
-        )
+        concrete_diagnostic = _concrete_diagnostic(diagnostic, app_context)
+        description = (concrete_diagnostic.__doc__ or "") if concrete_diagnostic else ""
 
         resource_usage = resource_usage_for_diagnostic(app_context.session, diagnostic.id)
 
@@ -340,7 +335,7 @@ def has_values(value_type: StoredMetricValueType) -> bool:
             provider=ProviderSummary.build(diagnostic.provider),
             slug=diagnostic.slug,
             name=diagnostic.name,
-            description=concrete_diagnostic.__doc__ or "",
+            description=description,
             execution_groups=[e.id for e in diagnostic.execution_groups],
             has_metric_values=has_metric_values,
             has_scalar_values=has_scalar_values,
diff --git a/backend/tests/test_api/test_routes/test_diagnostics.py b/backend/tests/test_api/test_routes/test_diagnostics.py
index e66026f3..1101a6a4 100644
--- a/backend/tests/test_api/test_routes/test_diagnostics.py
+++ b/backend/tests/test_api/test_routes/test_diagnostics.py
@@ -8,6 +8,7 @@
 
 from climate_ref import models
 from climate_ref.database import Database
+from climate_ref.provider_registry import ProviderRegistry
 from ref_backend.api import deps
 from ref_backend.builder import build_app
 from ref_backend.core.config import get_settings
@@ -995,3 +996,16 @@ def counts(query: str = "") -> dict[str, tuple[int, int]]:
     assert sum(total for total, _ in model.values()) > 0
     assert sum(total for total, _ in model.values()) < sum(total for total, _ in everything.values())
     assert all(total == 0 for total, _ in nothing.values())
+
+
+def test_diagnostic_without_provider(app, client: TestClient, settings):
+    diagnostic = get_diagnostic(client, settings)
+    app.dependency_overrides[deps._provider_registry_dependency] = lambda: ProviderRegistry(providers=[])
+    try:
+        r = client.get(
+            f"{settings.API_V1_STR}/diagnostics/{diagnostic['provider']['slug']}/{diagnostic['slug']}"
+        )
+    finally:
+        del app.dependency_overrides[deps._provider_registry_dependency]
+
+    assert r.status_code == 200

From b24892f4af2aa8b06cd89dcf334165cc88a639fc Mon Sep 17 00:00:00 2001
From: Jared Lewis <jared@jared.kiwi.nz>
Date: Fri, 23 Oct 2026 13:38:55 +1100
Subject: [PATCH 2/5] test(perf): time the API endpoints against a synthetic
 database

Adds tests/perf, which requests common endpoints against a production-shaped synthetic database
from climate_ref.testing.synthetic.
Each endpoint fails when a query reads a whole large table it is not allowed to.
`make perf-compare` runs the same tests on a base ref and fails if a median is over 30% slower,
and CI runs it on every pull request.

The tests skip until the pinned climate-ref ships the generator.
---
 .github/workflows/ci.yml             |  23 +++++
 backend/Makefile                     |  11 +++
 backend/README.md                    |   3 +
 backend/pyproject.toml               |   1 +
 backend/scripts/perf-compare.sh      |  34 +++++++
 backend/tests/perf/README.md         |  40 ++++++++
 backend/tests/perf/conftest.py       | 133 +++++++++++++++++++++++++++
 backend/tests/perf/test_endpoints.py |  97 +++++++++++++++++++
 backend/uv.lock                      |  26 +++++-
 9 files changed, 367 insertions(+), 1 deletion(-)
 create mode 100755 backend/scripts/perf-compare.sh
 create mode 100644 backend/tests/perf/README.md
 create mode 100644 backend/tests/perf/conftest.py
 create mode 100644 backend/tests/perf/test_endpoints.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3485025f..8d797cc9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -92,3 +92,26 @@ jobs:
     - name: Run tests
       run: |
         uv run coverage run -m pytest tests/ && uv run coverage report --fail-under=90
+
+  backend-perf:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    defaults:
+      run:
+        working-directory: backend
+
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        fetch-depth: 0
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v10.3.0
+      with:
+        enable-cache: true
+
+    - name: Compare query performance against the base branch
+      env:
+        REF_PERF_ROUNDS: "10"
+      run: |
+        scripts/perf-compare.sh "origin/${{ github.base_ref }}"
diff --git a/backend/Makefile b/backend/Makefile
index 5f564983..d986a808 100644
--- a/backend/Makefile
+++ b/backend/Makefile
@@ -45,3 +45,14 @@ mypy: ## Run mypy type checker
 .PHONY: generate-metadata
 generate-metadata: ## Generate diagnostic metadata YAML from provider registry
 	uv run python scripts/generate_metadata.py
+
+REF_PERF_SCALE ?= 0.1
+BASE ?= origin/main
+
+.PHONY: test-perf
+test-perf: ## run the API performance tests against a synthetic database (REF_PERF_SCALE=1 for full size)
+	REF_PERF_SCALE=$(REF_PERF_SCALE) uv run pytest tests/perf -r a -v
+
+.PHONY: perf-compare
+perf-compare: ## fail if the API performance tests are slower than on BASE (default origin/main)
+	REF_PERF_SCALE=$(REF_PERF_SCALE) scripts/perf-compare.sh $(BASE)
diff --git a/backend/README.md b/backend/README.md
index aab98cd0..f5158c26 100644
--- a/backend/README.md
+++ b/backend/README.md
@@ -35,6 +35,9 @@ make dev
 make test
 ```
 
+`make test-perf` times the API against a large synthetic database.
+See [tests/perf/README.md](tests/perf/README.md).
+
 ### Code Quality
 
 The project uses:
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 662d3438..3354a659 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -28,6 +28,7 @@ dev = [
     "types-pyyaml>=6.0.12.20250915",
     "bump-my-version>=0.28.1",
     "towncrier>=24.8.0",
+    "pytest-benchmark>=5.3.0",
 ]
 
 [tool.uv.sources]
diff --git a/backend/scripts/perf-compare.sh b/backend/scripts/perf-compare.sh
new file mode 100755
index 00000000..94ff3612
--- /dev/null
+++ b/backend/scripts/perf-compare.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Time the API performance tests on a base ref and on this checkout, then fail if this checkout is slower.
+#
+# Usage: scripts/perf-compare.sh [base ref, default origin/main]
+#
+# REF_PERF_SCALE (default 0.1), REF_PERF_ROUNDS and REF_PERF_CACHE are passed through to the tests.
+# REF_PERF_THRESHOLD is the allowed slowdown of each endpoint's median, in percent (default 30).
+set -euo pipefail
+
+BASE="${1:-origin/main}"
+BACKEND="$(cd "$(dirname "$0")/.." && pwd)"
+WORK="$(mktemp -d)"
+trap 'git -C "$BACKEND" worktree remove --force "$WORK/base" >/dev/null 2>&1 || true; rm -rf "$WORK"' EXIT
+
+export REF_PERF_SCALE="${REF_PERF_SCALE:-0.1}"
+STORAGE="file://$WORK/benchmarks"
+
+git -C "$BACKEND" worktree add --quiet --detach "$WORK/base" "$BASE"
+
+# The base code runs this checkout's tests, so both sides time the same requests.
+# Endpoints missing on the base fail there and are left out of the comparison.
+(
+    cd "$WORK/base/backend"
+    uv sync --locked --quiet
+    uv run --with pytest-benchmark pytest "$BACKEND/tests/perf" \
+        --rootdir="$BACKEND" --confcutdir="$BACKEND/tests/perf" -p no:cacheprovider -q \
+        --benchmark-storage="$STORAGE" --benchmark-save=base
+) || echo "::warning::some performance tests failed on $BASE, comparing the rest"
+
+cd "$BACKEND"
+uv run pytest tests/perf -q \
+    --benchmark-storage="$STORAGE" --benchmark-compare \
+    --benchmark-compare-fail="median:${REF_PERF_THRESHOLD:-30}%" \
+    --benchmark-columns=median,iqr,rounds
diff --git a/backend/tests/perf/README.md b/backend/tests/perf/README.md
new file mode 100644
index 00000000..585417d0
--- /dev/null
+++ b/backend/tests/perf/README.md
@@ -0,0 +1,40 @@
+# API performance tests
+
+These tests time the API endpoints against a synthetic database shaped like production.
+The database comes from `climate_ref.testing.synthetic`, the same generator climate-ref uses for its reader tests.
+The data is read-only, so every request can be repeated against the same database.
+
+Each entry in `ENDPOINTS` in `test_endpoints.py` is one request the frontend makes.
+An entry fails when one of its queries reads every row of a large table, which `EXPLAIN QUERY PLAN` shows at any scale.
+Its `allowed_scans` lists the scans that are known and accepted.
+
+## Running
+
+```bash
+make test-perf                   # a tenth of production, quick
+make test-perf REF_PERF_SCALE=1  # full size, about 5 GB
+```
+
+The database is built on first use and cached in `~/.cache/ref-app/perf` (`REF_PERF_CACHE`).
+A change to the shape, the scale, the generator or the migrations builds a new one.
+Old databases are not removed, so clear the cache directory by hand.
+
+`REF_PERF_ROUNDS` sets the timed runs per endpoint, default 5.
+
+## Comparing against main
+
+```bash
+make perf-compare               # against origin/main
+make perf-compare BASE=v0.8.2   # against any ref
+```
+
+This checks out the base in a temporary worktree and runs this branch's tests against both.
+It fails when an endpoint's median is more than `REF_PERF_THRESHOLD` percent slower than on the base, default 30.
+Running the same code twice locally drifts by up to about 20%, so a smaller threshold is noise.
+CI runs this on every pull request.
+
+## Adding an endpoint
+
+Add an `Endpoint` to `ENDPOINTS`.
+The path and parameters are formatted with the `targets` fixture, e.g. `{provider_slug}`,
+which picks the busiest diagnostic and execution so each request reads its worst case.
diff --git a/backend/tests/perf/conftest.py b/backend/tests/perf/conftest.py
new file mode 100644
index 00000000..014710d8
--- /dev/null
+++ b/backend/tests/perf/conftest.py
@@ -0,0 +1,133 @@
+"""
+Fixtures for the API performance tests, which run against a large synthetic database.
+
+These only run when ``REF_PERF_SCALE`` is set (see ``make test-perf``).
+The database is built once per shape, scale and schema, and cached under ``REF_PERF_CACHE``.
+"""
+
+import os
+import sqlite3
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import pytest
+from starlette.testclient import TestClient
+
+from climate_ref.config import Config
+from climate_ref.database import Database
+from climate_ref.provider_registry import ProviderRegistry
+from ref_backend.api import deps
+from ref_backend.builder import build_app
+from ref_backend.core.config import Settings, get_settings
+
+
+@pytest.fixture(scope="session")
+def synthetic() -> Any:
+    """The synthetic database generator from climate-ref."""
+    if not os.environ.get("REF_PERF_SCALE"):
+        pytest.skip("set REF_PERF_SCALE to run the API performance tests")
+    return pytest.importorskip(
+        "climate_ref.testing.synthetic", reason="needs a climate-ref with the synthetic database generator"
+    )
+
+
+@pytest.fixture(scope="session")
+def perf_database_path(synthetic: Any) -> Path:
+    """The synthetic database for this scale and schema, built on first use."""
+    cache = Path(os.environ.get("REF_PERF_CACHE", Path.home() / ".cache" / "ref-app" / "perf"))
+    path: Path = synthetic.cached_synthetic_database(cache, scale=float(os.environ["REF_PERF_SCALE"]))
+    return path
+
+
+@pytest.fixture(scope="session")
+def perf_settings(tmp_path_factory: pytest.TempPathFactory) -> Settings:
+    # An empty configuration directory, so a developer's ref.toml cannot point the app elsewhere.
+    configuration = tmp_path_factory.mktemp("configuration")
+    return Settings(REF_CONFIGURATION=str(configuration), REF_READ_ONLY_DATABASE=True)
+
+
+@pytest.fixture(scope="session")
+def perf_ref_config(perf_database_path: Path, tmp_path_factory: pytest.TempPathFactory) -> Config:
+    config = Config()
+    config.db.database_url = f"sqlite:///{perf_database_path}"
+    config.paths.results = tmp_path_factory.mktemp("results")
+    return config
+
+
+@pytest.fixture(scope="session")
+def perf_database(perf_settings: Settings, perf_ref_config: Config) -> Database:
+    return deps._get_database_dependency(perf_settings, perf_ref_config)
+
+
+@pytest.fixture(scope="session")
+def perf_client(
+    perf_settings: Settings, perf_ref_config: Config, perf_database: Database
+) -> Iterator[TestClient]:
+    """A client for the app reading the synthetic database read-only, like the deployed API."""
+    app = build_app(settings=perf_settings, ref_config=perf_ref_config, database=perf_database)
+    app.dependency_overrides[get_settings] = lambda: perf_settings
+    app.dependency_overrides[deps._ref_config_dependency] = lambda: perf_ref_config
+    # Built once like `ref_backend.main`, and without registering, because the database is read-only.
+    registry = ProviderRegistry.build_from_config(perf_ref_config, perf_database, register=False)
+    app.dependency_overrides[deps._provider_registry_dependency] = lambda: registry
+    with TestClient(app) as client:
+        yield client
+
+
+@dataclass(frozen=True, kw_only=True)
+class Targets:
+    """Ids and slugs that the endpoints query, picked from the generated data."""
+
+    provider_slug: str
+    diagnostic_slug: str
+    """The diagnostic with the most metric values, the worst case for a diagnostic-scoped read."""
+
+    execution_group_id: int
+    """The group of the execution with the most metric values."""
+
+    dataset_slug: str
+    source_id: str
+    """The model with the most CMIP6 datasets."""
+
+
+@pytest.fixture(scope="session")
+def targets(perf_database_path: Path) -> Targets:
+    """Pick the busiest diagnostic and execution, so each endpoint reads its worst case."""
+    con = sqlite3.connect(f"file:{perf_database_path}?mode=ro", uri=True)
+    execution_id, _ = con.execute(
+        "SELECT execution_id, count(*) FROM metric_value GROUP BY execution_id ORDER BY 2 DESC LIMIT 1"
+    ).fetchone()
+    group_id, diagnostic_slug, provider_slug = con.execute(
+        "SELECT g.id, d.slug, p.slug FROM execution e JOIN execution_group g ON g.id = e.execution_group_id "
+        "JOIN diagnostic d ON d.id = g.diagnostic_id JOIN provider p ON p.id = d.provider_id WHERE e.id = ?",
+        (execution_id,),
+    ).fetchone()
+    (dataset_slug,) = con.execute("SELECT slug FROM dataset ORDER BY id DESC LIMIT 1").fetchone()
+    (source_id,) = con.execute(
+        "SELECT source_id FROM cmip6_dataset GROUP BY source_id ORDER BY count(*) DESC LIMIT 1"
+    ).fetchone()
+    con.close()
+    return Targets(
+        provider_slug=provider_slug,
+        diagnostic_slug=diagnostic_slug,
+        execution_group_id=group_id,
+        dataset_slug=dataset_slug,
+        source_id=source_id,
+    )
+
+
+@pytest.fixture(scope="session")
+def large_table_scans(
+    synthetic: Any, perf_database: Database, perf_database_path: Path
+) -> Callable[[Callable[[], Any]], frozenset[str]]:
+    """Run a call once and return the large tables that any of its queries reads in full."""
+
+    def _scans(call: Callable[[], Any]) -> frozenset[str]:
+        with synthetic.captured_statements(perf_database) as statements:
+            call()
+        scans: frozenset[str] = synthetic.large_table_scans(perf_database_path, statements)
+        return scans
+
+    return _scans
diff --git a/backend/tests/perf/test_endpoints.py b/backend/tests/perf/test_endpoints.py
new file mode 100644
index 00000000..1123d8ea
--- /dev/null
+++ b/backend/tests/perf/test_endpoints.py
@@ -0,0 +1,97 @@
+"""
+Performance of the API endpoints against a production-shaped synthetic database.
+
+Each endpoint is one request the frontend makes.
+It fails when a query reads a large table in full, which the query plan shows at any scale.
+The timings are compared against main by ``make perf-compare``.
+"""
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+
+import pytest
+
+
+@dataclass(frozen=True, kw_only=True)
+class Endpoint:
+    id: str
+    path: str
+    """Path below the API prefix, formatted with the ``targets`` fields, e.g. ``{provider_slug}``."""
+
+    params: Mapping[str, str] = field(default_factory=dict)
+    allowed_scans: frozenset[str] = frozenset()
+    """Large tables this request has to read in full, e.g. a summary over every execution."""
+
+
+_DIAGNOSTIC = "diagnostics/{provider_slug}/{diagnostic_slug}"
+_GROUP = "executions/{execution_group_id}"
+
+# The allowed scans record what each endpoint read when this suite was added, not what it should read.
+ENDPOINTS = [
+    # diagnostics: the landing page and a diagnostic's page
+    Endpoint(id="diagnostics.list", path="diagnostics/"),
+    Endpoint(
+        id="diagnostics.list.cmip6",
+        path="diagnostics/",
+        params={"mip_era": "CMIP6"},
+        allowed_scans=frozenset({"execution_group"}),
+    ),
+    Endpoint(id="diagnostics.catalog", path="diagnostics/catalog"),
+    Endpoint(id="diagnostics.value-flags", path="diagnostics/value-flags"),
+    Endpoint(id="diagnostics.facets", path="diagnostics/facets", allowed_scans=frozenset({"metric_value"})),
+    Endpoint(id="diagnostics.get", path=_DIAGNOSTIC),
+    Endpoint(id="diagnostics.execution-groups", path=f"{_DIAGNOSTIC}/execution_groups"),
+    Endpoint(id="diagnostics.values.scalar", path=f"{_DIAGNOSTIC}/values", params={"value_type": "scalar"}),
+    Endpoint(
+        id="diagnostics.values.scalar.source-id",
+        path=f"{_DIAGNOSTIC}/values",
+        params={"value_type": "scalar", "source_id": "{source_id}"},
+    ),
+    Endpoint(id="diagnostics.values.series", path=f"{_DIAGNOSTIC}/values", params={"value_type": "series"}),
+    # executions: the recent list and an execution group's page
+    Endpoint(id="executions.list", path="executions/", allowed_scans=frozenset({"execution_group"})),
+    Endpoint(
+        id="executions.list.failed",
+        path="executions/",
+        params={"successful": "false"},
+        allowed_scans=frozenset({"execution_group"}),
+    ),
+    Endpoint(
+        id="executions.statistics",
+        path="executions/statistics",
+        allowed_scans=frozenset({"execution", "dataset", "dataset_file"}),
+    ),
+    Endpoint(id="executions.get", path=_GROUP),
+    Endpoint(id="executions.values.scalar", path=f"{_GROUP}/values", params={"value_type": "scalar"}),
+    Endpoint(id="executions.values.series", path=f"{_GROUP}/values", params={"value_type": "series"}),
+    # models and datasets
+    Endpoint(
+        id="models.list",
+        path="models/",
+        allowed_scans=frozenset({"execution", "cmip6_dataset", "cmip7_dataset"}),
+    ),
+    Endpoint(id="models.get", path="models/{source_id}", allowed_scans=frozenset({"execution"})),
+    Endpoint(id="datasets.list", path="datasets/"),
+    Endpoint(id="datasets.get", path="datasets/{dataset_slug}"),
+]
+
+
+@pytest.mark.parametrize("endpoint", ENDPOINTS, ids=lambda e: e.id)
+def test_endpoint(  # noqa: PLR0913, PLR0917
+    endpoint: Endpoint, perf_client, perf_settings, targets, large_table_scans, benchmark
+):
+    fields = vars(targets)
+    url = f"{perf_settings.API_V1_STR}/{endpoint.path.format(**fields)}"
+    params = {k: v.format(**fields) for k, v in endpoint.params.items()}
+
+    def _get():
+        response = perf_client.get(url, params=params)
+        assert response.status_code == 200, response.text
+        return response
+
+    # This request also warms the caches, so the timed rounds need no warm-up.
+    unexpected = large_table_scans(_get) - endpoint.allowed_scans
+    benchmark.pedantic(_get, rounds=int(os.environ.get("REF_PERF_ROUNDS", "5")), warmup_rounds=0)
+
+    assert not unexpected, f"reads every row of {sorted(unexpected)}"
diff --git a/backend/uv.lock b/backend/uv.lock
index b504677b..edb90b1a 100644
--- a/backend/uv.lock
+++ b/backend/uv.lock
@@ -1,5 +1,5 @@
 version = 1
-revision = 3
+revision = 5
 requires-python = ">=3.12"
 resolution-markers = [
     "python_full_version >= '3.15' and platform_machine == 'ARM64' and sys_platform == 'win32'",
@@ -2666,6 +2666,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/20/be/b732c8418ffa5bcfda002890f5dc4c869fc17db66ff11f53b17cfe44afc0/psycopg2_binary-2.9.12-cp314-cp314-win_amd64.whl", hash = "sha256:f12ae41fcafadb39b2785e64a40f9db05d6de2ac114077457e0e7c597f3af980", size = 2848762, upload-time = "2026-04-20T23:35:46.421Z" },
 ]
 
+[[package]]
+name = "py-cpuinfo2"
+version = "10.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/dc/97/a8b1ddada14c8280a047c0746f95cb05d94a31b1a331cea22bcdc2b2a82d/py_cpuinfo2-10.1.1.tar.gz", hash = "sha256:7861133863663f16e06eca63b12904ef100b5760415e92372dac0162799a4771", size = 100840, upload-time = "2026-03-25T21:49:40.797Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/23/0a/ba69d2dde1ae12ef1d389ea5a216384c5ff6ef7a1e7a48d1e9b6686f6790/py_cpuinfo2-10.1.1-py3-none-any.whl", hash = "sha256:adc53396bfb206e6498d078ec2ab407f85799ecd819584ac36a8f80a2d4d762d", size = 23791, upload-time = "2026-03-25T21:49:39.574Z" },
+]
+
 [[package]]
 name = "py-multihash"
 version = "3.0.0"
@@ -3297,6 +3306,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
 ]
 
+[[package]]
+name = "pytest-benchmark"
+version = "5.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "py-cpuinfo2" },
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/63/8f/83a15e40dbc34a580ee56eb56983cae5394c6e94d50cf28fe268e457be25/pytest_benchmark-5.3.0.tar.gz", hash = "sha256:358444d4e89be901ee2b6404fb043ac3d7684002ad7f3563cc153fca6339c965", size = 375410, upload-time = "2026-08-23T17:45:08.891Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/eb/42/7e80f7cfa191e0a766d1de99b4661847415ad5db34f8209d81fd42175b59/pytest_benchmark-5.3.0-py3-none-any.whl", hash = "sha256:920ab1dfcffa718d49aa15ba144c7e357bda59216a0dc308016cc1c7236f719d", size = 48401, upload-time = "2026-08-23T17:45:07.094Z" },
+]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"
@@ -3457,6 +3479,7 @@ dev = [
     { name = "pandas-stubs" },
     { name = "pre-commit" },
     { name = "pytest" },
+    { name = "pytest-benchmark" },
     { name = "ruff" },
     { name = "towncrier" },
     { name = "types-pyyaml" },
@@ -3485,6 +3508,7 @@ dev = [
     { name = "pandas-stubs", specifier = ">=2.3.2.250827" },
     { name = "pre-commit", specifier = ">=4.0" },
     { name = "pytest", specifier = ">=9.0" },
+    { name = "pytest-benchmark", specifier = ">=5.3.0" },
     { name = "ruff", specifier = ">=0.15.0" },
     { name = "towncrier", specifier = ">=24.8.0" },
     { name = "types-pyyaml", specifier = ">=6.0.12.20250915" },

From 6d2c2a571fe472aa59a1bf6fb234a780cc83ef05 Mon Sep 17 00:00:00 2001
From: Jared Lewis <jared@jared.kiwi.nz>
Date: Fri, 23 Oct 2026 13:40:02 +1100
Subject: [PATCH 3/5] docs: add changelog for #145

---
 changelog/145.fix.md     | 1 +
 changelog/145.trivial.md | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changelog/145.fix.md
 create mode 100644 changelog/145.trivial.md

diff --git a/changelog/145.fix.md b/changelog/145.fix.md
new file mode 100644
index 00000000..cba144cc
--- /dev/null
+++ b/changelog/145.fix.md
@@ -0,0 +1 @@
+Fixed a 500 error when fetching a diagnostic whose provider is not installed.
diff --git a/changelog/145.trivial.md b/changelog/145.trivial.md
new file mode 100644
index 00000000..2b3ac300
--- /dev/null
+++ b/changelog/145.trivial.md
@@ -0,0 +1 @@
+Added API performance tests against a synthetic database, which CI compares against the base branch on every pull request.

From b3f2dde59b0d09a53e5125273e397ad5d0a6919e Mon Sep 17 00:00:00 2001
From: Jared Lewis <jared@jared.kiwi.nz>
Date: Fri, 23 Oct 2026 21:30:55 +1100
Subject: [PATCH 4/5] test(perf): compare against releases with an older schema

Adds scripts/perf-database.py, which builds the synthetic database and a copy migrated down to a given
Alembic revision. perf-compare.sh runs the base against the copy matching its climate-ref, so a
release such as v0.8.2 can be the baseline. REF_PERF_DATABASE points the tests at a prebuilt
database, and the scan check is skipped when the generator is missing.

CI caches the databases, keyed on both lock files.
---
 .github/workflows/ci.yml         | 13 +++++++
 backend/scripts/perf-compare.sh  | 19 +++++++--
 backend/scripts/perf-database.py | 66 ++++++++++++++++++++++++++++++++
 backend/tests/perf/README.md     |  7 +++-
 backend/tests/perf/conftest.py   | 32 ++++++++++------
 5 files changed, 120 insertions(+), 17 deletions(-)
 create mode 100644 backend/scripts/perf-database.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8d797cc9..1695baf9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -110,6 +110,19 @@ jobs:
       with:
         enable-cache: true
 
+    # The databases depend on the climate-ref version on each side, which the lock files pin.
+    - name: Key the synthetic database cache
+      id: perf-db
+      run: |
+        key=$(cat uv.lock <(git show "origin/${{ github.base_ref }}:backend/uv.lock") | sha256sum | cut -c1-16)
+        echo "key=perf-db-$key" >> "$GITHUB_OUTPUT"
+
+    - name: Cache the synthetic databases
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ref-app/perf
+        key: ${{ steps.perf-db.outputs.key }}
+
     - name: Compare query performance against the base branch
       env:
         REF_PERF_ROUNDS: "10"
diff --git a/backend/scripts/perf-compare.sh b/backend/scripts/perf-compare.sh
index 94ff3612..176d9e75 100755
--- a/backend/scripts/perf-compare.sh
+++ b/backend/scripts/perf-compare.sh
@@ -16,19 +16,30 @@ export REF_PERF_SCALE="${REF_PERF_SCALE:-0.1}"
 STORAGE="file://$WORK/benchmarks"
 
 git -C "$BACKEND" worktree add --quiet --detach "$WORK/base" "$BASE"
+(cd "$WORK/base/backend" && uv sync --locked --quiet)
+
+# The base may pin an older climate-ref, so it reads a copy migrated down to its schema.
+BASE_REVISION="$(cd "$WORK/base/backend" && uv run --no-sync python -c '
+import importlib.resources
+from alembic.config import Config
+from alembic.script import ScriptDirectory
+ini = importlib.resources.files("climate_ref") / "alembic.ini"
+print(ScriptDirectory.from_config(Config(str(ini))).get_current_head())
+')"
+cd "$BACKEND"
+HEAD_DATABASE="$(uv run python scripts/perf-database.py)"
+BASE_DATABASE="$(uv run python scripts/perf-database.py --revision "$BASE_REVISION")"
 
 # The base code runs this checkout's tests, so both sides time the same requests.
 # Endpoints missing on the base fail there and are left out of the comparison.
 (
     cd "$WORK/base/backend"
-    uv sync --locked --quiet
-    uv run --with pytest-benchmark pytest "$BACKEND/tests/perf" \
+    REF_PERF_DATABASE="$BASE_DATABASE" uv run --no-sync --with pytest-benchmark pytest "$BACKEND/tests/perf" \
         --rootdir="$BACKEND" --confcutdir="$BACKEND/tests/perf" -p no:cacheprovider -q \
         --benchmark-storage="$STORAGE" --benchmark-save=base
 ) || echo "::warning::some performance tests failed on $BASE, comparing the rest"
 
-cd "$BACKEND"
-uv run pytest tests/perf -q \
+REF_PERF_DATABASE="$HEAD_DATABASE" uv run pytest tests/perf -q \
     --benchmark-storage="$STORAGE" --benchmark-compare \
     --benchmark-compare-fail="median:${REF_PERF_THRESHOLD:-30}%" \
     --benchmark-columns=median,iqr,rounds
diff --git a/backend/scripts/perf-database.py b/backend/scripts/perf-database.py
new file mode 100644
index 00000000..1bd310f8
--- /dev/null
+++ b/backend/scripts/perf-database.py
@@ -0,0 +1,66 @@
+"""
+Print the path of a synthetic database for the API performance tests, building it on first use.
+
+Usage: uv run python scripts/perf-database.py [--revision REVISION]
+
+With ``--revision``, the database is a copy migrated down to that Alembic revision,
+so a release with an older schema can read it.
+Both are cached in ``REF_PERF_CACHE``, and ``REF_PERF_SCALE`` sets the size (default 0.1).
+"""
+
+import argparse
+import os
+import shutil
+from pathlib import Path
+
+import alembic.command
+from alembic.script import ScriptDirectory
+
+from climate_ref.config import Config
+from climate_ref.database import Database
+from climate_ref.testing.synthetic import cached_synthetic_database
+
+
+def _open(path: Path) -> tuple[Config, Database]:
+    config = Config()
+    config.db.database_url = f"sqlite:///{path}"
+    return config, Database.from_config(config, run_migrations=False)
+
+
+def migrated_down(path: Path, revision: str) -> Path:
+    """Return a copy of ``path`` migrated down to ``revision``, building it on first use."""
+    config, database = _open(path)
+    head = ScriptDirectory.from_config(database.alembic_config(config)).get_current_head()
+    database.close()
+    if revision == head:
+        return path
+
+    target = path.with_name(f"{path.stem}-{revision}.db")
+    if target.exists():
+        return target
+    tmp = target.with_name(f"{target.name}.{os.getpid()}.tmp")
+    shutil.copyfile(path, tmp)
+    config, database = _open(tmp)
+    alembic.command.downgrade(database.alembic_config(config), revision)
+    database.optimize()
+    database.close()
+    tmp.replace(target)
+    return target
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("--revision", help="Alembic revision to migrate the copy down to")
+    args = parser.parse_args()
+
+    cache = Path(os.environ.get("REF_PERF_CACHE", Path.home() / ".cache" / "ref-app" / "perf"))
+    path = cached_synthetic_database(cache, scale=float(os.environ.get("REF_PERF_SCALE", "0.1")))
+    if args.revision:
+        path = migrated_down(path, args.revision)
+    print(path)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/backend/tests/perf/README.md b/backend/tests/perf/README.md
index 585417d0..59cb8538 100644
--- a/backend/tests/perf/README.md
+++ b/backend/tests/perf/README.md
@@ -20,18 +20,23 @@ A change to the shape, the scale, the generator or the migrations builds a new o
 Old databases are not removed, so clear the cache directory by hand.
 
 `REF_PERF_ROUNDS` sets the timed runs per endpoint, default 5.
+`REF_PERF_DATABASE` runs against a database built already, e.g. by `scripts/perf-database.py`.
 
 ## Comparing against main
 
 ```bash
 make perf-compare               # against origin/main
 make perf-compare BASE=v0.8.2   # against any ref
+make perf-compare BASE=$(git describe --tags --abbrev=0)  # against the last release
 ```
 
 This checks out the base in a temporary worktree and runs this branch's tests against both.
+A base that pins an older climate-ref reads a copy of the database migrated down to its schema,
+which `scripts/perf-database.py` builds and caches next to the original.
+So a comparison against a release also measures the indexes added since.
 It fails when an endpoint's median is more than `REF_PERF_THRESHOLD` percent slower than on the base, default 30.
 Running the same code twice locally drifts by up to about 20%, so a smaller threshold is noise.
-CI runs this on every pull request.
+CI runs this on every pull request, and caches the databases until either lock file changes.
 
 ## Adding an endpoint
 
diff --git a/backend/tests/perf/conftest.py b/backend/tests/perf/conftest.py
index 014710d8..5eb0cbe5 100644
--- a/backend/tests/perf/conftest.py
+++ b/backend/tests/perf/conftest.py
@@ -1,10 +1,12 @@
 """
 Fixtures for the API performance tests, which run against a large synthetic database.
 
-These only run when ``REF_PERF_SCALE`` is set (see ``make test-perf``).
+These only run when ``REF_PERF_SCALE`` or ``REF_PERF_DATABASE`` is set (see ``make test-perf``).
 The database is built once per shape, scale and schema, and cached under ``REF_PERF_CACHE``.
+``REF_PERF_DATABASE`` points at a database built already, e.g. by ``scripts/perf-database.py``.
 """
 
+import importlib
 import os
 import sqlite3
 from collections.abc import Callable, Iterator
@@ -22,20 +24,23 @@
 from ref_backend.builder import build_app
 from ref_backend.core.config import Settings, get_settings
 
-
-@pytest.fixture(scope="session")
-def synthetic() -> Any:
-    """The synthetic database generator from climate-ref."""
-    if not os.environ.get("REF_PERF_SCALE"):
-        pytest.skip("set REF_PERF_SCALE to run the API performance tests")
-    return pytest.importorskip(
-        "climate_ref.testing.synthetic", reason="needs a climate-ref with the synthetic database generator"
-    )
+# Older climate-ref releases lack the generator, so they can only run against a REF_PERF_DATABASE.
+synthetic: Any
+try:
+    synthetic = importlib.import_module("climate_ref.testing.synthetic")
+except ImportError:
+    synthetic = None
 
 
 @pytest.fixture(scope="session")
-def perf_database_path(synthetic: Any) -> Path:
+def perf_database_path() -> Path:
     """The synthetic database for this scale and schema, built on first use."""
+    if database := os.environ.get("REF_PERF_DATABASE"):
+        return Path(database)
+    if not os.environ.get("REF_PERF_SCALE"):
+        pytest.skip("set REF_PERF_SCALE to run the API performance tests")
+    if synthetic is None:
+        pytest.skip("needs a climate-ref with the synthetic database generator")
     cache = Path(os.environ.get("REF_PERF_CACHE", Path.home() / ".cache" / "ref-app" / "perf"))
     path: Path = synthetic.cached_synthetic_database(cache, scale=float(os.environ["REF_PERF_SCALE"]))
     return path
@@ -120,11 +125,14 @@ def targets(perf_database_path: Path) -> Targets:
 
 @pytest.fixture(scope="session")
 def large_table_scans(
-    synthetic: Any, perf_database: Database, perf_database_path: Path
+    perf_database: Database, perf_database_path: Path
 ) -> Callable[[Callable[[], Any]], frozenset[str]]:
     """Run a call once and return the large tables that any of its queries reads in full."""
 
     def _scans(call: Callable[[], Any]) -> frozenset[str]:
+        if synthetic is None:
+            call()
+            return frozenset()
         with synthetic.captured_statements(perf_database) as statements:
             call()
         scans: frozenset[str] = synthetic.large_table_scans(perf_database_path, statements)

From 083ab9dbff9b5287ec7732895342be58555b098a Mon Sep 17 00:00:00 2001
From: Jared Lewis <jared@jared.kiwi.nz>
Date: Fri, 23 Oct 2026 23:35:50 +1100
Subject: [PATCH 5/5] test(perf): drop the comparison against older climate-ref
 schemas

Removes scripts/perf-database.py and REF_PERF_DATABASE, now that the v0.8.2 baseline is recorded.
A base without the synthetic database generator skips again. The CI cache for the databases stays.
---
 backend/scripts/perf-compare.sh  | 19 ++-------
 backend/scripts/perf-database.py | 66 --------------------------------
 backend/tests/perf/README.md     |  5 ---
 backend/tests/perf/conftest.py   | 32 ++++++----------
 4 files changed, 16 insertions(+), 106 deletions(-)
 delete mode 100644 backend/scripts/perf-database.py

diff --git a/backend/scripts/perf-compare.sh b/backend/scripts/perf-compare.sh
index 176d9e75..94ff3612 100755
--- a/backend/scripts/perf-compare.sh
+++ b/backend/scripts/perf-compare.sh
@@ -16,30 +16,19 @@ export REF_PERF_SCALE="${REF_PERF_SCALE:-0.1}"
 STORAGE="file://$WORK/benchmarks"
 
 git -C "$BACKEND" worktree add --quiet --detach "$WORK/base" "$BASE"
-(cd "$WORK/base/backend" && uv sync --locked --quiet)
-
-# The base may pin an older climate-ref, so it reads a copy migrated down to its schema.
-BASE_REVISION="$(cd "$WORK/base/backend" && uv run --no-sync python -c '
-import importlib.resources
-from alembic.config import Config
-from alembic.script import ScriptDirectory
-ini = importlib.resources.files("climate_ref") / "alembic.ini"
-print(ScriptDirectory.from_config(Config(str(ini))).get_current_head())
-')"
-cd "$BACKEND"
-HEAD_DATABASE="$(uv run python scripts/perf-database.py)"
-BASE_DATABASE="$(uv run python scripts/perf-database.py --revision "$BASE_REVISION")"
 
 # The base code runs this checkout's tests, so both sides time the same requests.
 # Endpoints missing on the base fail there and are left out of the comparison.
 (
     cd "$WORK/base/backend"
-    REF_PERF_DATABASE="$BASE_DATABASE" uv run --no-sync --with pytest-benchmark pytest "$BACKEND/tests/perf" \
+    uv sync --locked --quiet
+    uv run --with pytest-benchmark pytest "$BACKEND/tests/perf" \
         --rootdir="$BACKEND" --confcutdir="$BACKEND/tests/perf" -p no:cacheprovider -q \
         --benchmark-storage="$STORAGE" --benchmark-save=base
 ) || echo "::warning::some performance tests failed on $BASE, comparing the rest"
 
-REF_PERF_DATABASE="$HEAD_DATABASE" uv run pytest tests/perf -q \
+cd "$BACKEND"
+uv run pytest tests/perf -q \
     --benchmark-storage="$STORAGE" --benchmark-compare \
     --benchmark-compare-fail="median:${REF_PERF_THRESHOLD:-30}%" \
     --benchmark-columns=median,iqr,rounds
diff --git a/backend/scripts/perf-database.py b/backend/scripts/perf-database.py
deleted file mode 100644
index 1bd310f8..00000000
--- a/backend/scripts/perf-database.py
+++ /dev/null
@@ -1,66 +0,0 @@
-"""
-Print the path of a synthetic database for the API performance tests, building it on first use.
-
-Usage: uv run python scripts/perf-database.py [--revision REVISION]
-
-With ``--revision``, the database is a copy migrated down to that Alembic revision,
-so a release with an older schema can read it.
-Both are cached in ``REF_PERF_CACHE``, and ``REF_PERF_SCALE`` sets the size (default 0.1).
-"""
-
-import argparse
-import os
-import shutil
-from pathlib import Path
-
-import alembic.command
-from alembic.script import ScriptDirectory
-
-from climate_ref.config import Config
-from climate_ref.database import Database
-from climate_ref.testing.synthetic import cached_synthetic_database
-
-
-def _open(path: Path) -> tuple[Config, Database]:
-    config = Config()
-    config.db.database_url = f"sqlite:///{path}"
-    return config, Database.from_config(config, run_migrations=False)
-
-
-def migrated_down(path: Path, revision: str) -> Path:
-    """Return a copy of ``path`` migrated down to ``revision``, building it on first use."""
-    config, database = _open(path)
-    head = ScriptDirectory.from_config(database.alembic_config(config)).get_current_head()
-    database.close()
-    if revision == head:
-        return path
-
-    target = path.with_name(f"{path.stem}-{revision}.db")
-    if target.exists():
-        return target
-    tmp = target.with_name(f"{target.name}.{os.getpid()}.tmp")
-    shutil.copyfile(path, tmp)
-    config, database = _open(tmp)
-    alembic.command.downgrade(database.alembic_config(config), revision)
-    database.optimize()
-    database.close()
-    tmp.replace(target)
-    return target
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
-    )
-    parser.add_argument("--revision", help="Alembic revision to migrate the copy down to")
-    args = parser.parse_args()
-
-    cache = Path(os.environ.get("REF_PERF_CACHE", Path.home() / ".cache" / "ref-app" / "perf"))
-    path = cached_synthetic_database(cache, scale=float(os.environ.get("REF_PERF_SCALE", "0.1")))
-    if args.revision:
-        path = migrated_down(path, args.revision)
-    print(path)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/backend/tests/perf/README.md b/backend/tests/perf/README.md
index 59cb8538..13784f1a 100644
--- a/backend/tests/perf/README.md
+++ b/backend/tests/perf/README.md
@@ -20,20 +20,15 @@ A change to the shape, the scale, the generator or the migrations builds a new o
 Old databases are not removed, so clear the cache directory by hand.
 
 `REF_PERF_ROUNDS` sets the timed runs per endpoint, default 5.
-`REF_PERF_DATABASE` runs against a database built already, e.g. by `scripts/perf-database.py`.
 
 ## Comparing against main
 
 ```bash
 make perf-compare               # against origin/main
 make perf-compare BASE=v0.8.2   # against any ref
-make perf-compare BASE=$(git describe --tags --abbrev=0)  # against the last release
 ```
 
 This checks out the base in a temporary worktree and runs this branch's tests against both.
-A base that pins an older climate-ref reads a copy of the database migrated down to its schema,
-which `scripts/perf-database.py` builds and caches next to the original.
-So a comparison against a release also measures the indexes added since.
 It fails when an endpoint's median is more than `REF_PERF_THRESHOLD` percent slower than on the base, default 30.
 Running the same code twice locally drifts by up to about 20%, so a smaller threshold is noise.
 CI runs this on every pull request, and caches the databases until either lock file changes.
diff --git a/backend/tests/perf/conftest.py b/backend/tests/perf/conftest.py
index 5eb0cbe5..014710d8 100644
--- a/backend/tests/perf/conftest.py
+++ b/backend/tests/perf/conftest.py
@@ -1,12 +1,10 @@
 """
 Fixtures for the API performance tests, which run against a large synthetic database.
 
-These only run when ``REF_PERF_SCALE`` or ``REF_PERF_DATABASE`` is set (see ``make test-perf``).
+These only run when ``REF_PERF_SCALE`` is set (see ``make test-perf``).
 The database is built once per shape, scale and schema, and cached under ``REF_PERF_CACHE``.
-``REF_PERF_DATABASE`` points at a database built already, e.g. by ``scripts/perf-database.py``.
 """
 
-import importlib
 import os
 import sqlite3
 from collections.abc import Callable, Iterator
@@ -24,23 +22,20 @@
 from ref_backend.builder import build_app
 from ref_backend.core.config import Settings, get_settings
 
-# Older climate-ref releases lack the generator, so they can only run against a REF_PERF_DATABASE.
-synthetic: Any
-try:
-    synthetic = importlib.import_module("climate_ref.testing.synthetic")
-except ImportError:
-    synthetic = None
-
 
 @pytest.fixture(scope="session")
-def perf_database_path() -> Path:
-    """The synthetic database for this scale and schema, built on first use."""
-    if database := os.environ.get("REF_PERF_DATABASE"):
-        return Path(database)
+def synthetic() -> Any:
+    """The synthetic database generator from climate-ref."""
     if not os.environ.get("REF_PERF_SCALE"):
         pytest.skip("set REF_PERF_SCALE to run the API performance tests")
-    if synthetic is None:
-        pytest.skip("needs a climate-ref with the synthetic database generator")
+    return pytest.importorskip(
+        "climate_ref.testing.synthetic", reason="needs a climate-ref with the synthetic database generator"
+    )
+
+
+@pytest.fixture(scope="session")
+def perf_database_path(synthetic: Any) -> Path:
+    """The synthetic database for this scale and schema, built on first use."""
     cache = Path(os.environ.get("REF_PERF_CACHE", Path.home() / ".cache" / "ref-app" / "perf"))
     path: Path = synthetic.cached_synthetic_database(cache, scale=float(os.environ["REF_PERF_SCALE"]))
     return path
@@ -125,14 +120,11 @@ def targets(perf_database_path: Path) -> Targets:
 
 @pytest.fixture(scope="session")
 def large_table_scans(
-    perf_database: Database, perf_database_path: Path
+    synthetic: Any, perf_database: Database, perf_database_path: Path
 ) -> Callable[[Callable[[], Any]], frozenset[str]]:
     """Run a call once and return the large tables that any of its queries reads in full."""
 
     def _scans(call: Callable[[], Any]) -> frozenset[str]:
-        if synthetic is None:
-            call()
-            return frozenset()
         with synthetic.captured_statements(perf_database) as statements:
             call()
         scans: frozenset[str] = synthetic.large_table_scans(perf_database_path, statements)