test(perf): time the API endpoints against a synthetic database

.github/workflows/ci.yml

@@ -92,3 +92,39 @@ jobs:
     - name: Run tests
       run: |
         uv run coverage run -m pytest tests/ && uv run coverage report --fail-under=90
+
+  backend-perf:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    defaults:
+      run:
+        working-directory: backend
+
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        fetch-depth: 0
+
+    - name: Install uv
+      uses: astral-sh/[email protected]
+      with:
+        enable-cache: true
+
+    # The databases depend on the climate-ref version on each side, which the lock files pin.
+    - name: Key the synthetic database cache
+      id: perf-db
+      run: |
+        key=$(cat uv.lock <(git show "origin/${{ github.base_ref }}:backend/uv.lock") | sha256sum | cut -c1-16)
+        echo "key=perf-db-$key" >> "$GITHUB_OUTPUT"
+
+    - name: Cache the synthetic databases
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ref-app/perf
+        key: ${{ steps.perf-db.outputs.key }}
+
+    - name: Compare query performance against the base branch
+      env:
+        REF_PERF_ROUNDS: "10"
+      run: |
+        scripts/perf-compare.sh "origin/${{ github.base_ref }}"

backend/Makefile

@@ -45,3 +45,14 @@ mypy: ## Run mypy type checker
 .PHONY: generate-metadata
 generate-metadata: ## Generate diagnostic metadata YAML from provider registry
 	uv run python scripts/generate_metadata.py
+
+REF_PERF_SCALE ?= 0.1
+BASE ?= origin/main
+
+.PHONY: test-perf
+test-perf: ## run the API performance tests against a synthetic database (REF_PERF_SCALE=1 for full size)
+	REF_PERF_SCALE=$(REF_PERF_SCALE) uv run pytest tests/perf -r a -v
+
+.PHONY: perf-compare
+perf-compare: ## fail if the API performance tests are slower than on BASE (default origin/main)
+	REF_PERF_SCALE=$(REF_PERF_SCALE) scripts/perf-compare.sh $(BASE)

backend/README.md

@@ -35,6 +35,9 @@ make dev
 make test
 ```
 
+`make test-perf` times the API against a large synthetic database.
+See [tests/perf/README.md](tests/perf/README.md).
+
 ### Code Quality
 
 The project uses:

backend/pyproject.toml

@@ -28,6 +28,7 @@ dev = [
     "types-pyyaml>=6.0.12.20250915",
     "bump-my-version>=0.28.1",
     "towncrier>=24.8.0",
+    "pytest-benchmark>=5.3.0",
 ]
 
 [tool.uv.sources]

backend/scripts/perf-compare.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Time the API performance tests on a base ref and on this checkout, then fail if this checkout is slower.
+#
+# Usage: scripts/perf-compare.sh [base ref, default origin/main]
+#
+# REF_PERF_SCALE (default 0.1), REF_PERF_ROUNDS and REF_PERF_CACHE are passed through to the tests.
+# REF_PERF_THRESHOLD is the allowed slowdown of each endpoint's median, in percent (default 30).
+set -euo pipefail
+
+BASE="${1:-origin/main}"
+BACKEND="$(cd "$(dirname "$0")/.." && pwd)"
+WORK="$(mktemp -d)"
+trap 'git -C "$BACKEND" worktree remove --force "$WORK/base" >/dev/null 2>&1 || true; rm -rf "$WORK"' EXIT
+
+export REF_PERF_SCALE="${REF_PERF_SCALE:-0.1}"
+STORAGE="file://$WORK/benchmarks"
+
+git -C "$BACKEND" worktree add --quiet --detach "$WORK/base" "$BASE"
+
+# The base code runs this checkout's tests, so both sides time the same requests.
+# Endpoints missing on the base fail there and are left out of the comparison.
+(
+    cd "$WORK/base/backend"
+    uv sync --locked --quiet
+    uv run --with pytest-benchmark pytest "$BACKEND/tests/perf" \
+        --rootdir="$BACKEND" --confcutdir="$BACKEND/tests/perf" -p no:cacheprovider -q \
+        --benchmark-storage="$STORAGE" --benchmark-save=base
+) || echo "::warning::some performance tests failed on $BASE, comparing the rest"
+
+cd "$BACKEND"
+uv run pytest tests/perf -q \
+    --benchmark-storage="$STORAGE" --benchmark-compare \
+    --benchmark-compare-fail="median:${REF_PERF_THRESHOLD:-30}%" \
+    --benchmark-columns=median,iqr,rounds

backend/src/ref_backend/models/diagnostics.py

@@ -20,9 +20,19 @@
 from ref_backend.models.common import GroupBy, ProviderSummary
 
 if TYPE_CHECKING:
+    from climate_ref_core.diagnostics import Diagnostic
     from ref_backend.api.deps import AppContext
 
 
+def _concrete_diagnostic(diagnostic: models.Diagnostic, app_context: "AppContext") -> "Diagnostic | None":
+    """Find the registered diagnostic for a stored one, or None when its provider is not installed."""
+    try:
+        return app_context.provider_registry.get_metric(diagnostic.provider.slug, diagnostic.slug)
+    except KeyError:
+        logger.warning(f"Could not find concrete diagnostic for {diagnostic.provider.slug}/{diagnostic.slug}")
+        return None
+
+
 class DiagnosticCatalogEntry(BaseModel):
     """
     Summary information about a diagnostic, without whether it has metric values.
@@ -113,14 +123,8 @@ def _ensure_metadata_cache(app_context: "AppContext") -> Mapping[str, Diagnostic
     @staticmethod
     def _build_group_by_summary(diagnostic: models.Diagnostic, app_context: "AppContext") -> list[GroupBy]:
         """Extract and build group_by information from diagnostic data requirements."""
-        try:
-            concrete_diagnostic = app_context.provider_registry.get_metric(
-                diagnostic.provider.slug, diagnostic.slug
-            )
-        except KeyError:
-            logger.warning(
-                f"Could not find concrete diagnostic for {diagnostic.provider.slug}/{diagnostic.slug}"
-            )
+        concrete_diagnostic = _concrete_diagnostic(diagnostic, app_context)
+        if concrete_diagnostic is None:
             return []
         data_requirements = sorted(
             concrete_diagnostic.data_requirements,
@@ -196,16 +200,8 @@ def build_with_stats(  # noqa: PLR0913
         group_by_summary = DiagnosticCatalogEntry._build_group_by_summary(diagnostic, app_context)
         aft = DiagnosticCatalogEntry._get_aft_link(diagnostic)
 
-        try:
-            concrete_diagnostic = app_context.provider_registry.get_metric(
-                diagnostic.provider.slug, diagnostic.slug
-            )
-            description = concrete_diagnostic.__doc__ or ""
-        except KeyError:
-            logger.warning(
-                f"Could not find concrete diagnostic for {diagnostic.provider.slug}/{diagnostic.slug}"
-            )
-            description = ""
+        concrete_diagnostic = _concrete_diagnostic(diagnostic, app_context)
+        description = (concrete_diagnostic.__doc__ or "") if concrete_diagnostic else ""
 
         # Build the base diagnostic summary
         summary = DiagnosticCatalogEntry(
@@ -328,9 +324,8 @@ def has_values(value_type: StoredMetricValueType) -> bool:
             .count()
         )
 
-        concrete_diagnostic = app_context.provider_registry.get_metric(
-            diagnostic.provider.slug, diagnostic.slug
-        )
+        concrete_diagnostic = _concrete_diagnostic(diagnostic, app_context)
+        description = (concrete_diagnostic.__doc__ or "") if concrete_diagnostic else ""
 
         resource_usage = resource_usage_for_diagnostic(app_context.session, diagnostic.id)
 
@@ -340,7 +335,7 @@ def has_values(value_type: StoredMetricValueType) -> bool:
             provider=ProviderSummary.build(diagnostic.provider),
             slug=diagnostic.slug,
             name=diagnostic.name,
-            description=concrete_diagnostic.__doc__ or "",
+            description=description,
             execution_groups=[e.id for e in diagnostic.execution_groups],
             has_metric_values=has_metric_values,
             has_scalar_values=has_scalar_values,

backend/tests/perf/README.md

@@ -0,0 +1,40 @@
+# API performance tests
+
+These tests time the API endpoints against a synthetic database shaped like production.
+The database comes from `climate_ref.testing.synthetic`, the same generator climate-ref uses for its reader tests.
+The data is read-only, so every request can be repeated against the same database.
+
+Each entry in `ENDPOINTS` in `test_endpoints.py` is one request the frontend makes.
+An entry fails when one of its queries reads every row of a large table, which `EXPLAIN QUERY PLAN` shows at any scale.
+Its `allowed_scans` lists the scans that are known and accepted.
+
+## Running
+
+```bash
+make test-perf                   # a tenth of production, quick
+make test-perf REF_PERF_SCALE=1  # full size, about 5 GB
+```
+
+The database is built on first use and cached in `~/.cache/ref-app/perf` (`REF_PERF_CACHE`).
+A change to the shape, the scale, the generator or the migrations builds a new one.
+Old databases are not removed, so clear the cache directory by hand.
+
+`REF_PERF_ROUNDS` sets the timed runs per endpoint, default 5.
+
+## Comparing against main
+
+```bash
+make perf-compare               # against origin/main
+make perf-compare BASE=v0.8.2   # against any ref
+```
+
+This checks out the base in a temporary worktree and runs this branch's tests against both.
+It fails when an endpoint's median is more than `REF_PERF_THRESHOLD` percent slower than on the base, default 30.
+Running the same code twice locally drifts by up to about 20%, so a smaller threshold is noise.
+CI runs this on every pull request, and caches the databases until either lock file changes.
+
+## Adding an endpoint
+
+Add an `Endpoint` to `ENDPOINTS`.
+The path and parameters are formatted with the `targets` fixture, e.g. `{provider_slug}`,
+which picks the busiest diagnostic and execution so each request reads its worst case.

backend/tests/perf/conftest.py

@@ -0,0 +1,133 @@
+"""
+Fixtures for the API performance tests, which run against a large synthetic database.
+
+These only run when ``REF_PERF_SCALE`` is set (see ``make test-perf``).
+The database is built once per shape, scale and schema, and cached under ``REF_PERF_CACHE``.
+"""
+
+import os
+import sqlite3
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import pytest
+from starlette.testclient import TestClient
+
+from climate_ref.config import Config
+from climate_ref.database import Database
+from climate_ref.provider_registry import ProviderRegistry
+from ref_backend.api import deps
+from ref_backend.builder import build_app
+from ref_backend.core.config import Settings, get_settings
+
+
+@pytest.fixture(scope="session")
+def synthetic() -> Any:
+    """The synthetic database generator from climate-ref."""
+    if not os.environ.get("REF_PERF_SCALE"):
+        pytest.skip("set REF_PERF_SCALE to run the API performance tests")
+    return pytest.importorskip(
+        "climate_ref.testing.synthetic", reason="needs a climate-ref with the synthetic database generator"
+    )
+
+
+@pytest.fixture(scope="session")
+def perf_database_path(synthetic: Any) -> Path:
+    """The synthetic database for this scale and schema, built on first use."""
+    cache = Path(os.environ.get("REF_PERF_CACHE", Path.home() / ".cache" / "ref-app" / "perf"))
+    path: Path = synthetic.cached_synthetic_database(cache, scale=float(os.environ["REF_PERF_SCALE"]))
+    return path
+
+
+@pytest.fixture(scope="session")
+def perf_settings(tmp_path_factory: pytest.TempPathFactory) -> Settings:
+    # An empty configuration directory, so a developer's ref.toml cannot point the app elsewhere.
+    configuration = tmp_path_factory.mktemp("configuration")
+    return Settings(REF_CONFIGURATION=str(configuration), REF_READ_ONLY_DATABASE=True)
+
+
+@pytest.fixture(scope="session")
+def perf_ref_config(perf_database_path: Path, tmp_path_factory: pytest.TempPathFactory) -> Config:
+    config = Config()
+    config.db.database_url = f"sqlite:///{perf_database_path}"
+    config.paths.results = tmp_path_factory.mktemp("results")
+    return config
+
+
+@pytest.fixture(scope="session")
+def perf_database(perf_settings: Settings, perf_ref_config: Config) -> Database:
+    return deps._get_database_dependency(perf_settings, perf_ref_config)
+
+
+@pytest.fixture(scope="session")
+def perf_client(
+    perf_settings: Settings, perf_ref_config: Config, perf_database: Database
+) -> Iterator[TestClient]:
+    """A client for the app reading the synthetic database read-only, like the deployed API."""
+    app = build_app(settings=perf_settings, ref_config=perf_ref_config, database=perf_database)
+    app.dependency_overrides[get_settings] = lambda: perf_settings
+    app.dependency_overrides[deps._ref_config_dependency] = lambda: perf_ref_config
+    # Built once like `ref_backend.main`, and without registering, because the database is read-only.
+    registry = ProviderRegistry.build_from_config(perf_ref_config, perf_database, register=False)
+    app.dependency_overrides[deps._provider_registry_dependency] = lambda: registry
+    with TestClient(app) as client:
+        yield client
+
+
+@dataclass(frozen=True, kw_only=True)
+class Targets:
+    """Ids and slugs that the endpoints query, picked from the generated data."""
+
+    provider_slug: str
+    diagnostic_slug: str
+    """The diagnostic with the most metric values, the worst case for a diagnostic-scoped read."""
+
+    execution_group_id: int
+    """The group of the execution with the most metric values."""
+
+    dataset_slug: str
+    source_id: str
+    """The model with the most CMIP6 datasets."""
+
+
+@pytest.fixture(scope="session")
+def targets(perf_database_path: Path) -> Targets:
+    """Pick the busiest diagnostic and execution, so each endpoint reads its worst case."""
+    con = sqlite3.connect(f"file:{perf_database_path}?mode=ro", uri=True)
+    execution_id, _ = con.execute(
+        "SELECT execution_id, count(*) FROM metric_value GROUP BY execution_id ORDER BY 2 DESC LIMIT 1"
+    ).fetchone()
+    group_id, diagnostic_slug, provider_slug = con.execute(
+        "SELECT g.id, d.slug, p.slug FROM execution e JOIN execution_group g ON g.id = e.execution_group_id "
+        "JOIN diagnostic d ON d.id = g.diagnostic_id JOIN provider p ON p.id = d.provider_id WHERE e.id = ?",
+        (execution_id,),
+    ).fetchone()
+    (dataset_slug,) = con.execute("SELECT slug FROM dataset ORDER BY id DESC LIMIT 1").fetchone()
+    (source_id,) = con.execute(
+        "SELECT source_id FROM cmip6_dataset GROUP BY source_id ORDER BY count(*) DESC LIMIT 1"
+    ).fetchone()
+    con.close()
+    return Targets(
+        provider_slug=provider_slug,
+        diagnostic_slug=diagnostic_slug,
+        execution_group_id=group_id,
+        dataset_slug=dataset_slug,
+        source_id=source_id,
+    )
+
+
+@pytest.fixture(scope="session")
+def large_table_scans(
+    synthetic: Any, perf_database: Database, perf_database_path: Path
+) -> Callable[[Callable[[], Any]], frozenset[str]]:
+    """Run a call once and return the large tables that any of its queries reads in full."""
+
+    def _scans(call: Callable[[], Any]) -> frozenset[str]:
+        with synthetic.captured_statements(perf_database) as statements:
+            call()
+        scans: frozenset[str] = synthetic.large_table_scans(perf_database_path, statements)
+        return scans
+
+    return _scans