diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3485025f..1695baf9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,3 +92,39 @@ jobs: - name: Run tests run: | uv run coverage run -m pytest tests/ && uv run coverage report --fail-under=90 + + backend-perf: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + defaults: + run: + working-directory: backend + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v10.3.0 + with: + enable-cache: true + + # The databases depend on the climate-ref version on each side, which the lock files pin. + - name: Key the synthetic database cache + id: perf-db + run: | + key=$(cat uv.lock <(git show "origin/${{ github.base_ref }}:backend/uv.lock") | sha256sum | cut -c1-16) + echo "key=perf-db-$key" >> "$GITHUB_OUTPUT" + + - name: Cache the synthetic databases + uses: actions/cache@v4 + with: + path: ~/.cache/ref-app/perf + key: ${{ steps.perf-db.outputs.key }} + + - name: Compare query performance against the base branch + env: + REF_PERF_ROUNDS: "10" + run: | + scripts/perf-compare.sh "origin/${{ github.base_ref }}" diff --git a/backend/Makefile b/backend/Makefile index 5f564983..d986a808 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -45,3 +45,14 @@ mypy: ## Run mypy type checker .PHONY: generate-metadata generate-metadata: ## Generate diagnostic metadata YAML from provider registry uv run python scripts/generate_metadata.py + +REF_PERF_SCALE ?= 0.1 +BASE ?= origin/main + +.PHONY: test-perf +test-perf: ## run the API performance tests against a synthetic database (REF_PERF_SCALE=1 for full size) + REF_PERF_SCALE=$(REF_PERF_SCALE) uv run pytest tests/perf -r a -v + +.PHONY: perf-compare +perf-compare: ## fail if the API performance tests are slower than on BASE (default origin/main) + REF_PERF_SCALE=$(REF_PERF_SCALE) scripts/perf-compare.sh $(BASE) diff --git a/backend/README.md b/backend/README.md index aab98cd0..f5158c26 100644 --- a/backend/README.md +++ b/backend/README.md @@ -35,6 +35,9 @@ make dev make test ``` +`make test-perf` times the API against a large synthetic database. +See [tests/perf/README.md](tests/perf/README.md). + ### Code Quality The project uses: diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 662d3438..3354a659 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -28,6 +28,7 @@ dev = [ "types-pyyaml>=6.0.12.20250915", "bump-my-version>=0.28.1", "towncrier>=24.8.0", + "pytest-benchmark>=5.3.0", ] [tool.uv.sources] diff --git a/backend/scripts/perf-compare.sh b/backend/scripts/perf-compare.sh new file mode 100755 index 00000000..94ff3612 --- /dev/null +++ b/backend/scripts/perf-compare.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Time the API performance tests on a base ref and on this checkout, then fail if this checkout is slower. +# +# Usage: scripts/perf-compare.sh [base ref, default origin/main] +# +# REF_PERF_SCALE (default 0.1), REF_PERF_ROUNDS and REF_PERF_CACHE are passed through to the tests. +# REF_PERF_THRESHOLD is the allowed slowdown of each endpoint's median, in percent (default 30). +set -euo pipefail + +BASE="${1:-origin/main}" +BACKEND="$(cd "$(dirname "$0")/.." && pwd)" +WORK="$(mktemp -d)" +trap 'git -C "$BACKEND" worktree remove --force "$WORK/base" >/dev/null 2>&1 || true; rm -rf "$WORK"' EXIT + +export REF_PERF_SCALE="${REF_PERF_SCALE:-0.1}" +STORAGE="file://$WORK/benchmarks" + +git -C "$BACKEND" worktree add --quiet --detach "$WORK/base" "$BASE" + +# The base code runs this checkout's tests, so both sides time the same requests. +# Endpoints missing on the base fail there and are left out of the comparison. +( + cd "$WORK/base/backend" + uv sync --locked --quiet + uv run --with pytest-benchmark pytest "$BACKEND/tests/perf" \ + --rootdir="$BACKEND" --confcutdir="$BACKEND/tests/perf" -p no:cacheprovider -q \ + --benchmark-storage="$STORAGE" --benchmark-save=base +) || echo "::warning::some performance tests failed on $BASE, comparing the rest" + +cd "$BACKEND" +uv run pytest tests/perf -q \ + --benchmark-storage="$STORAGE" --benchmark-compare \ + --benchmark-compare-fail="median:${REF_PERF_THRESHOLD:-30}%" \ + --benchmark-columns=median,iqr,rounds diff --git a/backend/src/ref_backend/models/diagnostics.py b/backend/src/ref_backend/models/diagnostics.py index 04f473f9..75fc411f 100644 --- a/backend/src/ref_backend/models/diagnostics.py +++ b/backend/src/ref_backend/models/diagnostics.py @@ -20,9 +20,19 @@ from ref_backend.models.common import GroupBy, ProviderSummary if TYPE_CHECKING: + from climate_ref_core.diagnostics import Diagnostic from ref_backend.api.deps import AppContext +def _concrete_diagnostic(diagnostic: models.Diagnostic, app_context: "AppContext") -> "Diagnostic | None": + """Find the registered diagnostic for a stored one, or None when its provider is not installed.""" + try: + return app_context.provider_registry.get_metric(diagnostic.provider.slug, diagnostic.slug) + except KeyError: + logger.warning(f"Could not find concrete diagnostic for {diagnostic.provider.slug}/{diagnostic.slug}") + return None + + class DiagnosticCatalogEntry(BaseModel): """ Summary information about a diagnostic, without whether it has metric values. @@ -113,14 +123,8 @@ def _ensure_metadata_cache(app_context: "AppContext") -> Mapping[str, Diagnostic @staticmethod def _build_group_by_summary(diagnostic: models.Diagnostic, app_context: "AppContext") -> list[GroupBy]: """Extract and build group_by information from diagnostic data requirements.""" - try: - concrete_diagnostic = app_context.provider_registry.get_metric( - diagnostic.provider.slug, diagnostic.slug - ) - except KeyError: - logger.warning( - f"Could not find concrete diagnostic for {diagnostic.provider.slug}/{diagnostic.slug}" - ) + concrete_diagnostic = _concrete_diagnostic(diagnostic, app_context) + if concrete_diagnostic is None: return [] data_requirements = sorted( concrete_diagnostic.data_requirements, @@ -196,16 +200,8 @@ def build_with_stats( # noqa: PLR0913 group_by_summary = DiagnosticCatalogEntry._build_group_by_summary(diagnostic, app_context) aft = DiagnosticCatalogEntry._get_aft_link(diagnostic) - try: - concrete_diagnostic = app_context.provider_registry.get_metric( - diagnostic.provider.slug, diagnostic.slug - ) - description = concrete_diagnostic.__doc__ or "" - except KeyError: - logger.warning( - f"Could not find concrete diagnostic for {diagnostic.provider.slug}/{diagnostic.slug}" - ) - description = "" + concrete_diagnostic = _concrete_diagnostic(diagnostic, app_context) + description = (concrete_diagnostic.__doc__ or "") if concrete_diagnostic else "" # Build the base diagnostic summary summary = DiagnosticCatalogEntry( @@ -328,9 +324,8 @@ def has_values(value_type: StoredMetricValueType) -> bool: .count() ) - concrete_diagnostic = app_context.provider_registry.get_metric( - diagnostic.provider.slug, diagnostic.slug - ) + concrete_diagnostic = _concrete_diagnostic(diagnostic, app_context) + description = (concrete_diagnostic.__doc__ or "") if concrete_diagnostic else "" resource_usage = resource_usage_for_diagnostic(app_context.session, diagnostic.id) @@ -340,7 +335,7 @@ def has_values(value_type: StoredMetricValueType) -> bool: provider=ProviderSummary.build(diagnostic.provider), slug=diagnostic.slug, name=diagnostic.name, - description=concrete_diagnostic.__doc__ or "", + description=description, execution_groups=[e.id for e in diagnostic.execution_groups], has_metric_values=has_metric_values, has_scalar_values=has_scalar_values, diff --git a/backend/tests/perf/README.md b/backend/tests/perf/README.md new file mode 100644 index 00000000..13784f1a --- /dev/null +++ b/backend/tests/perf/README.md @@ -0,0 +1,40 @@ +# API performance tests + +These tests time the API endpoints against a synthetic database shaped like production. +The database comes from `climate_ref.testing.synthetic`, the same generator climate-ref uses for its reader tests. +The data is read-only, so every request can be repeated against the same database. + +Each entry in `ENDPOINTS` in `test_endpoints.py` is one request the frontend makes. +An entry fails when one of its queries reads every row of a large table, which `EXPLAIN QUERY PLAN` shows at any scale. +Its `allowed_scans` lists the scans that are known and accepted. + +## Running + +```bash +make test-perf # a tenth of production, quick +make test-perf REF_PERF_SCALE=1 # full size, about 5 GB +``` + +The database is built on first use and cached in `~/.cache/ref-app/perf` (`REF_PERF_CACHE`). +A change to the shape, the scale, the generator or the migrations builds a new one. +Old databases are not removed, so clear the cache directory by hand. + +`REF_PERF_ROUNDS` sets the timed runs per endpoint, default 5. + +## Comparing against main + +```bash +make perf-compare # against origin/main +make perf-compare BASE=v0.8.2 # against any ref +``` + +This checks out the base in a temporary worktree and runs this branch's tests against both. +It fails when an endpoint's median is more than `REF_PERF_THRESHOLD` percent slower than on the base, default 30. +Running the same code twice locally drifts by up to about 20%, so a smaller threshold is noise. +CI runs this on every pull request, and caches the databases until either lock file changes. + +## Adding an endpoint + +Add an `Endpoint` to `ENDPOINTS`. +The path and parameters are formatted with the `targets` fixture, e.g. `{provider_slug}`, +which picks the busiest diagnostic and execution so each request reads its worst case. diff --git a/backend/tests/perf/conftest.py b/backend/tests/perf/conftest.py new file mode 100644 index 00000000..014710d8 --- /dev/null +++ b/backend/tests/perf/conftest.py @@ -0,0 +1,133 @@ +""" +Fixtures for the API performance tests, which run against a large synthetic database. + +These only run when ``REF_PERF_SCALE`` is set (see ``make test-perf``). +The database is built once per shape, scale and schema, and cached under ``REF_PERF_CACHE``. +""" + +import os +import sqlite3 +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest +from starlette.testclient import TestClient + +from climate_ref.config import Config +from climate_ref.database import Database +from climate_ref.provider_registry import ProviderRegistry +from ref_backend.api import deps +from ref_backend.builder import build_app +from ref_backend.core.config import Settings, get_settings + + +@pytest.fixture(scope="session") +def synthetic() -> Any: + """The synthetic database generator from climate-ref.""" + if not os.environ.get("REF_PERF_SCALE"): + pytest.skip("set REF_PERF_SCALE to run the API performance tests") + return pytest.importorskip( + "climate_ref.testing.synthetic", reason="needs a climate-ref with the synthetic database generator" + ) + + +@pytest.fixture(scope="session") +def perf_database_path(synthetic: Any) -> Path: + """The synthetic database for this scale and schema, built on first use.""" + cache = Path(os.environ.get("REF_PERF_CACHE", Path.home() / ".cache" / "ref-app" / "perf")) + path: Path = synthetic.cached_synthetic_database(cache, scale=float(os.environ["REF_PERF_SCALE"])) + return path + + +@pytest.fixture(scope="session") +def perf_settings(tmp_path_factory: pytest.TempPathFactory) -> Settings: + # An empty configuration directory, so a developer's ref.toml cannot point the app elsewhere. + configuration = tmp_path_factory.mktemp("configuration") + return Settings(REF_CONFIGURATION=str(configuration), REF_READ_ONLY_DATABASE=True) + + +@pytest.fixture(scope="session") +def perf_ref_config(perf_database_path: Path, tmp_path_factory: pytest.TempPathFactory) -> Config: + config = Config() + config.db.database_url = f"sqlite:///{perf_database_path}" + config.paths.results = tmp_path_factory.mktemp("results") + return config + + +@pytest.fixture(scope="session") +def perf_database(perf_settings: Settings, perf_ref_config: Config) -> Database: + return deps._get_database_dependency(perf_settings, perf_ref_config) + + +@pytest.fixture(scope="session") +def perf_client( + perf_settings: Settings, perf_ref_config: Config, perf_database: Database +) -> Iterator[TestClient]: + """A client for the app reading the synthetic database read-only, like the deployed API.""" + app = build_app(settings=perf_settings, ref_config=perf_ref_config, database=perf_database) + app.dependency_overrides[get_settings] = lambda: perf_settings + app.dependency_overrides[deps._ref_config_dependency] = lambda: perf_ref_config + # Built once like `ref_backend.main`, and without registering, because the database is read-only. + registry = ProviderRegistry.build_from_config(perf_ref_config, perf_database, register=False) + app.dependency_overrides[deps._provider_registry_dependency] = lambda: registry + with TestClient(app) as client: + yield client + + +@dataclass(frozen=True, kw_only=True) +class Targets: + """Ids and slugs that the endpoints query, picked from the generated data.""" + + provider_slug: str + diagnostic_slug: str + """The diagnostic with the most metric values, the worst case for a diagnostic-scoped read.""" + + execution_group_id: int + """The group of the execution with the most metric values.""" + + dataset_slug: str + source_id: str + """The model with the most CMIP6 datasets.""" + + +@pytest.fixture(scope="session") +def targets(perf_database_path: Path) -> Targets: + """Pick the busiest diagnostic and execution, so each endpoint reads its worst case.""" + con = sqlite3.connect(f"file:{perf_database_path}?mode=ro", uri=True) + execution_id, _ = con.execute( + "SELECT execution_id, count(*) FROM metric_value GROUP BY execution_id ORDER BY 2 DESC LIMIT 1" + ).fetchone() + group_id, diagnostic_slug, provider_slug = con.execute( + "SELECT g.id, d.slug, p.slug FROM execution e JOIN execution_group g ON g.id = e.execution_group_id " + "JOIN diagnostic d ON d.id = g.diagnostic_id JOIN provider p ON p.id = d.provider_id WHERE e.id = ?", + (execution_id,), + ).fetchone() + (dataset_slug,) = con.execute("SELECT slug FROM dataset ORDER BY id DESC LIMIT 1").fetchone() + (source_id,) = con.execute( + "SELECT source_id FROM cmip6_dataset GROUP BY source_id ORDER BY count(*) DESC LIMIT 1" + ).fetchone() + con.close() + return Targets( + provider_slug=provider_slug, + diagnostic_slug=diagnostic_slug, + execution_group_id=group_id, + dataset_slug=dataset_slug, + source_id=source_id, + ) + + +@pytest.fixture(scope="session") +def large_table_scans( + synthetic: Any, perf_database: Database, perf_database_path: Path +) -> Callable[[Callable[[], Any]], frozenset[str]]: + """Run a call once and return the large tables that any of its queries reads in full.""" + + def _scans(call: Callable[[], Any]) -> frozenset[str]: + with synthetic.captured_statements(perf_database) as statements: + call() + scans: frozenset[str] = synthetic.large_table_scans(perf_database_path, statements) + return scans + + return _scans diff --git a/backend/tests/perf/test_endpoints.py b/backend/tests/perf/test_endpoints.py new file mode 100644 index 00000000..1123d8ea --- /dev/null +++ b/backend/tests/perf/test_endpoints.py @@ -0,0 +1,97 @@ +""" +Performance of the API endpoints against a production-shaped synthetic database. + +Each endpoint is one request the frontend makes. +It fails when a query reads a large table in full, which the query plan shows at any scale. +The timings are compared against main by ``make perf-compare``. +""" + +import os +from collections.abc import Mapping +from dataclasses import dataclass, field + +import pytest + + +@dataclass(frozen=True, kw_only=True) +class Endpoint: + id: str + path: str + """Path below the API prefix, formatted with the ``targets`` fields, e.g. ``{provider_slug}``.""" + + params: Mapping[str, str] = field(default_factory=dict) + allowed_scans: frozenset[str] = frozenset() + """Large tables this request has to read in full, e.g. a summary over every execution.""" + + +_DIAGNOSTIC = "diagnostics/{provider_slug}/{diagnostic_slug}" +_GROUP = "executions/{execution_group_id}" + +# The allowed scans record what each endpoint read when this suite was added, not what it should read. +ENDPOINTS = [ + # diagnostics: the landing page and a diagnostic's page + Endpoint(id="diagnostics.list", path="diagnostics/"), + Endpoint( + id="diagnostics.list.cmip6", + path="diagnostics/", + params={"mip_era": "CMIP6"}, + allowed_scans=frozenset({"execution_group"}), + ), + Endpoint(id="diagnostics.catalog", path="diagnostics/catalog"), + Endpoint(id="diagnostics.value-flags", path="diagnostics/value-flags"), + Endpoint(id="diagnostics.facets", path="diagnostics/facets", allowed_scans=frozenset({"metric_value"})), + Endpoint(id="diagnostics.get", path=_DIAGNOSTIC), + Endpoint(id="diagnostics.execution-groups", path=f"{_DIAGNOSTIC}/execution_groups"), + Endpoint(id="diagnostics.values.scalar", path=f"{_DIAGNOSTIC}/values", params={"value_type": "scalar"}), + Endpoint( + id="diagnostics.values.scalar.source-id", + path=f"{_DIAGNOSTIC}/values", + params={"value_type": "scalar", "source_id": "{source_id}"}, + ), + Endpoint(id="diagnostics.values.series", path=f"{_DIAGNOSTIC}/values", params={"value_type": "series"}), + # executions: the recent list and an execution group's page + Endpoint(id="executions.list", path="executions/", allowed_scans=frozenset({"execution_group"})), + Endpoint( + id="executions.list.failed", + path="executions/", + params={"successful": "false"}, + allowed_scans=frozenset({"execution_group"}), + ), + Endpoint( + id="executions.statistics", + path="executions/statistics", + allowed_scans=frozenset({"execution", "dataset", "dataset_file"}), + ), + Endpoint(id="executions.get", path=_GROUP), + Endpoint(id="executions.values.scalar", path=f"{_GROUP}/values", params={"value_type": "scalar"}), + Endpoint(id="executions.values.series", path=f"{_GROUP}/values", params={"value_type": "series"}), + # models and datasets + Endpoint( + id="models.list", + path="models/", + allowed_scans=frozenset({"execution", "cmip6_dataset", "cmip7_dataset"}), + ), + Endpoint(id="models.get", path="models/{source_id}", allowed_scans=frozenset({"execution"})), + Endpoint(id="datasets.list", path="datasets/"), + Endpoint(id="datasets.get", path="datasets/{dataset_slug}"), +] + + +@pytest.mark.parametrize("endpoint", ENDPOINTS, ids=lambda e: e.id) +def test_endpoint( # noqa: PLR0913, PLR0917 + endpoint: Endpoint, perf_client, perf_settings, targets, large_table_scans, benchmark +): + fields = vars(targets) + url = f"{perf_settings.API_V1_STR}/{endpoint.path.format(**fields)}" + params = {k: v.format(**fields) for k, v in endpoint.params.items()} + + def _get(): + response = perf_client.get(url, params=params) + assert response.status_code == 200, response.text + return response + + # This request also warms the caches, so the timed rounds need no warm-up. + unexpected = large_table_scans(_get) - endpoint.allowed_scans + benchmark.pedantic(_get, rounds=int(os.environ.get("REF_PERF_ROUNDS", "5")), warmup_rounds=0) + + assert not unexpected, f"reads every row of {sorted(unexpected)}" diff --git a/backend/tests/test_api/test_routes/test_diagnostics.py b/backend/tests/test_api/test_routes/test_diagnostics.py index e66026f3..1101a6a4 100644 --- a/backend/tests/test_api/test_routes/test_diagnostics.py +++ b/backend/tests/test_api/test_routes/test_diagnostics.py @@ -8,6 +8,7 @@ from climate_ref import models from climate_ref.database import Database +from climate_ref.provider_registry import ProviderRegistry from ref_backend.api import deps from ref_backend.builder import build_app from ref_backend.core.config import get_settings @@ -995,3 +996,16 @@ def counts(query: str = "") -> dict[str, tuple[int, int]]: assert sum(total for total, _ in model.values()) > 0 assert sum(total for total, _ in model.values()) < sum(total for total, _ in everything.values()) assert all(total == 0 for total, _ in nothing.values()) + + +def test_diagnostic_without_provider(app, client: TestClient, settings): + diagnostic = get_diagnostic(client, settings) + app.dependency_overrides[deps._provider_registry_dependency] = lambda: ProviderRegistry(providers=[]) + try: + r = client.get( + f"{settings.API_V1_STR}/diagnostics/{diagnostic['provider']['slug']}/{diagnostic['slug']}" + ) + finally: + del app.dependency_overrides[deps._provider_registry_dependency] + + assert r.status_code == 200 diff --git a/backend/uv.lock b/backend/uv.lock index 92644c72..304e7897 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -3214,6 +3214,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e8/30/3991c9fdcca90a5a1e55435292f4d74d176da2be15f3998f6858da3658cc/psycopg2_binary-2.9.13-cp315-cp315-win_amd64.whl", hash = "sha256:1752b9821f1377404d65ac43af03d59a1eccc57fb2c1eb8305f9a3fe8eb7a8ba", size = 2867246, upload-time = "2026-09-09T23:56:20.501Z" }, ] +[[package]] +name = "py-cpuinfo2" +version = "10.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/97/a8b1ddada14c8280a047c0746f95cb05d94a31b1a331cea22bcdc2b2a82d/py_cpuinfo2-10.1.1.tar.gz", hash = "sha256:7861133863663f16e06eca63b12904ef100b5760415e92372dac0162799a4771", size = 100840, upload-time = "2026-03-25T21:49:40.797Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/0a/ba69d2dde1ae12ef1d389ea5a216384c5ff6ef7a1e7a48d1e9b6686f6790/py_cpuinfo2-10.1.1-py3-none-any.whl", hash = "sha256:adc53396bfb206e6498d078ec2ab407f85799ecd819584ac36a8f80a2d4d762d", size = 23791, upload-time = "2026-03-25T21:49:39.574Z" }, +] + [[package]] name = "py-multihash" version = "3.0.0" @@ -3895,6 +3904,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-benchmark" +version = "5.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo2" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/8f/83a15e40dbc34a580ee56eb56983cae5394c6e94d50cf28fe268e457be25/pytest_benchmark-5.3.0.tar.gz", hash = "sha256:358444d4e89be901ee2b6404fb043ac3d7684002ad7f3563cc153fca6339c965", size = 375410, upload-time = "2026-08-23T17:45:08.891Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/42/7e80f7cfa191e0a766d1de99b4661847415ad5db34f8209d81fd42175b59/pytest_benchmark-5.3.0-py3-none-any.whl", hash = "sha256:920ab1dfcffa718d49aa15ba144c7e357bda59216a0dc308016cc1c7236f719d", size = 48401, upload-time = "2026-08-23T17:45:07.094Z" }, +] + [[package]] name = "python-dateutil" version = "3.0.0" @@ -4061,6 +4083,7 @@ dev = [ { name = "pandas-stubs" }, { name = "pre-commit" }, { name = "pytest" }, + { name = "pytest-benchmark" }, { name = "ruff" }, { name = "towncrier" }, { name = "types-pyyaml" }, @@ -4089,6 +4112,7 @@ dev = [ { name = "pandas-stubs", specifier = ">=2.3.2.250827" }, { name = "pre-commit", specifier = ">=4.0" }, { name = "pytest", specifier = ">=9.0" }, + { name = "pytest-benchmark", specifier = ">=5.3.0" }, { name = "ruff", specifier = ">=0.15.0" }, { name = "towncrier", specifier = ">=24.8.0" }, { name = "types-pyyaml", specifier = ">=6.0.12.20250915" }, diff --git a/changelog/145.fix.md b/changelog/145.fix.md new file mode 100644 index 00000000..cba144cc --- /dev/null +++ b/changelog/145.fix.md @@ -0,0 +1 @@ +Fixed a 500 error when fetching a diagnostic whose provider is not installed. diff --git a/changelog/145.trivial.md b/changelog/145.trivial.md new file mode 100644 index 00000000..2b3ac300 --- /dev/null +++ b/changelog/145.trivial.md @@ -0,0 +1 @@ +Added API performance tests against a synthetic database, which CI compares against the base branch on every pull request.