From bd84d95c1a93e72c19c001e913214c455e13d2d0 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 25 Oct 2026 09:48:41 +1100 Subject: [PATCH 1/8] refactor(api): build the model pages on climate-ref's filters The model list, model detail and ensemble pages now select through select_execution_group_ids, select_execution_groups and select_scalar_values. This drops their hand-built latest execution, promoted version and retraction joins, so the model pages count groups the same way as the diagnostics pages. The era scope comes from era_scope, as everywhere else. --- backend/src/ref_backend/core/mip_eras.py | 8 +- .../src/ref_backend/core/model_ensemble.py | 53 +++++----- backend/src/ref_backend/core/model_runs.py | 98 +++++++++---------- backend/tests/test_core/test_model_runs.py | 1 + 4 files changed, 69 insertions(+), 91 deletions(-) diff --git a/backend/src/ref_backend/core/mip_eras.py b/backend/src/ref_backend/core/mip_eras.py index ae4fa4a5..3d152944 100644 --- a/backend/src/ref_backend/core/mip_eras.py +++ b/backend/src/ref_backend/core/mip_eras.py @@ -3,12 +3,11 @@ from collections.abc import Collection from typing import Any, Literal -from sqlalchemy import ColumnElement, select +from sqlalchemy import select from sqlalchemy.orm import Session from climate_ref import models from climate_ref.models.execution import ExecutionGroupSourceType -from climate_ref.query.filters import ExecutionScope from climate_ref.results import InputDatasetFilter from climate_ref_core.datasets import SourceDatasetType @@ -59,11 +58,6 @@ def eras_in_scope(mip_era: MipEra | None) -> tuple[SourceDatasetType, ...]: return source_types_for(mip_era) or CMIP_ERAS -def era_group_clauses(mip_era: MipEra) -> list[ColumnElement[bool]]: - """Clauses over `ExecutionGroup` keeping the groups that draw on `mip_era`, or on no model data.""" - return ExecutionScope(**era_scope(mip_era)).group_clauses() - - def source_id_datasets(source_id: str | None) -> InputDatasetFilter | None: """Match executions that ran against a model, which selectors do not always record.""" return InputDatasetFilter(facets={"source_id": source_id}) if source_id else None diff --git a/backend/src/ref_backend/core/model_ensemble.py b/backend/src/ref_backend/core/model_ensemble.py index 98b91333..5d9cc071 100644 --- a/backend/src/ref_backend/core/model_ensemble.py +++ b/backend/src/ref_backend/core/model_ensemble.py @@ -5,12 +5,13 @@ from statistics import fmean, pstdev, quantiles from typing import Any, NamedTuple -from sqlalchemy import select +from sqlalchemy import Select, select from sqlalchemy.orm import Session from climate_ref import models -from climate_ref.query import select_latest_executions -from ref_backend.core.mip_eras import MipEra, cv_column, era_group_clauses +from climate_ref.query import select_scalar_values +from climate_ref.results import MetricValueFilter +from ref_backend.core.mip_eras import MipEra, cv_column, era_scope from ref_backend.models.model_runs import EnsembleComparison, EnsembleStatistics #: Dimensions that say which run produced a value rather than what the value measures. @@ -81,6 +82,13 @@ class ScalarRow(NamedTuple): dimensions: tuple[str | None, ...] +def _model_values(mip_era: MipEra | None, **kwargs: Any) -> Select[Any]: + """Select each group's latest model values, in the era's groups and those with no model input.""" + return select_scalar_values(MetricValueFilter(latest_only=True, **era_scope(mip_era), **kwargs)).order_by( + None + ) + + def _scalar_rows( session: Session, *, @@ -90,32 +98,22 @@ def _scalar_rows( ) -> list[ScalarRow]: value = models.ScalarMetricValue columns = [cv_column(value, name) for name in dimensions] - # A retried group holds a value per attempt, so only the unretracted execution that decided it counts. - latest = select_latest_executions().subquery() - statement = ( - select( + _model_values(mip_era, dimensions={"kind": "model"}) + .with_only_columns( models.ExecutionGroup.diagnostic_id, cv_column(value, "source_id"), value.value, value.attributes, *columns, + maintain_column_froms=True, ) - .join(models.Execution, value.execution_id == models.Execution.id) - .join(models.ExecutionGroup, models.Execution.execution_group_id == models.ExecutionGroup.id) - .join(models.Diagnostic, models.ExecutionGroup.diagnostic_id == models.Diagnostic.id) - .join(latest, latest.c.execution_id == models.Execution.id) .where( - models.ExecutionGroup.diagnostic_version == models.Diagnostic.promoted_version, models.ExecutionGroup.diagnostic_id.in_(diagnostic_ids), - models.Execution.retracted.is_(False), - cv_column(value, "kind") == "model", cv_column(value, "source_id").isnot(None), value.value.isnot(None), ) ) - if mip_era: - statement = statement.where(*era_group_clauses(mip_era)) return [ ScalarRow(row[0], row[1], row[2], _units(row[3]), tuple(row[4:])) @@ -123,18 +121,15 @@ def _scalar_rows( ] -def _diagnostics_for_model(session: Session, *, source_id: str) -> list[int]: +def _diagnostics_for_model( + session: Session, *, source_id: str, mip_era: MipEra | None, diagnostic_slug: str | None +) -> list[int]: """Find the diagnostics that recorded a scalar value for this model.""" statement = ( - select(models.ExecutionGroup.diagnostic_id) - .join(models.Execution, models.Execution.execution_group_id == models.ExecutionGroup.id) - .join(models.ScalarMetricValue, models.ScalarMetricValue.execution_id == models.Execution.id) - .join(models.Diagnostic, models.ExecutionGroup.diagnostic_id == models.Diagnostic.id) - .where( - models.ExecutionGroup.diagnostic_version == models.Diagnostic.promoted_version, - cv_column(models.ScalarMetricValue, "source_id") == source_id, - cv_column(models.ScalarMetricValue, "kind") == "model", + _model_values( + mip_era, diagnostic_slug=diagnostic_slug, dimensions={"kind": "model", "source_id": source_id} ) + .with_only_columns(models.ExecutionGroup.diagnostic_id, maintain_column_froms=True) .distinct() ) return list(session.scalars(statement)) @@ -156,17 +151,15 @@ def ensemble_comparisons( """ dimensions = grouping_dimensions() - model_diagnostics = _diagnostics_for_model(session, source_id=source_id) + model_diagnostics = _diagnostics_for_model( + session, source_id=source_id, mip_era=mip_era, diagnostic_slug=diagnostic_slug + ) diagnostics = { diagnostic.id: diagnostic for diagnostic in session.scalars( select(models.Diagnostic).where(models.Diagnostic.id.in_(model_diagnostics)) ) } - if diagnostic_slug: - diagnostics = { - key: diagnostic for key, diagnostic in diagnostics.items() if diagnostic.slug == diagnostic_slug - } if not diagnostics: return [] diff --git a/backend/src/ref_backend/core/model_runs.py b/backend/src/ref_backend/core/model_runs.py index 3e5bb65f..1ca44a09 100644 --- a/backend/src/ref_backend/core/model_runs.py +++ b/backend/src/ref_backend/core/model_runs.py @@ -5,11 +5,20 @@ from typing import Any, NamedTuple from sqlalchemy import CompoundSelect, Select, case, distinct, func, select, union -from sqlalchemy.orm import Session, aliased +from sqlalchemy.orm import Session from climate_ref import models -from climate_ref.query import select_latest_executions -from ref_backend.core.mip_eras import MipEra, cv_column, dataset_model_for, eras_in_scope, mip_era_for +from climate_ref.query import select_execution_group_ids, select_execution_groups +from climate_ref.results import ExecutionGroupFilter +from ref_backend.core.mip_eras import ( + MipEra, + cv_column, + dataset_model_for, + era_scope, + eras_in_scope, + mip_era_for, + source_id_datasets, +) from ref_backend.models.model_runs import RunCounts @@ -30,9 +39,21 @@ class ModelFacets(NamedTuple): dataset_count: int -def _model_groups(mip_era: MipEra | None, source_id: str | None) -> CompoundSelect[Any] | Select[Any] | None: +def _model_filter(mip_era: MipEra | None, **kwargs: Any) -> ExecutionGroupFilter: + return ExecutionGroupFilter(**era_scope(mip_era), **kwargs) + + +def _outcome(execution: Any) -> Any: + return case( + (execution.successful.is_(True), "successful"), + (execution.successful.is_(False), "failed"), + else_="running", + ) + + +def _model_groups(mip_era: MipEra | None, source_id: str | None) -> CompoundSelect[Any] | Select[Any]: """ - Select every (execution group, model) pairing at the promoted version of each diagnostic. + Select every (execution group, model) pairing the group filter keeps, with the group's outcome. A group that ran several models appears once per model, so the counts answer "what did this model take part in" rather than partitioning the groups. @@ -44,65 +65,42 @@ def _model_groups(mip_era: MipEra | None, source_id: str | None) -> CompoundSele Only `source_id` is carried alongside, so a group naming one model across two eras or two institutions still yields one row and is counted once. """ - latest = select_latest_executions().subquery() + groups = select_execution_group_ids(_model_filter(mip_era)) statements = [] for era in eras_in_scope(mip_era): dataset_model = dataset_model_for(era) model_source_id = cv_column(dataset_model, "source_id") statement = ( - select( + groups.with_only_columns( models.ExecutionGroup.id.label("group_id"), models.ExecutionGroup.diagnostic_id.label("diagnostic_id"), model_source_id.label("source_id"), models.Execution.id.label("execution_id"), + _outcome(models.Execution).label("outcome"), ) - .join(models.Diagnostic, models.ExecutionGroup.diagnostic_id == models.Diagnostic.id) - .join(latest, latest.c.execution_group_id == models.ExecutionGroup.id) - .join(models.Execution, models.Execution.id == latest.c.execution_id) .join(models.Execution.datasets.of_type(dataset_model)) - .where( - models.ExecutionGroup.diagnostic_version == models.Diagnostic.promoted_version, - models.Execution.retracted.is_(False), - model_source_id.isnot(None), - ) + .where(model_source_id.isnot(None)) .distinct() ) if source_id: statement = statement.where(model_source_id == source_id) statements.append(statement) - return union(*statements) if statements else None - - -def _outcome(execution: Any) -> Any: - return case( - (execution.successful.is_(True), "successful"), - (execution.successful.is_(False), "failed"), - else_="running", - ) + return union(*statements) if len(statements) > 1 else statements[0] def model_run_rows( session: Session, *, mip_era: MipEra | None = None, source_id: str | None = None ) -> list[ModelRunRow]: """Tally execution groups per model, diagnostic and outcome.""" - groups = _model_groups(mip_era, source_id) - if groups is None: - return [] - - subquery = groups.subquery("model_groups") - execution = aliased(models.Execution) - outcome = _outcome(execution) - + subquery = _model_groups(mip_era, source_id).subquery("model_groups") rows = session.execute( select( subquery.c.source_id, subquery.c.diagnostic_id, - outcome.label("outcome"), + subquery.c.outcome, func.count(distinct(subquery.c.group_id)).label("group_count"), - ) - .join(execution, execution.id == subquery.c.execution_id) - .group_by(subquery.c.source_id, subquery.c.diagnostic_id, outcome) + ).group_by(subquery.c.source_id, subquery.c.diagnostic_id, subquery.c.outcome) ).all() return [ModelRunRow(*row) for row in rows] @@ -167,22 +165,14 @@ def failed_runs( Returns the group, the id of the execution that decided the outcome, and the outcome itself. """ - groups = _model_groups(mip_era, source_id) - if groups is None: - return [] - - subquery = groups.subquery("model_groups") - execution = aliased(models.Execution) - outcome = _outcome(execution) - - rows = session.execute( - select(models.ExecutionGroup, execution.id, outcome.label("outcome")) - .select_from(subquery) - .join(models.ExecutionGroup, models.ExecutionGroup.id == subquery.c.group_id) - .join(execution, execution.id == subquery.c.execution_id) - .where(outcome != "successful") - .distinct() - .order_by(models.ExecutionGroup.updated_at.desc()) - ).all() + filters = _model_filter(mip_era, successful=False, input_datasets=source_id_datasets(source_id)) + statement: Select[Any] = ( + select_execution_groups(filters).order_by(None).order_by(models.ExecutionGroup.updated_at.desc()) + ) + rows = session.execute(statement).tuples().all() - return [(row[0], row[1], row[2]) for row in rows] + return [ + (group, execution.id, "failed" if execution.successful is False else "running") + for group, execution in rows + if execution + ] diff --git a/backend/tests/test_core/test_model_runs.py b/backend/tests/test_core/test_model_runs.py index b0c5e56c..22128315 100644 --- a/backend/tests/test_core/test_model_runs.py +++ b/backend/tests/test_core/test_model_runs.py @@ -118,6 +118,7 @@ def test_model_groups_carries_only_the_keys_it_counts_on(): "diagnostic_id", "source_id", "execution_id", + "outcome", ] From 762d893acec72f77904c9bd60482257bb1c1219c Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 25 Oct 2026 09:48:43 +1100 Subject: [PATCH 2/8] test(perf): time the ensemble page and record the model pages' scans --- backend/tests/perf/test_endpoints.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/tests/perf/test_endpoints.py b/backend/tests/perf/test_endpoints.py index 9cc2397c..56d82154 100644 --- a/backend/tests/perf/test_endpoints.py +++ b/backend/tests/perf/test_endpoints.py @@ -91,9 +91,16 @@ class Endpoint: Endpoint( id="models.list", path="models/", - allowed_scans=frozenset({"execution", "cmip6_dataset", "cmip7_dataset"}), + allowed_scans=frozenset({"execution", "execution_group", "cmip6_dataset", "cmip7_dataset"}), + ), + Endpoint( + id="models.get", + path="models/{source_id}", + allowed_scans=frozenset({"execution", "execution_group", "obs4mips_dataset"}), + ), + Endpoint( + id="models.ensemble", path="models/{source_id}/ensemble", allowed_scans=frozenset({"execution"}) ), - Endpoint(id="models.get", path="models/{source_id}", allowed_scans=frozenset({"execution"})), Endpoint(id="datasets.list", path="datasets/"), Endpoint(id="datasets.get", path="datasets/{dataset_slug}"), ] From 29ad3cc4e07f07f20aa950589051881bdd2afce1 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 25 Oct 2026 09:49:13 +1100 Subject: [PATCH 3/8] docs: add changelog for #156 --- changelog/156.improvement.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/156.improvement.md diff --git a/changelog/156.improvement.md b/changelog/156.improvement.md new file mode 100644 index 00000000..4e7e81e9 --- /dev/null +++ b/changelog/156.improvement.md @@ -0,0 +1 @@ +The `/models` endpoints now count execution groups the same way as the diagnostics endpoints. In the ensemble comparison a value with no `kind` now counts as a model value. From 6eb69bd646238e69ca65e3a66645a750d737f178 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 25 Oct 2026 09:49:32 +1100 Subject: [PATCH 4/8] refactor: name the ensemble's value filter --- backend/src/ref_backend/core/model_ensemble.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/src/ref_backend/core/model_ensemble.py b/backend/src/ref_backend/core/model_ensemble.py index 5d9cc071..23278c2c 100644 --- a/backend/src/ref_backend/core/model_ensemble.py +++ b/backend/src/ref_backend/core/model_ensemble.py @@ -84,9 +84,8 @@ class ScalarRow(NamedTuple): def _model_values(mip_era: MipEra | None, **kwargs: Any) -> Select[Any]: """Select each group's latest model values, in the era's groups and those with no model input.""" - return select_scalar_values(MetricValueFilter(latest_only=True, **era_scope(mip_era), **kwargs)).order_by( - None - ) + filters = MetricValueFilter(latest_only=True, **era_scope(mip_era), **kwargs) + return select_scalar_values(filters).order_by(None) def _scalar_rows( From 4fa5ca4b99968a2e13c062fc77a517432fe4fbd1 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 25 Oct 2026 10:16:21 +1100 Subject: [PATCH 5/8] perf: tally the model groups in one pass over the groups The group filter looks up each group's latest execution per group, so a select per era did that twice and made /models 54% slower at full scale. Joining once to the model datasets of every era runs in half the time main took. --- backend/src/ref_backend/core/model_runs.py | 55 +++++++++++++--------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/backend/src/ref_backend/core/model_runs.py b/backend/src/ref_backend/core/model_runs.py index 1ca44a09..e7509204 100644 --- a/backend/src/ref_backend/core/model_runs.py +++ b/backend/src/ref_backend/core/model_runs.py @@ -4,10 +4,11 @@ from collections.abc import Sequence from typing import Any, NamedTuple -from sqlalchemy import CompoundSelect, Select, case, distinct, func, select, union +from sqlalchemy import Select, Subquery, case, distinct, func, select, union from sqlalchemy.orm import Session from climate_ref import models +from climate_ref.models.execution import execution_datasets from climate_ref.query import select_execution_group_ids, select_execution_groups from climate_ref.results import ExecutionGroupFilter from ref_backend.core.mip_eras import ( @@ -51,7 +52,22 @@ def _outcome(execution: Any) -> Any: ) -def _model_groups(mip_era: MipEra | None, source_id: str | None) -> CompoundSelect[Any] | Select[Any]: +def _model_datasets(mip_era: MipEra | None, source_id: str | None) -> Subquery: + """Select the id and `source_id` of every model dataset in scope, across the eras' dataset tables.""" + statements = [] + for era in eras_in_scope(mip_era): + dataset_model = dataset_model_for(era) + model_source_id = cv_column(dataset_model, "source_id") + statement = select(dataset_model.id.label("dataset_id"), model_source_id.label("source_id")).where( + model_source_id.isnot(None) + ) + if source_id: + statement = statement.where(model_source_id == source_id) + statements.append(statement) + return (union(*statements) if len(statements) > 1 else statements[0]).subquery("model_datasets") + + +def _model_groups(mip_era: MipEra | None, source_id: str | None) -> Select[Any]: """ Select every (execution group, model) pairing the group filter keeps, with the group's outcome. @@ -65,28 +81,21 @@ def _model_groups(mip_era: MipEra | None, source_id: str | None) -> CompoundSele Only `source_id` is carried alongside, so a group naming one model across two eras or two institutions still yields one row and is counted once. """ - groups = select_execution_group_ids(_model_filter(mip_era)) - statements = [] - for era in eras_in_scope(mip_era): - dataset_model = dataset_model_for(era) - model_source_id = cv_column(dataset_model, "source_id") - statement = ( - groups.with_only_columns( - models.ExecutionGroup.id.label("group_id"), - models.ExecutionGroup.diagnostic_id.label("diagnostic_id"), - model_source_id.label("source_id"), - models.Execution.id.label("execution_id"), - _outcome(models.Execution).label("outcome"), - ) - .join(models.Execution.datasets.of_type(dataset_model)) - .where(model_source_id.isnot(None)) - .distinct() + # Joined once, because a select per era would look up every group's latest execution again. + datasets = _model_datasets(mip_era, source_id) + return ( + select_execution_group_ids(_model_filter(mip_era)) + .with_only_columns( + models.ExecutionGroup.id.label("group_id"), + models.ExecutionGroup.diagnostic_id.label("diagnostic_id"), + datasets.c.source_id, + models.Execution.id.label("execution_id"), + _outcome(models.Execution).label("outcome"), ) - if source_id: - statement = statement.where(model_source_id == source_id) - statements.append(statement) - - return union(*statements) if len(statements) > 1 else statements[0] + .join(execution_datasets, execution_datasets.c.execution_id == models.Execution.id) + .join(datasets, datasets.c.dataset_id == execution_datasets.c.dataset_id) + .distinct() + ) def model_run_rows( From 606b945f5a89777eed50dd47b3267422c9cf06b9 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 25 Oct 2026 10:18:17 +1100 Subject: [PATCH 6/8] chore(deps): bump climate-ref to 0.20.1 for the synthetic group source types --- backend/uv.lock | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/backend/uv.lock b/backend/uv.lock index 151d185b..82101e4d 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -649,7 +649,7 @@ wheels = [ [[package]] name = "climate-ref" -version = "0.20.0" +version = "0.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alembic" }, @@ -670,9 +670,9 @@ dependencies = [ { name = "typer" }, { name = "xarray" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e6/6a/8dea166923be6ed91aa1462f4909fcc20ed9e1c752aa227736350412933a/climate_ref-0.20.0.tar.gz", hash = "sha256:cc699f8dd646f81f0a6fec0a54b5473ff69d706d0d39e6f0c54ec1117a087e3a", size = 649749, upload-time = "2026-10-23T23:50:18.03Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/8b/f712029080403de032d453c5c9809e91199062f2eb878d87a8c5a8ef0740/climate_ref-0.20.1.tar.gz", hash = "sha256:e4021c0e3099140cb8dd5c52f651c3b9792d89e501b8a873d0dd722f7c6d777a", size = 650858, upload-time = "2026-10-24T23:15:14.903Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/34/e80f37ba0cb7e83e0cffd1545a6a57fb0d27a4202a10a0699d0dd1276cdc/climate_ref-0.20.0-py3-none-any.whl", hash = "sha256:b0baaa05e1193c10d6fd3e387f32ec0ce8ba894c40b23ed841714af080738101", size = 405969, upload-time = "2026-10-23T23:50:10.553Z" }, + { url = "https://files.pythonhosted.org/packages/65/9f/8711d52949230a086ad7524eb887b1ebf1e41bc783d7f84cf49ea032c597/climate_ref-0.20.1-py3-none-any.whl", hash = "sha256:bedb8f93ea485c14090f2409627442cdf8f0351c159b3882f50ea8a8fddfc491", size = 406337, upload-time = "2026-10-24T23:15:07.7Z" }, ] [package.optional-dependencies] @@ -688,7 +688,7 @@ postgres = [ [[package]] name = "climate-ref-core" -version = "0.20.0" +version = "0.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -710,14 +710,14 @@ dependencies = [ { name = "setuptools" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/10/9b/29b1f011faf4ecb2b004d97b8e133470168e1ef58f5bc10b1b43c4d5db57/climate_ref_core-0.20.0.tar.gz", hash = "sha256:3b95ee4aba60aa484685155bb2f2acf24744d355bf4dcd100d88e984e0d934ac", size = 242866, upload-time = "2026-10-23T23:50:19.651Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/d5/0b426f0d92c313e24813a1bf482a8ee3b5ec0d066e727817e9a14acacf39/climate_ref_core-0.20.1.tar.gz", hash = "sha256:b90ae0e26b81773db8428665a0a18a385b5d0a7d42fa4d26aba6178ee7974985", size = 242871, upload-time = "2026-10-24T23:15:16.655Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/09/5b/5c4b584584fd9552df832b9642a1983aac0dc5224aa49bca247de2ca0183/climate_ref_core-0.20.0-py3-none-any.whl", hash = "sha256:f8f04abfbaa5f9c3d468db540cb309e947e6a2341b432d8beea7ce0d4c4b7293", size = 161500, upload-time = "2026-10-23T23:50:12.858Z" }, + { url = "https://files.pythonhosted.org/packages/e7/1e/2235fbacf0e15c8c3763edb015c6dbfb241121a9336513dd5d92e94e61f4/climate_ref_core-0.20.1-py3-none-any.whl", hash = "sha256:0ea66f30af0f6947447fd47f334344691759b451a25a8b920354521f6a0cd132", size = 161500, upload-time = "2026-10-24T23:15:10.078Z" }, ] [[package]] name = "climate-ref-esmvaltool" -version = "0.20.0" +version = "0.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cftime" }, @@ -727,34 +727,34 @@ dependencies = [ { name = "pyyaml" }, { name = "xarray" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/05/95/17148dffdee6b125c1707707c8f0b6ce031d7a8f7cad7e7dae094ea4bc77/climate_ref_esmvaltool-0.20.0.tar.gz", hash = "sha256:cb3083c6907aec0ef17fcc9a9f7c69492b2e16d47322dd55fd1cbbca218dcd02", size = 544315, upload-time = "2026-10-23T23:50:20.542Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/10/829d752d6b173a117f21afbeac57c618ac6c4bc85b0922f6d6700b7c6dc7/climate_ref_esmvaltool-0.20.1.tar.gz", hash = "sha256:811809e7ad929f4521332773cbbe6018a35eba3c6f7f9fea716ffe776cd5cd5e", size = 544306, upload-time = "2026-10-24T23:15:17.475Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/6f/58ff5743107843200c263abf6096fa861dfc52c35df871be73c5250e5045/climate_ref_esmvaltool-0.20.0-py3-none-any.whl", hash = "sha256:338b82b3ae1b6f64359d2b89cc41c4c0a90441bcc8280e64a53057cfc0ac7c54", size = 162536, upload-time = "2026-10-23T23:50:14.066Z" }, + { url = "https://files.pythonhosted.org/packages/70/29/b2d3b4406202b65553c16d78faac4b28a9fc4438d2af69214023e2ae59c3/climate_ref_esmvaltool-0.20.1-py3-none-any.whl", hash = "sha256:aea02c1c45f6d004571844a84c36964dcd21124688746f5ec5194607bd0b4c65", size = 162537, upload-time = "2026-10-24T23:15:11.145Z" }, ] [[package]] name = "climate-ref-ilamb" -version = "0.20.0" +version = "0.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "climate-ref-core" }, { name = "ilamb3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/90/5c/728443255e80076bb74478ea5f70628f47d4cb570384c973728103b95fec/climate_ref_ilamb-0.20.0.tar.gz", hash = "sha256:d6d215b152b922bb605df0d58f0da477874c35e35e35134b07d44b07cc921f75", size = 259039, upload-time = "2026-10-23T23:50:22.246Z" } +sdist = { url = "https://files.pythonhosted.org/packages/15/f4/bdb4209812c3bb050d8e468a646467f3c48aa6dfa392b1901a8fd4f153f6/climate_ref_ilamb-0.20.1.tar.gz", hash = "sha256:465144863f053d99755b9db07927c22b20e7ef5338cd0e06d78a9a42a6bce92e", size = 259052, upload-time = "2026-10-24T23:15:19.242Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/02/9962f7db782ec453d3fbbbc2e50a98596714299c0fcbd87ccd6f6ba81e23/climate_ref_ilamb-0.20.0-py3-none-any.whl", hash = "sha256:1d48ff80aa6bc866d8720a6e21148cb26a642562a0bd6729998c0e6b95ff9c03", size = 26900, upload-time = "2026-10-23T23:50:15.994Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/59e25f2bb2da547ed1f7dae94500eb0d93b4489940b769e426bd3b532bcf/climate_ref_ilamb-0.20.1-py3-none-any.whl", hash = "sha256:10afc1a2f030f3afe66ed4ab434e5e7bf314e62912650e1ed472eb1b822fd7df", size = 26901, upload-time = "2026-10-24T23:15:13.025Z" }, ] [[package]] name = "climate-ref-pmp" -version = "0.20.0" +version = "0.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "climate-ref-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/12/65/8b7fb55e0aa9a1189bb494e511f6dc6393b5069baa57e79123f170e9beaf/climate_ref_pmp-0.20.0.tar.gz", hash = "sha256:ed4423836cfa8a4a4248188265d9e2045dc17ff75e7502d5f57becb876a7416a", size = 1344937, upload-time = "2026-10-23T23:50:23.256Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/ae/28f6544b6458cef783d241eaeec7235c8293e2675f8abc9ebf7f8949b707/climate_ref_pmp-0.20.1.tar.gz", hash = "sha256:866cdcbba03de399725bf19b3ccda8c3974948716dc00ae761e32c9434806a2b", size = 1344933, upload-time = "2026-10-24T23:15:20.015Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/9f/fa3b325c09c36a8e8d70178828eb84fa6bc6549ce124f69b2016260601ce/climate_ref_pmp-0.20.0-py3-none-any.whl", hash = "sha256:fa00805bc2aa93bc373718fc0baa7751fc45987d7588b54e72cfcb61b6427066", size = 116110, upload-time = "2026-10-23T23:50:16.898Z" }, + { url = "https://files.pythonhosted.org/packages/0f/8e/e688cb8d506d9f7912ee5a23cdc8b79215446a5fd0e10480ceaf65cfd503/climate_ref_pmp-0.20.1-py3-none-any.whl", hash = "sha256:139668940f735a76a13e7f299450ce11f705f2d337035399bc9fae20d9918852", size = 116108, upload-time = "2026-10-24T23:15:13.885Z" }, ] [[package]] From 748552cef0524437c5c507ce32da81b5d09e5b39 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 25 Oct 2026 10:18:17 +1100 Subject: [PATCH 7/8] test(perf): time the CMIP6 model pages --- backend/tests/perf/test_endpoints.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/backend/tests/perf/test_endpoints.py b/backend/tests/perf/test_endpoints.py index 56d82154..1c3412ab 100644 --- a/backend/tests/perf/test_endpoints.py +++ b/backend/tests/perf/test_endpoints.py @@ -101,6 +101,24 @@ class Endpoint: Endpoint( id="models.ensemble", path="models/{source_id}/ensemble", allowed_scans=frozenset({"execution"}) ), + Endpoint( + id="models.list.cmip6", + path="models/", + params={"mip_era": "CMIP6"}, + allowed_scans=frozenset({"execution_group", "cmip6_dataset"}), + ), + Endpoint( + id="models.get.cmip6", + path="models/{source_id}", + params={"mip_era": "CMIP6"}, + allowed_scans=frozenset({"execution_group"}), + ), + Endpoint( + id="models.ensemble.cmip6", + path="models/{source_id}/ensemble", + params={"mip_era": "CMIP6"}, + allowed_scans=frozenset({"execution"}), + ), Endpoint(id="datasets.list", path="datasets/"), Endpoint(id="datasets.get", path="datasets/{dataset_slug}"), ] From 3b5110fcf34aa82a7cb9ca7775d32499ee9d294b Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 25 Oct 2026 10:44:27 +1100 Subject: [PATCH 8/8] docs: note the models list speed-up in the changelog --- changelog/156.improvement.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/156.improvement.md b/changelog/156.improvement.md index 4e7e81e9..82cfe351 100644 --- a/changelog/156.improvement.md +++ b/changelog/156.improvement.md @@ -1 +1 @@ -The `/models` endpoints now count execution groups the same way as the diagnostics endpoints. In the ensemble comparison a value with no `kind` now counts as a model value. +The `/models` endpoints now count execution groups the same way as the diagnostics endpoints, and the models list is about twice as fast on a large database. In the ensemble comparison a value with no `kind` now counts as a model value.