diff --git a/openml/evaluations/__init__.py b/openml/evaluations/__init__.py index b56d0c2d5..29344b03a 100644 --- a/openml/evaluations/__init__.py +++ b/openml/evaluations/__init__.py @@ -1,10 +1,16 @@ # License: BSD 3-Clause from .evaluation import OpenMLEvaluation -from .functions import list_evaluation_measures, list_evaluations, list_evaluations_setups +from .functions import ( + list_estimation_procedures, + list_evaluation_measures, + list_evaluations, + list_evaluations_setups, +) __all__ = [ "OpenMLEvaluation", + "list_estimation_procedures", "list_evaluation_measures", "list_evaluations", "list_evaluations_setups", diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index f4e07c1b8..7516cfa27 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -2,6 +2,7 @@ # ruff: noqa: PLR0913 from __future__ import annotations +import warnings from functools import partial from itertools import chain from typing import TYPE_CHECKING, Literal @@ -15,6 +16,7 @@ import openml.utils if TYPE_CHECKING: + from openml.estimation_procedures import OpenMLEstimationProcedure from openml.evaluations import OpenMLEvaluation @@ -156,18 +158,65 @@ def list_evaluation_measures() -> list[str]: return openml._backend.evaluation_measure.list() -def list_estimation_procedures() -> list[str]: - """Return list of evaluation procedures available. +@overload +def list_estimation_procedures( + output_format: Literal["dataframe"], +) -> pd.DataFrame: ... + + +@overload +def list_estimation_procedures( + output_format: Literal["object"], +) -> dict[int, OpenMLEstimationProcedure]: ... + + +@overload +def list_estimation_procedures( + output_format: None = None, +) -> list[str]: ... + + +def list_estimation_procedures( + output_format: Literal["object", "dataframe"] | None = None, +) -> list[str] | dict[int, OpenMLEstimationProcedure] | pd.DataFrame: + """Return the estimation procedures available on OpenML. The function performs an API call to retrieve the entire list of - evaluation procedures' names that are available. + evaluation procedures. + + Parameters + ---------- + output_format : {"object", "dataframe"}, optional + The format of the returned procedures. ``"object"`` returns a + dictionary mapping procedure IDs to ``OpenMLEstimationProcedure`` + instances. ``"dataframe"`` returns one row per procedure. + If omitted, a list of procedure names is returned for backwards + compatibility and a warning is emitted. Returns ------- - list + list[str], dict[int, OpenMLEstimationProcedure], or pandas.DataFrame + The available estimation procedures in the requested format. """ result = openml._backend.estimation_procedure.list() - return [i.name for i in result] + + if output_format is None: + warnings.warn( + "The default output will change from a list of names to a dictionary of " + "OpenMLEstimationProcedure objects in a future release. Set " + "output_format='object' to use the new format.", + FutureWarning, + stacklevel=2, + ) + return [procedure.name for procedure in result] + + if output_format == "dataframe": + return pd.DataFrame.from_records(procedure._to_dict() for procedure in result) + + if output_format == "object": + return {procedure.id: procedure for procedure in result} + + raise ValueError("Invalid output format. Only 'object' and 'dataframe' are applicable.") def list_evaluations_setups( diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index e15556d7b..a616ea799 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -1,10 +1,14 @@ # License: BSD 3-Clause from __future__ import annotations +from unittest.mock import patch + import pytest import openml import openml.evaluations +from openml.estimation_procedures import OpenMLEstimationProcedure +from openml.tasks import TaskType from openml.testing import TestBase @@ -239,6 +243,55 @@ def test_list_evaluation_measures(self): assert isinstance(measures, list) is True assert all(isinstance(s, str) for s in measures) is True + def test_list_estimation_procedures_default_warns_about_future_change(self): + procedures = [ + OpenMLEstimationProcedure( + id=5, + task_type_id=TaskType.SUPERVISED_CLASSIFICATION, + name="10-fold Crossvalidation", + type="crossvalidation", + ) + ] + with patch.object(openml._backend.estimation_procedure, "list", return_value=procedures): + with pytest.warns(FutureWarning, match="output will change"): + result = openml.evaluations.list_estimation_procedures() + + assert result == ["10-fold Crossvalidation"] + + def test_list_estimation_procedures_object(self): + procedure = OpenMLEstimationProcedure( + id=5, + task_type_id=TaskType.SUPERVISED_CLASSIFICATION, + name="10-fold Crossvalidation", + type="crossvalidation", + ) + with patch.object(openml._backend.estimation_procedure, "list", return_value=[procedure]): + result = openml.evaluations.list_estimation_procedures(output_format="object") + + assert result == {5: procedure} + + def test_list_estimation_procedures_dataframe(self): + procedures = [ + OpenMLEstimationProcedure( + id=5, + task_type_id=TaskType.SUPERVISED_CLASSIFICATION, + name="10-fold Crossvalidation", + type="crossvalidation", + ) + ] + with patch.object(openml._backend.estimation_procedure, "list", return_value=procedures): + result = openml.evaluations.list_estimation_procedures(output_format="dataframe") + + assert list(result.columns) == ["id", "task_type_id", "name", "type"] + assert result.to_dict("records") == [ + { + "id": 5, + "task_type_id": TaskType.SUPERVISED_CLASSIFICATION, + "name": "10-fold Crossvalidation", + "type": "crossvalidation", + } + ] + @pytest.mark.production_server() def test_list_evaluations_setups_filter_flow(self): self.use_production_server()