diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06f1accda..14fce4642 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,12 +54,13 @@ pixi run -e test pytest -m postgresql # PostgreSQL tests only export DOCKER_HOST=unix://$HOME/.docker/run/docker.sock ``` -### PostgreSQL Backend +### Database Backends -DataJoint supports MySQL 8.0.13+ and PostgreSQL 15+ as production database backends. To install the PostgreSQL driver: +DataJoint supports PostgreSQL 15+ and MySQL 8.0.13+ as production database backends. Each database driver is an optional extra, and the `test` and `dev` extras install both: ```bash pip install -e ".[postgres]" # Installs psycopg2-binary +pip install -e ".[mysql]" # Installs pymysql ``` Tests automatically spin up both MySQL and PostgreSQL containers via testcontainers. Backend-parameterized tests (those using the `backend` fixture in `tests/conftest.py`) run against both backends to ensure feature parity. diff --git a/Dockerfile b/Dockerfile index 88c046345..28c94f18a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,6 @@ USER root RUN \ chown -R ${HOST_UID:-1000}:mambauser /main && \ chown -R ${HOST_UID:-1000}:mambauser /src && \ - ${CONDA_BIN} run -n base pip install -q --no-cache-dir /main && \ + ${CONDA_BIN} run -n base pip install -q --no-cache-dir "/main[postgres,mysql]" && \ rm -r /main/* USER ${MAMBA_USER} diff --git a/README.md b/README.md index 7b68d185b..099864cc3 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,11 @@ The **DataJoint Library** is a framework for scientific data pipelines based on ## Installation +Install DataJoint with the driver for your database: + ```bash -pip install datajoint +pip install "datajoint[postgres]" # PostgreSQL +pip install "datajoint[mysql]" # MySQL ``` or with Conda: diff --git a/RELEASE_MEMO.md b/RELEASE_MEMO.md index 1a3623330..951efde55 100644 --- a/RELEASE_MEMO.md +++ b/RELEASE_MEMO.md @@ -143,7 +143,9 @@ If the bot doesn't create a PR, manually update the feedstock: curl -sL https://pypi.org/pypi/datajoint/2.1.0/json | jq -r '.urls[] | select(.packagetype=="sdist") | .digests.sha256' ``` -4. **Check dependencies** match `pyproject.toml`: +4. **Check dependencies** match `pyproject.toml`. conda has no extras, so `run` + also lists the database drivers that `pyproject.toml` keeps in the `mysql` and + `postgres` extras: ```yaml requirements: host: @@ -154,6 +156,7 @@ If the bot doesn't create a PR, manually update the feedstock: - python >={{ python_min }} - numpy - pymysql >=0.7.2 + - psycopg2 >=2.9.0 - deepdiff - pyparsing - pandas diff --git a/pixi.lock b/pixi.lock index e287ed2ae..3984f1eb7 100644 --- a/pixi.lock +++ b/pixi.lock @@ -111,7 +111,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/b8/fbab973592e23ae313042d450fc26fa24282ebffba21ba373786e1ce63b4/pyparsing-3.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl @@ -236,7 +235,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl @@ -316,7 +314,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl @@ -4495,8 +4492,8 @@ packages: requires_python: '>=3.8' - pypi: ./ name: datajoint - version: 2.3.5.dev28+ge3d2a553e.d20261024 - sha256: 9ac2b52fcb44ab2d1c54ec75a4c390fc2a8d53b0480a2490d897c2dc22f593a3 + version: 2.3.5.dev44+g215175ada + sha256: 93aa78078608348c30e7d6a88ad78f218220990a716b35a4a52f62fb5d7955d6 requires_dist: - deepdiff - deepdiff<9 ; python_full_version >= '3.15' @@ -4507,7 +4504,6 @@ packages: - pandas - pydantic-settings>=2.0.0 - pydot - - pymysql>=0.7.2 - pyparsing - tqdm - pyarrow>=14.0.0 ; extra == 'arrow' @@ -4515,11 +4511,14 @@ packages: - codespell ; extra == 'dev' - polars>=0.20.0 ; extra == 'dev' - pre-commit ; extra == 'dev' + - psycopg2-binary>=2.9.0 ; extra == 'dev' - pyarrow>=14.0.0 ; extra == 'dev' + - pymysql>=0.7.2 ; extra == 'dev' - pytest ; extra == 'dev' - pytest-cov ; extra == 'dev' - ruff ; extra == 'dev' - gcsfs>=2023.1.0 ; extra == 'gcs' + - pymysql>=0.7.2 ; extra == 'mysql' - polars>=0.20.0 ; extra == 'polars' - psycopg2-binary>=2.9.0 ; extra == 'postgres' - s3fs>=2023.1.0 ; extra == 's3' @@ -4529,6 +4528,7 @@ packages: - polars>=0.20.0 ; extra == 'test' - psycopg2-binary>=2.9.0 ; extra == 'test' - pyarrow>=14.0.0 ; extra == 'test' + - pymysql>=0.7.2 ; extra == 'test' - pytest ; extra == 'test' - pytest-cov ; extra == 'test' - requests ; extra == 'test' diff --git a/pyproject.toml b/pyproject.toml index 47c1019e7..dae3c45ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,6 @@ name = "datajoint" dynamic = ["version"] dependencies = [ "numpy", - "pymysql>=0.7.2", "deepdiff", # deepdiff 9 requires cachebox<6, which ships no Python 3.15 wheels and cannot # be built for 3.15 (PyO3 0.27). Drop once deepdiff accepts cachebox 6. @@ -97,6 +96,8 @@ test = [ "ipython", "graphviz", "testcontainers[mysql,postgres]>=4.0", + "pymysql>=0.7.2", + "psycopg2-binary>=2.9.0", "polars>=0.20.0", "pyarrow>=14.0.0", ] @@ -105,7 +106,9 @@ test = [ s3 = ["s3fs>=2023.1.0"] gcs = ["gcsfs>=2023.1.0"] azure = ["adlfs>=2023.1.0"] +# Database drivers: install the extra for each backend in use. postgres = ["psycopg2-binary>=2.9.0"] +mysql = ["pymysql>=0.7.2"] polars = ["polars>=0.20.0"] arrow = ["pyarrow>=14.0.0"] viz = ["matplotlib", "ipython"] @@ -118,6 +121,7 @@ test = [ "ipython", "s3fs>=2023.1.0", "testcontainers[mysql,postgres]>=4.0", + "pymysql>=0.7.2", "psycopg2-binary>=2.9.0", "polars>=0.20.0", "pyarrow>=14.0.0", @@ -129,6 +133,8 @@ dev = [ # including test "pytest", "pytest-cov", + "pymysql>=0.7.2", + "psycopg2-binary>=2.9.0", "polars>=0.20.0", "pyarrow>=14.0.0", ] diff --git a/src/datajoint/__init__.py b/src/datajoint/__init__.py index 69c6df269..e6b0c6f66 100644 --- a/src/datajoint/__init__.py +++ b/src/datajoint/__init__.py @@ -278,7 +278,7 @@ def FreeTable(conn_or_name, full_table_name: str | None = None) -> _FreeTable: # ============================================================================= # Lazy imports — heavy dependencies loaded on first access # ============================================================================= -# These modules import heavy dependencies (networkx, matplotlib, click, pymysql) +# These modules import heavy dependencies (networkx, matplotlib, click) # that slow down `import datajoint`. They are loaded on demand. _lazy_modules = { diff --git a/src/datajoint/adapters/base.py b/src/datajoint/adapters/base.py index 1aa12f52f..fbe56cddf 100644 --- a/src/datajoint/adapters/base.py +++ b/src/datajoint/adapters/base.py @@ -7,9 +7,48 @@ from __future__ import annotations +import importlib from abc import ABC, abstractmethod +from types import ModuleType from typing import Any +from ..errors import DataJointError + + +def import_driver(module: str, backend: str, extra: str) -> ModuleType: + """ + Import a database driver, or name the extra that installs it. + + Database drivers are optional extras, so adapters import them when they are + created rather than when ``datajoint`` is imported. + + Parameters + ---------- + module : str + Import name of the driver, e.g. ``'pymysql'`` or ``'psycopg2'``. + backend : str + Backend name for the error message, e.g. ``'MySQL'``. + extra : str + The ``datajoint`` extra that installs the driver, e.g. ``'mysql'``. + + Returns + ------- + ModuleType + The imported driver module. + + Raises + ------ + DataJointError + If the driver is not installed. + """ + try: + return importlib.import_module(module) + except ImportError as error: + raise DataJointError( + f"The {backend} backend requires the {module} driver, which is not installed. " + f'Install it with: pip install "datajoint[{extra}]"' + ) from error + class DatabaseAdapter(ABC): """ diff --git a/src/datajoint/adapters/mysql.py b/src/datajoint/adapters/mysql.py index 38e3ad1b1..3b8738830 100644 --- a/src/datajoint/adapters/mysql.py +++ b/src/datajoint/adapters/mysql.py @@ -9,10 +9,12 @@ from typing import Any -import pymysql as client - from .. import errors -from .base import DatabaseAdapter +from .base import DatabaseAdapter, import_driver + +# The pymysql driver (the ``datajoint[mysql]`` extra), imported when the first +# MySQLAdapter is created so that ``import datajoint`` loads no database driver. +client: Any = None # Core type mapping: DataJoint core types → MySQL types CORE_TYPE_MAP = { @@ -48,6 +50,18 @@ class MySQLAdapter(DatabaseAdapter): """MySQL database adapter implementation.""" + def __init__(self) -> None: + """ + Initialize MySQL adapter. + + Raises + ------ + DataJointError + If pymysql is not installed (``pip install "datajoint[mysql]"``). + """ + global client + client = import_driver("pymysql", backend="MySQL", extra="mysql") + # ========================================================================= # Connection Management # ========================================================================= @@ -175,9 +189,7 @@ def get_cursor(self, connection: Any, as_dict: bool = False) -> Any: Any pymysql cursor object. """ - import pymysql - - cursor_class = pymysql.cursors.DictCursor if as_dict else pymysql.cursors.Cursor + cursor_class = client.cursors.DictCursor if as_dict else client.cursors.Cursor return connection.cursor(cursor=cursor_class) # ========================================================================= diff --git a/src/datajoint/adapters/postgres.py b/src/datajoint/adapters/postgres.py index 91cc5311b..7abe8aba6 100644 --- a/src/datajoint/adapters/postgres.py +++ b/src/datajoint/adapters/postgres.py @@ -10,15 +10,13 @@ import re from typing import Any -try: - import psycopg2 as client - from psycopg2 import sql -except ImportError: - client = None # type: ignore - sql = None # type: ignore - from .. import errors -from .base import DatabaseAdapter +from .base import DatabaseAdapter, import_driver + +# The psycopg2 driver (the ``datajoint[postgres]`` extra), imported when the first +# PostgreSQLAdapter is created so that ``import datajoint`` loads no database driver. +# Either the psycopg2 or the psycopg2-binary distribution provides it. +client: Any = None # Core type mapping: DataJoint core types → PostgreSQL types CORE_TYPE_MAP = { @@ -56,11 +54,16 @@ class PostgreSQLAdapter(DatabaseAdapter): """PostgreSQL database adapter implementation.""" def __init__(self) -> None: - """Initialize PostgreSQL adapter.""" - if client is None: - raise ImportError( - "psycopg2 is required for PostgreSQL support. " "Install it with: pip install 'datajoint[postgres]'" - ) + """ + Initialize PostgreSQL adapter. + + Raises + ------ + DataJointError + If psycopg2 is not installed (``pip install "datajoint[postgres]"``). + """ + global client + client = import_driver("psycopg2", backend="PostgreSQL", extra="postgres") # ========================================================================= # Connection Management diff --git a/tests/unit/test_adapters.py b/tests/unit/test_adapters.py index 1e0678150..405fbc482 100644 --- a/tests/unit/test_adapters.py +++ b/tests/unit/test_adapters.py @@ -4,9 +4,14 @@ Tests adapter functionality without requiring actual database connections. """ +import re +import sys + import pytest from datajoint.adapters import DatabaseAdapter, MySQLAdapter, PostgreSQLAdapter, get_adapter +from datajoint.connection import Connection +from datajoint.errors import DataJointError class TestAdapterRegistry: @@ -48,6 +53,45 @@ def test_get_adapter_invalid(self): get_adapter("sqlite") +# Each backend, the driver module its adapter imports, and the extra that installs it. +MISSING_DRIVER_CASES = [ + ("mysql", "pymysql", "mysql"), + ("postgresql", "psycopg2", "postgres"), + ("postgres", "psycopg2", "postgres"), +] + + +class TestMissingDriver: + """A missing database driver is reported with the extra that installs it.""" + + @pytest.mark.parametrize("backend, driver, extra", MISSING_DRIVER_CASES) + def test_get_adapter_names_extra(self, monkeypatch, backend, driver, extra): + """Creating the adapter without its driver raises a DataJointError naming the extra.""" + # A None entry in sys.modules makes `import ` raise ImportError. + monkeypatch.setitem(sys.modules, driver, None) + with pytest.raises(DataJointError, match=re.escape(f'pip install "datajoint[{extra}]"')) as info: + get_adapter(backend) + assert driver in str(info.value) + assert isinstance(info.value.__cause__, ImportError) + + @pytest.mark.parametrize("backend, driver, extra", MISSING_DRIVER_CASES) + def test_connection_names_extra(self, monkeypatch, backend, driver, extra): + """Connecting without the driver fails before contacting any server.""" + monkeypatch.setitem(sys.modules, driver, None) + with pytest.raises(DataJointError, match=re.escape(f'pip install "datajoint[{extra}]"')): + Connection("localhost", "user", "password", backend=backend) + + def test_other_driver_not_required(self, monkeypatch): + """Each backend needs only its own driver.""" + pytest.importorskip("psycopg2") + with monkeypatch.context() as patch: + patch.setitem(sys.modules, "pymysql", None) + assert isinstance(get_adapter("postgresql"), PostgreSQLAdapter) + with monkeypatch.context() as patch: + patch.setitem(sys.modules, "psycopg2", None) + assert isinstance(get_adapter("mysql"), MySQLAdapter) + + class TestMySQLAdapter: """Test MySQL adapter implementation.""" diff --git a/tests/unit/test_lazy_imports.py b/tests/unit/test_lazy_imports.py index a87412151..36e33c68d 100644 --- a/tests/unit/test_lazy_imports.py +++ b/tests/unit/test_lazy_imports.py @@ -2,9 +2,11 @@ Tests for lazy import behavior. These tests verify that heavy dependencies (networkx, matplotlib, click) -are not loaded until their associated features are accessed. +are not loaded until their associated features are accessed, and that +``import datajoint`` loads no database driver. """ +import subprocess import sys @@ -85,3 +87,20 @@ def test_core_imports_available(): # Heavy modules should still not be loaded assert "datajoint.diagram" not in sys.modules assert "datajoint.cli" not in sys.modules + + +def test_import_loads_no_database_driver(): + """import datajoint should not import pymysql or psycopg2. + + Drivers are optional extras, so the import must succeed without them and + leave the driver to the adapter that connects. A fresh interpreter is used + so that drivers already imported by other tests do not hide the import. + """ + code = ( + "import sys, warnings\n" + "warnings.simplefilter('ignore')\n" + "import datajoint\n" + "print(sorted(m for m in sys.modules if m.split('.')[0] in ('pymysql', 'psycopg2')))\n" + ) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + assert result.stdout.strip() == "[]", f"import datajoint loaded database drivers: {result.stdout.strip()}"