diff --git a/HISTORY.rst b/HISTORY.rst index be72a238..9c0d9205 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -71,6 +71,24 @@ History from the C ``Reader.metadata()`` has the ``UnicodeDecodeError`` as its cause. +* Added ``maxminddb.Mode``, which the README already described. +* Type checkers now reject a subscript of ``maxminddb.types.Record`` or + ``Primitive``, such as ``Record[str]``. The subscript still works at + runtime. Pull request by Adam Hitchcock. GitHub #464. + +* Type hints: + + * ``Primitive`` includes ``bytearray``, which the C extension returns for + the ``bytes`` type. + * ``Reader.__iter__`` declares its item type. + * Added the ``StrOrBytesPath`` and ``DatabaseSource`` aliases to + ``maxminddb.types``. + * The ``maxminddb.extension.Reader`` stub accepts only a path, as the + extension does. + * ``MODE_FD`` accepts any object whose ``read()`` method returns ``bytes``, + such as a ``gzip.GzipFile``. ``maxminddb.types.SupportsRead`` describes + this type. + 3.2.0 (2026-09-10) ++++++++++++++++++ diff --git a/docs/index.rst b/docs/index.rst index b062a0d9..5764d27c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,24 @@ Database Reader :undoc-members: :show-inheritance: +===== +Types +===== + +.. automodule:: maxminddb.types + :members: + :undoc-members: + :exclude-members: Primitive, Record + +.. The runtime values of these aliases contain AnyStr, so give the types that + type checkers see. + +.. autodata:: maxminddb.types.Primitive + :annotation: = str | bytes | bytearray | bool | float | int + +.. autodata:: maxminddb.types.Record + :annotation: = Primitive | RecordList | RecordDict + ================== Indices and tables ================== diff --git a/maxminddb/__init__.py b/maxminddb/__init__.py index b3ef0ef6..49b8576c 100644 --- a/maxminddb/__init__.py +++ b/maxminddb/__init__.py @@ -3,7 +3,7 @@ from __future__ import annotations from importlib.metadata import version -from typing import IO, TYPE_CHECKING, cast +from typing import TYPE_CHECKING, cast from .const import ( MODE_AUTO, @@ -12,12 +12,13 @@ MODE_MEMORY, MODE_MMAP, MODE_MMAP_EXT, + Mode, ) -from .decoder import InvalidDatabaseError +from .errors import InvalidDatabaseError from .reader import Reader if TYPE_CHECKING: - import os + from .types import DatabaseSource try: from . import extension as _extension @@ -33,13 +34,14 @@ "MODE_MMAP", "MODE_MMAP_EXT", "InvalidDatabaseError", + "Mode", "Reader", "open_database", ] def open_database( - database: str | bytes | int | os.PathLike[str] | os.PathLike[bytes] | IO[bytes], + database: DatabaseSource, mode: int = MODE_AUTO, ) -> Reader: """Open a MaxMind DB database. @@ -84,8 +86,9 @@ def open_database( # The C type exposes the same API as the Python Reader, so for type # checking purposes, pretend it is one. (Ideally this would be a subclass # of, or share a common parent class with, the Python Reader - # implementation.) - return cast("Reader", _extension.Reader(database, mode)) + # implementation.) The extension accepts only a path. It raises TypeError + # for a file descriptor or a file object. + return cast("Reader", _extension.Reader(database, mode)) # type: ignore[arg-type] __version__ = version("maxminddb") diff --git a/maxminddb/extension.pyi b/maxminddb/extension.pyi index 39ae512f..19c8d4fe 100644 --- a/maxminddb/extension.pyi +++ b/maxminddb/extension.pyi @@ -2,12 +2,10 @@ from collections.abc import Iterator from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network -from os import PathLike -from typing import IO from typing_extensions import Self -from maxminddb.types import Record +from maxminddb.types import Record, StrOrBytesPath class Reader: """A C extension implementation of a reader for the MaxMind DB format. @@ -19,14 +17,14 @@ class Reader: def __init__( self, - database: str | bytes | int | PathLike[str] | PathLike[bytes] | IO[bytes], + database: StrOrBytesPath, mode: int = ..., ) -> None: """Reader for the MaxMind DB file format. Arguments: database: A path to a valid MaxMind DB file such as a GeoIP database - file, or a file descriptor in the case of MODE_FD. + file. mode: mode to open the database with. The only supported modes are MODE_AUTO and MODE_MMAP_EXT. @@ -59,7 +57,7 @@ class Reader: def __iter__(self) -> Iterator[tuple[IPv4Network | IPv6Network, Record]]: ... def __enter__(self) -> Self: ... - def __exit__(self, *args) -> None: ... # noqa: ANN002 + def __exit__(self, *args: object) -> None: ... class Metadata: """Metadata for the MaxMind DB reader.""" diff --git a/maxminddb/file.py b/maxminddb/file.py index 1901d9ba..1d7945ea 100644 --- a/maxminddb/file.py +++ b/maxminddb/file.py @@ -51,7 +51,7 @@ def close(self) -> None: """Close file.""" self._handle.close() - if hasattr(os, "pread"): # type: ignore[attr-defined] + if hasattr(os, "pread"): def _read(self, buffersize: int, offset: int) -> bytes: """Read that uses pread.""" diff --git a/maxminddb/reader.py b/maxminddb/reader.py index ac2b71ff..58802bbb 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -11,7 +11,7 @@ import ipaddress from dataclasses import dataclass from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network -from typing import IO, TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any from maxminddb.const import MODE_AUTO, MODE_FD, MODE_FILE, MODE_MEMORY, MODE_MMAP from maxminddb.decoder import Decoder @@ -20,11 +20,10 @@ if TYPE_CHECKING: from collections.abc import Iterator - from os import PathLike from typing_extensions import Self - from maxminddb.types import Record, RecordDict + from maxminddb.types import DatabaseSource, Record, RecordDict _IPV4_MAX_NUM = 2**32 _REOPENED = "Attempt to iterate over a reopened MaxMind DB. Create a new iterator." @@ -58,7 +57,7 @@ class Reader: def __init__( self, - database: str | bytes | int | PathLike[str] | PathLike[bytes] | IO[bytes], + database: DatabaseSource, mode: int = MODE_AUTO, ) -> None: """Reader for the MaxMind DB file format. @@ -95,7 +94,7 @@ def __init__( def _load( self, - database: str | bytes | int | PathLike[str] | PathLike[bytes] | IO[bytes], + database: DatabaseSource, mode: int, ) -> None: # TRY301 is suppressed because the handler only closes the buffer and @@ -229,10 +228,12 @@ def get_with_prefix_len( return self._resolve_data_pointer(pointer), prefix_len return None, prefix_len - def __iter__(self) -> Iterator: + def __iter__(self) -> Iterator[tuple[IPv4Network | IPv6Network, Record]]: return self._iterate(self._generation) - def _iterate(self, generation: int) -> Iterator: + def _iterate( + self, generation: int + ) -> Iterator[tuple[IPv4Network | IPv6Network, Record]]: children = self._generate_children(0, 0, 0) while True: # Check before the walk resumes and reads more nodes, as the C @@ -247,7 +248,12 @@ def _iterate(self, generation: int) -> Iterator: return yield record - def _generate_children(self, node: int, depth: int, ip_acc: int) -> Iterator: + def _generate_children( + self, + node: int, + depth: int, + ip_acc: int, + ) -> Iterator[tuple[IPv4Network | IPv6Network, Record]]: node_count = self._node_count bits = self._address_bits if node > node_count: @@ -344,7 +350,7 @@ def _resolve_data_pointer(self, pointer: int) -> Record: def _load_buffer( self, - database: str | bytes | int | PathLike[str] | PathLike[bytes] | IO[bytes], + database: DatabaseSource, mode: int = MODE_AUTO, ) -> str: filename: Any @@ -393,7 +399,7 @@ def close(self) -> None: self.closed = True - def __exit__(self, *_) -> None: # noqa: ANN002 + def __exit__(self, *_: object) -> None: self.close() def __enter__(self) -> Self: diff --git a/maxminddb/types.py b/maxminddb/types.py index dcb7b61e..0dd2b191 100644 --- a/maxminddb/types.py +++ b/maxminddb/types.py @@ -1,10 +1,18 @@ -"""Types representing database records.""" +"""Types for database records and database arguments.""" from __future__ import annotations -from typing import TypeAlias +import os +from typing import TYPE_CHECKING, AnyStr, Protocol, TypeAlias -Primitive: TypeAlias = str | bytes | bool | float | int +if TYPE_CHECKING: + Primitive: TypeAlias = str | bytes | bytearray | bool | float | int + """Primitive is a type for a scalar value in a database record.""" +else: + # The 3.2.0 definition, so that a subscript such as Record[str] works + # and means the same at runtime. Keep it when the static alias changes. + # Type checkers reject the subscript. + Primitive: TypeAlias = AnyStr | bool | float | int RecordList: TypeAlias = list["Record"] """RecordList is a type for lists in a database record.""" @@ -13,3 +21,18 @@ """RecordDict is a type for dicts in a database record.""" Record: TypeAlias = Primitive | RecordList | RecordDict +"""Record is a type for a value in a database record.""" + +StrOrBytesPath: TypeAlias = str | bytes | os.PathLike[str] | os.PathLike[bytes] +"""StrOrBytesPath is a type for a path to a database file.""" + + +class SupportsRead(Protocol): + """SupportsRead is a type for a binary file object for MODE_FD.""" + + def read(self) -> bytes: + """Return the remaining bytes.""" + + +DatabaseSource: TypeAlias = StrOrBytesPath | int | SupportsRead +"""DatabaseSource is a type for the database argument of a reader.""" diff --git a/pyproject.toml b/pyproject.toml index c51facb6..7123736b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,11 @@ ignore = [ "setup.py" = ["ALL"] "tests/*" = ["ANN201", "D"] +[tool.mypy] +# mypy 2 enables this by default. Without it, mypy 1 accepts bytearray as +# bytes, and the bytearray check in tests/typing_test.py cannot fail. +strict_bytes = true + [tool.pytest.ini_options] # Use importlib mode to prevent pytest from adding the project root to sys.path. # This ensures tests import from the tox virtualenv's installed package (which includes diff --git a/tests/reader_test.py b/tests/reader_test.py index 45db4876..2fd1308e 100644 --- a/tests/reader_test.py +++ b/tests/reader_test.py @@ -18,7 +18,7 @@ import threading import tracemalloc import unittest -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, cast, get_args from unittest import mock import maxminddb @@ -28,6 +28,7 @@ except ImportError: maxminddb.extension = None # type: ignore[assignment] +import maxminddb.types from maxminddb import InvalidDatabaseError, open_database from maxminddb.const import ( MODE_AUTO, @@ -1418,6 +1419,21 @@ class TestExtensionReaderWithIPObjects(BaseTestReader): reader_class = maxminddb.extension.Reader +class TestModule(unittest.TestCase): + def test_mode_is_exported(self) -> None: + self.assertIs(maxminddb.Mode, maxminddb.const.Mode) + + def test_record_subscript_works_at_runtime(self) -> None: + # 3.2.0 accepted Record[str] at runtime. typing_test.py checks that + # type checkers reject it. + maxminddb.types.Record[str] # type: ignore[misc, type-arg] + # As in 3.2.0, Primitive[str] does not include bytes. + self.assertEqual( + get_args(maxminddb.types.Primitive[str]), # type: ignore[misc, type-arg] + (str, bool, float, int), + ) + + @unittest.skipIf( not has_maxminddb_extension() and not os.environ.get("MM_FORCE_EXT_TESTS"), "No C extension module found. Skipping tests", diff --git a/tests/typing_test.py b/tests/typing_test.py new file mode 100644 index 00000000..f7883659 --- /dev/null +++ b/tests/typing_test.py @@ -0,0 +1,52 @@ +# mypy: warn-unused-ignores +"""Static type checks for the public API. + +mypy checks this file in the lint environment. The code does not run. Each +type: ignore marks a line that must fail the type check. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import gzip + import io + from ipaddress import IPv4Network, IPv6Network + from pathlib import Path + + from typing_extensions import assert_type + + import maxminddb + import maxminddb.extension + from maxminddb.types import Primitive, Record + + reader = maxminddb.open_database("GeoIP2-City.mmdb") + assert_type(reader.get("1.1.1.1"), Record | None) + assert_type(reader.get_with_prefix_len("1.1.1.1"), tuple[Record | None, int]) + for network, record in reader: + assert_type(network, IPv4Network | IPv6Network) + assert_type(record, Record) + assert_type(reader.metadata().search_tree_size, int) + with maxminddb.open_database(Path("GeoIP2-City.mmdb")) as path_reader: + assert_type(path_reader, maxminddb.Reader) + + # Record includes the bytearray that the C extension returns for the + # bytes type. + value: Record = bytearray(b"\x00") + + # A TypeVar in Primitive would give Record members of type Any. mypy + # rejects Record[str] even then, so only this check catches it. + def check_not_generic(primitive: Primitive[str]) -> None: # type: ignore[type-arg] + pass + + def check_mode_fd(gzip_file: gzip.GzipFile, text_file: io.TextIOWrapper) -> None: + maxminddb.open_database(gzip_file, maxminddb.Mode.FD) + maxminddb.open_database(text_file, maxminddb.Mode.FD) # type: ignore[arg-type] + + extension_reader = maxminddb.extension.Reader("GeoIP2-City.mmdb") + for network, record in extension_reader: + assert_type(network, IPv4Network | IPv6Network) + assert_type(record, Record) + assert_type(extension_reader.metadata().node_byte_size, int) + maxminddb.extension.Reader(3) # type: ignore[arg-type]