From d1052108cf508e966c6c44ebbbabf273dfab9c0e Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sat, 31 Oct 2026 22:45:25 +0100 Subject: [PATCH 01/10] ci(examples): run example scripts once in their own job (#4586) * ci(examples): run example scripts once in their own job Each example's PEP 723 header defines its environment, so running tests/test_examples.py in every pytest matrix job repeated the same work. Its header-rewriting helper also depended on the test environment's packaging and tomlkit, which broke the min_deps job on Python 3.12.3. A new `just examples` recipe uses `uv add --script` to point each header at the local checkout, and one CI job runs it. `uv add` resolves the header, and that showed two examples declaring requires-python >=3.11 although zarr requires 3.12. Those headers now say >=3.12. tomlkit was only used by the removed test, so it is dropped from the test dependency group. Assisted-by: ClaudeCode:claude-opus-5-5 * test(examples): run the example scripts through pytest The previous commit ran the examples from a bash loop, which stops at the first failing script, so one broken example hid the others. The examples now run from tests/test_examples.py again, one test per script. The tests are marked `examples` and skipped unless pytest gets --run-examples, as slow_hypothesis tests are with --run-slow-hypothesis, so the matrix jobs skip them. `just examples` passes the option, and the CI job runs it in the test.py3.13-minimal hatch environment. Assisted-by: ClaudeCode:claude-opus-5-5 * chore(just): rename the examples recipe to test-examples `just examples` could mean listing or building the examples. `test-examples` says what the recipe does, and follows the `-` naming of recipes like `coverage-html` and `docs-build`. Assisted-by: ClaudeCode:claude-opus-5-5 --- .github/workflows/test.yml | 29 +++++- Justfile | 4 + examples/README.md | 2 +- .../codec_pipeline_performance.py | 2 +- .../sharding_coalescing.py | 2 +- pyproject.toml | 2 +- tests/conftest.py | 26 +++-- tests/test_examples.py | 98 ++++++------------- uv.lock | 15 --- 9 files changed, 84 insertions(+), 96 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4141d7356..3f995e8b5e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -201,6 +201,32 @@ jobs: run: | HATCH_ENV=test.py3.13-minimal just benchmark + examples: + name: Example scripts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # required for hatch version discovery when uv builds zarr + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.13' + cache: 'pip' + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + - name: Install Hatch + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 + - name: Run Examples + run: | + HATCH_ENV=test.py3.13-minimal just test-examples + test-complete: name: Test complete @@ -209,7 +235,8 @@ jobs: test, test-upstream-and-min-deps, doctests, - benchmarks + benchmarks, + examples ] if: always() runs-on: ubuntu-latest diff --git a/Justfile b/Justfile index b3f187e231..df96003eef 100644 --- a/Justfile +++ b/Justfile @@ -64,6 +64,10 @@ hypothesis *args: doctest *args: hatch run doctest:pytest tests/test_docs.py -v "$@" +# Run each example script against the local checkout, in its PEP 723 environment +test-examples *args: + hatch run {{ quote(hatch_env) }}:pytest --run-examples tests/test_examples.py -v "$@" + # Run the benchmark suite benchmark *args: hatch run {{ quote(hatch_env) }}:pytest --benchmark-enable tests/benchmarks "$@" diff --git a/examples/README.md b/examples/README.md index a6b5fa2179..fa664d2c58 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,7 +19,7 @@ examples/ To add a new example: 1. Create a new subdirectory: `examples/my_example/` -2. Add your Python code: `examples/my_example/my_example.py` +2. Add your Python code: `examples/my_example/my_example.py`. Start it with a [PEP 723](https://peps.python.org/pep-0723/) header that lists its dependencies, including `zarr`. `just test-examples` runs every example against your local checkout, and CI runs it on every pull request. 3. Create documentation: `examples/my_example/README.md` 4. Create a documentation page at `docs/user-guide/examples/my_example.md`. The documentation page should simply link to the `README.md` and the source code, e.g.: diff --git a/examples/codec_pipeline_performance/codec_pipeline_performance.py b/examples/codec_pipeline_performance/codec_pipeline_performance.py index b821f3e6a7..d146c51f4a 100644 --- a/examples/codec_pipeline_performance/codec_pipeline_performance.py +++ b/examples/codec_pipeline_performance/codec_pipeline_performance.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", diff --git a/examples/sharding_coalescing/sharding_coalescing.py b/examples/sharding_coalescing/sharding_coalescing.py index 5da62ed806..66838d2255 100644 --- a/examples/sharding_coalescing/sharding_coalescing.py +++ b/examples/sharding_coalescing/sharding_coalescing.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", diff --git a/pyproject.toml b/pyproject.toml index 2554180f38..136f757dac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,6 @@ test = [ "pytest-xdist==3.8.0", "pytest-benchmark==5.3.0", "pytest-codspeed==5.0.3", - "tomlkit==0.15.2", "uv==0.13.5", ] remote-tests = [ @@ -463,6 +462,7 @@ markers = [ "gpu: mark a test as requiring CuPy and GPU", "s3: mark a test as requiring a (mock) S3 backend via moto", "slow_hypothesis: slow hypothesis tests", + "examples: runs a script in examples/; skipped unless --run-examples is passed", ] [tool.repo-review] diff --git a/tests/conftest.py b/tests/conftest.py index 3e87a724e0..89eb90a1d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -262,15 +262,29 @@ def pytest_addoption(parser: Any) -> None: default=False, help="run slow hypothesis tests", ) + parser.addoption( + "--run-examples", + action="store_true", + default=False, + help="run the scripts in examples/", + ) + + +# Tests with one of these markers are skipped unless the matching option is passed. +_OPT_IN_MARKERS = { + "slow_hypothesis": "--run-slow-hypothesis", + "examples": "--run-examples", +} def pytest_collection_modifyitems(config: Any, items: Any) -> None: - if config.getoption("--run-slow-hypothesis"): - return - skip_slow_hyp = pytest.mark.skip(reason="need --run-slow-hypothesis option to run") - for item in items: - if "slow_hypothesis" in item.keywords: - item.add_marker(skip_slow_hyp) + for marker, option in _OPT_IN_MARKERS.items(): + if config.getoption(option): + continue + skip = pytest.mark.skip(reason=f"need {option} option to run") + for item in items: + if marker in item.keywords: + item.add_marker(skip) settings.register_profile( diff --git a/tests/test_examples.py b/tests/test_examples.py index a6634e8cc6..2421a3d0ca 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,87 +1,45 @@ from __future__ import annotations -import re import subprocess -import sys from pathlib import Path -from typing import Final import pytest -import tomlkit -from packaging.requirements import Requirement -examples_dir = "examples" -script_paths = tuple(Path(examples_dir).rglob("*.py")) +ZARR_PROJECT_PATH = Path(__file__).parent.parent +EXAMPLE_PATHS = tuple(sorted((ZARR_PROJECT_PATH / "examples").glob("*/*.py"))) -PEP_723_REGEX: Final = r"(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$" -# This is the absolute path to the local Zarr installation. Moving this test to a different directory will break it. -ZARR_PROJECT_PATH = Path(".").absolute() +def test_example_paths() -> None: + """The example scripts are found, so the parametrized test below is not empty.""" + assert len(EXAMPLE_PATHS) > 0 -def set_dep(script: str, dependency: str) -> str: +@pytest.mark.examples +@pytest.mark.parametrize("script_path", EXAMPLE_PATHS, ids=lambda path: path.stem) +def test_example_runs(script_path: Path, tmp_path: Path) -> None: """ - Set a dependency in a PEP-723 script header. - If the package is already in the list, it will be replaced. - If the package is not already in the list, it will be added. + Each example runs against this checkout, in the environment its PEP 723 header defines. - Source code modified from - https://packaging.python.org/en/latest/specifications/inline-script-metadata/#reference-implementation + The test copies the script and uses `uv add` to point the copy's zarr requirement at + this checkout. `uv add` also resolves the header, so a header that cannot be + satisfied fails here. """ - match = re.search(PEP_723_REGEX, script) - - if match is None: - raise ValueError(f"PEP-723 header not found in {script}") - - content = "".join( - line[2:] if line.startswith("# ") else line[1:] - for line in match.group("content").splitlines(keepends=True) - ) - - config = tomlkit.parse(content) - for idx, dep in enumerate(tuple(config["dependencies"])): - if Requirement(dep).name == Requirement(dependency).name: - config["dependencies"][idx] = dependency - - new_content = "".join( - f"# {line}" if line.strip() else f"#{line}" - for line in tomlkit.dumps(config).splitlines(keepends=True) - ) - - start, end = match.span("content") - return script[:start] + new_content + script[end:] - - -def resave_script(source_path: Path, dest_path: Path) -> None: - """ - Read a script from source_path and save it to dest_path after inserting the absolute path to the - local Zarr project directory in the PEP-723 header. - """ - source_text = source_path.read_text() - dest_text = set_dep(source_text, f"zarr @ file:///{ZARR_PROJECT_PATH}") - dest_path.write_text(dest_text) - - -def test_script_paths() -> None: - """ - Test that our test fixture is working properly and collecting script paths. - """ - assert len(script_paths) > 0 - - -@pytest.mark.skipif( - sys.platform == "win32", reason="This test fails for unknown reasons on Windows in CI." -) -@pytest.mark.parametrize("script_path", script_paths) -def test_scripts_can_run(script_path: Path, tmp_path: Path) -> None: dest_path = tmp_path / script_path.name - # We resave the script after inserting the absolute path to the local Zarr project directory, - # and then test its behavior. - # This allows the example to be useful to users who don't have Zarr installed, but also testable. - resave_script(script_path, dest_path) - result = subprocess.run( - ["uv", "run", "--refresh", str(dest_path)], capture_output=True, text=True, check=False + dest_path.write_text(script_path.read_text()) + + add = subprocess.run( + ["uv", "add", "--quiet", "--script", str(dest_path), str(ZARR_PROJECT_PATH)], + capture_output=True, + text=True, + check=False, ) - assert result.returncode == 0, ( - f"Script at {script_path} failed to run. Output: {result.stdout} Error: {result.stderr}" + assert add.returncode == 0, f"uv add failed for {script_path}:\n{add.stderr}" + + # `--reinstall-package` rebuilds zarr, so the run sees the current source. + run = subprocess.run( + ["uv", "run", "--reinstall-package", "zarr", str(dest_path)], + capture_output=True, + text=True, + check=False, ) + assert run.returncode == 0, f"{script_path} failed:\n{run.stdout}\n{run.stderr}" diff --git a/uv.lock b/uv.lock index f8094ec90e..e78182c154 100644 --- a/uv.lock +++ b/uv.lock @@ -3435,15 +3435,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl", hash = "sha256:3a49cf47b7675da0b15d0c6e1df8df4ebd96e9394bb905a5775adb0d884c5289", size = 26610, upload-time = "2024-10-24T14:58:28.029Z" }, ] -[[package]] -name = "tomlkit" -version = "0.15.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, -] - [[package]] name = "towncrier" version = "26.9.0" @@ -3846,7 +3837,6 @@ dev = [ { name = "requests" }, { name = "ruff" }, { name = "s3fs" }, - { name = "tomlkit" }, { name = "towncrier" }, { name = "universal-pathlib" }, { name = "uv" }, @@ -3888,7 +3878,6 @@ remote-tests = [ { name = "pytest-xdist" }, { name = "requests" }, { name = "s3fs" }, - { name = "tomlkit" }, { name = "uv" }, ] test = [ @@ -3903,7 +3892,6 @@ test = [ { name = "pytest-cov" }, { name = "pytest-reportlog" }, { name = "pytest-xdist" }, - { name = "tomlkit" }, { name = "uv" }, ] @@ -3956,7 +3944,6 @@ dev = [ { name = "requests", specifier = "==2.34.2" }, { name = "ruff", specifier = "==0.17.1" }, { name = "s3fs", specifier = ">=2023.10.0" }, - { name = "tomlkit", specifier = "==0.15.2" }, { name = "towncrier", specifier = "==26.9.0" }, { name = "universal-pathlib" }, { name = "uv", specifier = "==0.13.5" }, @@ -3996,7 +3983,6 @@ remote-tests = [ { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "requests", specifier = "==2.34.2" }, { name = "s3fs", specifier = ">=2023.10.0" }, - { name = "tomlkit", specifier = "==0.15.2" }, { name = "uv", specifier = "==0.13.5" }, ] test = [ @@ -4011,6 +3997,5 @@ test = [ { name = "pytest-cov", specifier = "==7.1.0" }, { name = "pytest-reportlog", specifier = "==1.0.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, - { name = "tomlkit", specifier = "==0.15.2" }, { name = "uv", specifier = "==0.13.5" }, ] From 4a72dd42a5b1cbaf0d8b3248dd3723df73708268 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 1 Nov 2026 09:42:38 +0100 Subject: [PATCH 02/10] feat(zarr-metadata): add Hypothesis strategies for metadata fields (#4575) * feat(zarr-metadata): add Hypothesis strategies for metadata fields Add `zarr_metadata.testing.strategies`, behind a new `testing` extra. It ships Hypothesis strategies that draw valid Zarr v3 metadata fields, one for every codec, data type, chunk grid and chunk key encoding the package defines that holds no other field. - Each kind's strategy takes what its configurations depend on: a data type's fill values take its configuration, a chunk grid takes the array shape, and a codec takes the `Chunk` it is handed. - `fields` draws every envelope spelling the spec allows, or only the canonical one. - Strategies are a mapping keyed by definition, so an extension author can add strategies for their own definitions. The tests read every draw back in `CORE_AND_EXTENSIONS` and require no problems. They also check: - that every definition in scope has a strategy, apart from the three that hold nested fields; - reach, with targets derived from each configuration's TypedDict: each optional key present and absent, each `Literal` value, and a non-canonical spelling where one exists; - fill-value spellings and envelope forms. Strategies follow the spec where the package is looser. Core data types are drawn only as bare strings, and regular-grid chunk lengths are never zero ("Chunk sizes must be greater than zero"), although the package's rules accept both today. Assisted-by: ClaudeCode:claude-opus-5-5 * chore(zarr-metadata): number the strategies changelog fragment after #414 Assisted-by: ClaudeCode:claude-opus-5-5 * chore(zarr-metadata): number the strategies changelog fragment after PR 4575 The fragment was numbered after fork PR 414, so the changelog linked to an unrelated upstream PR. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): draw a fresh empty configuration on every draw `st.just({})` (crc32c) and `st.just(EmptyConfiguration())` (every data type with nothing to configure) handed out one dict for every draw, and `fields` puts the configuration in the field as it is. A test that changed the configuration it drew changed every later draw, which Hypothesis reports as a flaky strategy. Both are now `st.builds(EmptyConfiguration)`. Export `Configurations` and `NOTHING_KNOWN` from `zarr_metadata.testing.strategies`: they appear in the signatures of `DefinitionStrategy.unconstrained` and `codec_configurations`. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): check every drawn field, and draw hex scale_offset values `fields` read a field only when asked for the canonical spelling, so a caller's configurations the definition refuses were drawn silently in the default spelling. Every field is now read, in a scope built once per call rather than once per draw, and a refused one raises in both spellings. A definition no Zarr v3 scope holds, such as a v2 codec, is a `TypeError` when `fields` is called, not a `StopIteration` on a draw. A float `scale_offset` codec's `offset` and `scale` are fill values of its type, so they are now also drawn as the hex string of their bits. The docstring no longer claims a non-zero scale makes decoding defined: a tiny one still overflows. The codec chunk test now draws chunks of an unknown data type and with unknown lengths, which every codec takes. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): draw scale_offset for every number type, and widen Strategies keys `scale_offset` drew nothing for complex numbers, numpy times and data types the package does not define, though its rules refuse only data types whose values are not numbers, and a numpy time "is compatible with any codec that supports arrays of signed 64-bit integers". A numpy time is now drawn as that integer, a complex number as a pair of finite floats, and a type the package does not define takes the identity configuration. None now means only what `CodecStrategy` says it means: no configuration takes the chunk. `Strategies` is keyed by `Any`: a mapping's key type is invariant, so a mapping an extension author builds, keyed by a `CodecDefinition`, was not a `Mapping[Definition[Any], ...]` to pyright. A rectilinear dimension of unknown length may now be drawn with no edges, which its rules accept. Assisted-by: ClaudeCode:claude-opus-5-5 * refactor(zarr-metadata): share the strategies' empty configuration and rank cap `EMPTY_CONFIGURATIONS` and `MAX_RANK` are defined once, in `_strategy`, rather than once in each module that draws them. A numpy time's `scale_offset` values are now drawn finite, never the integer numpy reads as `NaT`, as the docstring says. The `fields` docstring now says that a codec's chunk rules are not checked, since no chunk is known. Assisted-by: ClaudeCode:claude-opus-5-5 * test(zarr-metadata): cover scale_offset over a data type the package does not define The identity configuration `scale_offset` draws for a data type the package does not define had no test, since the chunk test draws only the package's own data types. A test now reads an extension `bfloat16` in a scope of its own and checks the codec takes every draw. Also splits the `_scope_of` error message across two literals and drops a doubled blank line in `_data_type`. Assisted-by: ClaudeCode:claude-opus-5-5 --------- Co-authored-by: Claude Opus 5.5 --- .../zarr-metadata/changes/4575.feature.md | 1 + packages/zarr-metadata/docs/api/testing.md | 8 + packages/zarr-metadata/mkdocs.yml | 1 + packages/zarr-metadata/pyproject.toml | 5 + .../src/zarr_metadata/testing/__init__.py | 6 + .../src/zarr_metadata/testing/_chunk_grid.py | 105 +++++ .../testing/_chunk_key_encoding.py | 40 ++ .../src/zarr_metadata/testing/_codec.py | 179 ++++++++ .../src/zarr_metadata/testing/_data_type.py | 200 ++++++++ .../src/zarr_metadata/testing/_strategy.py | 147 ++++++ .../src/zarr_metadata/testing/strategies.py | 299 ++++++++++++ .../zarr-metadata/tests/test_public_api.py | 8 + .../zarr-metadata/tests/testing/__init__.py | 0 .../tests/testing/test_strategies.py | 432 ++++++++++++++++++ packages/zarr-metadata/uv.lock | 7 + 15 files changed, 1438 insertions(+) create mode 100644 packages/zarr-metadata/changes/4575.feature.md create mode 100644 packages/zarr-metadata/docs/api/testing.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_codec.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/testing/strategies.py create mode 100644 packages/zarr-metadata/tests/testing/__init__.py create mode 100644 packages/zarr-metadata/tests/testing/test_strategies.py diff --git a/packages/zarr-metadata/changes/4575.feature.md b/packages/zarr-metadata/changes/4575.feature.md new file mode 100644 index 0000000000..bd1d87ab40 --- /dev/null +++ b/packages/zarr-metadata/changes/4575.feature.md @@ -0,0 +1 @@ +Added `zarr_metadata.testing.strategies`, Hypothesis strategies that draw valid Zarr v3 metadata fields, behind a new `testing` extra (`pip install zarr-metadata[testing]`). There is one strategy for every codec, data type, chunk grid and chunk key encoding the package defines that holds no other field. `fields` draws every envelope spelling the spec allows, or only the canonical one with `spellings="canonical"`. Codecs can be drawn for the `Chunk` they are handed, chunk grids for an array shape, and fill values for a data type. The strategies are a mapping keyed by definition, so an extension author can add strategies for their own definitions. diff --git a/packages/zarr-metadata/docs/api/testing.md b/packages/zarr-metadata/docs/api/testing.md new file mode 100644 index 0000000000..a5d6145258 --- /dev/null +++ b/packages/zarr-metadata/docs/api/testing.md @@ -0,0 +1,8 @@ +--- +title: testing.strategies +--- + +::: zarr_metadata.testing.strategies + options: + inherited_members: false + show_if_no_docstring: false diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 10464e3cf0..b19c0932d2 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -28,6 +28,7 @@ nav: - ' zarr_metadata.model': api/model.md - ' zarr_metadata.problems': api/problems.md - ' zarr_metadata.pydantic': api/pydantic.md + - ' zarr_metadata.testing.strategies': api/testing.md - ' zarr_metadata.typed_json': api/typed_json.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index abd93bc729..25d1c7aead 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -43,6 +43,11 @@ dependencies = [ "typing_extensions>=4.16", ] +[project.optional-dependencies] +# `zarr_metadata.testing.strategies`. 6.85: first release with +# `characters(exclude_categories=...)`. +testing = ["hypothesis>=6.85"] + [project.urls] Homepage = "https://github.com/zarr-developers/zarr-python" Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py new file mode 100644 index 0000000000..271a4dfe51 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py @@ -0,0 +1,6 @@ +"""Testing utilities for code that reads or writes Zarr metadata. + +`zarr_metadata.testing.strategies` holds Hypothesis strategies that draw +valid metadata. It needs the `testing` extra: +`pip install zarr-metadata[testing]`. +""" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py new file mode 100644 index 0000000000..b7528a3907 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -0,0 +1,105 @@ +"""Strategies for each chunk grid this package defines. + +Over an array of a known shape, a grid is drawn to fit it: a chunk +length for each dimension, never zero ("Chunk sizes must be greater +than zero", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40), +and rectilinear edges that cover each dimension. Over no shape, what is +drawn is what the grid's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import MAX_RANK, ChunkGridStrategy +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import ChunkGridDefinition + +_MAX_LENGTH: Final = 16 +"""The longest chunk drawn along a dimension whose length is unknown.""" + +_MAX_RUNS: Final = 5 +"""The most runs of equal lengths a rectilinear dimension is drawn with.""" + +_MAX_COUNT: Final = 4 +"""The most chunks a run is drawn with.""" + + +def _length(extent: int | None) -> st.SearchStrategy[int]: + """A chunk length along a dimension of `extent`: at least 1, and up to twice the extent when it is known and not 0.""" + if extent is None or extent == 0: + return st.integers(1, _MAX_LENGTH) + return st.integers(1, 2 * extent) + + +def _regular(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A chunk length for each dimension.""" + if shape is None: + lengths = st.lists(_length(None), max_size=MAX_RANK).map(tuple) + else: + lengths = st.tuples(*(_length(extent) for extent in shape)) + return lengths.map(lambda chunk_shape: {"chunk_shape": chunk_shape}) + + +def _spelled_run(draw: st.DrawFn, size: int, count: int) -> list[JSONValue]: + """`count` chunks of `size`: one `[size, count]` pair, or the size written once for each.""" + if draw(st.booleans()): + return [(size, count)] + return [size] * count + + +@st.composite +def _edges(draw: st.DrawFn, extent: int | None) -> JSONValue: + """Edge lengths along a dimension, covering `extent`, in runs spelled as pairs or written out. + + Runs are drawn apart, so two of one size can sit side by side and a + pair can count one chunk: spellings the canonical form merges. A + dimension of length zero, or of a length not known, may be covered by + no edges at all. + """ + min_runs = 0 if extent is None or extent == 0 else 1 + runs = draw( + st.lists( + st.tuples(st.integers(1, _MAX_LENGTH), st.integers(1, _MAX_COUNT)), + min_size=min_runs, + max_size=_MAX_RUNS, + ) + ) + covered = sum(size * count for size, count in runs) + if extent is not None and covered < extent: + runs.append((extent - covered, 1)) + spelled: list[JSONValue] = [] + for size, count in runs: + spelled.extend(_spelled_run(draw, size, count)) + return tuple(spelled) + + +def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[JSONValue]: + """A bare length, which repeats until it covers the dimension, or edges that cover it.""" + return _length(extent) | _edges(extent) + + +def _rectilinear(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """Inline chunk shapes, one entry for each dimension.""" + if shape is None: + dimensions = st.lists(_rectilinear_dimension(None), max_size=MAX_RANK).map(tuple) + else: + dimensions = st.tuples(*(_rectilinear_dimension(extent) for extent in shape)) + return dimensions.map(lambda chunk_shapes: {"kind": "inline", "chunk_shapes": chunk_shapes}) + + +CHUNK_GRID_STRATEGIES: Final[Mapping[ChunkGridDefinition[Any], ChunkGridStrategy]] = { + REGULAR_CHUNK_GRID: ChunkGridStrategy(configurations=_regular), + RECTILINEAR_CHUNK_GRID: ChunkGridStrategy(configurations=_rectilinear), +} +"""A strategy for each chunk grid.""" + +__all__ = ["CHUNK_GRID_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py new file mode 100644 index 0000000000..9c525118c9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py @@ -0,0 +1,40 @@ +"""Strategies for each chunk key encoding this package defines.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import ChunkKeyEncodingStrategy +from zarr_metadata.v3.chunk_key_encoding.default import ( + DEFAULT_CHUNK_KEY_ENCODING, + DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, +) +from zarr_metadata.v3.chunk_key_encoding.v2 import ( + V2_CHUNK_KEY_ENCODING, + V2_CHUNK_KEY_ENCODING_SEPARATOR, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition + +CHUNK_KEY_ENCODING_STRATEGIES: Final[ + Mapping[ChunkKeyEncodingDefinition[Any], ChunkKeyEncodingStrategy] +] = { + DEFAULT_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), + V2_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(V2_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), +} +"""A strategy for each chunk key encoding: a `separator`, or none, which means the encoding's default.""" + +__all__ = ["CHUNK_KEY_ENCODING_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py new file mode 100644 index 0000000000..caf7b71a05 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -0,0 +1,179 @@ +"""Strategies for each codec this package defines that holds no other field. + +A codec handed an array draws what fits the chunk: a `bytes` codec an +`endian` for a type whose values take several bytes, a `transpose` an +`order` with an entry for each axis. Handed `Chunk()`, nothing is known +of the chunk, and what is drawn is what the codec's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import AcceptedField +from zarr_metadata.testing._data_type import ( + COMPLEX_WIDTHS, + FLOAT_WIDTHS, + INTEGER_RANGES, + FloatWidth, + finite_floats, + float_spellings, +) +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy +from zarr_metadata.v3._definition import Chunk, storage_of +from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS +from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, ENDIANNESS +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import CodecDefinition + +_BLOSC_MAX_TYPESIZE: Final = 255 +"""The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" + +_BLOSC_MAX_BLOCKSIZE: Final = 2**31 - 1 +"""The largest block c-blosc takes, a signed 32-bit size; 0 lets it choose.""" + +_ENDIAN: Final = st.sampled_from(ENDIANNESS) + +_NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) +"""The numpy time types, each stored as a signed 64-bit integer.""" + +_NUMPY_TIME_RANGE: Final = (-(2**63) + 1, 2**63 - 1) +"""The signed 64-bit integers but the least, which numpy reads as `NaT`: the finite numpy times.""" + + +def _bytes(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `endian`, required for a data type whose values take several bytes; None for one whose values vary in size.""" + storage = None if chunk.data_type is None else storage_of(chunk.data_type) + if storage == "variable_length": + return None + if storage == "multi_byte": + return st.fixed_dictionaries({"endian": _ENDIAN}) + return st.fixed_dictionaries({}, optional={"endian": _ENDIAN}) + + +@st.composite +def _blosc(draw: st.DrawFn) -> Mapping[str, JSONValue]: + """A compressor, a level, a shuffle and a block size; and a `typesize`, which only `noshuffle` may leave out. + + Under `noshuffle` the spec says `typesize` "is ignored", so it is + drawn there too, a spelling that is not the simplest. + """ + shuffle = draw(st.sampled_from(BLOSC_SHUFFLE)) + typesize = st.integers(1, _BLOSC_MAX_TYPESIZE) + required: dict[str, st.SearchStrategy[JSONValue]] = { + "cname": st.sampled_from(BLOSC_CNAME), + "clevel": st.integers(0, 9), + "shuffle": st.just(shuffle), + "blocksize": st.integers(0, _BLOSC_MAX_BLOCKSIZE), + } + if shuffle == BLOSC_NO_SHUFFLE: + return draw(st.fixed_dictionaries(required, optional={"typesize": typesize})) + return draw(st.fixed_dictionaries({**required, "typesize": typesize})) + + +def _order(rank: int) -> st.SearchStrategy[Mapping[str, JSONValue]]: + return st.permutations(range(rank)).map(lambda order: {"order": tuple(order)}) + + +def _transpose(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A permutation of the chunk's axes; of some number of axes, when the chunk does not say.""" + if chunk.rank is not None: + return _order(chunk.rank) + return st.integers(0, MAX_RANK).flatmap(_order) + + +def _finite_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite float of `width` bits, as a number or the hex string of its bits.""" + return finite_floats(width).flatmap(lambda value: float_spellings(value, width)) + + +def _non_zero_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite, non-zero float of `width` bits, as a number or the hex string of its bits.""" + return ( + finite_floats(width) + .filter(lambda value: value != 0) + .flatmap(lambda value: float_spellings(value, width)) + ) + + +def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `offset` and a `scale`, each a finite value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. + + Each is a fill value of the chunk's data type + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), + a float's as a number or the hex string of its bits, a complex + number's as a pair of them. Each is drawn finite, and the scale + non-zero, so that decoding never divides by zero; a scale small + enough that decoding overflows is still drawn. A numpy time is drawn + as the signed 64-bit integer it is stored as, never `NaT`, since it + "is compatible with any codec that supports arrays of signed 64-bit + integers" + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). + A data type this package does not define, whose values it cannot + draw, takes only the configuration with neither, the identity. + Handed a chunk whose data type is unknown, the values are small + non-negative integers, which every integer and float type holds. + """ + data_type = chunk.data_type + definition = data_type.definition if isinstance(data_type, AcceptedField) else None + if definition is None: + offset: st.SearchStrategy[JSONValue] = st.integers(0, 127) + scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) + elif definition.name in NOT_NUMBERS: + return None + elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: + low, high = INTEGER_RANGES.get(definition, _NUMPY_TIME_RANGE) + offset = st.integers(low, high) + scale = st.integers(low, high).filter(lambda value: value != 0) + elif definition in FLOAT_WIDTHS: + offset = _finite_float_values(FLOAT_WIDTHS[definition]) + scale = _non_zero_float_values(FLOAT_WIDTHS[definition]) + elif definition in COMPLEX_WIDTHS: + width = COMPLEX_WIDTHS[definition] + offset = st.tuples(_finite_float_values(width), _finite_float_values(width)) + # Non-zero in either part is a non-zero complex number. + scale = st.one_of( + st.tuples(_non_zero_float_values(width), _finite_float_values(width)), + st.tuples(_finite_float_values(width), _non_zero_float_values(width)), + ) + else: + return EMPTY_CONFIGURATIONS + return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) + + +def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> CodecStrategy: + """A codec whose configurations do not depend on the chunk it is handed.""" + return CodecStrategy(configurations=lambda _chunk: configurations) + + +CODEC_STRATEGIES: Final[Mapping[CodecDefinition[Any], CodecStrategy]] = { + BLOSC_CODEC: _unchunked(_blosc()), + BYTES_CODEC: CodecStrategy(configurations=_bytes), + CRC32C_CODEC: _unchunked(EMPTY_CONFIGURATIONS), + GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), + SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), + TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), + ZSTD_CODEC: _unchunked( + st.fixed_dictionaries( + {"level": st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL)}, + optional={"checksum": st.booleans()}, + ) + ), +} +"""A strategy for each codec that holds no other field.""" + +__all__ = ["CODEC_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py new file mode 100644 index 0000000000..dd06df28a0 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -0,0 +1,200 @@ +"""Strategies for each data type this package defines that holds no other field. + +A fill value is drawn in every spelling the data type's spec gives it: +a float's as a number, a named non-finite value or a hex string of its +bits, a `bytes` value's as an array of byte values or base64. +""" + +from __future__ import annotations + +import base64 +import struct +from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args + +from hypothesis import strategies as st + +from zarr_metadata._numpy import NUMPY_TIME_MAX_SCALE_FACTOR +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, DataTypeStrategy +from zarr_metadata.v3.data_type._float import FloatSpecialFillValue +from zarr_metadata.v3.data_type._numpy_time import NUMPY_TIME_UNIT +from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE +from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE +from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE +from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE +from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE +from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration +from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE +from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE +from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE +from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE +from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import DataTypeDefinition + +FloatWidth = Literal[16, 32, 64] + +INTEGER_RANGES: Final[Mapping[DataTypeDefinition[Any], tuple[int, int]]] = { + INT8_DATA_TYPE: (-(2**7), 2**7 - 1), + INT16_DATA_TYPE: (-(2**15), 2**15 - 1), + INT32_DATA_TYPE: (-(2**31), 2**31 - 1), + INT64_DATA_TYPE: (-(2**63), 2**63 - 1), + UINT8_DATA_TYPE: (0, 2**8 - 1), + UINT16_DATA_TYPE: (0, 2**16 - 1), + UINT32_DATA_TYPE: (0, 2**32 - 1), + UINT64_DATA_TYPE: (0, 2**64 - 1), +} +"""The values each integer data type holds.""" + +FLOAT_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + FLOAT16_DATA_TYPE: 16, + FLOAT32_DATA_TYPE: 32, + FLOAT64_DATA_TYPE: 64, +} +"""The width in bits of each floating-point data type.""" + +COMPLEX_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + COMPLEX64_DATA_TYPE: 32, + COMPLEX128_DATA_TYPE: 64, +} +"""The width in bits of each part of each complex data type.""" + +_MAX_RAW_BYTES: Final = 32 +"""The widest raw bits drawn, in bytes: `r256`.""" + +_MAX_BYTES: Final = 16 +"""The longest `bytes` fill value drawn.""" + + +def finite_floats(width: FloatWidth) -> st.SearchStrategy[float]: + """Finite numbers a float of `width` bits holds exactly.""" + return st.floats(width=width, allow_nan=False, allow_infinity=False) + + +_PACKED: Final[Mapping[FloatWidth, str]] = {16: ">e", 32: ">f", 64: ">d"} +"""How `struct` packs a float of each width, most significant byte first.""" + + +def _hex_spellings(digits: str) -> st.SearchStrategy[str]: + """`0x` and the digits, the letters in either case.""" + return st.sampled_from((f"0x{digits}", f"0x{digits.upper()}")) + + +def float_spellings(value: float, width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """`value`, a float of `width` bits, as a number or as the hex string of its bits.""" + return st.just(value) | _hex_spellings(struct.pack(_PACKED[width], value).hex()) + + +def _float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A number, an integer the type holds exactly, a named non-finite value, or the bits in hex. + + A float's fill value is any of these + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79). + Numbers are drawn from those the type holds, so none is rounded: a + `float16` takes `1e300` by the letter of the spec, and a reader is + free to make of it what it will. + """ + # Every integer of magnitude up to 2**11 is a float16, the narrowest type. + numbers = finite_floats(width) | st.integers(-(2**11), 2**11) + named = st.sampled_from(get_args(FloatSpecialFillValue)) + bits = st.integers(0, 2**width - 1).flatmap( + lambda value: _hex_spellings(f"{value:0{width // 4}x}") + ) + return st.one_of(numbers, named, bits) + + +def _fill_values_of(values: st.SearchStrategy[JSONValue]) -> DataTypeStrategy: + """A data type with nothing to configure, whose fill values are `values`.""" + return DataTypeStrategy( + configurations=EMPTY_CONFIGURATIONS, fill_values=lambda _configuration: values + ) + + +def _byte_values(size: int) -> st.SearchStrategy[JSONValue]: + """`size` byte values, each an integer in `[0, 255]`, as a JSON array.""" + return st.lists(st.integers(0, 255), min_size=size, max_size=size).map( + lambda values: cast("JSONValue", tuple(values)) + ) + + +def _raw_bytes_fill_values(configuration: Mapping[str, JSONValue]) -> st.SearchStrategy[JSONValue]: + """One byte value for each 8 bits of the size.""" + return _byte_values(cast("RawBytesConfiguration", configuration)["bits"] // 8) + + +def _base64(value: bytes) -> JSONValue: + return base64.b64encode(value).decode("ascii") + + +def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A pair of float fill values of `width` bits, real then imaginary.""" + component = _float_fill_values(width) + return st.tuples(component, component).map(lambda pair: cast("JSONValue", pair)) + + +_BYTES_FILL_VALUES: Final = st.integers(0, _MAX_BYTES).flatmap(_byte_values) | st.binary( + max_size=_MAX_BYTES +).map(_base64) +"""A `bytes` fill value: byte values, or the bytes in standard-alphabet base64.""" + +_TEXT: Final = st.text(st.characters(exclude_categories=("Cs",)), max_size=16) +"""Text a JSON document can hold: no lone surrogate, which is no Unicode scalar value.""" + +_NUMPY_TIME: Final = DataTypeStrategy( + configurations=st.fixed_dictionaries( + { + "unit": st.sampled_from(NUMPY_TIME_UNIT), + "scale_factor": st.integers(1, NUMPY_TIME_MAX_SCALE_FACTOR), + } + ), + fill_values=lambda _configuration: st.integers(-(2**63), 2**63 - 1) | st.just("NaT"), +) +"""Both numpy time types: a unit, and how many of it one tick is; a signed 64-bit count of ticks, or `"NaT"`.""" + +DATA_TYPE_STRATEGIES: Final[Mapping[DataTypeDefinition[Any], DataTypeStrategy]] = { + BOOL_DATA_TYPE: _fill_values_of(st.booleans()), + **{ + definition: _fill_values_of(st.integers(low, high)) + for definition, (low, high) in INTEGER_RANGES.items() + }, + **{ + definition: _fill_values_of(_float_fill_values(width)) + for definition, width in FLOAT_WIDTHS.items() + }, + **{ + definition: _fill_values_of(_complex_fill_values(width)) + for definition, width in COMPLEX_WIDTHS.items() + }, + RAW_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=st.integers(1, _MAX_RAW_BYTES).map( + lambda size: RawBytesConfiguration(bits=8 * size) + ), + fill_values=_raw_bytes_fill_values, + ), + BYTES_DATA_TYPE: _fill_values_of(_BYTES_FILL_VALUES), + STRING_DATA_TYPE: _fill_values_of(_TEXT), + NUMPY_DATETIME64_DATA_TYPE: _NUMPY_TIME, + NUMPY_TIMEDELTA64_DATA_TYPE: _NUMPY_TIME, +} +"""A strategy for each data type that holds no other field.""" + +__all__ = [ + "COMPLEX_WIDTHS", + "DATA_TYPE_STRATEGIES", + "FLOAT_WIDTHS", + "INTEGER_RANGES", + "FloatWidth", + "finite_floats", + "float_spellings", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py new file mode 100644 index 0000000000..73de0f8f5f --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -0,0 +1,147 @@ +"""What draws one definition's configurations: a record for each kind of definition. + +Each record holds what its kind's configurations depend on, the way +each kind of definition does: a data type's fill values depend on its +configuration, a chunk grid's on the shape of the array, a codec's on +the chunk it is handed. A chunk key encoding's depend on nothing. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, cast + +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata.v3._definition import ( + Chunk, + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + DataTypeDefinition, +) + +if TYPE_CHECKING: + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.definition._definition import Definition + +Configurations: TypeAlias = "SearchStrategy[Mapping[str, JSONValue]]" +"""Configurations of one definition, each the mapping every configuration is.""" + +NOTHING_KNOWN: Final = Chunk() +"""A chunk nothing is known of, which every codec takes.""" + +MAX_RANK: Final = 5 +"""The most axes drawn where the number of axes is not known.""" + +EMPTY_CONFIGURATIONS: Final = st.builds(EmptyConfiguration) +"""A fresh empty configuration for each draw, so a test that changes one changes no other.""" + + +class DefinitionStrategy(ABC): + """How to draw the configurations of one kind of definition.""" + + __slots__ = () + + kind: ClassVar[type[Definition[Any]]] + """The kind of definition whose configurations it draws.""" + + @abstractmethod + def unconstrained(self) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DataTypeStrategy(DefinitionStrategy): + """How to draw a data type's configurations, and the fill values an array of one takes.""" + + kind: ClassVar[type[Definition[Any]]] = DataTypeDefinition + + configurations: Configurations + """Configurations its definition accepts.""" + fill_values: Callable[[Mapping[str, JSONValue]], SearchStrategy[JSONValue]] + """Fill values of the data type a configuration describes.""" + + def unconstrained(self) -> Configurations: + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkGridStrategy(DefinitionStrategy): + """How to draw a chunk grid's configurations over an array of a shape.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkGridDefinition + + configurations: Callable[[tuple[int, ...] | None], Configurations] + """Configurations of grids that fit an array of the shape. + + Handed None, no shape is known, and what is drawn is what the grid's + rules accept, whatever shape it is put over. + """ + + def unconstrained(self) -> Configurations: + return self.configurations(None) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkKeyEncodingStrategy(DefinitionStrategy): + """How to draw a chunk key encoding's configurations.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkKeyEncodingDefinition + + configurations: Configurations + """Configurations its definition accepts.""" + + def unconstrained(self) -> Configurations: + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CodecStrategy(DefinitionStrategy): + """How to draw a codec's configurations, handed a chunk. + + Every codec takes `Chunk()`, a chunk nothing is known of, so a + strategy that draws nothing for it is refused when it is built. + """ + + kind: ClassVar[type[Definition[Any]]] = CodecDefinition + + configurations: Callable[[Chunk], Configurations | None] + """Configurations of the codec handed the chunk; None when no configuration takes it. + + A `bytes` codec handed a `string` chunk is None, since the spec gives + that data type a codec of its own. + """ + + def __post_init__(self) -> None: + if self.configurations(NOTHING_KNOWN) is None: + msg = "a codec strategy draws configurations for Chunk(), a chunk nothing is known of; got None" + raise ValueError(msg) + + def unconstrained(self) -> Configurations: + return cast("Configurations", self.configurations(NOTHING_KNOWN)) + + +Strategies: TypeAlias = Mapping[Any, DefinitionStrategy] +"""How to draw each definition's configurations, by the definition. + +The key is `Any`, not `Definition[Any]`: a mapping's key type is +invariant, so `{MY_CODEC: CodecStrategy(...)}`, a mapping keyed by a +`CodecDefinition`, would not be a `Mapping[Definition[Any], ...]`. +""" + +__all__ = [ + "NOTHING_KNOWN", + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "Configurations", + "DataTypeStrategy", + "DefinitionStrategy", + "Strategies", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py new file mode 100644 index 0000000000..fd13c43839 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -0,0 +1,299 @@ +"""Hypothesis strategies for valid Zarr v3 metadata fields. + +Every strategy here draws metadata its definition accepts, and that the +spec allows: a reader, writer or converter can test itself against the +whole space of a codec's, data type's, chunk grid's or chunk key +encoding's configurations instead of the handful its authors thought to +write down. + +```python +from hypothesis import given +from zarr_metadata.testing.strategies import fields +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC + +@given(fields(BLOSC_CODEC)) +def test_my_reader_reads_every_blosc_codec(field): + my_reader.read_codec(field) +``` + +What a configuration depends on, each kind's function takes: +`fill_values` a data type's configuration, `chunk_grid_configurations` +the shape of the array, `codec_configurations` the `Chunk` the codec is +handed. Left out, nothing is known of it, and what is drawn is what the +definition's rules accept. `fields` wraps configurations in the +envelopes a document writes them in. + +How each definition's configurations are drawn is a value, +`Strategies`, keyed by the definition: `DEFAULT_STRATEGIES` holds one +for each definition this package defines that holds no other field. An +extension author adds their own with +`DEFAULT_STRATEGIES | {MY_CODEC: CodecStrategy(...)}`. + +Configurations are drawn at the data level the TypedDicts declare, JSON +arrays as tuples; `json.dumps` takes them as they are. + +Requires the `testing` extra (`pip install zarr-metadata[testing]`). +""" + +from __future__ import annotations + +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast + +from hypothesis import strategies as st + +from zarr_metadata._sentinel import UNSET +from zarr_metadata.definition._definition import Definition, as_kind, canonicalize, kind_of +from zarr_metadata.testing._chunk_grid import CHUNK_GRID_STRATEGIES +from zarr_metadata.testing._chunk_key_encoding import CHUNK_KEY_ENCODING_STRATEGIES +from zarr_metadata.testing._codec import CODEC_STRATEGIES +from zarr_metadata.testing._data_type import DATA_TYPE_STRATEGIES +from zarr_metadata.testing._strategy import ( + NOTHING_KNOWN, + ChunkGridStrategy, + ChunkKeyEncodingStrategy, + CodecStrategy, + Configurations, + DataTypeStrategy, + DefinitionStrategy, + Strategies, +) +from zarr_metadata.v3._context import ZarrV3Context +from zarr_metadata.v3._core import CORE +from zarr_metadata.v3._definition import Chunk, DataTypeDefinition + +if TYPE_CHECKING: + from collections.abc import Mapping + + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3 import ZarrV3MetadataFieldJSON + from zarr_metadata.v3._context import ZarrV3ContextSlots + from zarr_metadata.v3._definition import ( + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + ) + +C = TypeVar("C") + +Spellings = Literal["all", "canonical"] +"""How fields are written: every spelling the spec allows, or only the simplest. + +`"all"` draws each envelope the spec gives a field -- a bare name, an +object without a configuration, an empty configuration, a +`must_understand` of `true` -- and configurations that say what a +simpler one says, as a `blosc` `typesize` under `noshuffle` does. +`"canonical"` draws each in the spelling `canonicalize` gives it, which +is what a writer that writes the fewest words writes. +""" + +DEFAULT_STRATEGIES: Final[MappingProxyType[Definition[Any], DefinitionStrategy]] = MappingProxyType( + { + **DATA_TYPE_STRATEGIES, + **CHUNK_GRID_STRATEGIES, + **CHUNK_KEY_ENCODING_STRATEGIES, + **CODEC_STRATEGIES, + } +) +"""A strategy for each definition this package defines that holds no other field.""" + + +def _strategy_for(definition: Definition[Any], strategies: Strategies) -> DefinitionStrategy: + """The strategy `strategies` holds for `definition`, which draws configurations of its kind.""" + found = strategies.get(definition) + if found is None: + msg = f"no strategy for {definition.name!r}; pass one in `strategies`" + raise LookupError(msg) + if not isinstance(definition, found.kind): + msg = ( + f"the strategy for {definition.name!r}, a {type(definition).__name__}, draws " + f"configurations of a {found.kind.__name__}: {found!r}" + ) + raise TypeError(msg) + return found + + +def data_type_configurations( + definition: DataTypeDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the data type `definition` defines.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def fill_values( + definition: DataTypeDefinition[C], + configuration: C, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[JSONValue]: + """Fill values of the data type `definition` and `configuration` describe, in every spelling the spec gives them.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return found.fill_values(cast("Mapping[str, JSONValue]", configuration)) + + +def chunk_grid_configurations( + definition: ChunkGridDefinition[C], + shape: tuple[int, ...] | None = None, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C]: + """Configurations of the chunk grid `definition` defines, that fit an array of `shape`. + + With no shape, what is drawn is what the grid's rules accept. + """ + found = cast("ChunkGridStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations(shape)) + + +def chunk_key_encoding_configurations( + definition: ChunkKeyEncodingDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the chunk key encoding `definition` defines.""" + found = cast("ChunkKeyEncodingStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def codec_configurations( + definition: CodecDefinition[C], + chunk: Chunk = NOTHING_KNOWN, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C] | None: + """Configurations of the codec `definition` defines, handed `chunk`; None when none takes it. + + With no chunk, nothing is known of it, and what is drawn is what the + codec's rules accept. + """ + found = cast("CodecStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C] | None", found.configurations(chunk)) + + +def _written_bare(definition: Definition[Any]) -> bool: + """Whether the spec has the field written as its bare name: a data type it defines itself. + + "If the data type is defined in this specification, then the value + must be the data type identifier provided as a string" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L509-L512). + """ + if not isinstance(definition, DataTypeDefinition): + return False + return CORE.claimant(DataTypeDefinition, definition.name) is definition + + +@st.composite +def _any_spelling( + draw: st.DrawFn, definition: Definition[Any], configurations: SearchStrategy[Any] +) -> ZarrV3MetadataFieldJSON: + """A configuration of `definition` in one of the envelopes the spec allows it. + + A configuration with nothing in it is written as the bare name, an + object without one, or an object with an empty one; any object may + say `must_understand` is `true`, which is what absence means. A name + that carries the configuration, as `r16` does, and a data type the + spec defines itself, are written as the bare name. + """ + configuration = cast("Mapping[str, JSONValue]", draw(configurations)) + carried = definition._carrying_name(configuration) # pyright: ignore[reportPrivateUsage] + if carried is not None: + return carried + if _written_bare(definition): + return definition.name + empty = len(configuration) == 0 + if empty and draw(st.booleans()): + return definition.name + field: dict[str, JSONValue] = {"name": definition.name} + if not empty or draw(st.booleans()): + field["configuration"] = configuration + if draw(st.booleans()): + field["must_understand"] = True + return cast("ZarrV3MetadataFieldJSON", field) + + +def _scope_of(definition: Definition[Any]) -> ZarrV3Context: + """The scope that claims `definition` and nothing else; a `TypeError` for a definition no slot of a Zarr v3 scope holds.""" + for slot, kind in ZarrV3Context.kinds.items(): + if isinstance(definition, kind): + return ZarrV3Context(**cast("ZarrV3ContextSlots", {slot: (definition,)})) + msg = ( + f"{definition.name!r} is a {type(definition).__name__}, " + "which no slot of a Zarr v3 scope holds" + ) + raise TypeError(msg) + + +def _checked( + field: ZarrV3MetadataFieldJSON, + *, + definition: Definition[Any], + kind: type[Definition[Any]], + scope: ZarrV3Context, + spellings: Spellings, +) -> ZarrV3MetadataFieldJSON: + """`field`, or its canonical spelling, once it reads without a problem in `scope`; a `ValueError` when it does not.""" + canonical, problems = canonicalize(field, kind, scope) + if len(problems) != 0: + msg = f"{field!r}, drawn for {definition.name!r}, which is refused: {problems!r}" + raise ValueError(msg) + if spellings == "canonical": + return cast("ZarrV3MetadataFieldJSON", canonical) + return field + + +def fields( + definition: Definition[C], + configurations: SearchStrategy[C] | UNSET = UNSET, + *, + spellings: Spellings = "all", + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[ZarrV3MetadataFieldJSON]: + """Metadata fields of `definition`: its name and a configuration, in an envelope a document writes. + + `configurations` is what to draw the configuration from, by default + the definition's own with nothing known of what they depend on: pass + `codec_configurations(definition, chunk)` for a codec handed a chunk, + once it is known not to be None. `spellings` says whether every + spelling the spec allows is drawn, or only the simplest. + + Each field is read in a scope of its definition alone, with no chunk + known, and one its definition's rules refuse, as they refuse one + drawn from `configurations` they do not accept, raises a + `ValueError`. A codec's chunk rules are not checked: what fits the + chunk is what `codec_configurations` draws. + """ + drawn: Configurations | SearchStrategy[C] = ( + _strategy_for(definition, strategies).unconstrained() + if configurations is UNSET + else configurations + ) + checked = partial( + _checked, + definition=definition, + kind=as_kind(kind_of(definition)), + scope=_scope_of(definition), + spellings=spellings, + ) + return _any_spelling(definition, drawn).map(checked) + + +__all__ = [ + "DEFAULT_STRATEGIES", + "NOTHING_KNOWN", + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "Configurations", + "DataTypeStrategy", + "DefinitionStrategy", + "Spellings", + "Strategies", + "chunk_grid_configurations", + "chunk_key_encoding_configurations", + "codec_configurations", + "data_type_configurations", + "fields", + "fill_values", +] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index e0095aec31..0baad413c2 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -277,6 +277,9 @@ def test_all_is_grouped_and_unique() -> None: # example, `ZarrV3CodecField`, read in the same scope as the field that # contains it. "Field", + # How to draw one kind of definition's configurations, in + # `zarr_metadata.testing.strategies`: `CodecStrategy`. + "Strategy", ) _EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") @@ -294,6 +297,9 @@ def test_all_is_grouped_and_unique() -> None: # two definitions of one name. "ClaimKey", "Conflict", + # What one definition's configurations are drawn from, in + # `zarr_metadata.testing.strategies`. + "Configurations", "ScopeConflictError", # A reader's problems, as the signatures of every reader name them. "Problems", @@ -339,6 +345,8 @@ def test_all_is_grouped_and_unique() -> None: "ScalarMap", "ScalarMapEntry", "ShardingIndexLocation", + "Spellings", + "Strategies", "Struct", "StructField", "ValidationProblem", diff --git a/packages/zarr-metadata/tests/testing/__init__.py b/packages/zarr-metadata/tests/testing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py new file mode 100644 index 0000000000..c0d8aa0e3d --- /dev/null +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -0,0 +1,432 @@ +"""The strategies against the definitions they draw for, and against what they ought to reach. + +Every draw is read in `CORE_AND_EXTENSIONS` and must read without a +problem, by the definition it was drawn for: the field alone, a fill +value beside its data type, a chunk grid over the shape it was drawn +for, a codec handed the chunk it was drawn for. A strategy that only +drew valid metadata could draw almost none, so reach is checked too, +with targets worked out from each configuration's TypedDict rather than +from the strategies: every optional key present and absent, every +`Literal` value, and for a definition with a simplest spelling, a +configuration it simplifies. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Literal, cast, get_args, get_origin + +import pytest +from hypothesis import event, find, given, settings +from hypothesis import strategies as st + +from zarr_metadata._typed_json import typeddict_keys +from zarr_metadata.definition import ( + AcceptedField, + Definition, + EmptyConfiguration, + fill_value_problems, + resolve, +) +from zarr_metadata.definition._definition import as_kind, canonicalize, kind_of, unchanged +from zarr_metadata.testing.strategies import ( + DEFAULT_STRATEGIES, + ChunkKeyEncodingStrategy, + CodecStrategy, + Spellings, + chunk_grid_configurations, + codec_configurations, + data_type_configurations, + fields, + fill_values, +) +from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v3._definition import chunk_grid_lengths +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC +from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC +from zarr_metadata.v3.data_type._float import FloatSpecialFillValue +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE +from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE +from zarr_metadata.v3.definition import ( + CORE, + CORE_AND_EXTENSIONS, + Chunk, + ChunkGridDefinition, + CodecDefinition, + DataTypeDefinition, + ZarrV3Context, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3 import ZarrV3MetadataFieldJSON + +_HOLDING_FIELDS = frozenset({SHARDING_INDEXED_CODEC, CAST_VALUE_CODEC, STRUCT_DATA_TYPE}) +"""The definitions whose configurations hold other fields, which have no strategy yet.""" + +_IN_SCOPE = frozenset( + ( + *CORE_AND_EXTENSIONS.data_types.values(), + *CORE_AND_EXTENSIONS.chunk_grids.values(), + *CORE_AND_EXTENSIONS.chunk_key_encodings.values(), + *CORE_AND_EXTENSIONS.codecs.values(), + *CORE_AND_EXTENSIONS.storage_transformers.values(), + ) +) +"""The definitions at the spec's extension points that `CORE_AND_EXTENSIONS` holds; members of a document it defines are not fields.""" +_CORE_DATA_TYPES = frozenset(CORE.data_types.values()) + +_DEFINITIONS = sorted(DEFAULT_STRATEGIES, key=lambda definition: definition.name) +_DATA_TYPES = [d for d in _DEFINITIONS if isinstance(d, DataTypeDefinition)] +_CHUNK_GRIDS = [d for d in _DEFINITIONS if isinstance(d, ChunkGridDefinition)] +_CODECS = [d for d in _DEFINITIONS if isinstance(d, CodecDefinition)] + +_SHAPES = st.lists(st.integers(0, 12), max_size=4).map(tuple) + + +def _id(value: object) -> str: + if isinstance(value, Definition): + return f"{type(value).__name__}:{value.name}" + return str(value) + + +def _read(definition: Definition[Any], field: object) -> Any: # noqa: ANN401 + """`field` read in `CORE_AND_EXTENSIONS`, asserted to read without a problem by `definition`.""" + resolved, problems = resolve(field, as_kind(kind_of(definition)), CORE_AND_EXTENSIONS) + assert problems == () + assert isinstance(resolved, AcceptedField) + assert resolved.definition is definition + return resolved + + +def test_every_definition_has_a_strategy() -> None: + """Each definition in scope has a strategy, but those that hold other fields; and none is out of scope.""" + assert _IN_SCOPE - frozenset(DEFAULT_STRATEGIES) == _HOLDING_FIELDS + assert frozenset(DEFAULT_STRATEGIES) <= _IN_SCOPE + + +@pytest.mark.parametrize("spellings", get_args(Spellings)) +@pytest.mark.parametrize("definition", _DEFINITIONS, ids=_id) +@given(data=st.data()) +def test_fields_read( + definition: Definition[Any], spellings: Spellings, data: st.DataObject +) -> None: + """A drawn field reads by its definition, through JSON too; a canonical one is its own simplest spelling.""" + field = data.draw(fields(definition, spellings=spellings)) + _read(definition, field) + _read(definition, json.loads(json.dumps(field))) + if definition in _CORE_DATA_TYPES: + assert isinstance(field, str) + if spellings == "canonical": + canonical, _ = canonicalize(field, as_kind(kind_of(definition)), CORE_AND_EXTENSIONS) + assert canonical == field + + +@pytest.mark.parametrize("definition", _DEFINITIONS, ids=_id) +@given(data=st.data()) +def test_a_changed_draw_changes_no_other(definition: Definition[Any], data: st.DataObject) -> None: + """Each draw is a fresh value: a test that changes the configuration it drew does not change the next draw, which Hypothesis would report as flaky.""" + field = data.draw(fields(definition)) + if isinstance(field, dict) and "configuration" in field: + configuration = cast("dict[str, JSONValue]", field["configuration"]) + assert "changed" not in configuration + configuration["changed"] = True + + +@pytest.mark.parametrize("definition", _DATA_TYPES, ids=_id) +@given(data=st.data()) +def test_fill_values_fit_their_data_type( + definition: DataTypeDefinition[Any], data: st.DataObject +) -> None: + configuration = data.draw(data_type_configurations(definition)) + data_type = _read(definition, data.draw(fields(definition, st.just(configuration)))) + value = data.draw(fill_values(definition, configuration)) + assert fill_value_problems(data_type, value) == () + assert fill_value_problems(data_type, json.loads(json.dumps(value))) == () + + +@pytest.mark.parametrize("definition", _CHUNK_GRIDS, ids=_id) +@given(data=st.data(), shape=_SHAPES) +def test_chunk_grids_fit_their_shape( + definition: ChunkGridDefinition[Any], data: st.DataObject, shape: tuple[int, ...] +) -> None: + configurations = chunk_grid_configurations(definition, shape) + chunk_grid = _read(definition, data.draw(fields(definition, configurations))) + lengths, problems = chunk_grid_lengths(chunk_grid, shape) + assert problems == () + # "Chunk sizes must be greater than zero", which the regular grid's rules do not yet refuse. + assert all(0 not in axis for axis in lengths if axis is not None) + + +@st.composite +def _chunks(draw: st.DrawFn) -> Chunk: + """A chunk of a data type drawn from the strategies, or of none known, with lengths known along some axes, all, or none.""" + definition = draw(st.none() | st.sampled_from(_DATA_TYPES)) + data_type = None if definition is None else _read(definition, draw(fields(definition))) + axis = st.none() | st.integers(1, 8).map(lambda length: frozenset({length})) + lengths = draw(st.none() | st.lists(axis, max_size=4).map(tuple)) + return Chunk(lengths, data_type) + + +@pytest.mark.parametrize("definition", _CODECS, ids=_id) +@given(data=st.data(), chunk=_chunks()) +def test_codecs_take_their_chunk( + definition: CodecDefinition[Any], data: st.DataObject, chunk: Chunk +) -> None: + """A codec drawn for a chunk takes it; one drawn for none is one the spec does not give it, which needs the data type known.""" + configurations = codec_configurations(definition, chunk) + if configurations is None: + assert chunk.data_type is not None + assert chunk.data_type.definition is not None + data_type_name = chunk.data_type.definition.name + event(f"takes no {data_type_name}") + codec = _read(definition, data.draw(fields(definition))) + assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) != [] + return + codec = _read(definition, data.draw(fields(definition, configurations))) + assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) == [] + + +Target = tuple[Literal["present", "absent"], str] | tuple[Literal["value"], str, object] +FieldForm = Literal["bare", "object", "configured", "must_understand"] + + +def _configuration_targets( + definition: Definition[Any], +) -> list[Target | tuple[Literal["simplified"]]]: + """What a strategy for `definition` ought to reach, by its configuration's TypedDict.""" + targets: list[Target | tuple[Literal["simplified"]]] = [] + for key, (annotation, required) in typeddict_keys(definition.configuration).members.items(): + if not required: + targets.extend([("present", key), ("absent", key)]) + if get_origin(annotation) is Literal: + targets.extend(("value", key, value) for value in get_args(annotation)) + if definition.canonical is not unchanged: + targets.append(("simplified",)) + return targets + + +def _field_forms(definition: Definition[Any]) -> list[FieldForm]: + """The envelopes the spec allows a field of `definition` to be written in.""" + if definition in _CORE_DATA_TYPES: + return [] # only ever bare, which `test_fields_read` asserts + if definition.requires_configuration: + return ["configured", "must_understand"] + return ["bare", "object", "configured", "must_understand"] + + +def _form_of(field: ZarrV3MetadataFieldJSON) -> set[FieldForm]: + if isinstance(field, str): + return {"bare"} + forms: set[FieldForm] = {"configured" if "configuration" in field else "object"} + if "must_understand" in field: + forms.add("must_understand") + return forms + + +def _hits( + definition: Definition[Any], + target: Target | tuple[Literal["simplified"]], + configuration: Mapping[str, JSONValue], +) -> bool: + match target: + case ("present", key): + return key in configuration + case ("absent", key): + return key not in configuration + case ("value", key, value): + return configuration.get(key) == value + case ("simplified",): + return definition.canonical(configuration) != configuration + + +_REACH = settings(max_examples=2000, database=None) + + +@pytest.mark.parametrize( + ("definition", "target"), + [(d, target) for d in _DEFINITIONS for target in _configuration_targets(d)], + ids=_id, +) +def test_configurations_reach( + definition: Definition[Any], target: Target | tuple[Literal["simplified"]] +) -> None: + configurations = fields(definition).map(lambda field: _read(definition, field).configuration) + find( + configurations, + lambda configuration: _hits(definition, target, configuration), + settings=_REACH, + ) + + +@pytest.mark.parametrize( + ("definition", "form"), + [(d, form) for d in _DEFINITIONS for form in _field_forms(d)], + ids=_id, +) +def test_fields_reach(definition: Definition[Any], form: FieldForm) -> None: + find(fields(definition), lambda field: form in _form_of(field), settings=_REACH) + + +def test_an_added_strategy_is_drawn_from() -> None: + """A strategy joined to the defaults, as an extension author adds one, is the one drawn from.""" + level_three = CodecStrategy(configurations=lambda _chunk: st.just({"level": 3})) + strategies = DEFAULT_STRATEGIES | {GZIP_CODEC: level_three} + field = find(fields(GZIP_CODEC, strategies=strategies), lambda _field: True, settings=_REACH) + assert _read(GZIP_CODEC, field).configuration == {"level": 3} + + +def test_a_definition_without_a_strategy_is_a_lookup_error() -> None: + with pytest.raises(LookupError, match="no strategy for 'sharding_indexed'"): + fields(SHARDING_INDEXED_CODEC) + + +def test_a_strategy_of_another_kind_is_a_type_error() -> None: + # Unannotated, as an extension author writes it: a mapping keyed by a + # `CodecDefinition` is a `Strategies`. + strategies = {GZIP_CODEC: ChunkKeyEncodingStrategy(configurations=st.just({}))} + with pytest.raises(TypeError, match="draws configurations of a ChunkKeyEncodingDefinition"): + codec_configurations(GZIP_CODEC, strategies=strategies) + + +def test_a_codec_strategy_refusing_an_unknown_chunk_is_a_value_error() -> None: + with pytest.raises(ValueError, match=r"Chunk\(\), a chunk nothing is known of"): + CodecStrategy(configurations=lambda _chunk: None) + + +@pytest.mark.parametrize("spellings", get_args(Spellings)) +def test_a_refused_draw_is_a_value_error(spellings: Spellings) -> None: + refused = fields(GZIP_CODEC, st.just({"level": 99}), spellings=spellings) + with pytest.raises(ValueError, match="which is refused"): + find(refused, lambda _field: True, settings=settings(database=None)) + + +_BFLOAT16 = DataTypeDefinition(name="bfloat16", configuration=EmptyConfiguration) +"""A number type this package does not define, as an extension author would.""" + + +@given(data=st.data()) +def test_scale_offset_takes_a_data_type_the_package_does_not_define(data: st.DataObject) -> None: + """Values of a type the package does not define cannot be drawn, so the configuration is the identity, which its rules take.""" + data_type, problems = resolve( + "bfloat16", DataTypeDefinition, ZarrV3Context(data_types=[_BFLOAT16]) + ) + assert problems == () + chunk = Chunk(None, data_type) + configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) + assert configurations is not None + configuration = data.draw(configurations) + assert configuration == {} + assert list(SCALE_OFFSET_CODEC.chunk_rules(configuration, {}, chunk)) == [] + + +def test_a_definition_no_v3_scope_holds_is_a_type_error() -> None: + zlib = CORE_V2.codecs["zlib"] + with pytest.raises(TypeError, match="which no slot of a Zarr v3 scope holds"): + fields(zlib, st.just({})) + + +@pytest.mark.parametrize( + "definition", [FLOAT16_DATA_TYPE, FLOAT32_DATA_TYPE, FLOAT64_DATA_TYPE], ids=_id +) +@pytest.mark.parametrize("member", ["offset", "scale"]) +def test_scale_offset_reaches_a_value_written_in_hex( + definition: DataTypeDefinition[Any], member: str +) -> None: + """A float's `offset` and `scale` are fill values of its type, which the bits in hex spell too.""" + chunk = Chunk(None, _read(definition, definition.name)) + configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) + assert configurations is not None + find( + configurations, + lambda configuration: isinstance(configuration.get(member), str), + settings=_REACH, + ) + + +def _has_pair(configuration: Mapping[str, Any]) -> bool: + return any( + isinstance(entry, tuple) + for dimension in configuration["chunk_shapes"] + if isinstance(dimension, tuple) + for entry in cast("tuple[object, ...]", dimension) + ) + + +def test_rectilinear_edges_reach_a_run_written_as_a_pair() -> None: + """The `[size, count]` spelling, which no TypedDict target asks for: it is one arm of a union.""" + find(chunk_grid_configurations(RECTILINEAR_CHUNK_GRID), _has_pair, settings=_REACH) + + +FillValueSpelling = Literal[ + "number", "named", "hex lowercase", "hex uppercase", "array", "base64", "NaT" +] + +_FILL_VALUE_SPELLINGS: list[tuple[DataTypeDefinition[Any], FillValueSpelling]] = [ + *( + (definition, spelling) + for definition in ( + FLOAT16_DATA_TYPE, + FLOAT32_DATA_TYPE, + FLOAT64_DATA_TYPE, + COMPLEX64_DATA_TYPE, + COMPLEX128_DATA_TYPE, + ) + for spelling in get_args(FillValueSpelling)[:4] + ), + (BYTES_DATA_TYPE, "array"), + (BYTES_DATA_TYPE, "base64"), + (NUMPY_DATETIME64_DATA_TYPE, "number"), + (NUMPY_DATETIME64_DATA_TYPE, "NaT"), + (NUMPY_TIMEDELTA64_DATA_TYPE, "number"), + (NUMPY_TIMEDELTA64_DATA_TYPE, "NaT"), +] +"""The spellings the specs give a fill value, by the data types whose fill values take them.""" + + +def _spellings_of(value: object) -> set[FillValueSpelling]: + """How a fill value is spelled; a complex one, how either component is.""" + if isinstance(value, bool): + return set() + if isinstance(value, int | float): + return {"number"} + if isinstance(value, tuple): + parts = cast("tuple[object, ...]", value) + if all(isinstance(part, int) for part in parts): + return {"array"} + return {spelling for part in parts for spelling in _spellings_of(part)} + if not isinstance(value, str): + return set() + if value == "NaT": + return {"NaT"} + if value in get_args(FloatSpecialFillValue): + return {"named"} + if value.startswith("0x"): + digits = value[2:] + return { + *(("hex lowercase",) if any(c in "abcdef" for c in digits) else ()), + *(("hex uppercase",) if any(c in "ABCDEF" for c in digits) else ()), + } + return {"base64"} + + +@pytest.mark.parametrize(("definition", "spelling"), _FILL_VALUE_SPELLINGS, ids=_id) +def test_fill_values_reach( + definition: DataTypeDefinition[Any], spelling: FillValueSpelling +) -> None: + values = data_type_configurations(definition).flatmap( + lambda configuration: fill_values(definition, configuration) + ) + find(values, lambda value: spelling in _spellings_of(value), settings=_REACH) diff --git a/packages/zarr-metadata/uv.lock b/packages/zarr-metadata/uv.lock index 9c21f5a4c9..1679e33309 100644 --- a/packages/zarr-metadata/uv.lock +++ b/packages/zarr-metadata/uv.lock @@ -1233,6 +1233,11 @@ dependencies = [ { name = "typing-extensions" }, ] +[package.optional-dependencies] +testing = [ + { name = "hypothesis" }, +] + [package.dev-dependencies] docs = [ { name = "griffe-inherited-docstrings" }, @@ -1254,8 +1259,10 @@ test = [ [package.metadata] requires-dist = [ { name = "annotated-types", specifier = ">=0.6" }, + { name = "hypothesis", marker = "extra == 'testing'", specifier = ">=6.85" }, { name = "typing-extensions", specifier = ">=4.16" }, ] +provides-extras = ["testing"] [package.metadata.requires-dev] docs = [ From be4d9d4316fb81b259efede10dcf2967433fc317 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 1 Nov 2026 09:42:49 +0100 Subject: [PATCH 03/10] chore(deps): bump `cast-value.rs` (#4584) * chore(deps): bump cast-value.rs to >=0.5.2 * chore(deps): raise the runtime cast-value-rs floor to 0.5.1 The previous commit raised the packaging floor to 0.5.1 but left the runtime check at 0.4.2. cast-value-rs 0.5.0 makes casts exact at integer and float range limits; 0.5.0 was never published to PyPI, so the floor is 0.5.1. Also update the error message, the version-check test cases, and uv.lock. Assisted-by: ClaudeCode:claude-opus-5-5 * test(cast_value): derive version-check cases from the minimum The cases hard-coded versions near the floor, so every floor bump meant rewriting them. Build them from CAST_VALUE_RS_MIN_VERSION instead, and drop cases that repeated the same comparison. A floor bump now touches only pyproject.toml and the constant. Assisted-by: ClaudeCode:claude-opus-5-5 --- pyproject.toml | 2 +- src/zarr/codecs/cast_value.py | 14 +-- tests/test_codecs/test_cast_value.py | 17 ++-- uv.lock | 142 ++++++++++++++++----------- 4 files changed, 102 insertions(+), 73 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 136f757dac..c95ecb6ee8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,7 @@ remote = [ gpu = [ "cupy-cuda12x; sys_platform != 'darwin'", ] -cast-value-rs = ["cast-value-rs>=0.4.2"] +cast-value-rs = ["cast-value-rs>=0.5.1"] cli = ["typer"] optional = ["universal-pathlib"] diff --git a/src/zarr/codecs/cast_value.py b/src/zarr/codecs/cast_value.py index 629b4e7a27..3d465070b8 100644 --- a/src/zarr/codecs/cast_value.py +++ b/src/zarr/codecs/cast_value.py @@ -5,7 +5,7 @@ rounding, out-of-range handling, and explicit scalar mappings. Requires the optional ``cast-value-rs`` package for the actual casting -logic. Install it with: ``pip install 'cast-value-rs>=0.4.2'``. +logic. Install it with: ``pip install 'cast-value-rs>=0.5.1'``. """ from __future__ import annotations @@ -125,10 +125,12 @@ def parse_scalar_map(obj: ScalarMapJSON | ScalarMap) -> ScalarMap: # Backend: cast-value-rs # --------------------------------------------------------------------------- -# Versions below this silently transpose input arrays that are not row-major - +# Versions below 0.4.2 silently transpose input arrays that are not row-major - # the layout `transpose` hands to the next codec - writing corrupted data with -# no error. Keep in sync with the `cast-value-rs` extra in pyproject.toml. -CAST_VALUE_RS_MIN_VERSION: Final = "0.4.2" +# no error. Versions below 0.5.0 give wrong results for some casts at integer +# and float range limits. 0.5.0 was never published to PyPI, so the floor is +# 0.5.1. Keep in sync with the `cast-value-rs` extra in pyproject.toml. +CAST_VALUE_RS_MIN_VERSION: Final = "0.5.1" _INSTALL_HINT: Final = f"Install it with: pip install 'cast-value-rs>={CAST_VALUE_RS_MIN_VERSION}'" @@ -144,8 +146,8 @@ def _check_backend_version() -> str | None: if Version(installed) < Version(CAST_VALUE_RS_MIN_VERSION): return ( f"The cast_value codec requires cast-value-rs >= {CAST_VALUE_RS_MIN_VERSION}, " - f"but version {installed} is installed. Earlier versions silently corrupt data " - f"when the input array is not row-major. {_INSTALL_HINT}" + f"but version {installed} is installed. Earlier versions can write wrong " + f"values without raising an error. {_INSTALL_HINT}" ) return None diff --git a/tests/test_codecs/test_cast_value.py b/tests/test_codecs/test_cast_value.py index c2e78770d9..d6c703767f 100644 --- a/tests/test_codecs/test_cast_value.py +++ b/tests/test_codecs/test_cast_value.py @@ -9,7 +9,7 @@ import zarr from tests.conftest import Expect, ExpectFail from zarr.codecs import BytesCodec, TransposeCodec -from zarr.codecs.cast_value import CastValue +from zarr.codecs.cast_value import CAST_VALUE_RS_MIN_VERSION, CastValue from zarr.storage import MemoryStore try: @@ -490,15 +490,14 @@ def test_parse_scalar_map(case: Expect[Any, Any]) -> None: @pytest.mark.parametrize( "case", [ - Expect(input="0.4.2", output=None, id="exactly-minimum"), - Expect(input="0.4.3", output=None, id="newer-patch"), - Expect(input="0.5.0", output=None, id="newer-minor"), - Expect(input="1.0.0", output=None, id="newer-major"), - Expect(input="0.4.2.post1", output=None, id="post-release"), + Expect(input=CAST_VALUE_RS_MIN_VERSION, output=None, id="exactly-minimum"), + Expect(input=f"{CAST_VALUE_RS_MIN_VERSION}.post1", output=None, id="post-release"), + Expect( + input=f"{CAST_VALUE_RS_MIN_VERSION}.dev1", + output=f"{CAST_VALUE_RS_MIN_VERSION}.dev1", + id="pre-release-of-minimum", + ), Expect(input="0.4.0", output="0.4.0", id="known-corrupting"), - Expect(input="0.4.1", output="0.4.1", id="one-patch-below"), - Expect(input="0.3.0", output="0.3.0", id="older-minor"), - Expect(input="0.4.2.dev1", output="0.4.2.dev1", id="pre-release-of-minimum"), ], ids=lambda c: c.id, ) diff --git a/uv.lock b/uv.lock index e78182c154..05d88533ec 100644 --- a/uv.lock +++ b/uv.lock @@ -441,63 +441,91 @@ wheels = [ [[package]] name = "cast-value-rs" -version = "0.4.2" +version = "0.5.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/56/a4/634d1917ac49ea69291471e87eee9c1aa45d71d6da9b2cd5e1bfe82729de/cast_value_rs-0.4.2.tar.gz", hash = "sha256:fd621b8dd4f7e93bbffdb119882d85e2731c2f0fb1d30de85f29e8e04c044822", size = 86862, upload-time = "2026-08-13T09:16:59.199Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/c7/9db59d415df2d1c9f905c4da951fe023c6259a162584ef68bd81c171d448/cast_value_rs-0.4.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6d8e1c6d7b0fec6b1060f83867d20ebd0cb1ac6fa2954cd82f418a31a96ac504", size = 494354, upload-time = "2026-08-13T09:14:41.627Z" }, - { url = "https://files.pythonhosted.org/packages/8f/53/10ece0b4ba4a0c156674c2db7e5d05ce9b04efe72e191417546e111d645c/cast_value_rs-0.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8c953321ff7954e0fe1b25c1baf96f0a9cae0961b8a56ce63dc7b94e2ea7d85c", size = 458463, upload-time = "2026-08-13T09:14:43.371Z" }, - { url = "https://files.pythonhosted.org/packages/91/0b/e50ac2eb271eb6c34402b4abe317c795bc7d650d332d0bf422ae868aa2cd/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68492eef745691c80754cd027e216dbd5f7a4ce2e07fddfded3ee59bd947afb5", size = 487628, upload-time = "2026-08-13T09:14:44.86Z" }, - { url = "https://files.pythonhosted.org/packages/a9/0f/69befc7905af2e73a2739b26444787cd528331c041f37235f3853bf15f75/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fbf145ed6f724fa7f5ddc9715f78e0d450ac141fc05299c83b2e7d627793ddaf", size = 555538, upload-time = "2026-08-13T09:14:46.305Z" }, - { url = "https://files.pythonhosted.org/packages/4a/1b/4105fabec3268139bda3555422808211ec29428e17fa56566c1af7957c04/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73fb23256dda16e4e9f49bfedebe0e068f00ece85091b526c102618f66144216", size = 650896, upload-time = "2026-08-13T09:14:48.049Z" }, - { url = "https://files.pythonhosted.org/packages/83/2a/452cf37675af02c3ac8bfa01fc962cb744a1392c51bccfb6d1faca423067/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8184135c78ebcabb904574a1b3bc5704c131140feae1b58d65c888fe08b46029", size = 561036, upload-time = "2026-08-13T09:14:49.431Z" }, - { url = "https://files.pythonhosted.org/packages/d4/87/edb215d17da304a6b36489fc51b8eb785c5b041c2fcc5f200f16adde3133/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:686ea86aad1ffebd0b6793549043108600593a85032457eb88a3a0513a4100f5", size = 549187, upload-time = "2026-08-13T09:14:50.801Z" }, - { url = "https://files.pythonhosted.org/packages/63/82/0f9963ef0bcb434cd1244cb2620123c2d71763d429d017ceb98fbb6faf48/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d6c425d2803e13408755a0022faa9efd0ab0becdf3573a1a8a7af64b1adee4ba", size = 591027, upload-time = "2026-08-13T09:14:52.281Z" }, - { url = "https://files.pythonhosted.org/packages/90/97/b489f8048491f2ece21bf99d5a1fea3bbfaf3cd0e9f555ac4a0352c11d81/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:92556f1bb9d6b222736d967228565ec745c640c29bc291d1cb730a1011d3dcac", size = 665201, upload-time = "2026-08-13T09:14:53.769Z" }, - { url = "https://files.pythonhosted.org/packages/ce/9f/fabffd9788f0b2d3f49ca4582b8859ab58b929b1b8837ec0be1263399279/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1b0881c23a8f5ff1dc6c70542179f2fc9191e0b7fa7a02f65316fb48904cd8d1", size = 831085, upload-time = "2026-08-13T09:14:55.244Z" }, - { url = "https://files.pythonhosted.org/packages/b5/66/ca653bf9b51eb624615623611db28ec9428a351259fcf5625c5520faf759/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8a67648bf47d9b740388efaf6f05185f22dea783e97eba895efdd1dbafe8e3cc", size = 788462, upload-time = "2026-08-13T09:14:56.666Z" }, - { url = "https://files.pythonhosted.org/packages/f1/54/3a9280f791c8765a54f7814a010d2f460c7d5f5483be3e5634f4d9b38d75/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:decf297bc576675fff2139885fb88f84844e27b99e5d28eaa01909db7a327c4b", size = 753055, upload-time = "2026-08-13T09:14:58.302Z" }, - { url = "https://files.pythonhosted.org/packages/06/26/09f4618dbab2de8ea427e817c349b8391b4d559316de899e9b54dde6143e/cast_value_rs-0.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:79a8c13af39bf65014dd6567dc5fa7c42dfda246e4335ab52814f3dd6fb42cac", size = 442068, upload-time = "2026-08-13T09:15:00.414Z" }, - { url = "https://files.pythonhosted.org/packages/fc/49/7def5933625ff3c98e7d20cbd7de0aa1e83098ffa425a9df087764657767/cast_value_rs-0.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:b8af7de3639271ad9e62aba4d15ddd22819c706bd7c7859663fdbb022397bdad", size = 397846, upload-time = "2026-08-13T09:15:02.022Z" }, - { url = "https://files.pythonhosted.org/packages/6f/fc/1f48ae9ea1f10caf3bbc31b288d08137835d8965188f6ddcccdb3c7d398e/cast_value_rs-0.4.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:297445abc64891a2e62af93e4f35bb280f874ac548646d0a6b04a86d45e636a5", size = 494394, upload-time = "2026-08-13T09:15:03.604Z" }, - { url = "https://files.pythonhosted.org/packages/7f/f3/d633218979cd8839fcfbade64a277c0ff9d8ff59d21bbd8a59f1467085b4/cast_value_rs-0.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b8b6e0781b4ce61a559cc050d836a52157bd30b1f59d63fdeee79fbc19ab7e57", size = 458677, upload-time = "2026-08-13T09:15:04.947Z" }, - { url = "https://files.pythonhosted.org/packages/97/e4/96d72ccf5e00dd7af7bf55c96313df2ffba1f34a224f10430716232775c0/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8399dc89570c01537c07adc03fc2fc0e43a517be8a198e2f9e3f8a70dd003be0", size = 487289, upload-time = "2026-08-13T09:15:06.493Z" }, - { url = "https://files.pythonhosted.org/packages/93/30/18313de8c1d3826e9a6abd883b7a7981b19391f674dcd52ff2635d19f8ee/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:939417d72c9945ce24eb959e6b2cd32f7aa7d0a50a06a0fac484056b10b650dd", size = 555221, upload-time = "2026-08-13T09:15:08.28Z" }, - { url = "https://files.pythonhosted.org/packages/e9/1f/ed22b2e536ccbd4c5b234d869fc8f478347a45f7b940273023a0d6013134/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:101318e594ee998a5ccbc5ea7ed045cbd9fd22d45cac467c745fbd30309f1343", size = 650972, upload-time = "2026-08-13T09:15:10.103Z" }, - { url = "https://files.pythonhosted.org/packages/d7/d2/39f9ff9b678dfb01d206aec6be0f486137d5e9669ebadf0d1f356d2a7324/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d23bac26b2c2fe6a2368d10fbb072a71671f6e0d4d8cf9e893f62ed3530bbd1b", size = 560869, upload-time = "2026-08-13T09:15:11.602Z" }, - { url = "https://files.pythonhosted.org/packages/60/18/f63e8378db66b5a99e96c1422c68ba02a4e35de34355b90cf6d169d99ae6/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9d799a20d7517974d61d7c4dd54e5d7ef439e726012c087bada31bc644b1535", size = 548885, upload-time = "2026-08-13T09:15:13.18Z" }, - { url = "https://files.pythonhosted.org/packages/98/af/12349007b670b11e16d3be51c29ab2398550b2653dae692a5d788d3dda87/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9f0f04815efcc47592474e91f61f24810139396696f0690f3d0bf0e6ff3bb4a", size = 590846, upload-time = "2026-08-13T09:15:14.67Z" }, - { url = "https://files.pythonhosted.org/packages/a3/cf/6a39df104923bc397ae58f90e21d1e87edd62d17f24fe3d6777e6c4d3e5f/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e69f81dab0b47c74c1f30c79198c412938b6c41510b8124dbd08fb36d13a8807", size = 665064, upload-time = "2026-08-13T09:15:16.527Z" }, - { url = "https://files.pythonhosted.org/packages/36/df/0a9ac5bae4e1d3b7b4a79795054d822c26bdbc3bedad71fe13e8ce666c4b/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:59e15ec495e8707447180e0111e65e083464da60236edd5c3387efedce3da997", size = 830909, upload-time = "2026-08-13T09:15:18.088Z" }, - { url = "https://files.pythonhosted.org/packages/2f/7d/9e99c969dd76087dbf731667982d8bb6b390d29c8f37c09dd330657b0487/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f9e787397588fabb28a146a28159150f44e8ef85a25abe364f989044f1b7159d", size = 788217, upload-time = "2026-08-13T09:15:19.706Z" }, - { url = "https://files.pythonhosted.org/packages/21/a9/5d7410d37be26bed1dc740c885a0c7fdb213b93e9b13ac76fb3282aa50ea/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c66ca6ad21758993830a9f4d24263065727bf767e4524d79ded0c0b61e3da9d", size = 752817, upload-time = "2026-08-13T09:15:21.197Z" }, - { url = "https://files.pythonhosted.org/packages/65/10/9c18cb01104bb211c9b0d9a956fae3f1a40f743c35bbc6763d0114ec36a7/cast_value_rs-0.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:26fe305d5bd35d21c5ee72c5651892895447e175ba3019a24713c565d965c0ef", size = 442025, upload-time = "2026-08-13T09:15:22.575Z" }, - { url = "https://files.pythonhosted.org/packages/64/49/42a0b48a71aca3ebb0abd0948fa9d484723c3562af30a8d6904b874abb2f/cast_value_rs-0.4.2-cp313-cp313-win_arm64.whl", hash = "sha256:9ce6678fc4b1bdc459178225347acd8019f1d22bca1e13ab86cc2a3e4ecb2f46", size = 397871, upload-time = "2026-08-13T09:15:23.906Z" }, - { url = "https://files.pythonhosted.org/packages/be/e9/baf39cd3991962705ffd055fce071e23e7707f62df3fb44af2ffe0e84d11/cast_value_rs-0.4.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cefdcfd4d2b95e3b35fd98b3d3d865be9b3ef1bc1264db3bc1b8c50c5e8ac999", size = 495672, upload-time = "2026-08-13T09:15:25.168Z" }, - { url = "https://files.pythonhosted.org/packages/ed/9f/7ac6bf95d160f3fd89020aa952ffb4a49216b85a3fda02f84bc2c2612a2f/cast_value_rs-0.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbcec2cfd199c1e404f5bda71b273b4a7470762f4b55c1c21f09d60d2762e339", size = 458480, upload-time = "2026-08-13T09:15:26.647Z" }, - { url = "https://files.pythonhosted.org/packages/65/e5/46c2985ccbae41c8140f04f2acc9080761f10e8d2f8101c46083e632f812/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c224a9b30c2521cd06eac4bb1ac3860f41c902695e6f806344bc039d7a8a1d93", size = 486807, upload-time = "2026-08-13T09:15:28.129Z" }, - { url = "https://files.pythonhosted.org/packages/18/13/97851c45b275fbc9d42b5d5a340fd5ea430525b60c2eacfe6a36399d69ef/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1eaa7c3d5fe418a74928f78514f9baeaeb8e31b93db96b1761edd6144f0d2eff", size = 555335, upload-time = "2026-08-13T09:15:29.648Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b3/c84c1e820786ddb062b162112b80e5dfc1b5b54b11125618f38c1e18d075/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bddec46b36f29964c8d4bbe82b8270575fbe1c0fb41c404bb3da1778f390a24", size = 650347, upload-time = "2026-08-13T09:15:31.044Z" }, - { url = "https://files.pythonhosted.org/packages/60/38/52b4cc31570384e0286e28a03164a357d53a8aba7421ef83e500843d5449/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab3fd77144d26e2da2a113f862c05ea1cf465df347daf5deb6cf1689253fddfa", size = 561498, upload-time = "2026-08-13T09:15:32.485Z" }, - { url = "https://files.pythonhosted.org/packages/41/c8/313e1d016b0adfccd1ead2bb28d2c94313ab62534bd6e11db19a480d3b55/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4affe8668a2f17764ed978c2dddb2b49554f36a71389e6928a8a5523c1a4173", size = 548676, upload-time = "2026-08-13T09:15:33.912Z" }, - { url = "https://files.pythonhosted.org/packages/81/8b/a11f8e514598ebfc084507b0db390ecbdc6bfcffbf91ae666e0de93e6c25/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a67316b00ca01a78865d222badef2b1cb6c60ad5ebd360bf4fa3b224e87b8ffa", size = 590803, upload-time = "2026-08-13T09:15:35.384Z" }, - { url = "https://files.pythonhosted.org/packages/94/1e/ed86da6fe69eff312cc83e348cf4715d4168ab2699783febd24bfd477cd8/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9946054c77d7c5df579e9e99ebebd4d8a220cb04a6af1b6467112dfe07eee72d", size = 664118, upload-time = "2026-08-13T09:15:36.797Z" }, - { url = "https://files.pythonhosted.org/packages/0b/01/58a3eadff1233bd98d5809e2f6b5dc6d1b8c4c770664145f57fd0bdce756/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:95b064458233a7ef0525481eccfd2d37e86620d877ab1535ae809d37355cf131", size = 830686, upload-time = "2026-08-13T09:15:38.645Z" }, - { url = "https://files.pythonhosted.org/packages/e6/40/a234d9d06013a0bee08f662ab083b3a55decfa2dbb315094fdf06693d425/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:d7de068a1dcc65d1aaf4e93228198b588776bd25b62ef42478ab7efe5245a6f3", size = 787835, upload-time = "2026-08-13T09:15:40.078Z" }, - { url = "https://files.pythonhosted.org/packages/d6/72/93f906345d03972de52e3e74dd4ed12b56aab53bec85cc96020690ba6dbe/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae0cbdd44290a74e34519eb990b33bf0f8e96f0fde95ffc799913ff1aab27719", size = 753039, upload-time = "2026-08-13T09:15:41.702Z" }, - { url = "https://files.pythonhosted.org/packages/e3/a5/47b6db4f68da086db3f3768775f545b9b7f660cab8f61039c91de1f813c4/cast_value_rs-0.4.2-cp314-cp314-win32.whl", hash = "sha256:d438517530d3a8d6bc810ab6dee831c48b45287435513fea1b0476b5d4fda07d", size = 387092, upload-time = "2026-08-13T09:15:43.218Z" }, - { url = "https://files.pythonhosted.org/packages/d3/9c/5f71a5e94875bb999646227a583792dbbe9a2b257df0d961e8a2da879f9e/cast_value_rs-0.4.2-cp314-cp314-win_amd64.whl", hash = "sha256:7e7bb9faa4820b7eb0d99074fd61efff3a7392e72b6abe4b41c9d6b6d14fe457", size = 442126, upload-time = "2026-08-13T09:15:44.623Z" }, - { url = "https://files.pythonhosted.org/packages/95/1c/8e7546a044da8e92eafbb12594169301cc94e2a071fd3e34204595d0a05f/cast_value_rs-0.4.2-cp314-cp314-win_arm64.whl", hash = "sha256:1f57e762a06102d61e940548a6ab841e05ad1f51f1c4e2aa26bc27a8c59a5abc", size = 398200, upload-time = "2026-08-13T09:15:46.117Z" }, - { url = "https://files.pythonhosted.org/packages/a0/09/527646deeb622e5a597e4271e9e06d9d7cd24d64de77b8cb4ce250128c18/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:230f6440db74d2d11e513060d0e72aa3f897546465bc778d30aaeac90f8e733e", size = 485197, upload-time = "2026-08-13T09:15:47.622Z" }, - { url = "https://files.pythonhosted.org/packages/b9/79/aaa713cb151a902439b670a4774ea76d0a5dadf1f03954f6a34af2a6cb1e/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:37538170dc1fedbfd62dd56670e443b881474e7eda7f583a3c989c83a9ff15fc", size = 538669, upload-time = "2026-08-13T09:15:49.148Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b1/2cf4f4f13f74caaa4dab4ddf4924fdeeffb39920d5ec30a80112aa041efa/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d00b9cef0d38d0c5631ba63270c38c5d7c85aa87c609644f204919f939f0c9", size = 649341, upload-time = "2026-08-13T09:15:50.614Z" }, - { url = "https://files.pythonhosted.org/packages/39/8b/1faa6b27cda3dd793635c7945118e49188b6714d6bf23bb072efd718e467/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89434eebb7a12f66e99382e08c32648f055e24d9fce66274adef6620af2aebbd", size = 555253, upload-time = "2026-08-13T09:15:52.252Z" }, - { url = "https://files.pythonhosted.org/packages/8b/c2/d31f177025b51a3fda70c5888b92f5fd30ec53bc2df2c046fc9a2717cd36/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:704171aec75f417f5a70a3eb5f107fbfc79183620093c9d9074290920a5981f1", size = 547964, upload-time = "2026-08-13T09:15:53.669Z" }, - { url = "https://files.pythonhosted.org/packages/64/99/521c2d6347ca39deef237909dfee61c6d2cb806dcc20d74636e209bc646e/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2add2599d299436ec161661a8e119a69742c763a0936e542db92b1ce9d6e0c04", size = 574778, upload-time = "2026-08-13T09:15:55.479Z" }, - { url = "https://files.pythonhosted.org/packages/f0/70/35c7cb37839454aeb93b9576b42daebd8249dd59e3834cd1be020b4b0660/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:da5df79a926bdcc1f5494612f621027970a9702eb49d556068025166d4c0ace2", size = 662461, upload-time = "2026-08-13T09:15:56.97Z" }, - { url = "https://files.pythonhosted.org/packages/24/4f/a71bfaea1286bd12cc912733d650bd89a07c4430ee715202551c6e3eb0b0/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:5552e3c80d43de7a8a821793cb57da0dd3720a377051a5bf86bcabe094e87777", size = 814743, upload-time = "2026-08-13T09:15:58.652Z" }, - { url = "https://files.pythonhosted.org/packages/94/a3/b04e0f42dafeae30604abca079cce303ebb69ba24f3c4debbf211ecc781c/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:73978c6dd2ce45a05a59ec7090df8b3f2a579ee86db9b1c7df21ede17f32189e", size = 782340, upload-time = "2026-08-13T09:16:00.135Z" }, - { url = "https://files.pythonhosted.org/packages/3a/6c/913de75054092de796509dc04b92fc6ac1bf83dc86901f1f482667dead16/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:45d630592afb18880d4df0d7afb28b8cfc0ba6aa742d8db115f616df284b453a", size = 754068, upload-time = "2026-08-13T09:16:01.572Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/a2/25/151a7ac3e68a7bfa76d91dcbe318828dfa227841cd3ca1809890c052b0d6/cast_value_rs-0.5.1.tar.gz", hash = "sha256:67220f38cf791489b93ecee1e33370428326a523a211760faa36dd6f9b60a07b", size = 91796, upload-time = "2026-10-31T08:45:39.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/01/3b5bd177228cfcaf72a08a42f0b2184d029845edabb337b4b32c825be5ac/cast_value_rs-0.5.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f6410d5f5ca3b1c5a5ddb9e54f0064d2a1486ec8cdb187d6df344cb8294bba51", size = 514726, upload-time = "2026-10-31T08:43:15.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/fe/b0863f6769c8396a247ba30a728c041c094634890c9fd698349711fb5498/cast_value_rs-0.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d10b5ff6fd0d07133f5143384837f692d4034a113947e8b0bc4309aa39ab66f1", size = 474619, upload-time = "2026-10-31T08:43:17.336Z" }, + { url = "https://files.pythonhosted.org/packages/9b/0b/3820f21a9f5c9d71cad9dba22274cee693962bc9a503a5abbe0d94c80704/cast_value_rs-0.5.1-cp312-cp312-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:a43ca46ebdff1aaa9260b61b4527db72040123010aab1ec281fe8dcfced2ad1f", size = 563294, upload-time = "2026-10-31T08:43:18.812Z" }, + { url = "https://files.pythonhosted.org/packages/65/f9/3bcb27ea7ee0766eaffee2e75461a416beafccafde35c6816d152eebd906/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b167288f74bc2aa1201ed9b3595c767cdb67abe289cd36c1c78b5f394d94eda3", size = 477658, upload-time = "2026-10-31T08:43:20.248Z" }, + { url = "https://files.pythonhosted.org/packages/f0/04/187ff973140d1cd77d0a8067f2ac96e4693d50f3cbd206ca8127019edef8/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:7aecf24a3444ea02e082f94a4c469ff527c2f580eb71cb702f091ebc3c19c5d2", size = 536492, upload-time = "2026-10-31T08:43:21.733Z" }, + { url = "https://files.pythonhosted.org/packages/48/02/b2dbf5e4ed5caecc4f1686d5511bdff3769db5f28f262acf8f22d62feba8/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4351299e8fdeb83cbd34c0827a29fd8c8b160a3226cae774d3e682a0d9acc3e1", size = 632370, upload-time = "2026-10-31T08:43:23.687Z" }, + { url = "https://files.pythonhosted.org/packages/db/44/f168d64613bb9a42595c4e55cedcf94b81fa4c47bb142c71970a7d6359e8/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a8bfb2dd64d8cf51cddff9abb396f0d5bac118a8065c69f9c0a5b223bcd3a2cd", size = 558721, upload-time = "2026-10-31T08:43:25.189Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b3/383147f4e9e3dc04ab8e392c282653bedc871fb96344f6d773a23d6345f9/cast_value_rs-0.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:891fc3844dc25980a688dee92f7042d332ca6966ca952347cd335339b0ad1504", size = 525962, upload-time = "2026-10-31T08:43:26.784Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d3/7e84f237a128047398cb786fc2ea1e4b0bc52f07cbdb6e8bf47d9d0df635/cast_value_rs-0.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f973626e9d6d31da4784dbc53d30c8c200724bf4f244d6dc890c79a479c01871", size = 654291, upload-time = "2026-10-31T08:43:28.259Z" }, + { url = "https://files.pythonhosted.org/packages/8d/5c/3e9442c4aa6d6b1e45e901bbd4f591df1729acc7842b02eb4c4a7c5429f6/cast_value_rs-0.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cd7c41110ffe110f5364a04e2b2a82a7dbaa57166e5b7c2893b9970d4e871c33", size = 812705, upload-time = "2026-10-31T08:43:29.696Z" }, + { url = "https://files.pythonhosted.org/packages/43/7c/3e1b5742839ba9a97ba4e2cffb02758c8209b462f00474b08dc69537edc6/cast_value_rs-0.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2d56b5b7cfeeea8737772bf1ccb19835111cbf669d773591826ec53fb551dbeb", size = 766586, upload-time = "2026-10-31T08:43:31.22Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e0/ffa2b2fbe125f9f0fd3f2c3cd4c818317d7523a2c02a9b6ab0d48d76644c/cast_value_rs-0.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b14d256361bb449562194c4f12c51f9b7745900412f37c84c1f759e6eab362fc", size = 730423, upload-time = "2026-10-31T08:43:32.782Z" }, + { url = "https://files.pythonhosted.org/packages/40/7a/42c1ac8202610e48204a07f98f7055515c2b7d860c6fd8de5101e45c7974/cast_value_rs-0.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:ee3e305f27dd0f2c3334eb80c874bdd5262f0a707ad60b4ff2857a3c9a07b5a3", size = 401775, upload-time = "2026-10-31T08:43:34.402Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e0/9b85c3da33c11f0ef38b03878bfe4b55ab06d4282bc3b2db0be94f30f2c6/cast_value_rs-0.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:6bc12f62e3a890243322999596d7001cf730cc59599538ad284f5e4508101bbc", size = 370508, upload-time = "2026-10-31T08:43:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/42/ce/e881f4ca411676ee6918ad74ba57639a5a0b0fa0ef43f3c6dc037c1dd5e2/cast_value_rs-0.5.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:32ff4e7e5c2bcbc3fe85314bd15553321ad43f4869e7015a45e05f1d131964d0", size = 514776, upload-time = "2026-10-31T08:43:37.558Z" }, + { url = "https://files.pythonhosted.org/packages/80/cb/37300e5c5e2bf2396883aa9fdfd90e2c9cea6b33dfc1a69b04b4e723a973/cast_value_rs-0.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2689fe1558d5ecbd115bbd2bb97f3a1f2a458f3ebc656f268ceb94624b4dda26", size = 474544, upload-time = "2026-10-31T08:43:39.12Z" }, + { url = "https://files.pythonhosted.org/packages/b8/e5/c45635cb3da6b1ab9d06e57bd63e294362948d9ad6380a75570c4686ffb8/cast_value_rs-0.5.1-cp313-cp313-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:71e419e6f4799c187fffbe8c6a96bc1c07d17ccc63a8d77f1fcf74f01ce6f98e", size = 563316, upload-time = "2026-10-31T08:43:40.906Z" }, + { url = "https://files.pythonhosted.org/packages/cf/5d/da4920da8731987c016730cb527688d5eccfd0660dec1ad387ac5e97a996/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a0fa4372466e59197be1c0ab17a909d16e9108fc5327ca38d8519e014fa6bf70", size = 476817, upload-time = "2026-10-31T08:43:42.322Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b0/afabc7883058a18f6e3849a260e6c9faacbfa864d4ec78cadf2d7c8db49d/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2a3acfbcc7e4cf29ea45cbf2f481199dd19be6f101e5a519614708cfb8c9d9ce", size = 536511, upload-time = "2026-10-31T08:43:43.788Z" }, + { url = "https://files.pythonhosted.org/packages/98/d5/0806712a64a7b117bbd24e7b2aface573f31a6b8977b0ae50d5a449d0621/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f7701b7d680689b901c2480289af095c3065c28731066a98d0bf4058ff1d7ec9", size = 631482, upload-time = "2026-10-31T08:43:45.193Z" }, + { url = "https://files.pythonhosted.org/packages/0d/81/b24b7c75eab58e0f653c62a037955868907f8d782963d88b8c3cd1ab8fba/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:f895ffcf8c24870a92e8697619ef8b091d87159cb354ca3168ed72b092e81907", size = 558991, upload-time = "2026-10-31T08:43:46.766Z" }, + { url = "https://files.pythonhosted.org/packages/14/b3/f7c034c10bc88ae99ae00c282690aeed534a978fa591fb5ba7d7947adca2/cast_value_rs-0.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c5acd38e2e8d0aa2873881efb624f076c6286b04af030e61aa124769fa75d06", size = 525925, upload-time = "2026-10-31T08:43:48.247Z" }, + { url = "https://files.pythonhosted.org/packages/ed/32/6bbbbbfe1352395243120ee77d364c54684613fa0e8c9e59b82571553816/cast_value_rs-0.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2373117423aee9df307b8ff87af9487faa6324124fc30eb358aa9f5630462f0f", size = 653682, upload-time = "2026-10-31T08:43:49.685Z" }, + { url = "https://files.pythonhosted.org/packages/14/2c/81f025c5ff0acb2ffa5ea3dccc69e7a6ba62fc432376fc1df72b33ddde5e/cast_value_rs-0.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:fa1cda7c516ecb5219b9b109603ba09193bdb1ca9b3a64599e9f597dda576327", size = 812907, upload-time = "2026-10-31T08:43:51.151Z" }, + { url = "https://files.pythonhosted.org/packages/3d/cf/9baf257796769718ee29e2c0e372d44dc9bf1682edc3e9b2053fe2a0d094/cast_value_rs-0.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b619762eace293242b43286c3fedb8b83660effaac016afca400f7852f71c0da", size = 766871, upload-time = "2026-10-31T08:43:52.97Z" }, + { url = "https://files.pythonhosted.org/packages/59/27/acbfb113c69afb86bac85b38ba872d95f7c61ecbea3ff350ab5e8e70c050/cast_value_rs-0.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18b3ddcf763d5208e2f8ac9db659837c67d77e22c44b7315e924b7351ed6e08c", size = 730422, upload-time = "2026-10-31T08:43:54.922Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e9/7a097d2844b3541b36946cdeb51430b2cf1f287026e054ac0240b46649ab/cast_value_rs-0.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:0fdc504d6f46b1f49b7c7fd1bf108b605d9d9ecb6054ba578d3f6a38843d6fce", size = 402047, upload-time = "2026-10-31T08:43:56.312Z" }, + { url = "https://files.pythonhosted.org/packages/c1/71/ad4dbde8cee35f86c660473f6bcb98641236c7b506a4b481bd4954a208b1/cast_value_rs-0.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:b0ef63e39dc55c5210d31e3fc2377668c12db86659af6212474338ab1168c0a8", size = 370621, upload-time = "2026-10-31T08:43:57.832Z" }, + { url = "https://files.pythonhosted.org/packages/61/7b/4f3aac163a600c3a70be75d9b5652b0482c71117daac7c0276c040e60cf6/cast_value_rs-0.5.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:7f52ddc9536a2319caa0358088cdb552bd5c3ea0d576dd4bef5378dd87605f94", size = 519536, upload-time = "2026-10-31T08:43:59.456Z" }, + { url = "https://files.pythonhosted.org/packages/8c/fd/b2b7456241e86e485de36bf7f659be592fe2eaf93aa0df4d2f4b60b37de9/cast_value_rs-0.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4048177e1200023661ef37c0dacedb40d39465cae41f6c8e0da70a64cd98ddb2", size = 476470, upload-time = "2026-10-31T08:44:01.182Z" }, + { url = "https://files.pythonhosted.org/packages/66/f3/74ce87a4064388862485d6d5394553a2764de3af5e1f7568cb19e19a0111/cast_value_rs-0.5.1-cp314-cp314-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:ef3fa404ed66e5f33c24cef003071f509b7e1f36a7f748182e62eda9e20941da", size = 563457, upload-time = "2026-10-31T08:44:02.813Z" }, + { url = "https://files.pythonhosted.org/packages/4b/1f/fc7a5e1b0e6af62ee5b48e57c50bcef62713900467b09bcb87a56fbf3fd2/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:946f69a6e586e0d1f8c395fc97e7a5556b64dbba87a8a34bef112dbbb9690373", size = 478965, upload-time = "2026-10-31T08:44:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/6c/b1/39ce0bfd2ba25cbda8113eed86afa77f6eedd348d711a067352a73cba2e2/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:18460ef2f1883238083c6bbd50c2b4be41afc5384de0ad815a7b1483e8b2e491", size = 537191, upload-time = "2026-10-31T08:44:06.348Z" }, + { url = "https://files.pythonhosted.org/packages/4f/13/8c3f1d0a9593a15add6e0973226ab9e8ddeafad4aeb464bdac3811eae5c3/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:44d6d057327b96e1fef29ce10510a6ecc9612874d4df30e3e43e0540844b1bf2", size = 633461, upload-time = "2026-10-31T08:44:07.787Z" }, + { url = "https://files.pythonhosted.org/packages/49/65/fb101465f33896a1454e5a959c9c50099e6d6a37db4d4d899cf924d766e1/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d3708705c38210d9ba5c6d75433532ca98744f36a5844b07e926a7511b7c0c3b", size = 559760, upload-time = "2026-10-31T08:44:09.529Z" }, + { url = "https://files.pythonhosted.org/packages/3c/24/9171671d9b14febdbc94810a61f33276bdc1dbc591c0c8d954d0f05a4086/cast_value_rs-0.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:20bb849f145eb1d7cff568e84dd2970481c25fcdbae92d3bfb6f041750e92a48", size = 527072, upload-time = "2026-10-31T08:44:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/9b/bc/f1e902a8b24b04564a45cea3471be06b4a1ba185f65553be4e26f7dd5a7e/cast_value_rs-0.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b6705f2c17704925a516392fbdc194fa1a3010829bd0800f205fd673d15b9751", size = 655678, upload-time = "2026-10-31T08:44:12.986Z" }, + { url = "https://files.pythonhosted.org/packages/31/50/6ddedc98d93acd6d0fde9bfe6bf17ec5fe83485a00e0dc946b026fde2b31/cast_value_rs-0.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7b98cf7330ab7378f5497c215a7597e974e6224ebaaa46744eb6ec10c7cc6b55", size = 813526, upload-time = "2026-10-31T08:44:14.439Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/b3ec914d461603d2d36dc1eb445710e58c90436b6efa1e9d99493adbbbe4/cast_value_rs-0.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:d4229f74c608b895ff7ebae51ea894384c4f8a639f1129df394ed86ac4f7fcf2", size = 767696, upload-time = "2026-10-31T08:44:16.427Z" }, + { url = "https://files.pythonhosted.org/packages/85/ce/fb4f4ec09a80ebcd5c9e15ef6bb9943edf394653c0f97762d67eab5fda3d/cast_value_rs-0.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:60f4d3fdf33b44a294942938f49618d1908af8051ae80a2bb3ac7a81bf8cb563", size = 731352, upload-time = "2026-10-31T08:44:18.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/88/41ce6ec2663e40a28e2e9c8e35e08962a904909fed7ad45d1f6ae1f54c47/cast_value_rs-0.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:d9b5f2a2a8e0c7ca9dec0741e0d7a011fff2e1f7c36273ce4edd806ad7f5b5be", size = 404653, upload-time = "2026-10-31T08:44:19.421Z" }, + { url = "https://files.pythonhosted.org/packages/b3/27/80a353c489247f3fcb1555e2bb8fa87cfa273fdb12f6bafecf6fcf39313b/cast_value_rs-0.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:94ca8cd384413459399f3de50e383007090e46d55a89e554dc5f8cf2df4633e1", size = 371183, upload-time = "2026-10-31T08:44:20.766Z" }, + { url = "https://files.pythonhosted.org/packages/63/d7/a1a6f2ad4112592a342e0291accd7d7503d00f7f77afba7bfa746568440b/cast_value_rs-0.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:85511ddd1e54f5d065b2cec047cb90765006b104fab44ce2394364b91257d3ff", size = 491276, upload-time = "2026-10-31T08:44:22.216Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d2/7a6f34c0a425f5b5264aa72982ae975394ff56af7667c4175da402538967/cast_value_rs-0.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e7588000b116674a94b62b09668afd4047ae1cfa6ca23fa217e2a0f9cc7c5d9", size = 659458, upload-time = "2026-10-31T08:44:23.938Z" }, + { url = "https://files.pythonhosted.org/packages/e7/95/14ebfec2e8bc9b5f4c4119f3bf341ac8d4af1e4f80cc469dc899627b3095/cast_value_rs-0.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5695711e9c5abe0bd1663e1cca65135ef6521c4aa9b0f66c3ea5dfa6a520d0d4", size = 578621, upload-time = "2026-10-31T08:44:25.344Z" }, + { url = "https://files.pythonhosted.org/packages/31/bc/79e87e32286543f83cdcc3922dd7346fc5f59391a7f296d20efa77c1b091/cast_value_rs-0.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3394e3114361b4f7898dfc6e06da75373ae1a37ce042d40bd87a2e9e0aed5201", size = 552219, upload-time = "2026-10-31T08:44:27.024Z" }, + { url = "https://files.pythonhosted.org/packages/e7/61/2fdad5258778bff2dd557030f1712e7f9fc82be3b4504235111d57111123/cast_value_rs-0.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:57f08679dbca2f45c15ffd21de75da7891be7bfb7fa04ffe4cadcc10740dde49", size = 668370, upload-time = "2026-10-31T08:44:28.511Z" }, + { url = "https://files.pythonhosted.org/packages/72/c7/e3fd8da9d62a6fafb7ac35b54753bbade989463588769d24d65147d10573/cast_value_rs-0.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea5de15f91e9b59c7077ddee0903de11982440e3ed261995e0173f0d4dfb35db", size = 756835, upload-time = "2026-10-31T08:44:29.986Z" }, + { url = "https://files.pythonhosted.org/packages/4a/02/61bdbf5c06fce0dcdda04fbd8a2e6b224c5329c2b05089df2ec8b87206f7/cast_value_rs-0.5.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:3bf1543670fd8a3e70b40bfe099848ad938e79c2583335be44328466e9452f60", size = 476694, upload-time = "2026-10-31T08:44:31.419Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e3/1d842641d5f2ddd7d8debcd26cc372cb048bb5a0c032e011608c3f4cf8e9/cast_value_rs-0.5.1-cp315-cp315-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:d64f290dc2f440a67717cb5051552b509403f3d8ef5b9690876dc514acd13486", size = 563843, upload-time = "2026-10-31T08:44:33.063Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0c/c3c1b448e97fe9684865bcd6534273cef3ae242d41706f8a05bfa8fab372/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7fafc0825b4f675fe55d9d22f480043da2b920ce474b4dc53162306c64bca3e4", size = 479009, upload-time = "2026-10-31T08:44:34.718Z" }, + { url = "https://files.pythonhosted.org/packages/20/5f/91e9a654ac1fbd299ec6c6b6e0dd43f244438debfeb810e519cb7322c6da/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d997464871e9c5ad2fa5a950de15390b134abd5a2071fd5e9fb2b27fe97abda", size = 537179, upload-time = "2026-10-31T08:44:36.188Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8c/844d1a4d7bd0f8a8908b1f311d14a414c11fb2ce8445eb9a88621d528b0b/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:ea69d3d3e1a0aeb428268bff34e2bd88b3ccf5d48200ef77e91e700b637c8380", size = 634858, upload-time = "2026-10-31T08:44:37.65Z" }, + { url = "https://files.pythonhosted.org/packages/23/c1/93384916c505767f270240cb7f84bb838d486bf2a95e7a12a70267a0c03f/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9c6d156cc51ba7cc6083bc6c13751df9865b40fb22cb01de0e0de510c3b1c60b", size = 559196, upload-time = "2026-10-31T08:44:39.205Z" }, + { url = "https://files.pythonhosted.org/packages/50/e8/17541402fe90d8241ce4c8c228d81dba4dc2cf15f9a447d0a3d42f15d6db/cast_value_rs-0.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:63ff7854dd69c00e3d113c1cf80e137ab62b703e1d5a3933a6f317e48629aac2", size = 527247, upload-time = "2026-10-31T08:44:40.665Z" }, + { url = "https://files.pythonhosted.org/packages/ed/f2/7791c4cc6348daf07ae32be323a387c3abf3693e2d34540810ab0bfa2ee8/cast_value_rs-0.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:306f35ad3cd2cf6dfad817d47ff4f1d72c30857654da264170d5ab0f4cd16ba3", size = 656202, upload-time = "2026-10-31T08:44:42.272Z" }, + { url = "https://files.pythonhosted.org/packages/20/0a/35f4cadb0012a1f03aa34b4c4dedbc040fa6f2a8ded51791aa54a911078b/cast_value_rs-0.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:01bb73fe09e64f6d7b98efed7e527870b7c5b506248728532c0bb6fb96eb870a", size = 813923, upload-time = "2026-10-31T08:44:43.751Z" }, + { url = "https://files.pythonhosted.org/packages/23/b9/cfc4993e242375217aaec5c4732adc537280538f99d229c92c1cc4a0bccf/cast_value_rs-0.5.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:79115de49db59bb715dfb97dd6cb565ed937bb0f4ab65bf7a85c115aeb279865", size = 767624, upload-time = "2026-10-31T08:44:45.189Z" }, + { url = "https://files.pythonhosted.org/packages/86/73/b4ad932c157b66d95e94e325aa79669667fe9fcee82cd2d4f148c04a9a63/cast_value_rs-0.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:fdb09db5d2585407125ada3b5bb9025cadb2fe7fc467e50839a6ded43add14de", size = 731376, upload-time = "2026-10-31T08:44:46.594Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e2/9e4153d8efa644059126a03ed1957174baf9b8453de6f1ddabc97436dc09/cast_value_rs-0.5.1-cp315-cp315-win32.whl", hash = "sha256:0176678830df81c30cc931624b0a352544bf40a89883b0651471f76389900114", size = 347664, upload-time = "2026-10-31T08:44:48.142Z" }, + { url = "https://files.pythonhosted.org/packages/81/c8/54e5e9a13ab37bb26b9e769892ac79dd7bae30dca847353d5a9948bf5a21/cast_value_rs-0.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:50b335dd32f4ab9da85c0b07627bf2dfd9c123e6c281eb67c3ad812ecb96cc66", size = 404766, upload-time = "2026-10-31T08:44:49.632Z" }, + { url = "https://files.pythonhosted.org/packages/31/9c/2b8d62dd4720aa96e88b5a5b499eba91dea6fb6d5f2c88ccca77563af4a1/cast_value_rs-0.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:39a2017df8ac0ac80fc6049920a569c7ef9ffb452958e51d7690003d241f3b41", size = 371573, upload-time = "2026-10-31T08:44:51.008Z" }, + { url = "https://files.pythonhosted.org/packages/d2/61/a1bc81db4af793e7268854065f1e238d548ebf107a12c276cf1296525647/cast_value_rs-0.5.1-cp315-cp315t-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:95f1d08db5067bef51ac65b9addcec901132d4dc51bee089f5a50513c07c4e16", size = 594666, upload-time = "2026-10-31T08:44:52.621Z" }, + { url = "https://files.pythonhosted.org/packages/ac/bf/cd460b4f80d0c7ca531f3bda9596a9902b8847b986fed351210b9e1a3d63/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bc4f9995a6c9629c1b625c81309a22584a38738d8a4bf19d53d703e877d0a625", size = 491256, upload-time = "2026-10-31T08:44:53.958Z" }, + { url = "https://files.pythonhosted.org/packages/55/f9/82558521a5be44dca82f14d32ed39e268aaca7a8c55d855e07cbe1839d01/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:d1fa3b07d705a625ff1effb6cf7068290a04b6fb342ffdb2734666563c4a5842", size = 555869, upload-time = "2026-10-31T08:44:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/0345a1d127f86672383abcf201f465915b1c7551726da61d0c7a7a39a075/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:82fb5958e1bddd1b6b59e5f320b7060a02c28ea4dfba959cf0199e92c8451fc5", size = 658025, upload-time = "2026-10-31T08:44:57.02Z" }, + { url = "https://files.pythonhosted.org/packages/16/32/0884b5a9debc5d45fee87732aec96e720ec9e312b14f352c85d811d5ecb6/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:0aee9b55d721ab9aca8e9ca882e904d25d43562139d228ca8190605b19b77d4d", size = 578598, upload-time = "2026-10-31T08:44:58.622Z" }, + { url = "https://files.pythonhosted.org/packages/fb/01/da585bb0fb303495779123d1f681fcbc219b00e33c23b0055ad803e586da/cast_value_rs-0.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:76fcd3c6322bdbd24be9772c66c7aec314ad48e801c14857f8553ad42884733a", size = 552414, upload-time = "2026-10-31T08:45:00.145Z" }, + { url = "https://files.pythonhosted.org/packages/5c/71/28da1e78625ae61110f03688d92a4c77f1e1e5a72dfd266901f4ba79c3a0/cast_value_rs-0.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:eef6fa6aa0dbbf0a31a50bd3e4482f95a662a9a1bbcc2cb4d47adb68189ed94c", size = 668386, upload-time = "2026-10-31T08:45:01.72Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/5016bec95d999a884fb2f49746e8f2b705f522677701051111d1442cc7c2/cast_value_rs-0.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:c27411eeb713daf0afe0a3d8025fdc1cc32191d15df0c0ac09f229cf3cd97d1b", size = 831458, upload-time = "2026-10-31T08:45:03.37Z" }, + { url = "https://files.pythonhosted.org/packages/44/db/cb3369d143c5eb5f998fd78e2772f985a3f73b30452a0be043ee83e62916/cast_value_rs-0.5.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:fdb1bdca4c472199a410ade27c3123b9c8aca4c5dd6a91597a5ca740a00d8eca", size = 796713, upload-time = "2026-10-31T08:45:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/f6/2b/73ea6609be5583e648a6fcc723dc66296b0753bd98cbd10a3afca49c8da3/cast_value_rs-0.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:825d422a0e4f79a1103e8fae70f45a3244ca437ce317bb0d49bec6b5a4fccb6a", size = 757395, upload-time = "2026-10-31T08:45:06.343Z" }, + { url = "https://files.pythonhosted.org/packages/c3/4a/317620763de8d2061638cb30a10871100f2585e45a65a97bb54e1f0d1a29/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:5a0eed236a5f65a5a4c84598b5cbfb77d6b295785684ef6912b47c2267429768", size = 587381, upload-time = "2026-10-31T08:45:25.09Z" }, + { url = "https://files.pythonhosted.org/packages/9c/bb/449c2fbcc5238cf24fd4c74d05d8d457287eee8ed7e6436dd8f730745f17/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d072bc28e8f9fa25c2bd7fc73ed5566a2f752af38e38203ef142a9258353ff4", size = 485865, upload-time = "2026-10-31T08:45:26.455Z" }, + { url = "https://files.pythonhosted.org/packages/e5/cf/8113580800507c4c5a35a7876c4f2dccd2d05bbd84fd0820ad74489223c4/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:55a474d9495588216625f960eb9ac5d281d5463222c5dc717745020f914444e8", size = 547762, upload-time = "2026-10-31T08:45:28.065Z" }, + { url = "https://files.pythonhosted.org/packages/34/28/3119cdb31d40ea7fa5165a5bcb9e141db0d2a4737c393a09a473ab608a73/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:57bbaa75c2b19e4e276cb26c98a94a08e80c657398be4dd28e149aa3e99af671", size = 650458, upload-time = "2026-10-31T08:45:30.123Z" }, + { url = "https://files.pythonhosted.org/packages/34/04/40206336236ccbc59638a0e87383f1bf4c76a8feea89e48f0dac2b44d463/cast_value_rs-0.5.1-pp312-pypy312_pp80-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1f2fcb8e0487f5a0f9cbca9bb2d5ab15f227f77acaa7ea23b5dbda7e58179084", size = 568208, upload-time = "2026-10-31T08:45:31.545Z" }, + { url = "https://files.pythonhosted.org/packages/e4/31/8a26430d6a1e0f6b6f47b2c5faf30dc30a098d9db2e0734ad8fd44f4ed99/cast_value_rs-0.5.1-pp312-pypy312_pp80-musllinux_1_2_aarch64.whl", hash = "sha256:740e7fadb7d4884451d5b75ad4410ae2508c7a09f682797462c910abb285cc95", size = 663319, upload-time = "2026-10-31T08:45:33.133Z" }, + { url = "https://files.pythonhosted.org/packages/73/d2/705418aa24017bd6bf574f96d42132fd61cb0a69dbf495129e7dfb1ca1d7/cast_value_rs-0.5.1-pp312-pypy312_pp80-musllinux_1_2_armv7l.whl", hash = "sha256:7ec196b6debda0ae9cbded8528951e0f8d2be57eb464028568733afd32aa5b2c", size = 823564, upload-time = "2026-10-31T08:45:34.543Z" }, + { url = "https://files.pythonhosted.org/packages/e3/e4/3c29de8fa19ddf9f97acfd9f44d93423da9596b5c2bcc27a9f22e2840026/cast_value_rs-0.5.1-pp312-pypy312_pp80-musllinux_1_2_i686.whl", hash = "sha256:8d69bddf06b53ca6f543d49ec64e99aeda569d208808f4b2b0829044a10e6299", size = 787628, upload-time = "2026-10-31T08:45:36.069Z" }, + { url = "https://files.pythonhosted.org/packages/cd/9f/f5bd9736239a12106e726de7b1eac0341d4b0a3e611e120452fd4c827999/cast_value_rs-0.5.1-pp312-pypy312_pp80-musllinux_1_2_x86_64.whl", hash = "sha256:a3b162f91f060845743ef8519e86643b3cfa92fd4404ab7e988c2a09c2b45432", size = 745735, upload-time = "2026-10-31T08:45:37.706Z" }, ] [[package]] @@ -3897,7 +3925,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "cast-value-rs", marker = "extra == 'cast-value-rs'", specifier = ">=0.4.2" }, + { name = "cast-value-rs", marker = "extra == 'cast-value-rs'", specifier = ">=0.5.1" }, { name = "cupy-cuda12x", marker = "sys_platform != 'darwin' and extra == 'gpu'" }, { name = "donfig", specifier = ">=0.8" }, { name = "fsspec", marker = "extra == 'remote'", specifier = ">=2023.10.0" }, From bc177b3fa2142f184e1a5ee04084622616635a07 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 1 Nov 2026 18:40:38 +0100 Subject: [PATCH 04/10] feat(zarr-metadata)!: read what zarr-python writes in a ZARR_PYTHON_CONTEXT scope (#4583) * feat(zarr-metadata)!: read what zarr-python writes in a ZARR_PYTHON_CONTEXT scope Add definitions for the extensions zarr-python writes. zarr-extensions registers `fixed_length_utf32` and the `vlen-utf8` and `vlen-bytes` codecs, so they join CORE_AND_EXTENSIONS_CONTEXT. zarr-python's own `null_terminated_bytes` and `variable_length_bytes`, which no specification registers, are only in the new ZARR_PYTHON_CONTEXT, so the default scope stays spec-pure. Rename CORE to CORE_CONTEXT and CORE_AND_EXTENSIONS to CORE_AND_EXTENSIONS_CONTEXT, so that a scope's name says it is a context. Fixes d-v-b/zarr-python#423 Assisted-by: ClaudeCode:claude-opus-5-5 * docs(zarr-metadata): number the changelog fragments for PR 426 Assisted-by: ClaudeCode:claude-opus-5-5 * docs(zarr-metadata): number the changelog fragments for PR 4583 Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): keep the old scope names and tighten the string rules Keep `CORE` and `CORE_AND_EXTENSIONS` importable from `zarr_metadata.v3` and `zarr_metadata.v3.definition` as the same scopes with a DeprecationWarning, so a consumer such as pydantic-zarr keeps working across the rename. Check that `vlen-utf8` encodes `string` and `vlen-bytes` encodes `bytes` (or zarr-python's `variable_length_bytes`), as zarr-extensions says, instead of accepting any data type whose values vary in size. Do not count trailing null bytes against the size of a `null_terminated_bytes` fill value, since they are padding. Spell a `fixed_length_utf32` fill value without its trailing U+0000 padding, so `"a"` and `"a\u0000"` have one canonical spelling. Assisted-by: ClaudeCode:claude-opus-5-5 * feat(zarr-metadata): read zarr-python's raw_bytes and rename CORE_V2 Add zarr-python's `raw_bytes` data type (NumPy's `V`) to ZARR_PYTHON_CONTEXT, so the scope reads every data type zarr-python writes. Its names start with `ZarrPython`, since `RawBytes` already names the spec's raw bits. A fill value longer than `length_bytes` is a problem, because zarr-python cuts it short, and the canonical spelling pads it with null bytes, as zarr-python writes it. Rename CORE_V2 to CORE_V2_CONTEXT, so every scope's name ends in `_CONTEXT`. The old name keeps working with a DeprecationWarning, through the same `renamed_scope` helper as the v3 names, which moves to `zarr_metadata._renamed`. Share the base64 decoding and spelling of fill values in `_byte.py`, and say that the vlen codecs' `configuration` key is optional, since zarr-python writes an empty object. Assisted-by: ClaudeCode:claude-opus-5-5 * refactor(zarr-metadata): declare the v2 scope's old name once Move CORE_V2_CONTEXT and its table of old names into `v2/_core.py`, as `v3/_core.py` holds the v3 scopes, so `zarr_metadata.v2` and `zarr_metadata.v2.definition` share one table. Say why a `fixed_length_utf32` fill value counts its trailing U+0000 code points, as the spec requires, while `null_terminated_bytes` follows zarr-python and does not, and test a value that only its padding makes too long. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): give main's strategies the renamed scopes and new definitions The Hypothesis strategies merged from main (#4575) imported `CORE`, `CORE_AND_EXTENSIONS` and `CORE_V2` from modules that now hold only the `_CONTEXT` names, and their coverage test requires a strategy for every definition in `CORE_AND_EXTENSIONS_CONTEXT`. Rename the imports, and add strategies for `fixed_length_utf32`, `vlen-utf8` and `vlen-bytes`; each vlen codec draws nothing for a data type it does not read. Assisted-by: ClaudeCode:claude-opus-5-5 * fix(zarr-metadata): draw zarr-python's own data types in the testing strategies `DEFAULT_STRATEGIES` holds a strategy for each definition the package defines, but `null_terminated_bytes`, `variable_length_bytes` and `raw_bytes` had none, so `fields()` raised `LookupError` for them. Draw each, and check the strategies against `ZARR_PYTHON_CONTEXT`, which holds every definition. Say that the documents in test_zarr_python_scope.py leave out the compressors zarr-python adds. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * test(zarr-metadata): pin the registered extensions to CORE_AND_EXTENSIONS_CONTEXT The strategies are now checked against `ZARR_PYTHON_CONTEXT`, which no longer proves that `fixed_length_utf32`, `vlen-utf8` and `vlen-bytes` are in `CORE_AND_EXTENSIONS_CONTEXT`. Assert it directly. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * refactor(zarr-metadata): let the vlen strategies ask their codec's rules Each vlen codec strategy named its compatible data types a second time; draw nothing for exactly the chunks the codec's own chunk rules refuse, so the two cannot drift. Draw `fixed_length_utf32` fill values with trailing U+0000 padding too, which the canonical spelling removes. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- packages/zarr-metadata/README.md | 18 +- .../zarr-metadata/changes/4583.feature.md | 1 + .../zarr-metadata/changes/4583.removal.md | 1 + packages/zarr-metadata/docs/api/index.md | 4 +- packages/zarr-metadata/docs/api/v3/codec.md | 4 + .../zarr-metadata/docs/api/v3/data_type.md | 8 + packages/zarr-metadata/docs/index.md | 18 +- .../zarr-metadata/docs/parsing-validation.md | 12 +- .../src/zarr_metadata/__init__.py | 24 ++ .../src/zarr_metadata/_renamed.py | 26 ++ .../src/zarr_metadata/definition/__init__.py | 12 +- .../zarr_metadata/definition/_definition.py | 2 +- .../src/zarr_metadata/json_schema.py | 2 +- .../src/zarr_metadata/model/__init__.py | 2 +- .../src/zarr_metadata/model/_array_v2.py | 18 +- .../src/zarr_metadata/model/_array_v3.py | 12 +- .../src/zarr_metadata/model/_group_v2.py | 30 +-- .../src/zarr_metadata/model/_group_v3.py | 38 +-- .../src/zarr_metadata/model/_repair.py | 10 +- .../src/zarr_metadata/pydantic.py | 30 +-- .../src/zarr_metadata/testing/_codec.py | 17 +- .../src/zarr_metadata/testing/_data_type.py | 56 +++++ .../src/zarr_metadata/testing/strategies.py | 4 +- .../src/zarr_metadata/v2/__init__.py | 18 +- .../src/zarr_metadata/v2/_core.py | 17 ++ .../src/zarr_metadata/v2/definition.py | 24 +- .../src/zarr_metadata/v3/__init__.py | 38 ++- .../src/zarr_metadata/v3/_core.py | 60 ++++- .../src/zarr_metadata/v3/_json_schema.py | 10 +- .../src/zarr_metadata/v3/codec/__init__.py | 2 +- .../src/zarr_metadata/v3/codec/_vlen.py | 27 +++ .../src/zarr_metadata/v3/codec/vlen_bytes.py | 77 ++++++ .../src/zarr_metadata/v3/codec/vlen_utf8.py | 71 ++++++ .../src/zarr_metadata/v3/consolidated.py | 2 +- .../zarr_metadata/v3/data_type/__init__.py | 9 +- .../src/zarr_metadata/v3/data_type/_byte.py | 35 ++- .../src/zarr_metadata/v3/data_type/bytes.py | 16 +- .../v3/data_type/fixed_length_utf32.py | 124 ++++++++++ .../v3/data_type/null_terminated_bytes.py | 121 ++++++++++ .../v3/data_type/variable_length_bytes.py | 62 +++++ .../v3/data_type/zarr_python_raw_bytes.py | 123 ++++++++++ .../src/zarr_metadata/v3/definition.py | 27 ++- .../zarr-metadata/tests/model/test_array.py | 38 +-- .../tests/model/test_consolidating.py | 69 +++--- .../tests/model/test_construction.py | 20 +- .../tests/model/test_extension_points.py | 23 +- .../tests/model/test_extra_fields.py | 31 ++- .../zarr-metadata/tests/model/test_group.py | 26 +- .../zarr-metadata/tests/model/test_pair.py | 88 +++---- .../zarr-metadata/tests/model/test_pair_v2.py | 18 +- .../tests/model/test_pydantic_module.py | 101 ++++---- .../tests/model/test_read_array_metadata.py | 27 ++- .../model/test_read_array_metadata_v2.py | 6 +- .../tests/model/test_scope_threading.py | 18 +- .../tests/model/test_scope_threading_v2.py | 16 +- .../tests/model/test_unclaimed_fields.py | 38 +-- packages/zarr-metadata/tests/test_contexts.py | 17 +- .../zarr-metadata/tests/test_json_schema.py | 103 ++++---- .../zarr-metadata/tests/test_problem_data.py | 14 +- .../zarr-metadata/tests/test_public_api.py | 13 + .../tests/test_renamed_scopes.py | 45 ++++ .../tests/testing/test_strategies.py | 30 +-- .../zarr-metadata/tests/v2/test_definition.py | 30 +-- .../tests/v3/test_definitions.py | 124 ++++++---- .../tests/v3/test_every_definition.py | 69 ++++-- .../tests/v3/test_fill_values.py | 26 +- .../tests/v3/test_grid_shapes.py | 18 +- packages/zarr-metadata/tests/v3/test_kinds.py | 20 +- .../zarr-metadata/tests/v3/test_pipelines.py | 12 +- .../tests/v3/test_rule_locations.py | 24 +- packages/zarr-metadata/tests/v3/test_scope.py | 127 +++++----- .../zarr-metadata/tests/v3/test_sharding.py | 6 +- .../zarr-metadata/tests/v3/test_storage.py | 16 +- .../tests/v3/test_zarr_python_scope.py | 226 ++++++++++++++++++ 74 files changed, 1987 insertions(+), 664 deletions(-) create mode 100644 packages/zarr-metadata/changes/4583.feature.md create mode 100644 packages/zarr-metadata/changes/4583.removal.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/_renamed.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v2/_core.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/codec/_vlen.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_bytes.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/codec/vlen_utf8.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py create mode 100644 packages/zarr-metadata/tests/test_renamed_scopes.py create mode 100644 packages/zarr-metadata/tests/v3/test_zarr_python_scope.py diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 3c5513d1dd..12583e0c96 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -41,7 +41,7 @@ closely model the content of the Zarr specifications, such as: `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS`. + `CORE_AND_EXTENSIONS_CONTEXT`. - **JSON type checking** ([`zarr_metadata.typed_json`](https://zarr-metadata.readthedocs.io/en/latest/api/typed_json/)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. @@ -104,7 +104,9 @@ of context-free consistency rules, e.g.: In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. +a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. +`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no +specification registers. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader @@ -232,13 +234,13 @@ each field: ```python test="true" session="readme" from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.definition import canonical_of, with_problems -from zarr_metadata.v3.definition import CORE +from zarr_metadata.v3.definition import CORE_CONTEXT reading = read_array_metadata_v3(raw) beyond_core = [ loc for loc, field in reading.fields() - if field.name is not None and CORE.claimant(field.read_as, field.name) is None + if field.name is not None and CORE_CONTEXT.claimant(field.read_as, field.name) is None ] simplest = { loc: canonical_of(field, problems) @@ -283,7 +285,7 @@ scope claims is an `UnclaimedField` too, also when it sits inside another field, such as a codec inside a shard. A model's `unclaimed_fields` lists every field nothing in scope claims, at any depth, by its location, and leaves out the members that waive `must_understand`: a reader can open -the node when it is empty. `CORE` holds +the node when it is empty. `CORE_CONTEXT` holds `CONSOLIDATED_METADATA_EXTRA_FIELD`, which reads a group's `consolidated_metadata` as the envelope the spec fixes. A group's `consolidated()` reads the documents that member lists, in the group's @@ -291,7 +293,7 @@ own scope. A v3 model consists of its document and the scope it was read in. `ZarrV3ArrayMetadata(document, context=None)` reads the document in the -scope (`CORE_AND_EXTENSIONS` when none is given) and raises +scope (`CORE_AND_EXTENSIONS_CONTEXT` when none is given) and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as it was written, and `to_key_value` writes that same document to store bytes. @@ -322,7 +324,7 @@ all of them with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, which returns problems instead of raising them. A document with a problem then fails only when it is read. Pass `context=group.context` to read the documents in the group's own -scope; with no `context`, they are read in `CORE_AND_EXTENSIONS`. +scope; with no `context`, they are read in `CORE_AND_EXTENSIONS_CONTEXT`. To write consolidated metadata, build a `ZarrV3ConsolidatedMetadata` and pass it to the group's `update`. The group reads the listed models in its @@ -332,7 +334,7 @@ models as entries. Such an entry is accepted when its claims refine into the model's scope, and refused at its path otherwise. To consolidate children that were read in several scopes, consolidate them in the `union` of those scopes. Every reader takes `context=None` for the -default scope, `CORE_AND_EXTENSIONS`. +default scope, `CORE_AND_EXTENSIONS_CONTEXT`. Two models are equal when they represent the same document, even if each is spelled differently. Values the package interprets (each field, and the diff --git a/packages/zarr-metadata/changes/4583.feature.md b/packages/zarr-metadata/changes/4583.feature.md new file mode 100644 index 0000000000..cbbe44f174 --- /dev/null +++ b/packages/zarr-metadata/changes/4583.feature.md @@ -0,0 +1 @@ +Add the `ZARR_PYTHON_CONTEXT` scope, which reads what zarr-python writes: `CORE_AND_EXTENSIONS_CONTEXT` plus zarr-python's own data types `null_terminated_bytes` (NumPy's `S`), `raw_bytes` (NumPy's `V`) and `variable_length_bytes`, which no specification registers. `CORE_AND_EXTENSIONS_CONTEXT` now also reads the `fixed_length_utf32` data type (NumPy's `U`) and the `vlen-utf8` and `vlen-bytes` codecs that zarr-extensions registers, so the strings zarr-python writes are checked instead of read as unclaimed. diff --git a/packages/zarr-metadata/changes/4583.removal.md b/packages/zarr-metadata/changes/4583.removal.md new file mode 100644 index 0000000000..9faa94ee31 --- /dev/null +++ b/packages/zarr-metadata/changes/4583.removal.md @@ -0,0 +1 @@ +Rename the scopes `CORE`, `CORE_AND_EXTENSIONS` and `CORE_V2` to `CORE_CONTEXT`, `CORE_AND_EXTENSIONS_CONTEXT` and `CORE_V2_CONTEXT`, so that a scope's name says it is a context. The old names, imported from `zarr_metadata.v3`, `zarr_metadata.v3.definition`, `zarr_metadata.v2` or `zarr_metadata.v2.definition`, still refer to the same scopes with a `DeprecationWarning`, and will be removed in a later release. diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 4a23c5c2fb..e95a186284 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -22,7 +22,7 @@ The package is organized to mirror the structure of the Zarr specifications: - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents (`.zarray`, `.zgroup`, `.zattrs`, `.zmetadata`), and `zarr_metadata.v2.definition`: the v2 kinds of metadata, the scope - `CORE_V2`, and the readers of one `dtype` or codec. Each dtype family's + `CORE_V2_CONTEXT`, and the readers of one `dtype` or codec. Each dtype family's definition is in `zarr_metadata.v2.data_type`, and each codec's in `zarr_metadata.v2.codec` - [`zarr_metadata.v3`](v3/index.md) — `TypedDict` shapes for Zarr v3 @@ -35,7 +35,7 @@ The package is organized to mirror the structure of the Zarr specifications: scope type `Context`. Its module docstring is the guide - [`zarr_metadata.v3.definition`](v3/definition.md) — the five v3 kinds of metadata (`CodecDefinition`, `DataTypeDefinition`, ...), the codec - pipeline types, and the v3 scopes `CORE` and `CORE_AND_EXTENSIONS` + pipeline types, and the v3 scopes `CORE_CONTEXT` and `CORE_AND_EXTENSIONS_CONTEXT` The document types, models, and spec vocabulary — including the store keys — are re-exported at the top level, so diff --git a/packages/zarr-metadata/docs/api/v3/codec.md b/packages/zarr-metadata/docs/api/v3/codec.md index cb96d2c7d5..ae5171acd0 100644 --- a/packages/zarr-metadata/docs/api/v3/codec.md +++ b/packages/zarr-metadata/docs/api/v3/codec.md @@ -22,4 +22,8 @@ title: codec ::: zarr_metadata.v3.codec.transpose +::: zarr_metadata.v3.codec.vlen_bytes + +::: zarr_metadata.v3.codec.vlen_utf8 + ::: zarr_metadata.v3.codec.zstd diff --git a/packages/zarr-metadata/docs/api/v3/data_type.md b/packages/zarr-metadata/docs/api/v3/data_type.md index 3f8444625a..2bdfe0ad2c 100644 --- a/packages/zarr-metadata/docs/api/v3/data_type.md +++ b/packages/zarr-metadata/docs/api/v3/data_type.md @@ -40,8 +40,16 @@ title: data_type ::: zarr_metadata.v3.data_type.string +::: zarr_metadata.v3.data_type.fixed_length_utf32 + ::: zarr_metadata.v3.data_type.numpy_datetime64 ::: zarr_metadata.v3.data_type.numpy_timedelta64 ::: zarr_metadata.v3.data_type.struct + +::: zarr_metadata.v3.data_type.null_terminated_bytes + +::: zarr_metadata.v3.data_type.variable_length_bytes + +::: zarr_metadata.v3.data_type.zarr_python_raw_bytes diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index b3936156d0..6b238b7017 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -39,7 +39,7 @@ closely model the content of the Zarr specifications, such as: `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS`. + `CORE_AND_EXTENSIONS_CONTEXT`. - **JSON type checking** ([`zarr_metadata.typed_json`](api/typed_json.md)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. @@ -102,7 +102,9 @@ of context-free consistency rules, e.g.: In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. +a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. +`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no +specification registers. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader @@ -234,13 +236,13 @@ each field: ```python test="true" session="index" from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.definition import canonical_of, with_problems -from zarr_metadata.v3.definition import CORE +from zarr_metadata.v3.definition import CORE_CONTEXT reading = read_array_metadata_v3(raw) beyond_core = [ loc for loc, field in reading.fields() - if field.name is not None and CORE.claimant(field.read_as, field.name) is None + if field.name is not None and CORE_CONTEXT.claimant(field.read_as, field.name) is None ] simplest = { loc: canonical_of(field, problems) @@ -285,7 +287,7 @@ scope claims is an `UnclaimedField` too, also when it sits inside another field, such as a codec inside a shard. A model's `unclaimed_fields` lists every field nothing in scope claims, at any depth, by its location, and leaves out the members that waive `must_understand`: a reader can open -the node when it is empty. `CORE` holds +the node when it is empty. `CORE_CONTEXT` holds `CONSOLIDATED_METADATA_EXTRA_FIELD`, which reads a group's `consolidated_metadata` as the envelope the spec fixes. A group's `consolidated()` reads the documents that member lists, in the group's @@ -293,7 +295,7 @@ own scope. A v3 model consists of its document and the scope it was read in. `ZarrV3ArrayMetadata(document, context=None)` reads the document in the -scope (`CORE_AND_EXTENSIONS` when none is given) and raises +scope (`CORE_AND_EXTENSIONS_CONTEXT` when none is given) and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as it was written, and `to_key_value` writes that same document to store bytes. @@ -324,7 +326,7 @@ all of them with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, which returns problems instead of raising them. A document with a problem then fails only when it is read. Pass `context=group.context` to read the documents in the group's own -scope; with no `context`, they are read in `CORE_AND_EXTENSIONS`. +scope; with no `context`, they are read in `CORE_AND_EXTENSIONS_CONTEXT`. To write consolidated metadata, build a `ZarrV3ConsolidatedMetadata` and pass it to the group's `update`. The group reads the listed models in its @@ -334,7 +336,7 @@ models as entries. Such an entry is accepted when its claims refine into the model's scope, and refused at its path otherwise. To consolidate children that were read in several scopes, consolidate them in the `union` of those scopes. Every reader takes `context=None` for the -default scope, `CORE_AND_EXTENSIONS`. +default scope, `CORE_AND_EXTENSIONS_CONTEXT`. Two models are equal when they represent the same document, even if each is spelled differently. Values the package interprets (each field, and the diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index adfe8e51c2..a4122d87ea 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -161,7 +161,7 @@ This procedure yields one of two values: `zarr-metadata` does not run each stage as a separate function. Instead, a single read of the document collects the problems from every stage, so a caller gets the full list of problems at once. -The **registry** from the resolve-and-validate stage is called a **scope** in this library. For Zarr V3 it is a [`ZarrV3Context`](api/v3/definition.md) object, with one slot for each kind of field (`data_types`, `codecs`, `chunk_grids`, ...), and every reader takes a `context` argument. The default scope is `CORE_AND_EXTENSIONS`, which holds the data types, codecs, chunk grids, etc. defined in the Zarr V3 spec and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/). `CORE` holds only the definitions from the Zarr V3 spec. +The **registry** from the resolve-and-validate stage is called a **scope** in this library. For Zarr V3 it is a [`ZarrV3Context`](api/v3/definition.md) object, with one slot for each kind of field (`data_types`, `codecs`, `chunk_grids`, ...), and every reader takes a `context` argument. The default scope is `CORE_AND_EXTENSIONS_CONTEXT`, which holds the data types, codecs, chunk grids, etc. defined in the Zarr V3 spec and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/). `CORE_CONTEXT` holds only the definitions from the Zarr V3 spec. `ZARR_PYTHON_CONTEXT` adds to `CORE_AND_EXTENSIONS_CONTEXT` the data types that zarr-python writes but no specification registers (`null_terminated_bytes` for NumPy's `S`, `raw_bytes` for NumPy's `V`, and `variable_length_bytes`), so a strict check of what zarr-python writes passes it. `read_array_metadata_v3` reads a JSON document and returns a "reading". `reading.problems` is a tuple of the problems, and `reading.metadata` is the validated model, or `None` if there are problems. Each problem has a `loc`, which is its position in the document as a tuple, a `kind`, like `missing_key` or `invalid_value`, and a human-readable `message`: @@ -193,10 +193,10 @@ To read one field on its own, such as a codec, use `resolve` from `zarr_metadata ```python test="true" session="parsing-validation" from zarr_metadata.definition import resolve -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition field, problems = resolve( - {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS + {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT ) for problem in problems: print(problem.loc, problem.kind) @@ -214,17 +214,17 @@ To make an unknown data type or codec known, add its definition to the scope's s ```python test="true" session="parsing-validation" from zarr_metadata.definition import EmptyConfiguration -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition # The definition of the codec that was unknown: here, a codec named # `acme.codec` that takes no configuration. MY_CODEC = CodecDefinition( name="acme.codec", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" ) -context = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), MY_CODEC]) +context = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), MY_CODEC]) reading = read_array_metadata_v3(raw, context=context) ``` -`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS.update(codecs={**CORE_AND_EXTENSIONS.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own. +`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS_CONTEXT.update(codecs={**CORE_AND_EXTENSIONS_CONTEXT.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own. [^1]: The one exception is the `r*` data type, which is identified by a templated string with no `configuration` object. This mixes the identifier with the configuration in the same JSON value, which is not good design. So this library normalizes `r*` into the same form as all the other metadata objects: the field `"r16"` keeps its name, and is read by the `r*` definition with the configuration `{"bits": 16}`. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index bc57d90fd0..5e8b1ffcba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -136,6 +136,16 @@ TransposeCodecMetadata, TransposeCodecName, ) +from zarr_metadata.v3.codec.vlen_bytes import ( + VLEN_BYTES_CODEC_NAME, + VlenBytesCodecMetadata, + VlenBytesCodecName, +) +from zarr_metadata.v3.codec.vlen_utf8 import ( + VLEN_UTF8_CODEC_NAME, + VlenUtf8CodecMetadata, + VlenUtf8CodecName, +) from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata, ZstdCodecName from zarr_metadata.v3.consolidated import ( ZARR_V3_CONSOLIDATED_METADATA_KEY, @@ -162,6 +172,11 @@ Complex128DataTypeName, Complex128FillValue, ) +from zarr_metadata.v3.data_type.fixed_length_utf32 import ( + FIXED_LENGTH_UTF32_DATA_TYPE_NAME, + FixedLengthUtf32DataTypeName, + FixedLengthUtf32FillValue, +) from zarr_metadata.v3.data_type.float16 import ( FLOAT16_DATA_TYPE_NAME, Float16DataTypeName, @@ -265,6 +280,7 @@ "DEFAULT_CHUNK_KEY_ENCODING_NAME", "DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR", "ENDIANNESS", + "FIXED_LENGTH_UTF32_DATA_TYPE_NAME", "FLOAT16_DATA_TYPE_NAME", "FLOAT32_DATA_TYPE_NAME", "FLOAT64_DATA_TYPE_NAME", @@ -291,6 +307,8 @@ "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", + "VLEN_BYTES_CODEC_NAME", + "VLEN_UTF8_CODEC_NAME", "ZARR_V2_ARRAY_DIMENSION_SEPARATOR", "ZARR_V2_ARRAY_METADATA_STORE_KEY", "ZARR_V2_ARRAY_ORDER", @@ -325,6 +343,8 @@ "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingSeparator", "Endianness", + "FixedLengthUtf32DataTypeName", + "FixedLengthUtf32FillValue", "Float16DataTypeName", "Float16FillValue", "Float32DataTypeName", @@ -378,6 +398,10 @@ "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", "ValidationProblem", + "VlenBytesCodecMetadata", + "VlenBytesCodecName", + "VlenUtf8CodecMetadata", + "VlenUtf8CodecName", "ZarrV2ArrayDimensionSeparator", "ZarrV2ArrayMetadata", "ZarrV2ArrayMetadataJSON", diff --git a/packages/zarr-metadata/src/zarr_metadata/_renamed.py b/packages/zarr-metadata/src/zarr_metadata/_renamed.py new file mode 100644 index 0000000000..d0267a70db --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/_renamed.py @@ -0,0 +1,26 @@ +"""Scopes this package renamed, which keep their old names for now with a `DeprecationWarning`.""" + +import warnings +from collections.abc import Mapping + + +def renamed_scope(module: str, name: str, renames: Mapping[str, tuple[str, object]]) -> object: + """The scope that `name`, an old name in `renames`, refers to, with a `DeprecationWarning`. + + `renames` maps each old name to the scope's name now and the scope. A + module that exported the old names calls this from its `__getattr__`, + so an import of an old name keeps working. Raises `AttributeError`, as + attribute lookup does, when `name` is not an old name. + """ + if name not in renames: + raise AttributeError(f"module {module!r} has no attribute {name!r}") + new, scope = renames[name] + warnings.warn( + f"`{name}` is deprecated and will be removed; use `{new}`, the same scope", + DeprecationWarning, + stacklevel=3, + ) + return scope + + +__all__ = ["renamed_scope"] diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 6dd9c86173..782807f4a8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -40,7 +40,7 @@ configuration, checks the envelope of each nested field (an extra member, a `must_understand` of `false`), and then runs the rules, for one configuration: `GZIP_CODEC.read_configuration({"level": 12})`. -3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS)` reads a whole +3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT)` reads a whole field in a scope. It checks the envelope, maps the name to a definition, checks the configuration, and reads each nested field the same way. It returns the result of the reading and every problem. The @@ -70,17 +70,17 @@ from zarr_metadata.definition import resolve from zarr_metadata.v3.codec.gzip import GZIP_CODEC - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition + from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition resolved, problems = resolve({"name": "gzip", "configuration": {"level": 12}}, - CodecDefinition, CORE_AND_EXTENSIONS) + CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) resolved # RefusedField(..., definition=CodecDefinition(name='gzip'), ...) problems[0].loc # ('configuration', 'level') problems[0].input # 12 dict(problems[0].ctx) # {'ge': 0, 'le': 9} resolved, problems = resolve({"name": "gzip", "configuration": {"level": 5}}, - CodecDefinition, CORE_AND_EXTENSIONS) + CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) resolved.definition is GZIP_CODEC # True resolved.configuration # {'level': 5} @@ -124,7 +124,7 @@ from zarr_metadata import ValidationProblem from zarr_metadata.definition import Nested - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition + from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition class AcmeLz4Configuration(TypedDict, closed=True): @@ -149,7 +149,7 @@ def acme_lz4_rules( size="dynamic", rules=acme_lz4_rules, ) - CONTEXT = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), ACME_LZ4]) + CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), ACME_LZ4]) The TypedDict declares how keys it does not declare are handled. With `closed=True`, such a key is a problem, as above. With `extra_items=`, diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 5d50f8e605..0c30ffd16b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1374,7 +1374,7 @@ def resolve( else raises `TypeError`. """ if not _is_scope(context): - msg = f"a field is read in a scope, such as CORE_AND_EXTENSIONS, got {described(context)}" + msg = f"a field is read in a scope, such as CORE_AND_EXTENSIONS_CONTEXT, got {described(context)}" raise TypeError(msg) asked = as_kind(kind) _check_format(asked, context) diff --git a/packages/zarr-metadata/src/zarr_metadata/json_schema.py b/packages/zarr-metadata/src/zarr_metadata/json_schema.py index 9b7443a8f4..ea80ac0242 100644 --- a/packages/zarr-metadata/src/zarr_metadata/json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/json_schema.py @@ -30,7 +30,7 @@ **A whole `zarr.json`.** `node_metadata_json_schema_v3(context=None)` writes a whole v3 `zarr.json`, array or group, as the scope reads it -(`CORE_AND_EXTENSIONS` when none is given). Its fields are written +(`CORE_AND_EXTENSIONS_CONTEXT` when none is given). Its fields are written as the scope reads them. For each definition, the schema accepts its field (its name, its configuration as its TypedDict declares it, including bounds, a `must_understand` of `true`, and its bare name when diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 98776c7c5d..ab6006ca2a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -9,7 +9,7 @@ `read_consolidated_metadata_v2`. A reader checks a document's JSON structure. In a v3 document it also reads each extension point (codecs, chunk grids, data types, ...) through the definition that claims its -name in a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` +name in a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. A v3 reader checks the fill value against the data type the document names, the chunk grid against the shape, and the codecs as a pipeline, each codec against the chunk it receives. Each reader does one diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index 7768e68210..a4336ea0a5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -70,7 +70,7 @@ from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v2.definition import ( - CORE_V2, + CORE_V2_CONTEXT, ZarrV2CodecDefinition, ZarrV2DataTypeDefinition, ) @@ -437,7 +437,7 @@ def in_files( def read_array_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ArrayMetadataReading: - """Read `value`, a v2 array document, in `context` (`CORE_V2` when none is given), whatever it holds. + """Read `value`, a v2 array document, in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds. The result holds everything the read finds: the dtype, the compressor and each filter as the scope read them (an `AcceptedField` read by @@ -447,7 +447,7 @@ def read_array_metadata_v2( problem. One that the scope does not claim is not a problem. `fill_value` is checked against the dtype the scope read. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) reading, members = read_array_v2(value, context) if members is None: return reading @@ -510,7 +510,7 @@ class ZarrV2ArrayMetadata(Keyed): node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) reading, members = read_array_v2(document, context) if members is None: raise MetadataValidationError(reading.problems) @@ -731,7 +731,7 @@ def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetad in `context`. The existing reading is reused when `context` reads every claim identically. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -745,7 +745,7 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat in the document. Raises `MetadataValidationError` when a definition that `context` claims rejects what was written. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) @@ -807,7 +807,9 @@ def create_default( if lengths is not None: given["chunks"] = lengths if "dtype" in given and "fill_value" not in given: - dtype, _ = resolve(given["dtype"], ZarrV2DataTypeDefinition, scoped(context, CORE_V2)) + dtype, _ = resolve( + given["dtype"], ZarrV2DataTypeDefinition, scoped(context, CORE_V2_CONTEXT) + ) if len(fill_value_problems(dtype, 0)) != 0: given["fill_value"] = None merged = {key: value for key, value in {**document, **given}.items() if value is not UNSET} @@ -836,7 +838,7 @@ def from_key_value( valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 53d12254bc..58fb7e30c5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -62,7 +62,7 @@ # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3._definition import ( ArrayExtraFieldDefinition, Chunk, @@ -499,7 +499,7 @@ class ZarrV3ArrayMetadata(Keyed): node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_array_v3(document, context) if members is None: raise MetadataValidationError(reading.problems) @@ -696,7 +696,7 @@ def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetad in `context`. The existing reading is reused when `context` reads every claim identically. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -713,7 +713,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat gain can expose a problem. `with_context` reads the document in any scope. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) @@ -811,7 +811,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) return cls(load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY), context=context) @@ -888,7 +888,7 @@ def read_array_metadata_v3( spec's fields), iterate over the reading's `fields()`. A value that is not an object has no fields. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_array_v3(value, context) if members is None: return reading diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index eb290a6cb7..1a0cdc5e48 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -63,7 +63,7 @@ from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.consolidated import ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY -from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v2.definition import CORE_V2_CONTEXT from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY, ZarrV2GroupMetadataJSON if TYPE_CHECKING: @@ -136,8 +136,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_group_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2GroupMetadataReading: - """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2` when none is given), whatever it holds: every problem, and the model when there are none. A group has no fields for a scope to read; `context` is taken for consistency with every other v2 reader.""" - context = scoped(context, CORE_V2) + """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds: every problem, and the model when there are none. A group has no fields for a scope to read; `context` is taken for consistency with every other v2 reader.""" + context = scoped(context, CORE_V2_CONTEXT) problems = _group_problems_v2(value) if len(problems) != 0: return ZarrV2GroupMetadataReading(problems=problems) @@ -173,7 +173,7 @@ def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: def read_node_metadata_v2( files: Mapping[str, object], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadataReading: - """Read the v2 node described by `files`, in `context` (`CORE_V2` when none is given), as the node type its files declare. + """Read the v2 node described by `files`, in `context` (`CORE_V2_CONTEXT` when none is given), as the node type its files declare. `files` maps each of the node's files to its decoded JSON: `.zarray` or `.zgroup`, and `.zattrs` when there is one. They can come from a @@ -194,7 +194,7 @@ def read_node_metadata_v2( v2 counterpart of `read_node_metadata_v3`, which reads one `zarr.json` and its `node_type`. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) is_array = ZARR_V2_ARRAY_METADATA_STORE_KEY in files is_group = ZARR_V2_GROUP_METADATA_STORE_KEY in files if is_array and is_group: @@ -234,7 +234,7 @@ def node_metadata_from_key_value_v2( mapping: Mapping[StoreKey, bytes], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadata: """Return the model of the v2 node whose files are in `mapping`, a store's keys and bytes: each file is decoded, then read as `read_node_metadata_v2` reads it. Raises `MetadataValidationError` with every problem, including a file whose bytes are not JSON, and `TypeError` for a scope of another format before any bytes are decoded.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2(mapping) if len(problems) != 0: raise MetadataValidationError(problems) @@ -278,7 +278,7 @@ class ZarrV2GroupMetadata(Keyed): node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) problems = _group_problems_v2(document) if len(problems) != 0: raise MetadataValidationError(problems) @@ -368,7 +368,7 @@ def update(self, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2GroupMet def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return self._of(self._document, context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: @@ -406,7 +406,7 @@ def from_key_value( valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_GROUP_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) @@ -449,8 +449,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_consolidated_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ConsolidatedMetadataReading: - """Read `value`, a v2 `.zmetadata` document, in `context` (`CORE_V2` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV2ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" - context = scoped(context, CORE_V2) + """Read `value`, a v2 `.zmetadata` document, in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV2ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" + context = scoped(context, CORE_V2_CONTEXT) document, nodes, files, problems = _read_consolidated_v2(value, context) if len(problems) != 0: return ZarrV2ConsolidatedMetadataReading(problems=problems) @@ -487,7 +487,7 @@ class ZarrV2ConsolidatedMetadata(Keyed): zarr_consolidated_format: Final = 1 def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) read, nodes, files, problems = _read_consolidated_v2(document, context) if len(problems) != 0: raise MetadataValidationError(problems) @@ -576,7 +576,7 @@ def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, Za def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: @@ -586,7 +586,7 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2Consolidated node's entry. Raises `MetadataValidationError` when a gain exposes a problem. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) conflicts: list[Conflict] = [] for path, node in self._metadata.items(): if not isinstance(node, ZarrV2ArrayMetadata): @@ -633,7 +633,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before the bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return cls( load_store_json(mapping, ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY), context=context ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 155d8e8019..a189856678 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -80,7 +80,7 @@ # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3._definition import GroupExtraFieldDefinition from zarr_metadata.v3._hierarchy import hierarchy_problems, path_faults from zarr_metadata.v3.consolidated import ( @@ -169,7 +169,7 @@ class ZarrV3GroupMetadata(Keyed): the document as written, refined, and `context` is the scope. `attributes` and `consolidated_metadata` are views of the values the read refined. Each member the spec does not define is an extra field, - in `extra_fields`, as the scope read it. `CORE` reads a + in `extra_fields`, as the scope read it. `CORE_CONTEXT` reads a `consolidated_metadata` member as the envelope the core spec fixes, and no definition reads the documents the member lists: they follow a convention of the reference implementation, not the core spec. To @@ -186,7 +186,7 @@ class ZarrV3GroupMetadata(Keyed): node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_group_v3(document, context) if members is None or len(reading.problems) != 0: raise MetadataValidationError(reading.problems) @@ -281,7 +281,7 @@ def attributes(self) -> Mapping[str, JSONValue]: def extra_fields( self, ) -> Mapping[str, AcceptedField[GroupExtraFieldDefinition[Any]] | UnclaimedField]: - """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key (as `CORE` claims `consolidated_metadata`), and an `UnclaimedField` otherwise.""" + """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key (as `CORE_CONTEXT` claims `consolidated_metadata`), and an `UnclaimedField` otherwise.""" return self._shown[3] @property @@ -365,7 +365,7 @@ def update(self, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3GroupMet def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -382,7 +382,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadat in `context`, and `ZarrV3ConsolidatedMetadata.refined_in` checks that they read the same there. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) @@ -409,7 +409,7 @@ def create_default( **overrides: Unpack[ZarrV3GroupMetadataUpdate], ) -> ZarrV3GroupMetadata: """Return a group with no attributes, with any document members in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a member and a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Raises `MetadataValidationError` when the resulting document has a problem.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) changes = member_changes( overrides, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_members ) @@ -426,7 +426,7 @@ def from_json( Raises `MetadataValidationError` with every problem the read finds. A `consolidated_metadata` of `null`, which zarr-python 3.0 and 3.1 wrote, is a value the document wrote that is not an object. In a - scope that claims the member, as `CORE` does, this is a problem, + scope that claims the member, as `CORE_CONTEXT` does, this is a problem, because the spec requires an object. Use `read_repaired_node_metadata_v3` to read such a store. A member the spec does not define is kept in `extra_fields`. @@ -443,7 +443,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) return cls(load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY), context=context) @@ -495,7 +495,7 @@ class ZarrV3ConsolidatedMetadata(Keyed): must_understand: Final = False def __init__(self, member: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) # The member is always under a group's key, even when read on its # own, so nesting depth is counted from there, as in the group. readings, members, document, problems = _read_consolidated_v3( @@ -561,7 +561,7 @@ def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, Za def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if all( disagreements(context, claims_of(node.reading.fields())).agrees for node in self._metadata.values() @@ -577,7 +577,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3Consolidated document's path. Raises `MetadataValidationError` when a gain exposes a problem. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) conflicts = [ dataclasses.replace( conflict, loc=("metadata", path, *(() if conflict.loc is None else conflict.loc)) @@ -681,8 +681,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_consolidated_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ConsolidatedMetadataReading: - """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" - context = scoped(context, CORE_AND_EXTENSIONS) + """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS_CONTEXT` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) readings, members, document, problems = _read_consolidated_v3( value, context, (ZARR_V3_CONSOLIDATED_METADATA_KEY,) ) @@ -737,7 +737,7 @@ def read_node_metadata_v3( read `node_type` from unvalidated JSON, and a document in another format is reported as not v3. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) node_type, problems = _node_type(value) if node_type == "array": return read_array_metadata_v3(value, context=context) @@ -761,7 +761,7 @@ def node_metadata_from_json_v3( `MetadataValidationError` with every problem `read_node_metadata_v3` finds, including a `node_type` that is neither array nor group. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading = read_node_metadata_v3(data, context=context) if reading.metadata is None: raise MetadataValidationError(reading.problems) @@ -776,7 +776,7 @@ def node_metadata_from_key_value_v3( Raises `MetadataValidationError` when the key is missing, its bytes are not JSON, or the document is not a valid array or group. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) # Array documents and group documents are both stored at `zarr.json`. document = load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY) return node_metadata_from_json_v3(document, context=context) @@ -853,13 +853,13 @@ def read_group_metadata_v3( reader must understand each one that does not declare `must_understand: false`; the model reports these as `must_understand_fields`. A `consolidated_metadata` member, if - present and claimed by the scope, as `CORE` claims it, is checked + present and claimed by the scope, as `CORE_CONTEXT` claims it, is checked against the envelope the core spec fixes. The documents it lists are not read; `read_consolidated_metadata_v3` reads them. This is the form of `ZarrV3GroupMetadata(value, context)` that returns problems instead of raising them. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_group_v3(value, context) if members is None or members.document is None: return reading diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py index d16663b1fe..679a584fc8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py @@ -45,9 +45,9 @@ ) from zarr_metadata.model._group_v3 import ZarrV3NodeMetadataReading, read_node_metadata_v3 from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY -from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v2.definition import CORE_V2_CONTEXT from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3.consolidated import ZARR_V3_CONSOLIDATED_METADATA_KEY if TYPE_CHECKING: @@ -278,7 +278,7 @@ def read_repaired_node_metadata_v3( Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v3` reports it. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) repaired, repairs = repair_node_metadata_v3(value) return ZarrV3RepairedNodeMetadataReading( read_node_metadata_v3(repaired, context=context), repairs @@ -482,7 +482,7 @@ def read_repaired_node_metadata_v2( Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v2` reports it. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) repaired, repairs = repair_node_metadata_v2(files) return ZarrV2RepairedNodeMetadataReading( read_node_metadata_v2(repaired, context=context), repairs @@ -510,7 +510,7 @@ def read_repaired_consolidated_metadata_v2( read as it is, and reported the same way `read_consolidated_metadata_v2` reports it. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) repaired, repairs = repair_consolidated_metadata_v2(value) return ZarrV2RepairedConsolidatedMetadataReading( read_consolidated_metadata_v2(repaired, context=context), repairs diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index f369f2beec..84ded7390e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -20,7 +20,7 @@ `"zarr_metadata_context_v2"` item. This lets a model with fields of both formats give a scope for each format. - If no scope is given for a format, its field types read in the default - scope for that format, `CORE_AND_EXTENSIONS` or `CORE_V2`. + scope for that format, `CORE_AND_EXTENSIONS_CONTEXT` or `CORE_V2_CONTEXT`. A mapping key that starts like these keys but is not one of them raises `TypeError`, so a misspelled key is not read as no scope. To fix a field's @@ -128,8 +128,8 @@ class ArrayManifest(BaseModel): from zarr_metadata.definition._context import scoped from zarr_metadata.definition._definition import Context from zarr_metadata.definition._scope import ScopeConflictError -from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, ZarrV3Context +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, ZarrV3Context if TYPE_CHECKING: from collections.abc import Callable, Mapping @@ -321,7 +321,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json, _model.ZarrV3ArrayMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ArrayMetadataSchema, @@ -337,7 +337,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json, _model.ZarrV2ArrayMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ArrayMetadataSchema, @@ -353,7 +353,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json, _model.ZarrV3GroupMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3GroupMetadataSchema, @@ -369,7 +369,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json, _model.ZarrV2GroupMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2GroupMetadataSchema, @@ -385,7 +385,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json, _model.ZarrV3ConsolidatedMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, @@ -404,7 +404,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json, _model.ZarrV2ConsolidatedMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ConsolidatedMetadataSchema, @@ -419,15 +419,15 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _FIELD_PARTS: Final[Mapping[type[Any], tuple[ZarrV3Context | ZarrV2Context, object]]] = ( MappingProxyType( { - _model.ZarrV3ArrayMetadata: (CORE_AND_EXTENSIONS, _ZarrV3ArrayMetadataSchema), - _model.ZarrV2ArrayMetadata: (CORE_V2, _ZarrV2ArrayMetadataSchema), - _model.ZarrV3GroupMetadata: (CORE_AND_EXTENSIONS, _ZarrV3GroupMetadataSchema), - _model.ZarrV2GroupMetadata: (CORE_V2, _ZarrV2GroupMetadataSchema), + _model.ZarrV3ArrayMetadata: (CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3ArrayMetadataSchema), + _model.ZarrV2ArrayMetadata: (CORE_V2_CONTEXT, _ZarrV2ArrayMetadataSchema), + _model.ZarrV3GroupMetadata: (CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3GroupMetadataSchema), + _model.ZarrV2GroupMetadata: (CORE_V2_CONTEXT, _ZarrV2GroupMetadataSchema), _model.ZarrV3ConsolidatedMetadata: ( - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3ConsolidatedMetadataSchema, ), - _model.ZarrV2ConsolidatedMetadata: (CORE_V2, _ZarrV2ConsolidatedMetadataSchema), + _model.ZarrV2ConsolidatedMetadata: (CORE_V2_CONTEXT, _ZarrV2ConsolidatedMetadataSchema), } ) ) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index caf7b71a05..e966cd9087 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -12,7 +12,7 @@ from hypothesis import strategies as st -from zarr_metadata.definition._definition import AcceptedField +from zarr_metadata.definition._definition import AcceptedField, EmptyConfiguration from zarr_metadata.testing._data_type import ( COMPLEX_WIDTHS, FLOAT_WIDTHS, @@ -30,6 +30,8 @@ from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC +from zarr_metadata.v3.codec.vlen_bytes import VLEN_BYTES_CODEC +from zarr_metadata.v3.codec.vlen_utf8 import VLEN_UTF8_CODEC from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE @@ -155,6 +157,17 @@ def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) +def _vlen(definition: CodecDefinition[Any]) -> CodecStrategy: + """A codec with nothing to configure that takes only some data types: none for a chunk of another, which its own chunk rules say.""" + return CodecStrategy( + configurations=lambda chunk: ( + None + if any(definition.chunk_rules(EmptyConfiguration(), {}, chunk)) + else EMPTY_CONFIGURATIONS + ) + ) + + def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> CodecStrategy: """A codec whose configurations do not depend on the chunk it is handed.""" return CodecStrategy(configurations=lambda _chunk: configurations) @@ -167,6 +180,8 @@ def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> Co GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), + VLEN_BYTES_CODEC: _vlen(VLEN_BYTES_CODEC), + VLEN_UTF8_CODEC: _vlen(VLEN_UTF8_CODEC), ZSTD_CODEC: _unchunked( st.fixed_dictionaries( {"level": st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL)}, diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py index dd06df28a0..b592b2d979 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -21,6 +21,10 @@ from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.fixed_length_utf32 import ( + FIXED_LENGTH_UTF32_DATA_TYPE, + FixedLengthUtf32Configuration, +) from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE @@ -28,6 +32,10 @@ from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE +from zarr_metadata.v3.data_type.null_terminated_bytes import ( + NULL_TERMINATED_BYTES_DATA_TYPE, + NullTerminatedBytesConfiguration, +) from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration @@ -36,6 +44,11 @@ from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE +from zarr_metadata.v3.data_type.variable_length_bytes import VARIABLE_LENGTH_BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.zarr_python_raw_bytes import ( + ZARR_PYTHON_RAW_BYTES_DATA_TYPE, + ZarrPythonRawBytesConfiguration, +) if TYPE_CHECKING: from collections.abc import Mapping @@ -148,9 +161,33 @@ def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: ).map(_base64) """A `bytes` fill value: byte values, or the bytes in standard-alphabet base64.""" +_LENGTH_BYTES: Final = st.integers(1, _MAX_BYTES) +"""The `length_bytes` of zarr-python's fixed-length byte types.""" + + +def _base64_fill_values( + configuration: Mapping[str, JSONValue], +) -> st.SearchStrategy[JSONValue]: + """At most `length_bytes` bytes in standard-alphabet base64, as zarr-python's fixed-length byte types take them.""" + length_bytes = cast("ZarrPythonRawBytesConfiguration", configuration)["length_bytes"] + return st.binary(max_size=length_bytes).map(_base64) + + _TEXT: Final = st.text(st.characters(exclude_categories=("Cs",)), max_size=16) """Text a JSON document can hold: no lone surrogate, which is no Unicode scalar value.""" +_MAX_CODE_POINTS: Final = 16 +"""The most code points a drawn `fixed_length_utf32` value holds.""" + + +def _fixed_length_utf32_fill_values( + configuration: Mapping[str, JSONValue], +) -> st.SearchStrategy[JSONValue]: + """Text of at most one code point for each 4 bytes of the size, some of them trailing `U+0000` padding.""" + capacity = cast("FixedLengthUtf32Configuration", configuration)["length_bytes"] // 4 + return st.text(st.characters(exclude_categories=("Cs",)) | st.just("\x00"), max_size=capacity) + + _NUMPY_TIME: Final = DataTypeStrategy( configurations=st.fixed_dictionaries( { @@ -184,6 +221,25 @@ def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: ), BYTES_DATA_TYPE: _fill_values_of(_BYTES_FILL_VALUES), STRING_DATA_TYPE: _fill_values_of(_TEXT), + NULL_TERMINATED_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=_LENGTH_BYTES.map( + lambda length_bytes: NullTerminatedBytesConfiguration(length_bytes=length_bytes) + ), + fill_values=_base64_fill_values, + ), + VARIABLE_LENGTH_BYTES_DATA_TYPE: _fill_values_of(st.binary(max_size=_MAX_BYTES).map(_base64)), + ZARR_PYTHON_RAW_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=_LENGTH_BYTES.map( + lambda length_bytes: ZarrPythonRawBytesConfiguration(length_bytes=length_bytes) + ), + fill_values=_base64_fill_values, + ), + FIXED_LENGTH_UTF32_DATA_TYPE: DataTypeStrategy( + configurations=st.integers(1, _MAX_CODE_POINTS).map( + lambda count: FixedLengthUtf32Configuration(length_bytes=4 * count) + ), + fill_values=_fixed_length_utf32_fill_values, + ), NUMPY_DATETIME64_DATA_TYPE: _NUMPY_TIME, NUMPY_TIMEDELTA64_DATA_TYPE: _NUMPY_TIME, } diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py index fd13c43839..8491c9c171 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -60,7 +60,7 @@ def test_my_reader_reads_every_blosc_codec(field): Strategies, ) from zarr_metadata.v3._context import ZarrV3Context -from zarr_metadata.v3._core import CORE +from zarr_metadata.v3._core import CORE_CONTEXT from zarr_metadata.v3._definition import Chunk, DataTypeDefinition if TYPE_CHECKING: @@ -181,7 +181,7 @@ def _written_bare(definition: Definition[Any]) -> bool: """ if not isinstance(definition, DataTypeDefinition): return False - return CORE.claimant(DataTypeDefinition, definition.name) is definition + return CORE_CONTEXT.claimant(DataTypeDefinition, definition.name) is definition @st.composite diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py index 741ff5cfb5..af6dda6c22 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py @@ -6,12 +6,16 @@ and `data_type`, and the v2 kinds and scope are in `definition`. This module also exports every v2 definition (`GZIP_CODEC_V2`, -`FLOAT_DATA_TYPE_V2`, ...), the scope `CORE_V2`, and the scope class +`FLOAT_DATA_TYPE_V2`, ...), the scope `CORE_V2_CONTEXT`, and the scope class `ZarrV2Context`, so that one import builds or updates a scope. Each is the same object as in its own submodule, where a definition's configuration types and constants are. """ +from typing import TYPE_CHECKING + +from zarr_metadata._renamed import renamed_scope +from zarr_metadata.v2._core import RENAMED_SCOPES_V2 from zarr_metadata.v2.codec.checksum import ( ADLER32_CODEC_V2, CRC32_CODEC_V2, @@ -59,7 +63,7 @@ DATETIME64_DATA_TYPE_V2, TIMEDELTA64_DATA_TYPE_V2, ) -from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context __all__ = [ "ADLER32_CODEC_V2", @@ -70,7 +74,7 @@ "BYTES_DATA_TYPE_V2", "BZ2_CODEC_V2", "COMPLEX_DATA_TYPE_V2", - "CORE_V2", + "CORE_V2_CONTEXT", "CRC32C_CODEC_V2", "CRC32_CODEC_V2", "DATETIME64_DATA_TYPE_V2", @@ -98,3 +102,11 @@ "ZSTD_CODEC_V2", "ZarrV2Context", ] + + +if not TYPE_CHECKING: + # Hidden from type checkers, which would otherwise read every missing + # name of this module as a scope. + def __getattr__(name: str) -> object: + """`CORE_V2`, the scope's name before it ended in `_CONTEXT`, with a `DeprecationWarning`.""" + return renamed_scope(__name__, name, RENAMED_SCOPES_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_core.py b/packages/zarr-metadata/src/zarr_metadata/v2/_core.py new file mode 100644 index 0000000000..0daaa6c344 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_core.py @@ -0,0 +1,17 @@ +"""The scope this package provides for Zarr v2.""" + +from __future__ import annotations + +from typing import Final + +from zarr_metadata.v2._context import ZarrV2Context +from zarr_metadata.v2.codec import CODECS_V2 +from zarr_metadata.v2.data_type import DATA_TYPES_V2 + +CORE_V2_CONTEXT: Final = ZarrV2Context(data_types=DATA_TYPES_V2, codecs=CODECS_V2) +"""The data types that zarr-python 2.x writes and the codecs that numcodecs 0.16 configures.""" + +RENAMED_SCOPES_V2: Final = {"CORE_V2": ("CORE_V2_CONTEXT", CORE_V2_CONTEXT)} +"""The v2 scope's name before it ended in `_CONTEXT`, with the scope's name now and the scope.""" + +__all__ = ["CORE_V2_CONTEXT", "RENAMED_SCOPES_V2"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py index 8cf4990992..d0da681c58 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py @@ -4,10 +4,10 @@ in `compressor` and `filters`. Zarr v2 has no extension registry, but it still has a scope: the data types that zarr-python 2.x writes (one definition per NumPy family), and the codecs that numcodecs 0.16 -configures (one per id that this package models). `CORE_V2` is that +configures (one per id that this package models). `CORE_V2_CONTEXT` is that scope. To read one field in it, call `zarr_metadata.definition.resolve` with the field's kind, as for a v3 field: -`resolve("