diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80cb8a5d4..f7c543387 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: run: | make build-py - - name: Check native Python codegen output + - name: Check Python codegen output if: ${{ matrix.python-version == '3.12' }} run: | make generate-codegen-output-py diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index d0ae825ed..8daa8d4cd 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -1,8 +1,13 @@ { "files": { - "src/example_types/_private/schemas.py": "1d708d275049f3f4a86b343c3f38104ef4a7e5bf749a27cd1a490c95c8f9db59", - "src/example_types/models.py": "51aadd47e9971f2b44fabd47e81e4b04d7156aa61b64454b39f4b6d47603ab45", - "src/example_types/models.pyi": "31c14804662d43c11fc6c61d0a49244e5e606b65aadb774df913f6a26e1b135a" + "README.md": "b070865ebb1dc46192ea8a45f0961e1b0f812dfccd069537cfc04311b3327660", + "pyproject.toml": "3b4edeff9fe5c4641d8729e49abfa1e7c1936484d0512ad273a3724ebb54f40a", + "src/example_types/__init__.py": "27ea4bc5ba93a6fc6e1f11e9784565de24e9249f0d3e77a9e2fb64fa47d11012", + "src/example_types/_private/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/example_types/_private/schemas.py": "b3c1dd3b1ab8511ef45cca22cf415726aec599edfd457b30cdb22482bcd5b799", + "src/example_types/models.py": "262a8c40dce4fb282aca6b17214296db0c4fc1da526a50a3155cde0fe7344b6b", + "src/example_types/models.pyi": "36be0dc7d1d9402394e531d782fab2392d55cc60a949a1bcae4c4f469de6cc11", + "src/example_types/py.typed": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "version": 1 } diff --git a/codegen-output-py/types/README.md b/codegen-output-py/types/README.md new file mode 100644 index 000000000..8e6571146 --- /dev/null +++ b/codegen-output-py/types/README.md @@ -0,0 +1,5 @@ + + +## example_types Types Package + +Example Python codegen output diff --git a/codegen-output-py/types/pyproject.toml b/codegen-output-py/types/pyproject.toml new file mode 100644 index 000000000..4df29b3b4 --- /dev/null +++ b/codegen-output-py/types/pyproject.toml @@ -0,0 +1,36 @@ +# Code generated by smithy-python-codegen DO NOT EDIT. + +[project] +name = "example_types" +version = "1.0.0" +description = "Example Python codegen output" +readme = "README.md" +requires-python = ">=3.12" +keywords = ["smithy", "example_types"] +license = {text = "Apache-2.0"} +dependencies = [ + "smithy-core~=0.10.0", +] + +[build-system] +requires = [ + "hatchling", +] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = [ + "src/example_types", +] + +[tool.pyright] +typeCheckingMode = "strict" + +[tool.ruff] +target-version = "py312" + +[tool.ruff.lint] +ignore = ["F841"] + +[tool.ruff.format] +skip-magic-trailing-comma = true diff --git a/codegen-output-py/types/src/example_types/__init__.py b/codegen-output-py/types/src/example_types/__init__.py new file mode 100644 index 000000000..3e7d37b5b --- /dev/null +++ b/codegen-output-py/types/src/example_types/__init__.py @@ -0,0 +1 @@ +__version__: str = "1.0.0" diff --git a/codegen-output-py/types/src/example_types/_private/__init__.py b/codegen-output-py/types/src/example_types/_private/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/codegen-output-py/types/src/example_types/_private/schemas.py b/codegen-output-py/types/src/example_types/_private/schemas.py index 91264ec48..c2c99e18e 100644 --- a/codegen-output-py/types/src/example_types/_private/schemas.py +++ b/codegen-output-py/types/src/example_types/_private/schemas.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from smithy_core._schema_compact import StaticSchema n0 = "example" diff --git a/codegen-output-py/types/src/example_types/models.py b/codegen-output-py/types/src/example_types/models.py index 6803f9606..6ef4b985d 100644 --- a/codegen-output-py/types/src/example_types/models.py +++ b/codegen-output-py/types/src/example_types/models.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from smithy_core._dynamic_models import ModelRegistry from smithy_core.exceptions import ModeledError @@ -43,7 +41,7 @@ class ServiceError(ModeledError): } _MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit") -__all__ = ("ServiceError", *_MODEL_NAMES) +__all__ = ("ServiceError", *_MODEL_NAMES) # pyright: ignore[reportUnsupportedDunderAll] _registry = ModelRegistry( diff --git a/codegen-output-py/types/src/example_types/models.pyi b/codegen-output-py/types/src/example_types/models.pyi index 0d72eca2a..6502961ba 100644 --- a/codegen-output-py/types/src/example_types/models.pyi +++ b/codegen-output-py/types/src/example_types/models.pyi @@ -1,5 +1,3 @@ -from __future__ import annotations - from dataclasses import dataclass from enum import IntEnum, StrEnum from typing import Self diff --git a/codegen-output-py/types/src/example_types/py.typed b/codegen-output-py/types/src/example_types/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/designs/codegen/types.md b/designs/codegen/types.md index 9f7135dc3..58e07fa24 100644 --- a/designs/codegen/types.md +++ b/designs/codegen/types.md @@ -112,7 +112,7 @@ The `types` command accepts: * `--package-version` * `--package-description` * `--selector` -* `--generate-inputs-and-outputs` +* `--include-operation-io` `package` and `package-version` are required. The package identifies the generated import package and follows the validation rules in `SymbolProvider`. @@ -129,7 +129,7 @@ Types generation never infers a service. The options apply in this order: 2. `--selector` narrows those declaration shapes. The default identity selector keeps every candidate. 3. Input and output structures are removed as roots unless - `--generate-inputs-and-outputs` is set. + `--include-operation-io` is set. 4. Member-target dependencies of the remaining declarations are added transitively. @@ -382,16 +382,17 @@ model names: ```python _MODEL_NAMES = tuple(_MODEL_INDEX) -__all__ = ("ServiceError", *_MODEL_NAMES) +__all__ = ("ServiceError", *_MODEL_NAMES) # pyright: ignore[reportUnsupportedDunderAll] def __dir__() -> list[str]: return sorted({*globals(), *__all__}) ``` -This keeps Ruff from reporting `F822` for lazy exports. Star imports read -`__all__` and resolve each model through module `__getattr__`. Computing -`__dir__` from the index does not construct model classes. +This keeps Ruff from reporting `F822` for lazy exports. The targeted Pyright +suppression replaces excluding the complete runtime module from analysis. Star +imports read `__all__` and resolve each model through module `__getattr__`. +Computing `__dir__` from the index does not construct model classes. Model member access uses the schema's dense member indexes where positional dispatch is required. Name maps remain dictionaries where lookup is by a wire diff --git a/designs/codegen/writer.md b/designs/codegen/writer.md index 1c581d1d1..a258446b1 100644 --- a/designs/codegen/writer.md +++ b/designs/codegen/writer.md @@ -10,11 +10,15 @@ decides which declarations, fields, and defaults to write. from smithy_python.symbols import TypeReference from smithy_python.writer import PythonWriter -node = TypeReference("Node", "example.models", nullable=True) +leaf = TypeReference("Leaf", "example.models", nullable=True) writer = PythonWriter("example.models") +writer.line("class Leaf:") +with writer.indent(): + writer.line("pass") +writer.line() writer.line("class Node:") with writer.indent(): - writer.line("children: ", TypeReference("list", "builtins", (node,))) + writer.line("children: ", TypeReference("list", "builtins", (leaf,))) writer.line("amount: ", TypeReference("Decimal", "decimal")) source = writer.render() ``` @@ -22,13 +26,14 @@ source = writer.render() The result is: ```python -from __future__ import annotations - from decimal import Decimal +class Leaf: + pass + class Node: - children: list[Node | None] + children: list[Leaf | None] amount: Decimal ``` @@ -56,9 +61,9 @@ symbol provider ensures that declaration and field names are unique. current indentation. `line()` writes a blank line without spaces. * `indent()` adds four spaces inside a context manager and restores indentation even when the block raises an exception. -* `render()` returns source ending in a newline, with a future-annotations - header, sorted imports and the body in written order. It does not modify the - writer. Adding another reference can change aliases in the next render. +* `render()` returns source ending in a newline, with sorted imports and the body + in written order. It does not modify the writer. Adding another reference can + change aliases in the next render. Raw text is trusted Python, not a template. To find names, `render()` parses the written lines with the Python 3.12 grammar, with a unique placeholder @@ -69,9 +74,8 @@ later Python versions will reject them. Callers supply blank lines between declarations. The analysis reads the syntax tree rather than `symtable`, whose results change -between Python versions and omit annotations under `from __future__ import -annotations`. A reference joined to raw text, such as `obj.` followed by a -reference, has no scope of its own. It is treated as visible to every local +between Python versions. A reference joined to raw text, such as `obj.` followed +by a reference, has no scope of its own. It is treated as visible to every local name. Pass every builtin or imported name as a type reference, not just annotations. @@ -104,9 +108,9 @@ level: `list[str | None]` differs from `list[str] | None`. The `None` literal stays `None`, including when marked nullable. Only that literal can omit its module. Other module-less references raise `CodegenError`. -Generated source targets Python 3.12+. The future import permits references to -classes defined later, including recursive types. These are annotations, not -expressions to evaluate while defining classes. +Generated source targets Python 3.12+. References in runtime annotations must +name classes that have already been defined. Stub annotations can refer to +classes defined later. * Builtins normally use `str`, `int`, `list[T]` and `dict[K, V]`. * Types in the current module use their bare names without imports. @@ -135,9 +139,9 @@ without recording dependencies. ## Names that collide -An import uses a module-derived alias when its name matches a module-level -name, a local name visible where it is referenced, a builtin, the future -import's `annotations` name, or another imported type. +An import uses a module-derived alias when its name matches a module-level name, +a local name visible where it is referenced, a builtin, or another imported +type. For example, `decimal.Decimal` becomes `_decimal_Decimal`. Dots in a module path become underscores. All imports sharing a short name receive aliases, regardless of the order they were written. diff --git a/packages/smithy-python/README.md b/packages/smithy-python/README.md index 79db2d4bc..2392b8f02 100644 --- a/packages/smithy-python/README.md +++ b/packages/smithy-python/README.md @@ -1,19 +1,17 @@ # smithy-python -> [!WARNING] -> This package is an experimental scaffold. It does not generate code yet. The -> Java generator in the repository's `codegen` directory remains authoritative. - -`smithy-python` will provide Python-native code generation for Smithy models. -The initial command-line interface exposes the planned client and types generation -commands so that their top-level shape can be developed independently from the -generator implementation. +`smithy-python` provides Python-native code generation for Smithy models. +Standalone types generation is available; client generation remains under +development. ```console smithy-python generate client [OPTIONS] smithy-python generate types [OPTIONS] ``` -After validating their invocation options, both generation commands currently exit -with an error explaining that generation has not been implemented. The package is -included in workspace builds to validate its packaging and entry points. +`generate types` produces an installable package containing compact schemas, lazy +runtime models, and a complete static typing surface. Run +`smithy-python generate types --help` for artifact settings. + +`generate client` currently exits with an error explaining that client generation +has not been implemented. diff --git a/packages/smithy-python/src/smithy_python/cli.py b/packages/smithy-python/src/smithy_python/cli.py index a94d90e03..5edb2eda4 100644 --- a/packages/smithy-python/src/smithy_python/cli.py +++ b/packages/smithy-python/src/smithy_python/cli.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse +import re import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass @@ -12,6 +13,15 @@ from typing import BinaryIO, Final from . import __version__ +from . import dependencies as _dependencies +from . import exceptions as _exceptions +from . import files as _files +from . import generation as _generation +from . import model as _model +from . import models as _models +from . import selection as _selection +from . import symbols as _symbols +from . import types as _types from .environment import PluginEnvironment from .exceptions import CodegenError, InvalidInvocationError from .model import load_model @@ -22,14 +32,20 @@ _GENERATION_NOT_IMPLEMENTED: Final = ( "smithy-python: error: {artifact} generation is not implemented yet\n" ) +_PACKAGE_VERSION: Final = re.compile( + r"v?[0-9]+(?:\.[0-9]+)*" + r"(?:(?:a|b|rc)[0-9]+)?" + r"(?:\.post[0-9]+)?" + r"(?:\.dev[0-9]+)?" + r"(?:\+[a-z0-9]+(?:[._-][a-z0-9]+)*)?", + re.IGNORECASE, +) @dataclass(frozen=True, slots=True) class _Invocation: - artifact: str model_source: bytes output_dir: Path - environment: PluginEnvironment def main( @@ -48,24 +64,19 @@ def main( try: invocation = _resolve_invocation(args, environ=environ, stdin=stdin) - except InvalidInvocationError as error: - sys.stderr.write(f"smithy-python: error: {error}\n") - return 2 - except (CodegenError, OSError) as error: - sys.stderr.write(f"smithy-python: error: {error}\n") - return 1 - - try: model = load_model(invocation.model_source) - selection = select_shapes( - model, - service_id=args.service, - require_service=invocation.artifact == "client", - ) + if args.artifact == "types": + selection = _generate_types(args, invocation.output_dir, model) + else: + selection = select_shapes( + model, + service_id=args.service, + require_service=True, + ) except InvalidInvocationError as error: sys.stderr.write(f"smithy-python: error: {error}\n") return 2 - except CodegenError as error: + except (CodegenError, OSError) as error: sys.stderr.write(f"smithy-python: error: {error}\n") return 1 @@ -74,6 +85,8 @@ def main( f"smithy-python: excluded {selection.excluded_count} eligible data shape(s) " f"outside the closure of service {selection.service.id}\n" ) + if args.artifact == "types": + return 0 sys.stderr.write(_GENERATION_NOT_IMPLEMENTED.format(artifact=args.artifact)) return 1 @@ -93,11 +106,37 @@ def _create_parser() -> argparse.ArgumentParser: generate = commands.add_parser("generate", help="Generate Python source") artifacts = generate.add_subparsers(dest="artifact", required=True) common = _common_artifact_options() - for name, help_text in ( - ("client", "Generate a client package"), - ("types", "Generate a standalone types package"), - ): - artifacts.add_parser(name, help=help_text, parents=[common]) + artifacts.add_parser("client", help="Generate a client package", parents=[common]) + types = artifacts.add_parser( + "types", help="Generate a standalone types package", parents=[common] + ) + types.add_argument( + "--package", + required=True, + help="Import package and distribution name for the generated artifact.", + ) + types.add_argument( + "--package-version", + required=True, + type=_package_version, + help="Version for the generated package.", + ) + types.add_argument( + "--package-description", + help='Package description; defaults to " types package".', + ) + types.add_argument( + "--selector", + type=_parse_selector, + default=_selection.Selector.identity(), + help="Smithy selector used to narrow generated declaration roots.", + ) + types.add_argument( + "--include-operation-io", + dest="generate_inputs_and_outputs", + action="store_true", + help="Include operation input and output structures as generation roots.", + ) return parser @@ -175,8 +214,65 @@ def _resolve_invocation( raise InvalidInvocationError("Expected a Smithy JSON AST model") return _Invocation( - artifact=args.artifact, model_source=model_source, output_dir=output_dir, - environment=environment, ) + + +def _generate_types( + args: argparse.Namespace, + output_dir: Path, + model: _model.Model, +) -> _selection.Selection: + description = args.package_description or f"{args.package} types package" + settings = _generation.TypesSettings( + package=args.package, + package_version=args.package_version, + package_description=description, + selector=args.selector, + generate_inputs_and_outputs=args.generate_inputs_and_outputs, + ) + selection = _selection.select_type_shapes( + model, + service_id=args.service, + selector=settings.selector, + generate_inputs_and_outputs=settings.generate_inputs_and_outputs, + ) + try: + symbols = _symbols.SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_models._model_reserved_member_names( # pyright: ignore[reportPrivateUsage] + model + ), + ) + except ValueError as error: + raise InvalidInvocationError(str(error)) from error + + dependencies = _dependencies.DependencySet() + context = _generation.GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=_generation.GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=_files.FileManifest(dependencies), + ) + _types.TypesGenerator().generate(context) + context.files.commit(output_dir) + return selection + + +def _parse_selector(value: str) -> _selection.Selector: + try: + return _selection.Selector.parse(value) + except _exceptions.SelectorSyntaxError as error: + raise argparse.ArgumentTypeError(str(error)) from error + + +def _package_version(value: str) -> str: + if _PACKAGE_VERSION.fullmatch(value) is None: + raise argparse.ArgumentTypeError(f"Invalid version: {value!r}") + return value[1:] if value[:1].lower() == "v" else value diff --git a/packages/smithy-python/src/smithy_python/models.py b/packages/smithy-python/src/smithy_python/models.py index 0e02263a0..29be7e6ea 100644 --- a/packages/smithy-python/src/smithy_python/models.py +++ b/packages/smithy-python/src/smithy_python/models.py @@ -129,7 +129,11 @@ def _write_runtime(writer: PythonWriter, plan: ModelPlan) -> None: ) else: writer.line("_MODEL_NAMES = tuple(_MODEL_INDEX)") - writer.line("__all__ = (", _quote(exports[0]), ", *_MODEL_NAMES)") + writer.line( + "__all__ = (", + _quote(exports[0]), + ", *_MODEL_NAMES) # pyright: ignore[reportUnsupportedDunderAll]", + ) writer.line() writer.line() writer.line("_registry = ", _MODEL_REGISTRY, "(") diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py index dbc7df1e4..d4db8e837 100644 --- a/packages/smithy-python/src/smithy_python/symbols.py +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -54,11 +54,13 @@ def __init__( package: str, reserved_member_names: Callable[[Shape], Collection[str]] | None = None, ) -> None: - if any( + if not _DISTRIBUTION_NAME.fullmatch(package) or any( not part.isidentifier() or unicodedata.normalize("NFKC", part) in _KEYWORDS for part in package.split(".") ): - raise ValueError(f"Invalid Python package name: {package!r}") + raise ValueError( + f"Invalid Python package or distribution name: {package!r}" + ) self._model = model self._selection = selection self._package = package @@ -287,6 +289,7 @@ def _shape(self, shape_id: ShapeId | str) -> Shape: _ENUMS = frozenset({ShapeType.ENUM, ShapeType.INT_ENUM}) _RESERVED_DECLARATIONS = frozenset({"ServiceError"}) _UNIT = ShapeId("smithy.api", "Unit") +_DISTRIBUTION_NAME = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?") _ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])") _WORD_BOUNDARY = re.compile(r"([a-z0-9])([A-Z])") diff --git a/packages/smithy-python/src/smithy_python/types.py b/packages/smithy-python/src/smithy_python/types.py new file mode 100644 index 000000000..5ab58acd8 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/types.py @@ -0,0 +1,127 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Generation of complete standalone types packages.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable + +from . import dependencies as _dependencies +from . import generation as _generation +from . import models as _models +from . import schemas as _schemas + +__all__ = ["TypesGenerator"] + + +class TypesGenerator: + """Generate a standalone package containing schemas and lazy models.""" + + def generate( + self, context: _generation.GenerationContext[_generation.TypesSettings] + ) -> None: + """Register every file and dependency in the generation context.""" + _schemas.SchemaGenerator().generate(context) + _models.ModelGenerator().generate(context) + _write_package(context) + + +_GENERATED_COMMENT = "# Code generated by smithy-python-codegen DO NOT EDIT." +_GENERATED_MARKDOWN_COMMENT = ( + "" +) +_PYPROJECT_SUFFIX = """ +[tool.pyright] +typeCheckingMode = "strict" + +[tool.ruff] +target-version = "py312" + +[tool.ruff.lint] +ignore = ["F841"] + +[tool.ruff.format] +skip-magic-trailing-comma = true +""" + + +def _write_package( + context: _generation.GenerationContext[_generation.TypesSettings], +) -> None: + settings = context.settings + package = settings.package + directory = package.replace(".", "/") + description = settings.package_description or f"{package} types package" + + context.dependencies.add(_dependencies.HATCHLING) + context.files.text_file( + "pyproject.toml", + _render_pyproject(settings, context.dependencies, description), + ) + context.files.text_file( + "README.md", + f"{_GENERATED_MARKDOWN_COMMENT}\n\n" + f"## {package} Types Package\n\n{description}\n", + ) + + init = context.files.python_module(f"src/{directory}/__init__.py", package) + init.line("__version__: str = ", repr(settings.package_version)) + context.files.python_module( + f"src/{directory}/_private/__init__.py", f"{package}._private" + ) + context.files.text_file(f"src/{directory}/py.typed", "") + + +def _render_pyproject( + settings: _generation.TypesSettings, + dependencies: _dependencies.DependencySet, + description: str, +) -> str: + package_root = settings.package.split(".", 1)[0] + groups = "\n".join( + f"{group} = {_toml_array(map(_requirement, values))}" + for group in ("test", "docs") + if (values := dependencies.for_group(group)) + ) + if groups: + groups = f"\n\n[dependency-groups]\n{groups}" + + return f"""{_GENERATED_COMMENT} + +[project] +name = {_toml_string(settings.package)} +version = {_toml_string(settings.package_version)} +description = {_toml_string(description)} +readme = "README.md" +requires-python = ">=3.12" +keywords = ["smithy", {_toml_string(settings.package)}] +license = {{text = "Apache-2.0"}} +dependencies = {_toml_array(map(_requirement, dependencies.for_group("runtime")))}{groups} + +[build-system] +requires = {_toml_array(map(_requirement, dependencies.for_group("build")))} +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = {_toml_array((f"src/{package_root}",))} + +{_PYPROJECT_SUFFIX.lstrip()}""" + + +def _toml_array(values: Iterable[str]) -> str: + items = tuple(values) + return ( + "[\n" + "\n".join(f" {_toml_string(value)}," for value in items) + "\n]" + if items + else "[]" + ) + + +def _requirement(dependency: _dependencies.Dependency) -> str: + extras = f"[{','.join(dependency.extras)}]" if dependency.extras else "" + return f"{dependency.package}{extras}{dependency.specifier}" + + +def _toml_string(value: str) -> str: + return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") diff --git a/packages/smithy-python/src/smithy_python/writer.py b/packages/smithy-python/src/smithy_python/writer.py index efcf45e4b..9e99e33ce 100644 --- a/packages/smithy-python/src/smithy_python/writer.py +++ b/packages/smithy-python/src/smithy_python/writer.py @@ -94,12 +94,12 @@ def indent(self) -> Generator[None]: def render(self) -> str: """Return complete source without changing the recorded lines.""" names, imports = self._plan_imports(*self._names()) - header = "from __future__ import annotations\n" - if imports: - header += "\n" + _render_imports(imports) + "\n" + header = _render_imports(imports) + "\n" if imports else "" body = self._body(lambda reference: self._annotation(reference, names)) separator = "\n" if self._stub else "\n\n" - return header + (separator + "\n".join(body) + "\n" if body else "") + return header + ( + (separator if header else "") + "\n".join(body) + "\n" if body else "" + ) def _body(self, reference: Callable[[TypeReference], str]) -> list[str]: body: list[str] = [] @@ -199,7 +199,6 @@ def _plan_imports( if binding in reserved or _is_local(binding, scoped) or binding in _BUILTINS - or binding == "annotations" or counts[binding] > 1 else name ) diff --git a/packages/smithy-python/tests/unit/test_cli.py b/packages/smithy-python/tests/unit/test_cli.py index 6f08e6178..f468a4ed7 100644 --- a/packages/smithy-python/tests/unit/test_cli.py +++ b/packages/smithy-python/tests/unit/test_cli.py @@ -3,8 +3,11 @@ from __future__ import annotations +import importlib +import json import subprocess import sys +import tomllib from io import BytesIO from pathlib import Path @@ -20,6 +23,27 @@ def isatty(self) -> bool: return True +def _types_args( + output: Path, + *options: str, + package: str = "example_types", + version: str = "1.2.3", +) -> tuple[str, ...]: + args = ("generate", "types", "--output", str(output)) + return (*args, "--package", package, "--package-version", version, *options) + + +def _run_types( + output: Path, + *options: str, + source: bytes = VALID_MODEL.encode(), + package: str = "example_types", + version: str = "1.2.3", +) -> int: + args = _types_args(output, *options, package=package, version=version) + return main(args, environ={}, stdin=BytesIO(source)) + + @pytest.mark.parametrize( ("argv", "expected"), [ @@ -34,9 +58,7 @@ def test_information_commands( assert capsys.readouterr().out.startswith(expected) -@pytest.mark.parametrize("artifact", ["client", "types"]) -def test_generation_commands_are_explicitly_unavailable( - artifact: str, +def test_client_generation_is_explicitly_unavailable( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: @@ -47,7 +69,7 @@ def test_generation_commands_are_explicitly_unavailable( main( ( "generate", - artifact, + "client", "--model", str(model), "--output", @@ -58,8 +80,147 @@ def test_generation_commands_are_explicitly_unavailable( == 1 ) assert capsys.readouterr().err == ( - f"smithy-python: error: {artifact} generation is not implemented yet\n" + "smithy-python: error: client generation is not implemented yet\n" + ) + + +def test_types_generation_writes_complete_package( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + model = tmp_path / "model.json" + model.write_text( + '{"smithy": "2.0", "shapes": {"example#Data": {"type": "structure"}}}' ) + output = tmp_path / "output" + + assert _run_types(output, "--model", str(model)) == 0 + assert capsys.readouterr().err == "" + assert { + path.relative_to(output).as_posix() + for path in output.rglob("*") + if path.is_file() + } == { + ".smithy-python-manifest.json", + "README.md", + "pyproject.toml", + "src/example_types/__init__.py", + "src/example_types/_private/__init__.py", + "src/example_types/_private/schemas.py", + "src/example_types/models.py", + "src/example_types/models.pyi", + "src/example_types/py.typed", + } + project = tomllib.loads((output / "pyproject.toml").read_text()) + assert project["project"] == { + "name": "example_types", + "version": "1.2.3", + "description": "example_types types package", + "readme": "README.md", + "requires-python": ">=3.12", + "keywords": ["smithy", "example_types"], + "license": {"text": "Apache-2.0"}, + "dependencies": ["smithy-core~=0.10.0"], + } + assert project["build-system"] == { + "requires": ["hatchling"], + "build-backend": "hatchling.build", + } + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(output / "src") + ) + package = importlib.import_module("example_types") + models = importlib.import_module("example_types.models") + assert package.__version__ == "1.2.3" + assert models.Data.__module__ == "example_types.models" + + +def test_types_generation_escapes_del_in_toml_metadata(tmp_path: Path) -> None: + output = tmp_path / "output" + description = "before\x7fafter" + + assert _run_types(output, "--package-description", description) == 0 + pyproject = (output / "pyproject.toml").read_text() + assert "\\u007f" in pyproject + assert tomllib.loads(pyproject)["project"]["description"] == description + + +def test_types_generation_applies_selector_and_input_output_settings( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + source = json.dumps( + { + "smithy": "2.0", + "shapes": { + "example#Operation": { + "type": "operation", + "input": {"target": "example#Input"}, + "output": {"target": "example#Output"}, + }, + "example#Input": {"type": "structure"}, + "example#Output": {"type": "structure"}, + "example#Selected": {"type": "structure"}, + "example#Excluded": {"type": "structure"}, + }, + } + ).encode() + selected = tmp_path / "selected" + assert ( + _run_types( + selected, + "--selector", + "[id|name=Selected]", + source=source, + ) + == 0 + ) + stub = (selected / "src/example_types/models.pyi").read_text() + assert "class Selected(" in stub + assert "class Excluded(" not in stub + assert "class Input(" not in stub + assert "class Output(" not in stub + + complete = tmp_path / "complete" + assert ( + _run_types( + complete, + "--include-operation-io", + source=source, + ) + == 0 + ) + stub = (complete / "src/example_types/models.pyi").read_text() + assert "class Input(" in stub + assert "class Output(" in stub + assert capsys.readouterr().err == "" + + +def test_types_generation_packages_dotted_modules_from_namespace_root( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + output = tmp_path / "output" + assert ( + _run_types( + output, + package="generated.nested_types", + version="1.0.0", + ) + == 0 + ) + project = tomllib.loads((output / "pyproject.toml").read_text()) + wheel = project["tool"]["hatch"]["build"]["targets"]["wheel"] + assert wheel["packages"] == ["src/generated"] + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(output / "src") + ) + package = importlib.import_module("generated.nested_types") + assert package.__version__ == "1.0.0" + assert capsys.readouterr().err == "" @pytest.mark.parametrize( @@ -127,26 +288,73 @@ def test_invalid_model_is_a_model_failure( model = tmp_path / "model.json" model.write_bytes(source) - assert ( - main( - ( - "generate", - "types", - "--model", - str(model), - "--output", - str(tmp_path / "output"), - ), - environ={}, - ) - == 1 - ) + assert _run_types(tmp_path / "output", "--model", str(model)) == 1 err = capsys.readouterr().err assert err.startswith("smithy-python: error: ") assert expected in err assert "not implemented" not in err +@pytest.mark.parametrize("option", ["--package", "--package-version"]) +def test_types_generation_requires_package_settings( + option: str, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + output = tmp_path / "output" + args = list(_types_args(output)) + index = args.index(option) + del args[index : index + 2] + + assert main(args, environ={}, stdin=BytesIO(VALID_MODEL.encode())) == 2 + assert f"the following arguments are required: {option}" in capsys.readouterr().err + assert not output.exists() + + +@pytest.mark.parametrize( + ("option", "value", "expected"), + [ + ( + "--package", + "invalid-package", + "Invalid Python package or distribution name", + ), + ("--package", "_example", "Invalid Python package or distribution name"), + ("--package", "example_", "Invalid Python package or distribution name"), + ("--package", "café", "Invalid Python package or distribution name"), + ("--package-version", "", "Invalid version"), + ("--package-version", "not-a-version", "Invalid version"), + ("--selector", "[", "Expected an attribute path segment"), + ], +) +def test_types_generation_rejects_invalid_settings( + option: str, + value: str, + expected: str, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + output = tmp_path / "output" + args = list(_types_args(output)) + index = args.index(option) if option in args else len(args) + if option in args: + args[index + 1] = value + else: + args.extend((option, value)) + + assert main(args, environ={}, stdin=BytesIO(VALID_MODEL.encode())) == 2 + assert expected in capsys.readouterr().err + assert not output.exists() + + +def test_types_generation_normalizes_package_version(tmp_path: Path) -> None: + output = tmp_path / "output" + + assert _run_types(output, version="v1.2") == 0 + project = tomllib.loads((output / "pyproject.toml").read_text()) + assert project["project"]["version"] == "1.2" + + @pytest.mark.parametrize("option", ["--model", "--output"]) def test_run_plugin_rejects_direct_invocation_options( option: str, diff --git a/packages/smithy-python/tests/unit/test_files.py b/packages/smithy-python/tests/unit/test_files.py index 098d0e54f..baa9da348 100644 --- a/packages/smithy-python/tests/unit/test_files.py +++ b/packages/smithy-python/tests/unit/test_files.py @@ -43,8 +43,7 @@ def test_file_manifest_reuses_python_writer_and_records_dependencies( assert ( output.joinpath("example/models.py").read_text(encoding="utf-8") - == "from __future__ import annotations\n\n" - "from smithy_core.documents import Document\n\n\n" + == "from smithy_core.documents import Document\n\n\n" "value: list[Document]\n" ) assert output.joinpath("example/py.typed").read_text(encoding="utf-8") == "" diff --git a/packages/smithy-python/tests/unit/test_selection.py b/packages/smithy-python/tests/unit/test_selection.py index a41eaf87e..b24bbeba0 100644 --- a/packages/smithy-python/tests/unit/test_selection.py +++ b/packages/smithy-python/tests/unit/test_selection.py @@ -10,6 +10,16 @@ import pytest from smithy_python.cli import main +_TYPES_OPTIONS = ("--package", "example_types", "--package-version", "1.0.0") + + +def _artifact_args(artifact: str, *options: str) -> list[str]: + args = ["generate", artifact] + if artifact == "types": + args.extend(_TYPES_OPTIONS) + args.extend(options) + return args + @pytest.mark.parametrize("artifact", ["client", "types"]) @pytest.mark.parametrize( @@ -53,19 +63,29 @@ def test_cli_service_selection( } ).encode() output = tmp_path / "output" - argv = ["generate", artifact, "--output", str(output)] + argv = _artifact_args(artifact, "--output", str(output)) if service is not None: argv.extend(("--service", service)) + if artifact == "types" and service in (None, "example#Second"): + expected = 0 + diagnostic = ( + "" + if service is None + else "excluded 1 eligible data shape(s) outside the closure" + ) assert main(argv, environ={}, stdin=BytesIO(source)) == expected captured = capsys.readouterr() assert captured.out == "" - assert diagnostic in captured.err + if diagnostic: + assert diagnostic in captured.err + else: + assert captured.err == "" assert "unrecognized arguments" not in captured.err if service is None: assert "example#Base" not in captured.err if expected == 2: assert "not implemented" not in captured.err - assert not output.exists() + assert output.exists() is (expected == 0) @pytest.mark.parametrize("artifact", ["client", "types"]) @@ -82,18 +102,22 @@ def test_cli_implicit_service( if has_service: shapes["example#Service"] = {"type": "service"} source = json.dumps({"smithy": "2.0", "shapes": shapes}).encode() - requires_service = artifact == "client" and not has_service - assert main( - ("generate", artifact), - environ={"SMITHY_PLUGIN_DIR": str(tmp_path / "output")}, - stdin=BytesIO(source), - ) == (2 if requires_service else 1) + argv = _artifact_args(artifact) + expected = 0 if artifact == "types" else 1 if has_service else 2 + assert ( + main( + argv, + environ={"SMITHY_PLUGIN_DIR": str(tmp_path / "output")}, + stdin=BytesIO(source), + ) + == expected + ) captured = capsys.readouterr() assert captured.out == "" - assert ("not implemented" in captured.err) is not requires_service - if requires_service: + assert ("not implemented" in captured.err) is (artifact == "client" and has_service) + if artifact == "client" and not has_service: assert "service" in captured.err - assert not (tmp_path / "output").exists() + assert (tmp_path / "output").exists() is (artifact == "types") @pytest.mark.parametrize("artifact", ["client", "types"]) @@ -124,19 +148,24 @@ def test_cli_reports_only_excluded_eligible_data( } ).encode() output = tmp_path / "output" - argv = ["generate", artifact, "--service", "example#Service"] + argv = _artifact_args(artifact, "--service", "example#Service") environ = {"SMITHY_PLUGIN_DIR": str(output)} if plugin else {} if not plugin: argv.extend(("--output", str(output))) - assert main(argv, environ=environ, stdin=BytesIO(source)) == 1 + assert main(argv, environ=environ, stdin=BytesIO(source)) == ( + 0 if artifact == "types" else 1 + ) captured = capsys.readouterr() assert captured.out == "" - assert captured.err == ( - "smithy-python: excluded 1 eligible data shape(s) " - "outside the closure of service example#Service\n" - f"smithy-python: error: {artifact} generation is not implemented yet\n" - ) - assert not output.exists() + expected = "" + if artifact == "client": + expected = ( + "smithy-python: excluded 1 eligible data shape(s) " + "outside the closure of service example#Service\n" + "smithy-python: error: client generation is not implemented yet\n" + ) + assert captured.err == expected + assert output.exists() is (artifact == "types") def test_cli_conflicts_are_model_errors( @@ -146,7 +175,7 @@ def test_cli_conflicts_are_model_errors( { "smithy": "2.0", "shapes": { - shape_id: {"type": "string"} + shape_id: {"type": "structure"} for shape_id in ("a#Name", "b#name", "c#NAME", "a#Other", "b#other") }, } @@ -154,7 +183,7 @@ def test_cli_conflicts_are_model_errors( output = tmp_path / "output" assert ( main( - ("generate", "types", "--output", str(output)), + _artifact_args("types", "--output", str(output)), environ={}, stdin=BytesIO(source), ) diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index 3fb2e4a4a..fbce403cb 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -514,6 +514,9 @@ def test_only_emitted_declarations_collide() -> None: "pkg.class", "None", "pkg/child", + "_example", + "example_", + "café", "pkg.\uff43\uff4c\uff41\uff53\uff53", ], ) diff --git a/packages/smithy-python/tests/unit/test_writer.py b/packages/smithy-python/tests/unit/test_writer.py index c7df26470..8ced66c1e 100644 --- a/packages/smithy-python/tests/unit/test_writer.py +++ b/packages/smithy-python/tests/unit/test_writer.py @@ -68,10 +68,18 @@ def load_written_module( return module -def test_nested_forward_and_mutual_annotations( +def test_nested_same_module_annotations( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: writer = PythonWriter("written_models") + writer.line("class Leaf:") + with writer.indent(): + writer.line("pass") + writer.line() + writer.line("class _2HTTPServer:") + with writer.indent(): + writer.line("pass") + writer.line() writer.line("class Node:") with writer.indent(): writer.line( @@ -84,7 +92,7 @@ def test_nested_forward_and_mutual_annotations( TypeReference( "list", "builtins", - (TypeReference("Node", "written_models", nullable=True),), + (TypeReference("Leaf", "written_models", nullable=True),), nullable=True, ), ), @@ -94,26 +102,20 @@ def test_nested_forward_and_mutual_annotations( writer.line("server: ", TypeReference("_2HTTPServer", "written_models")) writer.line("amount: ", TypeReference("Decimal", "decimal")) writer.line("nothing: ", TypeReference("None")) - writer.line() - writer.line("class _2HTTPServer:") - with writer.indent(): - writer.line("node: ", TypeReference("Node", "written_models")) source = writer.render() - assert source.startswith("from __future__ import annotations\n") assert "from written_models" not in source - assert "dict[str, list[Node | None] | None] | None" in source + assert "dict[str, list[Leaf | None] | None] | None" in source assert source == writer.render() assert source.endswith("\n") module = load_written_module(writer, tmp_path, monkeypatch) node = module.Node server = module._2HTTPServer assert get_type_hints(node) == { - "children": dict[str, list[node | None] | None] | None, + "children": dict[str, list[module.Leaf | None] | None] | None, "server": server, "amount": Decimal, "nothing": type(None), } - assert get_type_hints(server) == {"node": node} @pytest.mark.parametrize( @@ -608,13 +610,17 @@ def test_equivalent_current_module_names( module: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: writer = PythonWriter(module) + writer.line("class Child:") + with writer.indent(): + writer.line("pass") + writer.line() writer.line("class Node:") with writer.indent(): writer.line( - "child: ", TypeReference("Node", "\uff57ritten_models", nullable=True) + "child: ", TypeReference("Child", "\uff57ritten_models", nullable=True) ) result = load_written_module(writer, tmp_path, monkeypatch) - assert get_type_hints(result.Node) == {"child": result.Node | None} + assert get_type_hints(result.Node) == {"child": result.Child | None} def test_equivalent_external_identities_are_deduplicated( @@ -684,14 +690,6 @@ def test_deep_collection_rendering() -> None: ) -def test_future_import_name_is_reserved() -> None: - writer = PythonWriter("models") - writer.line("value: ", TypeReference("annotations", "external")) - assert ( - "from external import annotations as _external_annotations" in writer.render() - ) - - @pytest.mark.parametrize( "reference", [ @@ -767,7 +765,7 @@ def test_generation_without_runtime_packages() -> None: def test_empty_writer_and_trusted_text() -> None: writer = PythonWriter("models") - assert writer.render() == "from __future__ import annotations\n" + assert writer.render() == "" writer.line("value = '$T {not_a_template}'") assert writer.render().endswith("value = '$T {not_a_template}'\n") @@ -806,8 +804,6 @@ def test_imports_are_grouped_and_combined() -> None: writer.line("fourth: ", TypeReference("Thing", "._private")) assert writer.render().startswith( - "from __future__ import annotations\n" - "\n" "from enum import IntEnum, StrEnum\n" "\n" "from smithy_core.documents import Document\n" diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index b84b7ef58..b904a65cf 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -1700,6 +1700,9 @@ "SymbolProvider": "class", "TypeReference": "class" }, + "smithy_python.types": { + "TypesGenerator": "class" + }, "smithy_python.writer": { "CodegenError": "rex-exception", "PythonWriter": "class", diff --git a/scripts/codegen-output-py/generate.py b/scripts/codegen-output-py/generate.py index 4306ffda4..991806d4d 100644 --- a/scripts/codegen-output-py/generate.py +++ b/scripts/codegen-output-py/generate.py @@ -9,14 +9,7 @@ import sys from pathlib import Path -from smithy_python.dependencies import DependencySet -from smithy_python.files import FileManifest -from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings -from smithy_python.model import load_model -from smithy_python.models import ModelGenerator, _model_reserved_member_names -from smithy_python.schemas import SchemaGenerator -from smithy_python.selection import select_type_shapes -from smithy_python.symbols import SymbolProvider +from smithy_python.cli import main as generate _ROOT = Path(__file__).resolve().parents[2] _MODEL = Path(__file__).with_name("model.json") @@ -26,36 +19,28 @@ def main() -> None: - model = load_model(_MODEL.read_bytes()) - selection = select_type_shapes( - model, - service_id=_SERVICE, - generate_inputs_and_outputs=True, + result = generate( + ( + "generate", + "types", + "--model", + str(_MODEL), + "--output", + str(_OUTPUT), + "--service", + _SERVICE, + "--package", + _PACKAGE, + "--package-version", + "1.0.0", + "--package-description", + "Example Python codegen output", + "--include-operation-io", + ), + environ={}, ) - settings = TypesSettings( - package=_PACKAGE, - package_version="1.0.0", - package_description="Example Python codegen output", - ) - symbols = SymbolProvider( - model, - selection, - package=settings.package, - reserved_member_names=_model_reserved_member_names(model), - ) - dependencies = DependencySet() - context = GenerationContext( - model=model, - selection=selection, - settings=settings, - symbols=symbols, - plan=GenerationPlan.from_selection(model, selection, symbols), - dependencies=dependencies, - files=FileManifest(dependencies), - ) - SchemaGenerator().generate(context) - ModelGenerator().generate(context) - context.files.commit(_OUTPUT) + if result: + raise SystemExit(result) _format_output()