diff --git a/.chronus/changes/python-discriminator-string-literals-2026-10-22.md b/.chronus/changes/python-discriminator-string-literals-2026-10-22.md new file mode 100644 index 00000000000..c0a0775d430 --- /dev/null +++ b/.chronus/changes/python-discriminator-string-literals-2026-10-22.md @@ -0,0 +1,7 @@ +--- +changeKind: fix +packages: + - "@typespec/http-client-python" +--- + +Escape string literals and wire names in generated Python declarations, discriminator initializers, field metadata, and templates to preserve specification values as data. diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index f7d4a9c6e7a..56913d3c8ad 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -6,7 +6,7 @@ from enum import Enum from collections import OrderedDict from typing import Any, Literal, Optional, TYPE_CHECKING, cast -from .utils import add_to_pylint_disable, NamespaceType, LOCALS_LENGTH_LIMIT +from .utils import add_to_pylint_disable, NamespaceType, LOCALS_LENGTH_LIMIT, python_string_literal from .base import BaseType from .constant_type import ConstantType from .property import Property @@ -164,7 +164,7 @@ def get_json_template_representation( # don't add additional properties, because there's not really a concept of # additional properties in the template representation = { - f'"{prop.wire_name}"': prop.get_json_template_representation( + python_string_literal(prop.wire_name): prop.get_json_template_representation( client_default_value_declaration=client_default_value_declaration, ) for prop in [ @@ -172,12 +172,14 @@ def get_json_template_representation( ] } if self.discriminator and self.discriminator_value: - representation[f'"{self.discriminator.wire_name}"'] = f'"{self.discriminator_value}"' + representation[python_string_literal(self.discriminator.wire_name)] = python_string_literal( + self.discriminator_value + ) # once we've finished, we want to reset created_json_template_representation to false # so we can call it again self._created_json_template_representation = False - optional_keys = [f'"{p.wire_name}"' for p in self.properties if getattr(p, "optional", False)] + optional_keys = [python_string_literal(p.wire_name) for p in self.properties if getattr(p, "optional", False)] return OrderedDict( sorted( representation.items(), diff --git a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py index e8a1d28c1f9..7c9a3933647 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py +++ b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py @@ -9,7 +9,7 @@ from .base import BaseType from .imports import FileImport, ImportType, TypingSection -from .utils import NamespaceType +from .utils import NamespaceType, python_string_literal if TYPE_CHECKING: from .code_model import CodeModel @@ -332,8 +332,8 @@ def validation(self) -> Optional[dict[str, Union[bool, int, str]]]: validation["pattern"] = RawString(self.pattern) # type: ignore return validation or None - def get_declaration(self, value) -> str: - return f"'{value}'" if value == '"' else f'"{value}"' + def get_declaration(self, value: str) -> str: + return python_string_literal(value, quote="'" if value == '"' else '"') def serialization_type(self, **kwargs: Any) -> str: return "str" diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index 5415e70df48..35da0818a2f 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -13,6 +13,15 @@ OrderedSet = dict[T, None] +def python_string_literal(value: str, *, quote: str = '"') -> str: + literal = repr(value) + if literal[0] == quote: + return literal + # Change delimiter escaping without changing the escaped value. + body = literal[1:-1].replace("\\" + literal[0], literal[0]) + return quote + body.replace(quote, "\\" + quote) + quote + + def add_to_description(description: str, entry: str) -> str: if not description: return entry diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 8d26d6bc358..94e866079e5 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -32,7 +32,7 @@ ParameterListType, ByteArraySchema, ) -from ..models.utils import NamespaceType, escape_sphinx_field_name +from ..models.utils import NamespaceType, escape_sphinx_field_name, python_string_literal from .parameter_serializer import ParameterSerializer, PopKwargType, check_body_optional from ..models.parameter_list import ParameterType from ..models.response import ( @@ -99,7 +99,8 @@ def _escape_str(input_str: str) -> str: def _get_polymorphic_subtype_template(polymorphic_subtype: ModelType) -> list[str]: retval: list[str] = [] retval.append("") - retval.append(f'# JSON input template for discriminator value "{polymorphic_subtype.discriminator_value}":') + discriminator_value = python_string_literal(polymorphic_subtype.discriminator_value) + retval.append(f"# JSON input template for discriminator value {discriminator_value}:") subtype_template = utils.json_dumps_template( polymorphic_subtype.get_json_template_representation(), ) @@ -385,7 +386,7 @@ def _json_input_example_template(self, builder: BuilderType) -> list[str]: discriminator_name = cast(Property, polymorphic_subtypes[0].discriminator).wire_name template.append( "# The input is polymorphic. The following are possible polymorphic " - f'inputs based off discriminator "{discriminator_name}":' + f"inputs based off discriminator {python_string_literal(discriminator_name)}:" ) for idx in range( min( @@ -582,7 +583,7 @@ def example_template(self, builder: OperationType) -> list[str]: retval.append("") retval.append( "# The response is polymorphic. The following are possible polymorphic " - f'responses based off discriminator "{discriminator_name}":' + f"responses based off discriminator {python_string_literal(discriminator_name)}:" ) for idx in range( min( diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py index 89bda92ae25..6ff97164f6e 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py @@ -78,7 +78,6 @@ def property_descriptions(self, async_mode: bool) -> list[str]: ":keyword int polling_interval: Default waiting time between two polls for LRO operations " "if no Retry-After header is present." ) - retval = [s.replace("\\", "\\\\") for s in retval] retval.append('"""') return retval diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 14ab92efc37..6d16f0921e7 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -23,7 +23,7 @@ ) from .import_serializer import FileImportSerializer from .base_serializer import BaseSerializer -from ..models.utils import NamespaceType, add_to_pylint_disable, escape_sphinx_field_name +from ..models.utils import NamespaceType, add_to_pylint_disable, escape_sphinx_field_name, python_string_literal def _get_xml_deserializer_name(prop: Property) -> Optional[str]: # pylint: disable=too-many-return-statements @@ -82,15 +82,17 @@ def _documentation_string( ) # Escape names that Sphinx would otherwise misinterpret in the info field target # (e.g. a wire name with a leading "@" such as "@search.facets"). - doc_name = escape_sphinx_field_name(doc_name) + # Strip repr's delimiters and escape triple quotes for the enclosing docstring. + doc_name = repr(escape_sphinx_field_name(doc_name))[1:-1].replace('"""', '\\"\\"\\"') sphinx_prefix = f":{description_keyword} {doc_name}:" - description = prop.description(is_operation_file=False).replace("\\", "\\\\") + description = prop.description(is_operation_file=False).replace("\\", "\\\\").replace('"""', '\\"\\"\\"') retval.append(f"{sphinx_prefix} {description}" if description else sphinx_prefix) # In the types file, use type_annotation (the serialized form) for docstrings if kwargs.get("serialize_namespace_type") == NamespaceType.TYPES_FILE: doc_type = prop.type.type_annotation(**kwargs) else: doc_type = prop.type.docstring_type(**kwargs) + doc_type = doc_type.replace("\\", "\\\\").replace('"""', '\\"\\"\\"') retval.append(f":{docstring_type_keyword} {doc_name}: {doc_type}") return retval @@ -137,7 +139,9 @@ def super_call(self, model: ModelType) -> list[str]: @staticmethod def initialize_discriminator_property(model: ModelType, prop: Property) -> str: - discriminator_value = f"'{model.discriminator_value}'" if model.discriminator_value else None + discriminator_value = ( + python_string_literal(model.discriminator_value, quote="'") if model.discriminator_value else None + ) if not discriminator_value: typing = "Optional[str]" else: @@ -422,7 +426,8 @@ def declare_model(self, model: ModelType) -> str: if model.parents: basename = ", ".join([m.name for m in model.parents]) if model.discriminator_value: - basename += f", discriminator='{model.discriminator_value}'" + discriminator_value = python_string_literal(model.discriminator_value, quote="'") + basename += f", discriminator={discriminator_value}" return f"class {model.name}({basename}):{self.class_pylint_disable(model)}" def class_pylint_disable(self, model: ModelType) -> str: @@ -458,7 +463,7 @@ def get_properties_to_declare(model: ModelType) -> list[Property]: def declare_property(self, prop: Property) -> str: args = [] if prop.client_name != prop.wire_name or prop.is_discriminator: - args.append(f'name="{prop.wire_name}"') + args.append(f"name={python_string_literal(prop.wire_name)}") if prop.visibility: v_list = ", ".join(f'"{x}"' for x in prop.visibility) args.append(f"visibility=[{v_list}]") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index c6912aa570a..48fe45e96fa 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -10,7 +10,7 @@ from ..models import ModelType, CodeModel from ..models.enum_type import EnumType from ..models.imports import FileImport, ImportType -from ..models.utils import NamespaceType +from ..models.utils import NamespaceType, python_string_literal from ..models.property import Property from .model_serializer import _documentation_string from .import_serializer import FileImportSerializer @@ -388,9 +388,9 @@ def declare_functional_model(self, model: ModelType) -> str: type_annotation = _qualify_shadowed_builtins(type_annotation, shadowed) is_optional = prop.optional or prop.client_default_value is not None if is_optional: - entries.append(f' "{prop.wire_name}": {type_annotation},') + entries.append(f" {python_string_literal(prop.wire_name)}: {type_annotation},") else: - entries.append(f' "{prop.wire_name}": Required[{type_annotation}],') + entries.append(f" {python_string_literal(prop.wire_name)}: Required[{type_annotation}],") fields = "\n".join(entries) return f'{model.name} = TypedDict("{model.name}", {{\n{fields}\n}}, total=False)' diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py index 52ee4d62e57..3b7881ad509 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py @@ -4,6 +4,7 @@ # license information. # -------------------------------------------------------------------------- import json +import re from typing import Optional, Any from pathlib import Path @@ -43,19 +44,8 @@ def get_namespace_from_package_name(package_name: Optional[str]) -> str: def _improve_json_string(template_representation: str) -> Any: - origin = template_representation.split("\n") - final = [] - for line in origin: - idx0 = line.find("#") - idx1 = line.rfind('"') - modified_line = "" - if idx0 > -1 and idx1 > -1: - modified_line = line[:idx0] + line[idx1:] + " " + line[idx0:idx1] + "\n" - else: - modified_line = line + "\n" - modified_line = modified_line.replace('"', "").replace("\\", '"') - final.append(modified_line) - return "".join(final) + # Unwrap JSON strings without changing the Python expressions stored inside them. + return re.sub(r'"(?:[^"\\]|\\.)*"', lambda match: json.loads(match.group()), template_representation) + "\n" def json_dumps_template(template_representation: Any) -> Any: diff --git a/packages/http-client-python/generator/pygen/codegen/templates/operation_tools.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/operation_tools.jinja2 index 434662f4a21..b134c050bb8 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/operation_tools.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/operation_tools.jinja2 @@ -32,8 +32,13 @@ {{ normalized_string | replace("\\", "\\\\") | wordwrap(width=width, break_long_words=False, break_on_hyphens=False, wrapstring=wrapstring)}}{% endmacro %} {# Escape quotes after wrapping so wrapping cannot split an escape sequence. #} -{% macro wrap_docstring(string, wrapstring, width=95) -%} -{{ wrap_string(string, wrapstring, width) | replace('"', '\\"') }} +{% macro wrap_docstring(string, wrapstring, width=95, escape_all_quotes=true) -%} +{%- set wrapped = wrap_string(string, wrapstring, width) -%} +{%- if escape_all_quotes -%} +{{ wrapped | replace('"', '\\"') }} +{%- else -%} +{{ wrapped | replace('"""', '\\"\\"\\"') }} +{%- endif -%} {%- endmacro %} {% macro description(builder, serializer) %} @@ -55,7 +60,7 @@ {% endfor %} {% for description in param_description_and_response_docstring %} {% if description %} -{% set description = wrap_string(description, wrapstring='\n ') %} +{% set description = wrap_docstring(description, wrapstring='\n ', escape_all_quotes=false) %} {{ " " if description[0] != ":" and description[0] != " " else "" }}{{ description}} {% else %} @@ -67,8 +72,7 @@ Example: .. code-block:: python {% for template_line in example_template %} {% if template_line %} - {% set wrap_amount = (template_line | length) - (template_line.lstrip() | length) + 10 %} - {{ wrap_string(template_line, wrapstring='\n' + " " * wrap_amount, width=(95 - wrap_amount)) }} + {{ template_line | replace("\\", "\\\\") | replace('"""', '\\"\\"\\"') }} {% else %} {% endif %} @@ -89,7 +93,11 @@ Example: {% macro serialize_with_wrap(lines, wrapstring) %} {% for line in lines %} {% if line %} -{{ wrap_string(line, wrapstring=wrapstring) }} + {% if line == '"""' %} +{{ line }} + {% else %} +{{ wrap_docstring(line, wrapstring=wrapstring, escape_all_quotes=false) }} + {% endif %} {% else %} {% endif %} diff --git a/packages/http-client-python/tests/unit/test_enum_docstrings.py b/packages/http-client-python/tests/unit/test_enum_docstrings.py index dd8df551a38..c107279b5e2 100644 --- a/packages/http-client-python/tests/unit/test_enum_docstrings.py +++ b/packages/http-client-python/tests/unit/test_enum_docstrings.py @@ -27,6 +27,9 @@ 'A"""; print("This should remain documentation"); """B.', 'Quotes: " and "" and """.', 'Ends with a quote"', + 'Ends with two quotes""', + 'Ends with four quotes""""', + 'Ends with a backslash and quote\\"', r'Quotes """ with a regex \W and a path C:\new\test.', 'First paragraph.\n\nSecond paragraph with """ quotes.', ], diff --git a/packages/http-client-python/tests/unit/test_string_literals.py b/packages/http-client-python/tests/unit/test_string_literals.py new file mode 100644 index 00000000000..b1103800bc0 --- /dev/null +++ b/packages/http-client-python/tests/unit/test_string_literals.py @@ -0,0 +1,723 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""Specification strings must remain literal data in generated Python.""" + +import ast +import textwrap +from types import SimpleNamespace +from typing import Literal, Optional, get_args + +import pytest +from jinja2 import Environment, PackageLoader + +from pygen import OptionsDict +from pygen.codegen.models import ( + BodyParameter, + CodeModel, + Operation, + Parameter, + ParameterList, + RequestBuilder, + Response, + build_type, +) +from pygen.codegen.models.parameter_list import RequestBuilderParameterList +from pygen.codegen.models.primitive_types import StringType +from pygen.codegen.models.utils import python_string_literal +from pygen.codegen.serializers.enum_serializer import EnumSerializer +from pygen.codegen.serializers.builder_serializer import OperationSerializer, _get_polymorphic_subtype_template +from pygen.codegen.serializers.model_serializer import DpgModelSerializer +from pygen.codegen.serializers.general_serializer import GeneralSerializer +from pygen.codegen.serializers.types_serializer import TypesSerializer +from pygen.codegen.serializers.utils import json_dumps_template + +VALUES = [ + "ordinary", + "", + "'", + '"', + "Both 'single' and \"double\" quotes", + r"C:\new\test\\", + "first\nsecond\r\t\x00\x1f", + "\u96ea\U0001f600", + 'prefix" + marker() + "suffix', + "prefix' + marker() + 'suffix", + "before # after", + 'triple """ quotes', + "a long discriminator value " * 8, +] + +PROJECTS_TOOLS_DESCRIPTION = ( + "A list of tool definitions that the model should be allowed to call.\n" + " For the Responses API, the list of tool definitions might look like:\n\n" + ".. code-block:: json\n\n" + " [\n" + ' { "type": "function", "name": "get_weather" },\n' + ' { "type": "mcp", "server_label": "deepwiki" },\n' + ' { "type": "image_generation" }\n' + " ]" +) + + +@pytest.fixture(scope="session", autouse=True) +def testserver(): + yield + + +@pytest.fixture +def code_model(): + return CodeModel( + { + "namespace": "sample", + "clients": [ + { + "name": "client", + "namespace": "sample", + "moduleName": "sample", + "parameters": [], + "url": "", + "operationGroups": [], + } + ], + }, + OptionsDict({"models-mode": "dpg", "flavor": "unbranded", "tsp_file": True}), + ) + + +@pytest.fixture +def env(): + return Environment(loader=PackageLoader("pygen.codegen", "templates"), trim_blocks=True, lstrip_blocks=True) + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize( + "description, expected", + [ + pytest.param( + "Updates the agent by adding a new version if there are any changes to the agent definition.\n" + "If no changes, returns the existing agent version.", + "Updates the agent by adding a new version if there are any changes to the agent definition. If\n" + "no changes, returns the existing agent version.", + id="agent-update-multiline-prose", + ), + pytest.param( + "New event types may be added in the future. Clients should gracefully\n" + "ignore unrecognized event types.\n\n" + "Wire format:", + "New event types may be added in the future. Clients should gracefully\n" + "ignore unrecognized event types.\n\n" + "Wire format:", + id="session-log-paragraphs", + ), + pytest.param( + 'Each SSE frame contains:\n\n* `event`: always `"log"`\n* `data`: a plain-text log line.', + 'Each SSE frame contains:\n\n* `event`: always `"log"`\n* `data`: a plain-text log line.', + id="session-log-bullets", + ), + pytest.param( + PROJECTS_TOOLS_DESCRIPTION, + "A list of tool definitions that the model should be allowed to call. For the Responses API, the\n" + "list of tool definitions might look like:\n\n" + ".. code-block:: json\n\n" + " [\n" + ' { "type": "function", "name": "get_weather" },\n' + ' { "type": "mcp", "server_label": "deepwiki" },\n' + ' { "type": "image_generation" }\n' + " ]", + id="allowed-tools-json-block", + ), + ], +) +def test_projects_operation_preserves_description_layout(code_model, env, async_mode, description, expected): + # #9466 and Azure/azure-sdk-for-python#44885 cover these Projects description layouts. + code_model.options["flavor"] = "azure" + client = code_model.clients[0] + yaml_data = { + "url": "/example", + "method": "GET", + "groupName": "", + "isOverload": False, + "apiVersions": [], + "description": description, + } + operation = Operation( + yaml_data, + code_model, + client, + "example", + RequestBuilder( + yaml_data, + code_model, + client, + "build_example_request", + RequestBuilderParameterList({}, code_model, parameters=[]), + ), + ParameterList({}, code_model, parameters=[]), + responses=[Response({"statusCodes": [204]}, code_model)], + exceptions=[], + ) + serializer = OperationSerializer(code_model, async_mode=async_mode, client_namespace="sample") + + source = env.get_template("operation.py.jinja2").render(operation=operation, operation_serializer=serializer) + + function = ast.parse(source).body[0] + documentation = ast.get_docstring(function) + assert documentation.split(":return:", 1)[0].strip() == expected + + +def test_projects_model_preserves_allowed_tools_json_block(code_model, env): + code_model.options["flavor"] = "azure" + model = build_type( + { + "type": "model", + "base": "dpg", + "name": "ToolChoiceAllowed", + "snakeCaseName": "tool_choice_allowed", + "properties": [ + { + "wireName": "tools", + "clientName": "tools", + "optional": False, + "description": PROJECTS_TOOLS_DESCRIPTION, + "type": {"type": "list", "elementType": {"type": "dict", "elementType": {"type": "any-object"}}}, + } + ], + }, + code_model, + ) + + source = DpgModelSerializer(code_model, env, models=[model]).serialize() + + declaration = next(node for node in ast.parse(source).body if isinstance(node, ast.ClassDef)) + documentation = ast.get_docstring(declaration) + assert ":ivar tools: Required. A list of tool definitions" in documentation + block = documentation.split(".. code-block:: json\n", 1)[1].split(":vartype tools:", 1)[0] + assert ast.literal_eval(textwrap.dedent(block).strip()) == [ + {"type": "function", "name": "get_weather"}, + {"type": "mcp", "server_label": "deepwiki"}, + {"type": "image_generation"}, + ] + + +@pytest.mark.parametrize("value", VALUES) +def test_string_declarations_are_literals(code_model, value): + declaration = StringType({"type": "string"}, code_model).get_declaration(value) + assert ast.literal_eval(declaration) == value + + +@pytest.mark.parametrize( + "value, expected", + [ + ("ordinary", '"ordinary"'), + ("application/json", '"application/json"'), + ("don't", '"don\'t"'), + ("", '""'), + ("\u96ea\U0001f600", '"\u96ea\U0001f600"'), + ('"', "'\"'"), + ], +) +def test_string_declarations_preserve_quote_style(code_model, value, expected): + assert StringType({"type": "string"}, code_model).get_declaration(value) == expected + + +@pytest.mark.parametrize("value", VALUES + ["\\'", '\\\\"', "\ud800", chr(0xDFFF)]) +@pytest.mark.parametrize("quote", ['"', "'"]) +def test_python_string_literal_preserves_value(value, quote): + source = python_string_literal(value, quote=quote) + assert ast.literal_eval(source) == value + source.encode("utf-8") + + +@pytest.mark.parametrize( + "description", + [ + 'Known values are: "function", "file_search", "computer", and "web_search". ' * 3, + 'Ends with a quote"', + 'Ends with two quotes""', + ], +) +def test_docstring_wrapping_preserves_ordinary_quotes(env, description): + macros = env.get_template("operation_tools.jinja2").module + assert macros.wrap_docstring(description, "\n ", escape_all_quotes=False) == macros.wrap_string(description, "\n ") + + +def test_existing_docstrings_preserve_quote_escaping(env): + macros = env.get_template("operation_tools.jinja2").module + assert macros.wrap_docstring('"Internal" refers to actions.', "\n ") == '\\"Internal\\" refers to actions.' + + +def test_model_declarations_preserve_quote_style(code_model, env): + model = build_type( + { + "type": "model", + "base": "dpg", + "name": "Child", + "snakeCaseName": "child", + "discriminatorValue": "child", + "properties": [ + { + "wireName": "@kind", + "clientName": "kind", + "optional": False, + "isDiscriminator": True, + "type": {"type": "constant", "value": "child", "valueType": {"type": "string"}}, + } + ], + }, + code_model, + ) + serializer = DpgModelSerializer(code_model, env, models=[model]) + model.parents = [ + SimpleNamespace( + name="Base", properties=[], discriminated_subtypes={"child": model}, parents=[], snake_case_name="base" + ) + ] + assert "class Child(Base, discriminator='child'):" in serializer.declare_model(model) + assert serializer.declare_property(model.properties[0]).startswith( + 'kind: Literal["child"] = rest_discriminator(name="@kind")' + ) + assert serializer.initialize_discriminator_property(model, model.properties[0]) == "self.kind: str = 'child'" + assert model.get_json_template_representation() == {'"@kind"': '"child"'} + assert _get_polymorphic_subtype_template(model)[1] == '# JSON input template for discriminator value "child":' + typed_model = build_type({**model.yaml_data, "base": "typeddict"}, code_model) + types_source = TypesSerializer(code_model, env, models=[typed_model]).serialize() + assert '"@kind": Required[Literal["child"]]' in types_source + + +@pytest.mark.parametrize("value", VALUES) +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize("file_kind", ["client", "config"]) +def test_complete_client_preserves_parameter_defaults(code_model, env, value, async_mode, file_kind): + string_type = {"type": "string"} + code_model = CodeModel( + { + "namespace": "sample", + "types": [string_type], + "clients": [ + { + "name": "Client", + "namespace": "sample", + "moduleName": "sample", + "description": "Example client.", + "url": "https://example.test", + "operationGroups": [], + "parameters": [ + { + "wireName": "value", + "clientName": "value", + "location": "query", + "implementation": "Client", + "optional": True, + "clientDefaultValue": value, + "description": "Example value.", + "type": string_type, + } + ], + } + ], + }, + code_model.options, + ) + serializer = GeneralSerializer(code_model, env, async_mode=async_mode, client_namespace="sample") + if file_kind == "client": + source = serializer.serialize_service_client_file(code_model.clients) + else: + source = serializer.serialize_config_file(code_model.clients) + module = ast.parse(source) + declaration = next(node for node in module.body if isinstance(node, ast.ClassDef)) + documentation = " ".join(ast.get_docstring(declaration).split()) + literal = StringType({"type": "string"}, code_model).get_declaration(value) + assert f"Default value is {literal}." in documentation + + +@pytest.mark.parametrize("value", VALUES) +@pytest.mark.parametrize("async_mode", [False, True]) +def test_complete_operation_preserves_parameter_defaults(code_model, env, value, async_mode): + parameter = Parameter( + { + "wireName": "value", + "clientName": "value", + "location": "query", + "implementation": "Method", + "optional": True, + "clientDefaultValue": value, + "description": "Example value.", + }, + code_model, + type=StringType({"type": "string"}, code_model), + ) + client = code_model.clients[0] + yaml_data = { + "url": "/example", + "method": "GET", + "groupName": "", + "isOverload": False, + "apiVersions": [], + "description": "Example operation.", + } + operation = Operation( + yaml_data, + code_model, + client, + "example", + RequestBuilder( + yaml_data, + code_model, + client, + "build_example_request", + RequestBuilderParameterList({}, code_model, parameters=[]), + ), + ParameterList({}, code_model, parameters=[parameter]), + responses=[Response({"statusCodes": [204]}, code_model)], + exceptions=[], + ) + serializer = OperationSerializer(code_model, async_mode=async_mode, client_namespace="sample") + source = env.get_template("operation.py.jinja2").render(operation=operation, operation_serializer=serializer) + function = ast.parse(source).body[0] + assert isinstance(function, ast.AsyncFunctionDef if async_mode else ast.FunctionDef) + # cspell:ignore kwonlyargs + defaults = {argument.arg: default for argument, default in zip(function.args.kwonlyargs, function.args.kw_defaults)} + assert ast.literal_eval(defaults["value"]) == value + documentation = " ".join(ast.get_docstring(function).split()) + literal = StringType({"type": "string"}, code_model).get_declaration(value) + assert f"Default value is {literal}." in documentation + + +@pytest.mark.parametrize("value", VALUES) +@pytest.mark.parametrize("models_mode", ["dpg", "typeddict"]) +def test_complete_model_docstrings_preserve_literals(code_model, env, value, models_mode): + code_model.options["models-mode"] = models_mode + model = build_type( + { + "type": "model", + "base": models_mode, + "name": "Example", + "snakeCaseName": "example", + "properties": [ + { + "wireName": "value", + "clientName": "value", + "optional": False, + "type": {"type": "constant", "value": value, "valueType": {"type": "string"}}, + } + ], + }, + code_model, + ) + serializer_type = DpgModelSerializer if models_mode == "dpg" else TypesSerializer + source = serializer_type(code_model, env, models=[model]).serialize() + module = ast.parse(source) + declaration = next(node for node in module.body if isinstance(node, ast.ClassDef)) + documentation = " ".join(ast.get_docstring(declaration).split()) + literal = StringType({"type": "string"}, code_model).get_declaration(value) + assert literal in documentation + if value == "ordinary": + assert ':ivar value: Required. Default value is "ordinary".' in source + if models_mode == "typeddict": + assert f"Literal[{literal}]" in documentation + + +@pytest.mark.parametrize("wire_name", [name for name in VALUES if name]) +def test_complete_polymorphic_examples_preserve_wire_names(code_model, env, wire_name): + code_model.options["models-mode"] = "none" + base = build_type( + { + "type": "model", + "base": "json", + "name": "Base", + "snakeCaseName": "base", + "properties": [ + { + "wireName": wire_name, + "clientName": "kind", + "optional": False, + "isDiscriminator": True, + "type": {"type": "string"}, + } + ], + }, + code_model, + ) + child = build_type( + { + "type": "model", + "base": "json", + "name": "Child", + "snakeCaseName": "child", + "discriminatorValue": "child", + "properties": [ + { + "wireName": wire_name, + "clientName": "kind", + "optional": False, + "isDiscriminator": True, + "type": {"type": "constant", "value": "child", "valueType": {"type": "string"}}, + } + ], + }, + code_model, + ) + child.parents = [base] + base.discriminated_subtypes = {"child": child} + client = code_model.clients[0] + yaml_data = { + "url": "/example", + "method": "POST", + "groupName": "", + "isOverload": False, + "apiVersions": [], + "description": "Example operation.", + } + body = BodyParameter( + { + "wireName": "body", + "clientName": "body", + "location": "body", + "optional": False, + "contentTypes": ["application/json"], + "defaultContentType": "application/json", + "description": "Example body.", + }, + code_model, + type=base, + ) + operation = Operation( + yaml_data, + code_model, + client, + "example", + RequestBuilder( + yaml_data, + code_model, + client, + "build_example_request", + RequestBuilderParameterList({}, code_model, parameters=[]), + ), + ParameterList({}, code_model, parameters=[], body_parameter=body), + responses=[Response({"statusCodes": [200], "defaultContentType": "application/json"}, code_model, type=base)], + exceptions=[], + ) + serializer = OperationSerializer(code_model, async_mode=False, client_namespace="sample") + template = "\n".join(serializer.example_template(operation)) + for direction in ("input", "response"): + assert ( + f"# The {direction} is polymorphic. The following are possible polymorphic " + f"{direction}s based off discriminator {python_string_literal(wire_name)}:" + ) in template.splitlines() + ast.parse(template) + documentation = env.from_string( + '{% import "operation_tools.jinja2" as tools %}{{ tools.description(builder, serializer) }}' + ).render(builder=operation, serializer=serializer) + function = ast.parse("def example():\n" + textwrap.indent(documentation.strip(), " ")).body[0] + example = textwrap.dedent(ast.get_docstring(function).split(".. code-block:: python\n", 1)[1]) + assignments = [node for node in ast.parse(example).body if isinstance(node, ast.Assign)] + assert len(assignments) == 3 + for assignment in assignments[::2]: + assert ast.literal_eval(assignment.value) == {wire_name: "child"} + assert example.count(f"based off discriminator {python_string_literal(wire_name)}:") == 2 + + +def test_json_template_preserves_nested_python_expressions(): + representation = { + '"nested"': [ + { + '"text"': repr("str"), + '"count"': "0", + '"flag"': "True", + '"data"': "None", + } + ], + '"model"': "models.Example()", + } + template = json_dumps_template(representation) + expression = ast.parse(template, mode="eval").body + assert ast.literal_eval(expression.values[0]) == [{"text": "str", "count": 0, "flag": True, "data": None}] + assert ast.unparse(expression.values[1]) == "models.Example()" + + +@pytest.mark.parametrize( + "value, wire_name", + [(value, "kind") for value in VALUES] + [("child", wire_name) for wire_name in VALUES if wire_name], +) +def test_discriminator_generation_preserves_values(code_model, env, value, wire_name): + model = build_type( + { + "type": "model", + "base": "dpg", + "name": "Child", + "snakeCaseName": "child", + "discriminatorValue": value, + "properties": [ + { + "wireName": wire_name, + "clientName": "kind", + "optional": False, + "isDiscriminator": True, + "type": {"type": "constant", "value": value, "valueType": {"type": "string"}}, + } + ], + }, + code_model, + ) + serializer = DpgModelSerializer(code_model, env, models=[model]) + model.parents = [SimpleNamespace(name="Base", properties=[])] + prop = model.properties[0] + if value: + representation = model.get_json_template_representation() + assert {ast.literal_eval(k): ast.literal_eval(v) for k, v in representation.items()} == {wire_name: value} + template = json_dumps_template(representation) + assert ast.literal_eval(template) == {wire_name: value} + model.parents[0].discriminated_subtypes = {value: model} + model.parents[0].parents = [] + model.parents[0].snake_case_name = "base" + rendered = "\n".join(_get_polymorphic_subtype_template(model)) + assignments = ast.parse(rendered).body + assert len(assignments) == 1 + assert ast.literal_eval(assignments[0].value) == {wire_name: value} + documentation = env.from_string( + '{% import "operation_tools.jinja2" as tools %}{{ tools.description(builder, serializer) }}' + ).render( + builder=SimpleNamespace(), + serializer=SimpleNamespace( + example_template=lambda builder: rendered.splitlines(), + param_description_and_response_docstring=lambda builder: [], + description_and_summary=lambda builder: ["Example operation."], + ), + ) + function = ast.parse("def operation():\n" + textwrap.indent(documentation.strip(), " ")).body[0] + docstring = ast.get_docstring(function) + example = textwrap.dedent(docstring.split(".. code-block:: python\n", 1)[1]) + documented_assignments = ast.parse(example).body + assert len(documented_assignments) == 1 + assert ast.literal_eval(documented_assignments[0].value) == {wire_name: value} + calls = [] + + def marker(): + calls.append(True) + return "unexpected" + + source = "\n".join( + [ + "class Base(_Model):", + " __mapping__ = {}", + " " + serializer.declare_property(prop), + serializer.declare_model(model), + " " + serializer.declare_property(prop), + " def __init__(self, *args, **kwargs):", + *[" " + line for line in serializer.super_call(model)], + ] + ) + module = ast.parse(source) + annotation = module.body[1].body[0].annotation + assert ast.literal_eval(annotation.slice) == value + if value: + assert ast.literal_eval(module.body[1].keywords[0].value) == value + + runtime = env.get_template("model_base.py.jinja2").render(code_model=code_model) + namespace = {"__name__": "string_literal_test"} + exec(compile(runtime, "", "exec"), namespace) + namespace.update( + _Model=namespace["Model"], + Literal=Literal, + marker=marker, + ) + exec(compile(source, "", "exec"), namespace) + child = namespace["Child"] + assert get_args(child.__annotations__["kind"]) == (value,) + for _ in range(2): + instance = child() + assert instance.kind == value + assert instance.as_dict() == {wire_name: value} + if value: + selected = namespace["Base"]._deserialize({wire_name: value}, []) + assert isinstance(selected, child) + assert selected.as_dict() == {wire_name: value} + assert calls == [] + + +@pytest.mark.parametrize("wire_name", [wire_name for wire_name in VALUES if wire_name]) +@pytest.mark.parametrize("optional", [False, True]) +def test_property_wire_names_are_literals(code_model, env, wire_name, optional): + model = build_type( + { + "type": "model", + "base": "dpg", + "name": "Example", + "snakeCaseName": "example", + "properties": [ + { + "wireName": "required", + "clientName": "required", + "optional": False, + "type": {"type": "string"}, + }, + { + "wireName": wire_name, + "clientName": "value", + "optional": optional, + "type": {"type": "string"}, + }, + ], + }, + code_model, + ) + template = json_dumps_template(model.get_json_template_representation()) + assert ast.literal_eval(template) == {"required": "str", wire_name: "str"} + if optional: + assert list(ast.literal_eval(template)) == ["required", wire_name] + serializer = DpgModelSerializer(code_model, env, models=[model]) + source = "\n".join( + [serializer.declare_model(model), *[" " + serializer.declare_property(prop) for prop in model.properties]] + ) + runtime = env.get_template("model_base.py.jinja2").render(code_model=code_model) + namespace = {"__name__": "wire_name_test"} + exec(compile(runtime, "", "exec"), namespace) + namespace.update(_Model=namespace["Model"], Optional=Optional) + exec(compile(source, "", "exec"), namespace) + instance = namespace["Example"](required="first", value="example") + assert instance.as_dict() == {"required": "first", wire_name: "example"} + assert namespace["Example"]({"required": "first", wire_name: "example"}).value == "example" + typed_model = build_type({**model.yaml_data, "base": "typeddict"}, code_model) + types_source = TypesSerializer(code_model, env, models=[typed_model]).serialize() + types_namespace = {"__name__": "wire_name_types_test"} + exec(compile(types_source, "", "exec"), types_namespace) + assert set(types_namespace["Example"].__annotations__) == {"required", wire_name} + + +@pytest.mark.parametrize("value", VALUES[0:1] + VALUES[2:]) +def test_discriminator_initializer_is_literal(value): + model = SimpleNamespace(discriminator_value=value) + prop = SimpleNamespace(client_name="kind") + statement = DpgModelSerializer.initialize_discriminator_property(model, prop) + assignment = ast.parse(statement).body[0] + assert ast.literal_eval(assignment.value) == value + + +@pytest.mark.parametrize("value", VALUES) +def test_enum_values_are_literals(code_model, env, value): + enum_yaml = { + "type": "enum", + "name": "Kind", + "valueType": {"type": "string"}, + "values": [], + } + enum = build_type(enum_yaml, code_model) + enum.values.append( + build_type( + { + "type": "enumvalue", + "name": "VALUE", + "value": value, + "enumType": enum_yaml, + "valueType": {"type": "string"}, + }, + code_model, + ) + ) + source = EnumSerializer(code_model, env, enums=[enum]).serialize() + assignments = [node for node in ast.walk(ast.parse(source)) if isinstance(node, ast.Assign)] + assignment = next(node for node in assignments if any(getattr(t, "id", "") == "VALUE" for t in node.targets)) + assert ast.literal_eval(assignment.value) == value