Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: fix
packages:
- "@typespec/http-client-python"
---

Escape string literals and wire names in generated Python declarations, discriminator initializers, field metadata, and templates to preserve specification values as data.
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from enum import Enum
from collections import OrderedDict
from typing import Any, Literal, Optional, TYPE_CHECKING, cast
from .utils import add_to_pylint_disable, NamespaceType, LOCALS_LENGTH_LIMIT
from .utils import add_to_pylint_disable, NamespaceType, LOCALS_LENGTH_LIMIT, python_string_literal
from .base import BaseType
from .constant_type import ConstantType
from .property import Property
Expand Down Expand Up @@ -164,20 +164,22 @@ def get_json_template_representation(
# don't add additional properties, because there's not really a concept of
# additional properties in the template
representation = {
f'"{prop.wire_name}"': prop.get_json_template_representation(
python_string_literal(prop.wire_name): prop.get_json_template_representation(
client_default_value_declaration=client_default_value_declaration,
)
for prop in [
p for p in self.properties if not (p.is_discriminator or p.client_name == "additional_properties")
]
}
if self.discriminator and self.discriminator_value:
representation[f'"{self.discriminator.wire_name}"'] = f'"{self.discriminator_value}"'
representation[python_string_literal(self.discriminator.wire_name)] = python_string_literal(
self.discriminator_value
)

# once we've finished, we want to reset created_json_template_representation to false
# so we can call it again
self._created_json_template_representation = False
optional_keys = [f'"{p.wire_name}"' for p in self.properties if getattr(p, "optional", False)]
optional_keys = [python_string_literal(p.wire_name) for p in self.properties if getattr(p, "optional", False)]
return OrderedDict(
sorted(
representation.items(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

from .base import BaseType
from .imports import FileImport, ImportType, TypingSection
from .utils import NamespaceType
from .utils import NamespaceType, python_string_literal

if TYPE_CHECKING:
from .code_model import CodeModel
Expand Down Expand Up @@ -332,8 +332,8 @@ def validation(self) -> Optional[dict[str, Union[bool, int, str]]]:
validation["pattern"] = RawString(self.pattern) # type: ignore
return validation or None

def get_declaration(self, value) -> str:
return f"'{value}'" if value == '"' else f'"{value}"'
def get_declaration(self, value: str) -> str:
return python_string_literal(value, quote="'" if value == '"' else '"')

def serialization_type(self, **kwargs: Any) -> str:
return "str"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@
OrderedSet = dict[T, None]


def python_string_literal(value: str, *, quote: str = '"') -> str:
literal = repr(value)
if literal[0] == quote:
return literal
# Change delimiter escaping without changing the escaped value.
body = literal[1:-1].replace("\\" + literal[0], literal[0])
return quote + body.replace(quote, "\\" + quote) + quote


def add_to_description(description: str, entry: str) -> str:
if not description:
return entry
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
ParameterListType,
ByteArraySchema,
)
from ..models.utils import NamespaceType, escape_sphinx_field_name
from ..models.utils import NamespaceType, escape_sphinx_field_name, python_string_literal
from .parameter_serializer import ParameterSerializer, PopKwargType, check_body_optional
from ..models.parameter_list import ParameterType
from ..models.response import (
Expand Down Expand Up @@ -99,7 +99,8 @@ def _escape_str(input_str: str) -> str:
def _get_polymorphic_subtype_template(polymorphic_subtype: ModelType) -> list[str]:
retval: list[str] = []
retval.append("")
retval.append(f'# JSON input template for discriminator value "{polymorphic_subtype.discriminator_value}":')
discriminator_value = python_string_literal(polymorphic_subtype.discriminator_value)
retval.append(f"# JSON input template for discriminator value {discriminator_value}:")
subtype_template = utils.json_dumps_template(
polymorphic_subtype.get_json_template_representation(),
)
Expand Down Expand Up @@ -385,7 +386,7 @@ def _json_input_example_template(self, builder: BuilderType) -> list[str]:
discriminator_name = cast(Property, polymorphic_subtypes[0].discriminator).wire_name
template.append(
"# The input is polymorphic. The following are possible polymorphic "
f'inputs based off discriminator "{discriminator_name}":'
f"inputs based off discriminator {python_string_literal(discriminator_name)}:"
)
for idx in range(
min(
Expand Down Expand Up @@ -582,7 +583,7 @@ def example_template(self, builder: OperationType) -> list[str]:
retval.append("")
retval.append(
"# The response is polymorphic. The following are possible polymorphic "
f'responses based off discriminator "{discriminator_name}":'
f"responses based off discriminator {python_string_literal(discriminator_name)}:"
)
for idx in range(
min(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ def property_descriptions(self, async_mode: bool) -> list[str]:
":keyword int polling_interval: Default waiting time between two polls for LRO operations "
"if no Retry-After header is present."
)
retval = [s.replace("\\", "\\\\") for s in retval]
retval.append('"""')
return retval

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
)
from .import_serializer import FileImportSerializer
from .base_serializer import BaseSerializer
from ..models.utils import NamespaceType, add_to_pylint_disable, escape_sphinx_field_name
from ..models.utils import NamespaceType, add_to_pylint_disable, escape_sphinx_field_name, python_string_literal


def _get_xml_deserializer_name(prop: Property) -> Optional[str]: # pylint: disable=too-many-return-statements
Expand Down Expand Up @@ -82,15 +82,17 @@ def _documentation_string(
)
# Escape names that Sphinx would otherwise misinterpret in the info field target
# (e.g. a wire name with a leading "@" such as "@search.facets").
doc_name = escape_sphinx_field_name(doc_name)
# Strip repr's delimiters and escape triple quotes for the enclosing docstring.
doc_name = repr(escape_sphinx_field_name(doc_name))[1:-1].replace('"""', '\\"\\"\\"')
sphinx_prefix = f":{description_keyword} {doc_name}:"
description = prop.description(is_operation_file=False).replace("\\", "\\\\")
description = prop.description(is_operation_file=False).replace("\\", "\\\\").replace('"""', '\\"\\"\\"')
retval.append(f"{sphinx_prefix} {description}" if description else sphinx_prefix)
# In the types file, use type_annotation (the serialized form) for docstrings
if kwargs.get("serialize_namespace_type") == NamespaceType.TYPES_FILE:
doc_type = prop.type.type_annotation(**kwargs)
else:
doc_type = prop.type.docstring_type(**kwargs)
doc_type = doc_type.replace("\\", "\\\\").replace('"""', '\\"\\"\\"')
retval.append(f":{docstring_type_keyword} {doc_name}: {doc_type}")
return retval

Expand Down Expand Up @@ -137,7 +139,9 @@ def super_call(self, model: ModelType) -> list[str]:

@staticmethod
def initialize_discriminator_property(model: ModelType, prop: Property) -> str:
discriminator_value = f"'{model.discriminator_value}'" if model.discriminator_value else None
discriminator_value = (
python_string_literal(model.discriminator_value, quote="'") if model.discriminator_value else None
)
if not discriminator_value:
typing = "Optional[str]"
else:
Expand Down Expand Up @@ -422,7 +426,8 @@ def declare_model(self, model: ModelType) -> str:
if model.parents:
basename = ", ".join([m.name for m in model.parents])
if model.discriminator_value:
basename += f", discriminator='{model.discriminator_value}'"
discriminator_value = python_string_literal(model.discriminator_value, quote="'")
basename += f", discriminator={discriminator_value}"
return f"class {model.name}({basename}):{self.class_pylint_disable(model)}"

def class_pylint_disable(self, model: ModelType) -> str:
Expand Down Expand Up @@ -458,7 +463,7 @@ def get_properties_to_declare(model: ModelType) -> list[Property]:
def declare_property(self, prop: Property) -> str:
args = []
if prop.client_name != prop.wire_name or prop.is_discriminator:
args.append(f'name="{prop.wire_name}"')
args.append(f"name={python_string_literal(prop.wire_name)}")
if prop.visibility:
v_list = ", ".join(f'"{x}"' for x in prop.visibility)
args.append(f"visibility=[{v_list}]")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from ..models import ModelType, CodeModel
from ..models.enum_type import EnumType
from ..models.imports import FileImport, ImportType
from ..models.utils import NamespaceType
from ..models.utils import NamespaceType, python_string_literal
from ..models.property import Property
from .model_serializer import _documentation_string
from .import_serializer import FileImportSerializer
Expand Down Expand Up @@ -388,9 +388,9 @@ def declare_functional_model(self, model: ModelType) -> str:
type_annotation = _qualify_shadowed_builtins(type_annotation, shadowed)
is_optional = prop.optional or prop.client_default_value is not None
if is_optional:
entries.append(f' "{prop.wire_name}": {type_annotation},')
entries.append(f" {python_string_literal(prop.wire_name)}: {type_annotation},")
else:
entries.append(f' "{prop.wire_name}": Required[{type_annotation}],')
entries.append(f" {python_string_literal(prop.wire_name)}: Required[{type_annotation}],")
fields = "\n".join(entries)
return f'{model.name} = TypedDict("{model.name}", {{\n{fields}\n}}, total=False)'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
# license information.
# --------------------------------------------------------------------------
import json
import re
from typing import Optional, Any
from pathlib import Path

Expand Down Expand Up @@ -43,19 +44,8 @@ def get_namespace_from_package_name(package_name: Optional[str]) -> str:


def _improve_json_string(template_representation: str) -> Any:
origin = template_representation.split("\n")
final = []
for line in origin:
idx0 = line.find("#")
idx1 = line.rfind('"')
modified_line = ""
if idx0 > -1 and idx1 > -1:
modified_line = line[:idx0] + line[idx1:] + " " + line[idx0:idx1] + "\n"
else:
modified_line = line + "\n"
modified_line = modified_line.replace('"', "").replace("\\", '"')
final.append(modified_line)
return "".join(final)
# Unwrap JSON strings without changing the Python expressions stored inside them.
return re.sub(r'"(?:[^"\\]|\\.)*"', lambda match: json.loads(match.group()), template_representation) + "\n"


def json_dumps_template(template_representation: Any) -> Any:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,13 @@
{{ normalized_string | replace("\\", "\\\\") | wordwrap(width=width, break_long_words=False, break_on_hyphens=False, wrapstring=wrapstring)}}{% endmacro %}

{# Escape quotes after wrapping so wrapping cannot split an escape sequence. #}
{% macro wrap_docstring(string, wrapstring, width=95) -%}
{{ wrap_string(string, wrapstring, width) | replace('"', '\\"') }}
{% macro wrap_docstring(string, wrapstring, width=95, escape_all_quotes=true) -%}
{%- set wrapped = wrap_string(string, wrapstring, width) -%}
{%- if escape_all_quotes -%}
{{ wrapped | replace('"', '\\"') }}
{%- else -%}
{{ wrapped | replace('"""', '\\"\\"\\"') }}
{%- endif -%}
{%- endmacro %}

{% macro description(builder, serializer) %}
Expand All @@ -55,7 +60,7 @@
{% endfor %}
{% for description in param_description_and_response_docstring %}
{% if description %}
{% set description = wrap_string(description, wrapstring='\n ') %}
{% set description = wrap_docstring(description, wrapstring='\n ', escape_all_quotes=false) %}
{{ " " if description[0] != ":" and description[0] != " " else "" }}{{ description}}
{% else %}

Expand All @@ -67,8 +72,7 @@ Example:
.. code-block:: python
{% for template_line in example_template %}
{% if template_line %}
{% set wrap_amount = (template_line | length) - (template_line.lstrip() | length) + 10 %}
{{ wrap_string(template_line, wrapstring='\n' + " " * wrap_amount, width=(95 - wrap_amount)) }}

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

iscai-msft I remember the code has special effect for specific SDK doc string, so not sure whether it is ok to remove the logic

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we added it to prevent overlong lines in generated docstring examples, while keeping wrapped continuation lines indented inside the sphinx code block .. code-block:: python, it might be one of the agent sdks, i can't remember exactly which sdk it was

can you have copilot have a look to see if we can still preserve this ability and write it in a better way?

{{ template_line | replace("\\", "\\\\") | replace('"""', '\\"\\"\\"') }}
{% else %}

{% endif %}
Expand All @@ -89,7 +93,11 @@ Example:
{% macro serialize_with_wrap(lines, wrapstring) %}
{% for line in lines %}
{% if line %}
{{ wrap_string(line, wrapstring=wrapstring) }}
{% if line == '"""' %}
{{ line }}
{% else %}
{{ wrap_docstring(line, wrapstring=wrapstring, escape_all_quotes=false) }}
{% endif %}
{% else %}

{% endif %}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@
'A"""; print("This should remain documentation"); """B.',
'Quotes: " and "" and """.',
'Ends with a quote"',
'Ends with two quotes""',
'Ends with four quotes""""',
'Ends with a backslash and quote\\"',
r'Quotes """ with a regex \W and a path C:\new\test.',
'First paragraph.\n\nSecond paragraph with """ quotes.',
],
Expand Down
Loading
Loading