From 4e96885a85afcfdf3b0352dafdf215fe09f0bffb Mon Sep 17 00:00:00 2001 From: Sai Asish Y Date: Sun, 18 Oct 2026 05:11:15 -0700 Subject: [PATCH] Share PO message IDs across catalogs on conventional CPython Signed-off-by: Sai Asish Y --- babel/messages/pofile.py | 13 + tests/messages/test_pofile_id_sharing.py | 303 +++++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 tests/messages/test_pofile_id_sharing.py diff --git a/babel/messages/pofile.py b/babel/messages/pofile.py index 27f113ddf..ca79e5842 100644 --- a/babel/messages/pofile.py +++ b/babel/messages/pofile.py @@ -13,6 +13,8 @@ import os import re +import sys +import sysconfig from collections.abc import Iterable from typing import TYPE_CHECKING, Literal @@ -28,6 +30,12 @@ _unescape_re = re.compile(r'\\([\\trn"])') +# Free-threaded CPython keeps interned strings alive until shutdown. +# Limit sharing to conventional CPython, even when a free-threaded GIL is enabled. +_SHARE_MESSAGE_IDS = sys.implementation.name == 'cpython' and not sysconfig.get_config_var( + 'Py_GIL_DISABLED', +) + def unescape(string: str) -> str: r"""Reverse `escape` the given string. @@ -211,6 +219,11 @@ def _add_message(self) -> None: msgid = self.messages[0].denormalize() string = self.translations[0][1].denormalize() msgctxt = self.context.denormalize() if self.context else None + if _SHARE_MESSAGE_IDS and not (self.obsolete and self.ignore_obsolete): + if isinstance(msgid, tuple): + msgid = tuple(sys.intern(part) for part in msgid) + else: + msgid = sys.intern(msgid) message = Message( msgid, string, diff --git a/tests/messages/test_pofile_id_sharing.py b/tests/messages/test_pofile_id_sharing.py new file mode 100644 index 000000000..eb55f90ff --- /dev/null +++ b/tests/messages/test_pofile_id_sharing.py @@ -0,0 +1,303 @@ +import importlib.util +import sys +import sysconfig +from io import BytesIO, StringIO +from types import ModuleType, SimpleNamespace +from unittest.mock import patch + +import pytest + +from babel.messages import pofile + + +@pytest.fixture +def pofile_for_runtime(): + def load(implementation, gil_disabled, gil_enabled): + runtime_sys = ModuleType('sys') + runtime_sys.__dict__.update(vars(sys)) + runtime_sys.implementation = SimpleNamespace(**vars(sys.implementation)) + runtime_sys.implementation.name = implementation + runtime_sys._is_gil_enabled = lambda: gil_enabled + runtime_config = ModuleType('sysconfig') + runtime_config.__dict__.update(vars(sysconfig)) + + def get_config_var(name): + if name == 'Py_GIL_DISABLED': + return gil_disabled + return sysconfig.get_config_var(name) + + runtime_config.get_config_var = get_config_var + interned = [] + + def intern(value): + interned.append(value) + return sys.intern(value) + + runtime_sys.intern = intern + spec = importlib.util.spec_from_file_location( + 'babel.messages._pofile_runtime_test', + pofile.__file__, + ) + module = importlib.util.module_from_spec(spec) + with patch.dict(sys.modules, {'sys': runtime_sys, 'sysconfig': runtime_config}): + spec.loader.exec_module(module) + interned.clear() + return module, interned + + return load + + +@pytest.fixture +def conventional_pofile(pofile_for_runtime): + module, _ = pofile_for_runtime('cpython', 0, True) + return module + + +@pytest.mark.parametrize( + ('implementation', 'gil_disabled', 'gil_enabled'), + [ + ('cpython', 1, True), + ('cpython', 1, False), + ('pypy', None, True), + ('pypy', 0, True), + ('graalpy', None, True), + ], +) +@pytest.mark.parametrize('plural', [False, True]) +def test_read_po_avoids_interning_on_other_builds( + pofile_for_runtime, + implementation, + gil_disabled, + gil_enabled, + plural, +): + # A free-threaded build can enable its GIL without changing string lifetime. + module, interned = pofile_for_runtime(implementation, gil_disabled, gil_enabled) + po = '''\ +# Translator comment +#. Extracted comment +#: source.py:3 +#, fuzzy +msgctxt "menu" +msgid "A Unicode source ID: café, Ω, 日本語." +''' + if plural: + po += '''\ +msgid_plural "Several Unicode source IDs: café, Ω, 日本語." +msgstr[0] "Une traduction" +msgstr[1] "Des traductions" +''' + else: + po += 'msgstr "Une traduction"\n' + catalog = module.read_po(BytesIO(po.encode('utf-8')), locale='fr') + message = catalog.get('A Unicode source ID: café, Ω, 日本語.', context='menu') + if plural: + assert message.id == ( + 'A Unicode source ID: café, Ω, 日本語.', + 'Several Unicode source IDs: café, Ω, 日本語.', + ) + # French has three plural forms; the omitted translation stays empty. + assert message.string == ('Une traduction', 'Des traductions', '') + else: + assert message.id == 'A Unicode source ID: café, Ω, 日本語.' + assert message.string == 'Une traduction' + assert message.context == 'menu' + assert message.flags == {'fuzzy'} + assert message.locations == [('source.py', 3)] + assert message.user_comments == ['Translator comment'] + assert message.auto_comments == ['Extracted comment'] + assert interned == [] + + +@pytest.mark.parametrize('gil_disabled', [None, 0]) +@pytest.mark.parametrize('plural', [False, True]) +def test_read_po_interns_ids_on_conventional_cpython( + pofile_for_runtime, + gil_disabled, + plural, +): + module, interned = pofile_for_runtime('cpython', gil_disabled, True) + po = 'msgid "A shared Unicode source ID: café, Ω, 日本語."\n' + expected = ['A shared Unicode source ID: café, Ω, 日本語.'] + if plural: + po += '''\ +msgid_plural "Several shared Unicode source IDs: café, Ω, 日本語." +msgstr[0] "Une traduction" +msgstr[1] "Des traductions" +''' + expected.append('Several shared Unicode source IDs: café, Ω, 日本語.') + else: + po += 'msgstr "Une traduction"\n' + first = module.read_po(StringIO(po), locale='fr') + second = module.read_po(BytesIO(po.encode('utf-8')), locale='fr') + assert interned == expected * 2 + first_id = first['A shared Unicode source ID: café, Ω, 日本語.'].id + second_id = second['A shared Unicode source ID: café, Ω, 日本語.'].id + if plural: + assert first_id == second_id == tuple(expected) + assert first_id[0] is second_id[0] + assert first_id[1] is second_id[1] + else: + assert first_id == second_id == expected[0] + assert first_id is second_id + + +def test_read_po_does_not_intern_discarded_obsolete_ids(pofile_for_runtime): + module, interned = pofile_for_runtime('cpython', 0, True) + catalog = module.read_po( + StringIO('''\ +#~ msgctxt "menu" +#~ msgid "An obsolete Unicode source ID: café, Ω, 日本語." +#~ msgid_plural "Several obsolete Unicode source IDs: café, Ω, 日本語." +#~ msgstr[0] "Une traduction" +#~ msgstr[1] "Des traductions" +'''), + locale='fr', + ignore_obsolete=True, + ) + assert not catalog + assert not catalog.obsolete + assert interned == [] + + +def test_read_po_shares_source_ids_across_catalogs(conventional_pofile): + first = conventional_pofile.read_po( + StringIO('''\ +msgid "A source message shared by several independently loaded translation catalogs." +msgstr "First translation" +'''), + ) + second = conventional_pofile.read_po( + StringIO('''\ +msgid "A source message shared by several independently loaded translation catalogs." +msgstr "Second translation" +'''), + ) + identifier = ( + "A source message shared by several independently loaded translation catalogs." + ) + assert first[identifier].id == second[identifier].id == identifier + assert first[identifier].id is second[identifier].id + assert first[identifier].string == "First translation" + assert second[identifier].string == "Second translation" + + +def test_read_po_shares_each_plural_source_id(conventional_pofile): + po = '''\ +msgid "One source message shared by independently loaded translation catalogs." +msgid_plural "Several source messages shared by independently loaded translation catalogs." +msgstr[0] "One translated message" +msgstr[1] "Several translated messages" +''' + first = conventional_pofile.read_po(StringIO(po)) + second = conventional_pofile.read_po(StringIO(po)) + identifier = "One source message shared by independently loaded translation catalogs." + expected = ( + identifier, + "Several source messages shared by independently loaded translation catalogs.", + ) + assert first[identifier].id == second[identifier].id == expected + assert first[identifier].id[0] is second[identifier].id[0] + assert first[identifier].id[1] is second[identifier].id[1] + assert first[identifier].string == ( + "One translated message", + "Several translated messages", + ) + + +def test_read_po_shares_denormalized_unicode_ids(conventional_pofile): + first = conventional_pofile.read_po( + StringIO('''\ +msgid "Unicode source: 日本語, Ω, café and an escaped \\"quote\\"." +msgstr "first" +'''), + ) + second = conventional_pofile.read_po( + StringIO('''\ +msgid "" +"Unicode source: 日本語, Ω, " +"café and an escaped \\"quote\\"." +msgstr "second" +'''), + ) + identifier = 'Unicode source: 日本語, Ω, café and an escaped "quote".' + assert first[identifier].id == second[identifier].id == identifier + assert first[identifier].id is second[identifier].id + + +def test_read_po_keeps_messages_and_metadata_independent(): + po = '''\ +# Translator comment +#. Extracted comment +#: source.py:3 +#, fuzzy +msgid "An immutable source ID must not cause mutable message data to be shared." +msgstr "translation" +''' + first = pofile.read_po(StringIO(po)) + second = pofile.read_po(StringIO(po)) + identifier = "An immutable source ID must not cause mutable message data to be shared." + message = first[identifier] + message.string = "changed" + message.flags.add("custom") + message.locations.append(("other.py", 9)) + message.user_comments.append("additional translator comment") + message.auto_comments.append("additional extracted comment") + assert message is not second[identifier] + assert second[identifier].string == "translation" + assert second[identifier].flags == {"fuzzy"} + assert second[identifier].locations == [("source.py", 3)] + assert second[identifier].user_comments == ["Translator comment"] + assert second[identifier].auto_comments == ["Extracted comment"] + + +def test_read_po_preserves_distinct_contexts(): + catalog = pofile.read_po( + StringIO('''\ +msgid "A source ID occurring in distinct contexts." +msgstr "without context" + +msgctxt "" +msgid "A source ID occurring in distinct contexts." +msgstr "empty context" + +msgctxt "menu" +msgid "A source ID occurring in distinct contexts." +msgstr "menu context" +'''), + ) + identifier = "A source ID occurring in distinct contexts." + assert len(catalog) == 3 + for context, translation in ( + (None, "without context"), + ("", "empty context"), + ("menu", "menu context"), + ): + message = catalog.get(identifier, context=context) + assert message.id == identifier + assert message.context == context + assert message.string == translation + + +def test_read_po_preserves_obsolete_and_duplicate_messages(): + po = '''\ +#: first.py:1 +msgid "An active source ID repeated with a second reference." +msgstr "first translation" + +#: second.py:2 +msgid "An active source ID repeated with a second reference." +msgstr "second translation" + +#~ msgid "An obsolete source ID retained separately from active messages." +#~ msgstr "obsolete translation" +''' + catalog = pofile.read_po(StringIO(po)) + ignored = pofile.read_po(StringIO(po), ignore_obsolete=True) + identifier = "An active source ID repeated with a second reference." + assert len(catalog) == len(ignored) == 1 + assert catalog[identifier].string == "first translation" + assert catalog[identifier].locations == [("first.py", 1), ("second.py", 2)] + obsolete_id = "An obsolete source ID retained separately from active messages." + assert catalog.obsolete[obsolete_id].string == "obsolete translation" + assert not ignored.obsolete