Share PO message IDs across catalogs on conventional CPython

Signed-off-by: Sai Asish Y <say.apm35@gmail.com>

Author: SAY-5

babel/messages/pofile.py

@@ -13,6 +13,8 @@
 
 import os
 import re
+import sys
+import sysconfig
 from collections.abc import Iterable
 from typing import TYPE_CHECKING, Literal
 
@@ -28,6 +30,12 @@
 
 _unescape_re = re.compile(r'\\([\\trn"])')
 
+# Free-threaded CPython keeps interned strings alive until shutdown.
+# Limit sharing to conventional CPython, even when a free-threaded GIL is enabled.
+_SHARE_MESSAGE_IDS = sys.implementation.name == 'cpython' and not sysconfig.get_config_var(
+    'Py_GIL_DISABLED',
+)
+
 
 def unescape(string: str) -> str:
     r"""Reverse `escape` the given string.
@@ -211,6 +219,11 @@ def _add_message(self) -> None:
             msgid = self.messages[0].denormalize()
             string = self.translations[0][1].denormalize()
         msgctxt = self.context.denormalize() if self.context else None
+        if _SHARE_MESSAGE_IDS and not (self.obsolete and self.ignore_obsolete):
+            if isinstance(msgid, tuple):
+                msgid = tuple(sys.intern(part) for part in msgid)
+            else:
+                msgid = sys.intern(msgid)
         message = Message(
             msgid,
             string,

tests/messages/test_pofile_id_sharing.py

@@ -0,0 +1,303 @@
+import importlib.util
+import sys
+import sysconfig
+from io import BytesIO, StringIO
+from types import ModuleType, SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+from babel.messages import pofile
+
+
+@pytest.fixture
+def pofile_for_runtime():
+    def load(implementation, gil_disabled, gil_enabled):
+        runtime_sys = ModuleType('sys')
+        runtime_sys.__dict__.update(vars(sys))
+        runtime_sys.implementation = SimpleNamespace(**vars(sys.implementation))
+        runtime_sys.implementation.name = implementation
+        runtime_sys._is_gil_enabled = lambda: gil_enabled
+        runtime_config = ModuleType('sysconfig')
+        runtime_config.__dict__.update(vars(sysconfig))
+
+        def get_config_var(name):
+            if name == 'Py_GIL_DISABLED':
+                return gil_disabled
+            return sysconfig.get_config_var(name)
+
+        runtime_config.get_config_var = get_config_var
+        interned = []
+
+        def intern(value):
+            interned.append(value)
+            return sys.intern(value)
+
+        runtime_sys.intern = intern
+        spec = importlib.util.spec_from_file_location(
+            'babel.messages._pofile_runtime_test',
+            pofile.__file__,
+        )
+        module = importlib.util.module_from_spec(spec)
+        with patch.dict(sys.modules, {'sys': runtime_sys, 'sysconfig': runtime_config}):
+            spec.loader.exec_module(module)
+        interned.clear()
+        return module, interned
+
+    return load
+
+
+@pytest.fixture
+def conventional_pofile(pofile_for_runtime):
+    module, _ = pofile_for_runtime('cpython', 0, True)
+    return module
+
+
+@pytest.mark.parametrize(
+    ('implementation', 'gil_disabled', 'gil_enabled'),
+    [
+        ('cpython', 1, True),
+        ('cpython', 1, False),
+        ('pypy', None, True),
+        ('pypy', 0, True),
+        ('graalpy', None, True),
+    ],
+)
+@pytest.mark.parametrize('plural', [False, True])
+def test_read_po_avoids_interning_on_other_builds(
+    pofile_for_runtime,
+    implementation,
+    gil_disabled,
+    gil_enabled,
+    plural,
+):
+    # A free-threaded build can enable its GIL without changing string lifetime.
+    module, interned = pofile_for_runtime(implementation, gil_disabled, gil_enabled)
+    po = '''\
+# Translator comment
+#. Extracted comment
+#: source.py:3
+#, fuzzy
+msgctxt "menu"
+msgid "A Unicode source ID: café, Ω, 日本語."
+'''
+    if plural:
+        po += '''\
+msgid_plural "Several Unicode source IDs: café, Ω, 日本語."
+msgstr[0] "Une traduction"
+msgstr[1] "Des traductions"
+'''
+    else:
+        po += 'msgstr "Une traduction"\n'
+    catalog = module.read_po(BytesIO(po.encode('utf-8')), locale='fr')
+    message = catalog.get('A Unicode source ID: café, Ω, 日本語.', context='menu')
+    if plural:
+        assert message.id == (
+            'A Unicode source ID: café, Ω, 日本語.',
+            'Several Unicode source IDs: café, Ω, 日本語.',
+        )
+        # French has three plural forms; the omitted translation stays empty.
+        assert message.string == ('Une traduction', 'Des traductions', '')
+    else:
+        assert message.id == 'A Unicode source ID: café, Ω, 日本語.'
+        assert message.string == 'Une traduction'
+    assert message.context == 'menu'
+    assert message.flags == {'fuzzy'}
+    assert message.locations == [('source.py', 3)]
+    assert message.user_comments == ['Translator comment']
+    assert message.auto_comments == ['Extracted comment']
+    assert interned == []
+
+
+@pytest.mark.parametrize('gil_disabled', [None, 0])
+@pytest.mark.parametrize('plural', [False, True])
+def test_read_po_interns_ids_on_conventional_cpython(
+    pofile_for_runtime,
+    gil_disabled,
+    plural,
+):
+    module, interned = pofile_for_runtime('cpython', gil_disabled, True)
+    po = 'msgid "A shared Unicode source ID: café, Ω, 日本語."\n'
+    expected = ['A shared Unicode source ID: café, Ω, 日本語.']
+    if plural:
+        po += '''\
+msgid_plural "Several shared Unicode source IDs: café, Ω, 日本語."
+msgstr[0] "Une traduction"
+msgstr[1] "Des traductions"
+'''
+        expected.append('Several shared Unicode source IDs: café, Ω, 日本語.')
+    else:
+        po += 'msgstr "Une traduction"\n'
+    first = module.read_po(StringIO(po), locale='fr')
+    second = module.read_po(BytesIO(po.encode('utf-8')), locale='fr')
+    assert interned == expected * 2
+    first_id = first['A shared Unicode source ID: café, Ω, 日本語.'].id
+    second_id = second['A shared Unicode source ID: café, Ω, 日本語.'].id
+    if plural:
+        assert first_id == second_id == tuple(expected)
+        assert first_id[0] is second_id[0]
+        assert first_id[1] is second_id[1]
+    else:
+        assert first_id == second_id == expected[0]
+        assert first_id is second_id
+
+
+def test_read_po_does_not_intern_discarded_obsolete_ids(pofile_for_runtime):
+    module, interned = pofile_for_runtime('cpython', 0, True)
+    catalog = module.read_po(
+        StringIO('''\
+#~ msgctxt "menu"
+#~ msgid "An obsolete Unicode source ID: café, Ω, 日本語."
+#~ msgid_plural "Several obsolete Unicode source IDs: café, Ω, 日本語."
+#~ msgstr[0] "Une traduction"
+#~ msgstr[1] "Des traductions"
+'''),
+        locale='fr',
+        ignore_obsolete=True,
+    )
+    assert not catalog
+    assert not catalog.obsolete
+    assert interned == []
+
+
+def test_read_po_shares_source_ids_across_catalogs(conventional_pofile):
+    first = conventional_pofile.read_po(
+        StringIO('''\
+msgid "A source message shared by several independently loaded translation catalogs."
+msgstr "First translation"
+'''),
+    )
+    second = conventional_pofile.read_po(
+        StringIO('''\
+msgid "A source message shared by several independently loaded translation catalogs."
+msgstr "Second translation"
+'''),
+    )
+    identifier = (
+        "A source message shared by several independently loaded translation catalogs."
+    )
+    assert first[identifier].id == second[identifier].id == identifier
+    assert first[identifier].id is second[identifier].id
+    assert first[identifier].string == "First translation"
+    assert second[identifier].string == "Second translation"
+
+
+def test_read_po_shares_each_plural_source_id(conventional_pofile):
+    po = '''\
+msgid "One source message shared by independently loaded translation catalogs."
+msgid_plural "Several source messages shared by independently loaded translation catalogs."
+msgstr[0] "One translated message"
+msgstr[1] "Several translated messages"
+'''
+    first = conventional_pofile.read_po(StringIO(po))
+    second = conventional_pofile.read_po(StringIO(po))
+    identifier = "One source message shared by independently loaded translation catalogs."
+    expected = (
+        identifier,
+        "Several source messages shared by independently loaded translation catalogs.",
+    )
+    assert first[identifier].id == second[identifier].id == expected
+    assert first[identifier].id[0] is second[identifier].id[0]
+    assert first[identifier].id[1] is second[identifier].id[1]
+    assert first[identifier].string == (
+        "One translated message",
+        "Several translated messages",
+    )
+
+
+def test_read_po_shares_denormalized_unicode_ids(conventional_pofile):
+    first = conventional_pofile.read_po(
+        StringIO('''\
+msgid "Unicode source: 日本語, Ω, café and an escaped \\"quote\\"."
+msgstr "first"
+'''),
+    )
+    second = conventional_pofile.read_po(
+        StringIO('''\
+msgid ""
+"Unicode source: 日本語, Ω, "
+"café and an escaped \\"quote\\"."
+msgstr "second"
+'''),
+    )
+    identifier = 'Unicode source: 日本語, Ω, café and an escaped "quote".'
+    assert first[identifier].id == second[identifier].id == identifier
+    assert first[identifier].id is second[identifier].id
+
+
+def test_read_po_keeps_messages_and_metadata_independent():
+    po = '''\
+# Translator comment
+#. Extracted comment
+#: source.py:3
+#, fuzzy
+msgid "An immutable source ID must not cause mutable message data to be shared."
+msgstr "translation"
+'''
+    first = pofile.read_po(StringIO(po))
+    second = pofile.read_po(StringIO(po))
+    identifier = "An immutable source ID must not cause mutable message data to be shared."
+    message = first[identifier]
+    message.string = "changed"
+    message.flags.add("custom")
+    message.locations.append(("other.py", 9))
+    message.user_comments.append("additional translator comment")
+    message.auto_comments.append("additional extracted comment")
+    assert message is not second[identifier]
+    assert second[identifier].string == "translation"
+    assert second[identifier].flags == {"fuzzy"}
+    assert second[identifier].locations == [("source.py", 3)]
+    assert second[identifier].user_comments == ["Translator comment"]
+    assert second[identifier].auto_comments == ["Extracted comment"]
+
+
+def test_read_po_preserves_distinct_contexts():
+    catalog = pofile.read_po(
+        StringIO('''\
+msgid "A source ID occurring in distinct contexts."
+msgstr "without context"
+
+msgctxt ""
+msgid "A source ID occurring in distinct contexts."
+msgstr "empty context"
+
+msgctxt "menu"
+msgid "A source ID occurring in distinct contexts."
+msgstr "menu context"
+'''),
+    )
+    identifier = "A source ID occurring in distinct contexts."
+    assert len(catalog) == 3
+    for context, translation in (
+        (None, "without context"),
+        ("", "empty context"),
+        ("menu", "menu context"),
+    ):
+        message = catalog.get(identifier, context=context)
+        assert message.id == identifier
+        assert message.context == context
+        assert message.string == translation
+
+
+def test_read_po_preserves_obsolete_and_duplicate_messages():
+    po = '''\
+#: first.py:1
+msgid "An active source ID repeated with a second reference."
+msgstr "first translation"
+
+#: second.py:2
+msgid "An active source ID repeated with a second reference."
+msgstr "second translation"
+
+#~ msgid "An obsolete source ID retained separately from active messages."
+#~ msgstr "obsolete translation"
+'''
+    catalog = pofile.read_po(StringIO(po))
+    ignored = pofile.read_po(StringIO(po), ignore_obsolete=True)
+    identifier = "An active source ID repeated with a second reference."
+    assert len(catalog) == len(ignored) == 1
+    assert catalog[identifier].string == "first translation"
+    assert catalog[identifier].locations == [("first.py", 1), ("second.py", 2)]
+    obsolete_id = "An obsolete source ID retained separately from active messages."
+    assert catalog.obsolete[obsolete_id].string == "obsolete translation"
+    assert not ignored.obsolete