From 1396b505cea938c60266c19079c31465d853f946 Mon Sep 17 00:00:00 2001 From: Jun Date: Mon, 2 Nov 2026 03:00:43 +0900 Subject: [PATCH] [3.15] gh-88336: Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList (GH-26328) * bpo-44170: Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList * Round up the allocated size instead of rejecting the exact fit Allocating one extra alignment block for a value whose encoded size is a multiple of the alignment made a copy of the list wider than the original. Round the size up instead, so that it is the same for the same value, and keep allowing a value which fills the slot exactly: rejecting it broke setting an 8-byte string in a slot of an integer. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Serhiy Storchaka (cherry picked from commit 9348e131a0d97a4b6fd91ea8bff7bfdbaa7d7bd0) Co-authored-by: Jun --- Lib/multiprocessing/shared_memory.py | 14 +++++++---- Lib/test/_test_multiprocessing.py | 23 ++++++++++--------- .../2021-05-24-14-41-29.bpo-44170._v28mv.rst | 1 + 3 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 99a8ce3320ad4e..582a00a32facff 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -305,6 +305,13 @@ def _extract_recreation_code(value): else: return 3 # NoneType + @staticmethod + def _encode_value(value): + if not isinstance(value, str): + return value + else: + return value.encode(_encoding) + def __init__(self, sequence=None, *, name=None): if name is None or sequence is not None: sequence = sequence or () @@ -312,7 +319,7 @@ def __init__(self, sequence=None, *, name=None): self._types_mapping[type(item)] if not isinstance(item, (str, bytes)) else self._types_mapping[type(item)] % ( - self._alignment * (len(item) // self._alignment + 1), + self._alignment * ((len(self._encode_value(item)) - 1) // self._alignment + 1), ) for item in sequence ] @@ -353,7 +360,7 @@ def __init__(self, sequence=None, *, name=None): "".join(_formats), self.shm.buf, self._offset_data_start, - *(v.encode(_enc) if isinstance(v, str) else v for v in sequence) + *(self._encode_value(v) for v in sequence) ) struct.pack_into( self._format_packing_metainfo, @@ -463,8 +470,7 @@ def __setitem__(self, position, value): else: allocated_length = self._allocated_offsets[position + 1] - item_offset - encoded_value = (value.encode(_encoding) - if isinstance(value, str) else value) + encoded_value = self._encode_value(value) if len(encoded_value) > allocated_length: raise ValueError("bytes/str item exceeds available storage") if current_format[-1] == "s": diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index cda379cc797696..87e13b05e1bfe9 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4810,7 +4810,7 @@ def test_shared_memory_SharedMemoryManager_basics(self): def test_shared_memory_ShareableList_basics(self): sl = shared_memory.ShareableList( - ['howdy', b'HoWdY', -273.154, 100, None, True, 42] + ['howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥'] ) self.addCleanup(sl.shm.unlink) @@ -4820,11 +4820,11 @@ def test_shared_memory_ShareableList_basics(self): # Index Out of Range (get) with self.assertRaises(IndexError): - sl[7] + sl[8] # Index Out of Range (set) with self.assertRaises(IndexError): - sl[7] = 2 + sl[8] = 2 # Assign value without format change (str -> str) current_format = sl._get_packing_format(0) @@ -4832,10 +4832,10 @@ def test_shared_memory_ShareableList_basics(self): self.assertEqual(current_format, sl._get_packing_format(0)) # Verify attributes are readable. - self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q') + self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q16s') # Exercise len(). - self.assertEqual(len(sl), 7) + self.assertEqual(len(sl), 8) # Exercise index(). with warnings.catch_warnings(): @@ -4847,12 +4847,13 @@ def test_shared_memory_ShareableList_basics(self): # Exercise retrieving individual values. self.assertEqual(sl[0], 'howdy') - self.assertEqual(sl[-2], True) + self.assertEqual(sl[-3], True) + self.assertEqual(sl[-1], '💥 💥') # Exercise iterability. self.assertEqual( tuple(sl), - ('howdy', b'HoWdY', -273.154, 100, None, True, 42) + ('howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥') ) # Exercise modifying individual values. @@ -4860,17 +4861,17 @@ def test_shared_memory_ShareableList_basics(self): self.assertEqual(sl[3], 42) sl[4] = 'some' # Change type at a given position. self.assertEqual(sl[4], 'some') - self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q') + self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q16s') with self.assertRaisesRegex(ValueError, "exceeds available storage"): sl[4] = 'far too many' self.assertEqual(sl[4], 'some') - sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data - self.assertEqual(sl[0], 'encodés') + sl[0] = 'éncodé' # Exactly 8 bytes of UTF-8 data + self.assertEqual(sl[0], 'éncodé') self.assertEqual(sl[1], b'HoWdY') # no spillage with self.assertRaisesRegex(ValueError, "exceeds available storage"): - sl[0] = 'encodées' # Exactly 9 bytes of UTF-8 data + sl[0] = 'éncodés' # Exactly 9 bytes of UTF-8 data self.assertEqual(sl[1], b'HoWdY') with self.assertRaisesRegex(ValueError, "exceeds available storage"): diff --git a/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst b/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst new file mode 100644 index 00000000000000..db8366f6ef0f09 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst @@ -0,0 +1 @@ +Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList.