diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 2c6d6daa01994e..c295ed124ed183 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -314,6 +314,9 @@ extern int _Py_GetTicksPerSecond(long *ticks_per_second); // Export for '_testcapi' shared extension PyAPI_FUNC(int) _Py_IsValidFD(int fd); +// Export for '_remote_debugging' shared extension +PyAPI_FUNC(PyObject*) _Py_strerror(int code); + #ifdef __cplusplus } #endif diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 4f8ebf15d367d2..b93155131c62f5 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -435,9 +435,10 @@ def parse_config_h(fp, vars=None): if vars is None: vars = {} import re - define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") - undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") - quoted_re = re.compile('^"(.*)"$') + name_rx = '(?:[A-Z]|_Py_)[A-Za-z0-9_]+' + define_rx = re.compile(fr"#define ({name_rx}) (.*)\n") + undef_rx = re.compile(fr"/[*] #undef ({name_rx}) [*]/\n") + quoted_re = re.compile(r'"(.*)"') while True: line = fp.readline() @@ -446,7 +447,7 @@ def parse_config_h(fp, vars=None): m = define_rx.match(line) if m: n, v = m.group(1, 2) - if mq := quoted_re.match(v): + if mq := quoted_re.fullmatch(v): v = mq.group(1) try: if n in _ALWAYS_STR: diff --git a/Lib/test/test_free_threading/test_os.py b/Lib/test/test_free_threading/test_os.py new file mode 100644 index 00000000000000..72582da29015e9 --- /dev/null +++ b/Lib/test/test_free_threading/test_os.py @@ -0,0 +1,36 @@ +import errno +import os +import sysconfig +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestOs(unittest.TestCase): + @unittest.skipUnless(sysconfig.get_config_var('_Py_HAVE_STRERROR_R'), + 'need _Py_HAVE_STRERROR_R macro') + def test_strerror(self): + # gh-158893: os.strerror() is implemented with strerror_r() which is + # thread safe. Well, check if it's actually the case. + last_error = max([getattr(errno, name) for name in dir(errno) + if name.startswith('E')]) + test_errors = tuple(range(1, last_error + 1)) + loops = 20 + + def worker(): + for _ in range(loops): + for i in test_errors: + os.strerror(i) + + run_concurrently( + worker_func=worker, nthreads=NTHREADS + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 9bb3e326ff3e95..5263799a4c7088 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -29,7 +29,7 @@ get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, _expand_vars, _get_preferred_schemes, - is_python_build, _PROJECT_BASE) + is_python_build, _PROJECT_BASE, parse_config_h) from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name import _imp import _osx_support @@ -760,6 +760,72 @@ def test_sysconfig_config_vars_no_prefix_cache(self): self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) self.assertEqual(config_vars['platbase'], sys.exec_prefix) + def test_parse_config_h(self): + config = textwrap.dedent(''' + #ifndef Py_PYCONFIG_H + #define Py_PYCONFIG_H + + /* C comment */ + + #define ALIGNOF_LONG 8 + #define HAVE_ACCEPT 1 + #define _Py_HAVE_COSPI 1 + #define INVALID_NUMBER abc + #define ALT_SOABI "cpython-316t-x86_64-linux-gnu" + + // Undef macros must be written as "/* #undef NAME */": + // name must be valid and there is not value. + /* #undef ANDROID_API_LEVEL */ + #undef IGNORE_UNDEF + /* #undef IGNORE_VALUE 1 */ + + # _ALWAYS_STR: don't convert values to an integer, + # but quotes are removed + #define IPHONEOS_DEPLOYMENT_TARGET "13.0" + #define MACOSX_DEPLOYMENT_TARGET 10 + + // Spaces are tolerated after the name, not before + #define SPACES_AFTER 1 + #define IGNORED_SPACES_BEFORE 1 + + // Ignore macro without value + #define IGNORE_NO_VALUE + + // Ignore macros with an invalid name + #define _PRIVATE_IGNORED 1 + #define aLOWER_IGNORED 1 + #define 123IGNORED 1 + #define INVALID-NAME 1 + #define INVALID#NAME 1 + #define NONASCII_NAME_é 1 + + // Ignore single letter names + #define A 1 + /* #undef A */ + + #endif /*Py_PYCONFIG_H*/ + ''') + + filename = TESTFN + self.addCleanup(unlink, filename) + with open(filename, "w", encoding="utf-8") as fp: + fp.write(config) + vars = {} + with open(filename, encoding="utf-8") as fp: + parse_config_h(fp, vars) + expected = { + 'ALIGNOF_LONG': 8, + 'HAVE_ACCEPT': 1, + '_Py_HAVE_COSPI': 1, + 'INVALID_NUMBER': 'abc', + 'ALT_SOABI': 'cpython-316t-x86_64-linux-gnu', + 'ANDROID_API_LEVEL': 0, + 'IPHONEOS_DEPLOYMENT_TARGET': '13.0', + 'MACOSX_DEPLOYMENT_TARGET': '10', # str, not int + 'SPACES_AFTER': 1, + } + self.assertEqual(vars, expected) + class MakefileTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-10-06-20-03-01.gh-issue-158893.B6A53q.rst b/Misc/NEWS.d/next/Library/2026-10-06-20-03-01.gh-issue-158893.B6A53q.rst new file mode 100644 index 00000000000000..9345b46da0f476 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-06-20-03-01.gh-issue-158893.B6A53q.rst @@ -0,0 +1,2 @@ +Make :func:`os.strerror` thread-safe: use the reentrant ``strerror_r()`` +function if available. Patch by Victor Stinner. diff --git a/Modules/_remote_debugging/subprocess.c b/Modules/_remote_debugging/subprocess.c index 056cc6773a88b4..2c6248f9bfeb66 100644 --- a/Modules/_remote_debugging/subprocess.c +++ b/Modules/_remote_debugging/subprocess.c @@ -6,6 +6,7 @@ ******************************************************************************/ #include "_remote_debugging.h" +#include "pycore_fileutils.h" // _Py_strerror() #ifndef MS_WINDOWS #include @@ -229,9 +230,13 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) if (entry == NULL) { if (errno != 0) { int err = errno; - _set_debug_oserror_from_errno_with_filename(err, "/proc", - "Failed to read process directory '/proc': %s", - strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno_with_filename(err, "/proc", + "Failed to read process directory '/proc': %S", + message); + Py_DECREF(message); + } goto done; } break; @@ -259,9 +264,13 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) if (closedir(proc_dir) != 0) { int err = errno; proc_dir = NULL; - _set_debug_oserror_from_errno_with_filename(err, "/proc", - "Failed to close process directory '/proc': %s", - strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno_with_filename(err, "/proc", + "Failed to close process directory '/proc': %S", + message); + Py_DECREF(message); + } goto done; } proc_dir = NULL; diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 04c70cc96d6bd1..f77767c8328521 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -6,6 +6,7 @@ ******************************************************************************/ #include "_remote_debugging.h" +#include "pycore_fileutils.h" // _Py_strerror() #ifndef MS_WINDOWS #include @@ -680,9 +681,13 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st int err = errno; closedir(dir); _Py_RemoteDebug_InitThreadsState(unwinder, st); - _set_debug_oserror_from_errno_with_filename(err, task_path, - "Failed to read process task directory '%s': %s", - task_path, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno_with_filename(err, task_path, + "Failed to read process task directory '%s': %S", + task_path, message); + Py_DECREF(message); + } return -1; } break; @@ -713,9 +718,13 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st if (closedir(dir) != 0) { int err = errno; _Py_RemoteDebug_InitThreadsState(unwinder, st); - _set_debug_oserror_from_errno_with_filename(err, task_path, - "Failed to close process task directory '%s': %s", - task_path, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno_with_filename(err, task_path, + "Failed to close process task directory '%s': %S", + task_path, message); + Py_DECREF(message); + } return -1; } st->tids = unwinder->thread_tids; @@ -780,8 +789,12 @@ _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_T } if (ret < 0) { detach_threads(st, seized); - _set_debug_oserror_from_errno(err, - "Failed to seize thread %d: %s", tid, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno(err, + "Failed to seize thread %d: %S", tid, message); + Py_DECREF(message); + } _Py_RemoteDebug_InitThreadsState(unwinder, st); return -1; } @@ -791,8 +804,12 @@ _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_T err = errno; if (err != ESRCH) { detach_threads(st, seized); - _set_debug_oserror_from_errno(err, - "Failed to interrupt thread %d: %s", tid, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno(err, + "Failed to interrupt thread %d: %S", tid, message); + Py_DECREF(message); + } _Py_RemoteDebug_InitThreadsState(unwinder, st); return -1; } @@ -803,8 +820,12 @@ _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_T err = errno; if (err != ECHILD && err != ESRCH) { detach_threads(st, seized); - _set_debug_oserror_from_errno(err, - "waitpid failed for thread %d: %s", tid, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno(err, + "waitpid failed for thread %d: %S", tid, message); + Py_DECREF(message); + } _Py_RemoteDebug_InitThreadsState(unwinder, st); return -1; } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c81ec259d2bee9..842b100507a826 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13831,13 +13831,7 @@ static PyObject * os_strerror_impl(PyObject *module, int code) /*[clinic end generated code: output=baebf09fa02a78f2 input=75a8673d97915a91]*/ { - char *message = strerror(code); - if (message == NULL) { - PyErr_SetString(PyExc_ValueError, - "strerror() argument out of range"); - return NULL; - } - return PyUnicode_DecodeLocale(message, "surrogateescape"); + return _Py_strerror(code); } diff --git a/Python/errors.c b/Python/errors.c index 48b03e5fd714b1..9de35382ffb6df 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -829,8 +829,7 @@ PyErr_SetFromErrnoWithFilenameObjects(PyObject *exc, PyObject *filenameObject, P #ifndef MS_WINDOWS if (i != 0) { - const char *s = strerror(i); - message = PyUnicode_DecodeLocale(s, "surrogateescape"); + message = _Py_strerror(i); } else { /* Sometimes errno didn't get set */ diff --git a/Python/fileutils.c b/Python/fileutils.c index 7425a528f75570..324d672fcbfeb2 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -3172,3 +3172,97 @@ _Py_IsValidFD(int fd) return (fstat(fd, &st) == 0); #endif } + + +// Call strerror_r(code) if available, or use strerror() otherwise. Decode the +// result from the locale encoding using surrogateescape error handler. +// +// On success, return a Unicode string. On error, set an exception and return +// NULL. +PyObject* +_Py_strerror(int code) +/*[clinic end generated code: output=baebf09fa02a78f2 input=75a8673d97915a91]*/ +{ + const char *errors = "surrogateescape"; + +#ifdef _Py_HAVE_STRERROR_R + // Check which strerror_r() API is used +# if defined(__GLIBC__) && !((_POSIX_C_SOURCE >= 200112L) && !defined(_GNU_SOURCE)) +# define Py_STRERROR_R_GNU +# elif defined(__ANDROID__) && defined(_GNU_SOURCE) +# define Py_STRERROR_R_GNU +# endif +#endif + +#ifdef Py_STRERROR_R_GNU + // Implementation for the GNU flavor of strerror_r() + + // On Linux, the longest translated strerror() message is 86 bytes + // (including the NUL byte). + char buffer[100]; + char *message = strerror_r(code, buffer, Py_ARRAY_LENGTH(buffer)); + // The strerror_r() GNU flavor doesn't provide a way to check if the error + // message was truncated or not. + // + // When the buffer is used, a trailing NUL byte is always written. + assert(message != buffer || memchr(buffer, 0, Py_ARRAY_LENGTH(buffer)) != NULL); + return PyUnicode_DecodeLocale(message, errors); + +#elif defined(_Py_HAVE_STRERROR_R) + // Implementation for the XSI-compliant flavor of strerror_r() + + // On Linux and FreeBSD, the longest translated strerror() message is 86 + // bytes (including the NUL byte). + char small_buffer[100]; + size_t buflen = Py_ARRAY_LENGTH(small_buffer); + char *buffer = NULL; +#ifndef NDEBUG + // Make sure that strerror_r() writes a trailing null byte + small_buffer[buflen - 1] = '#'; +#endif + int len = strerror_r(code, small_buffer, buflen); + if (len == ERANGE) { + while (len == ERANGE) { + if (buflen > (size_t)PY_SSIZE_T_MAX / 2) { + PyMem_Free(buffer); + PyErr_NoMemory(); + return NULL; + } + buflen = buflen * 2; + + char *new_buffer = PyMem_Realloc(buffer, buflen); + if (new_buffer == NULL) { + PyMem_Free(buffer); + PyErr_NoMemory(); + return NULL; + } + buffer = new_buffer; +#ifndef NDEBUG + buffer[buflen - 1] = '#'; +#endif + len = strerror_r(code, buffer, buflen); + } + } + else { + buffer = small_buffer; + } + + // strerror_r() always writes a trailing NUL byte + assert(memchr(buffer, 0, buflen) != NULL); + PyObject *result = PyUnicode_DecodeLocale(buffer, errors); + if (buffer != small_buffer) { + PyMem_Free(buffer); + } + return result; + +#else + // strerror() implementation (usually not thread-safe) + char *message = strerror(code); + if (message == NULL) { + PyErr_SetString(PyExc_ValueError, + "strerror() argument out of range"); + return NULL; + } + return PyUnicode_DecodeLocale(message, errors); +#endif +} diff --git a/configure b/configure index b355c265ecd1f8..d23048a9496858 100755 --- a/configure +++ b/configure @@ -3598,6 +3598,8 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu + + if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then # If we're building out-of-tree, we need to make sure the following # resources get picked up before their $srcdir counterparts. @@ -21168,6 +21170,50 @@ then : fi + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for strerror_r" >&5 +printf %s "checking for strerror_r... " >&6; } +if test ${ac_cv_func_strerror_r+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +void *x=strerror_r + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_strerror_r=yes +else case e in #( + e) ac_cv_func_strerror_r=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_strerror_r" >&5 +printf "%s\n" "$ac_cv_func_strerror_r" >&6; } + if test "x$ac_cv_func_strerror_r" = xyes +then : + +printf "%s\n" "#define _Py_HAVE_STRERROR_R 1" >>confdefs.h + +fi + + + + + + # os.statx uses Linux's statx function. AIX also has a function named statx, # but it's unrelated. Check only on Linux (including Android). case $ac_sys_system in #( diff --git a/configure.ac b/configure.ac index 8a0b921bd750a1..ade9004e15b965 100644 --- a/configure.ac +++ b/configure.ac @@ -73,6 +73,12 @@ AC_DEFUN([PY_CHECK_FUNC], AS_VAR_POPDEF([py_define]) ]) +dnl PY_CHECK_FUNC_PRIVATE(FUNCTION, [INCLUDES], [AC_DEFINE-VAR]) +dnl Similar to PY_CHECK_FUNC but define macro _Py_HAVE_xxx instead of HAVE_xxx +AC_DEFUN([PY_CHECK_FUNC_PRIVATE], +[ PY_CHECK_FUNC([$1], [$2], m4_ifblank([$3], [[_Py_HAVE_]m4_toupper($1)], [$3])) +]) + dnl PY_CHECK_LIB(LIBRARY, FUNCTION, [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND], [OTHER-LIBRARIES]) dnl Like AC_CHECK_LIB() but does not modify LIBS AC_DEFUN([PY_CHECK_LIB], @@ -5516,6 +5522,9 @@ AC_CHECK_FUNCS([ \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) +PY_CHECK_FUNC_PRIVATE([strerror_r], [@%:@include ]) + + # os.statx uses Linux's statx function. AIX also has a function named statx, # but it's unrelated. Check only on Linux (including Android). AS_CASE([$ac_sys_system], diff --git a/pyconfig.h.in b/pyconfig.h.in index 0c21755680cf88..facf6435648e2d 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -2094,6 +2094,9 @@ /* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ #undef _Py_HAVE_PR_SET_VMA_ANON_NAME +/* Define if you have the 'strerror_r' function. */ +#undef _Py_HAVE_STRERROR_R + /* Thread stack size set by the linker (in bytes). */ #undef _Py_LINKER_THREAD_STACK_SIZE