From 3932e1db5353bbcf3e3c1133cc9d2cde654cb645 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 19 Nov 2024 17:44:53 +0300 Subject: [PATCH 1/8] gh-126980: Fix `bytearray.__buffer__` crash on `PyBUF_{READ,WRITE}` (#126981) Co-authored-by: Victor Stinner --- Lib/test/test_buffer.py | 8 ++++++++ .../2024-11-18-23-18-17.gh-issue-126980.r8QHdi.rst | 3 +++ Objects/bytearrayobject.c | 5 +++-- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-23-18-17.gh-issue-126980.r8QHdi.rst diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 332e49ce9f1..61921e93e85 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4439,6 +4439,14 @@ def test_issue_7385(self): x = ndarray([1,2,3], shape=[3], flags=ND_GETBUF_FAIL) self.assertRaises(BufferError, memoryview, x) + def test_bytearray_release_buffer_read_flag(self): + # See https://github.com/python/cpython/issues/126980 + obj = bytearray(b'abc') + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.READ) + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.WRITE) + @support.cpython_only def test_pybuffer_size_from_format(self): # basic tests diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-23-18-17.gh-issue-126980.r8QHdi.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-23-18-17.gh-issue-126980.r8QHdi.rst new file mode 100644 index 00000000000..84484e7c300 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-23-18-17.gh-issue-126980.r8QHdi.rst @@ -0,0 +1,3 @@ +Fix :meth:`~object.__buffer__` of :class:`bytearray` crashing when +:attr:`~inspect.BufferFlags.READ` or :attr:`~inspect.BufferFlags.WRITE` are +passed as flags. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 5a52b2f702a..871f99b6f88 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -52,8 +52,9 @@ bytearray_getbuffer(PyObject *self, Py_buffer *view, int flags) } void *ptr = (void *) PyByteArray_AS_STRING(obj); - /* cannot fail if view != NULL and readonly == 0 */ - (void)PyBuffer_FillInfo(view, (PyObject*)obj, ptr, Py_SIZE(obj), 0, flags); + if (PyBuffer_FillInfo(view, (PyObject*)obj, ptr, Py_SIZE(obj), 0, flags) < 0) { + return -1; + } obj->ob_exports++; return 0; } From 5fcc3a4cee76ab9bef4c2a76b0f5591cf576e2bf Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 19 Nov 2024 17:28:34 +0200 Subject: [PATCH 2/8] Update docs 'make serve' to suggest 'make htmllive' (#126969) --- Doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/Makefile b/Doc/Makefile index a090ee5ba92..22e43ee3e54 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -294,7 +294,7 @@ check: _ensure-pre-commit .PHONY: serve serve: - @echo "The serve target was removed, use htmlview instead (see bpo-36329)" + @echo "The serve target was removed, use htmllive instead (see gh-80510)" # Targets for daily automated doc build # By default, Sphinx only rebuilds pages where the page content has changed. From c5c9286804e38c95fe717f22ce1bf2f18eee5b17 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 19 Nov 2024 15:42:19 +0000 Subject: [PATCH 3/8] gh-118201: Simplify conv_confname (#126089) --- Lib/test/support/os_helper.py | 3 +- Lib/test/test_os.py | 2 +- Lib/test/test_posix.py | 34 +++- ...-10-28-19-49-18.gh-issue-118201.v41XXh.rst | 2 + Modules/clinic/posixmodule.c.h | 10 +- Modules/posixmodule.c | 151 ++++++------------ 6 files changed, 89 insertions(+), 113 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-10-28-19-49-18.gh-issue-118201.v41XXh.rst diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 891405943b7..8071c248b9b 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -632,8 +632,7 @@ def fd_count(): if hasattr(os, 'sysconf'): try: MAXFD = os.sysconf("SC_OPEN_MAX") - except (OSError, ValueError): - # gh-118201: ValueError is raised intermittently on iOS + except OSError: pass old_modes = None diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 919ed92ddb4..467da78f9e1 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2447,8 +2447,8 @@ def test_fchown(self): support.is_emscripten or support.is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) - @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_fpathconf(self): + self.assertIn("PC_NAME_MAX", os.pathconf_names) self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") self.check_bool(os.pathconf, "PC_NAME_MAX") diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index ef9d617f66f..c9cbe1541e7 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -568,10 +568,38 @@ def test_dup(self): @unittest.skipUnless(hasattr(posix, 'confstr'), 'test needs posix.confstr()') - @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_confstr(self): - self.assertRaises(ValueError, posix.confstr, "CS_garbage") - self.assertEqual(len(posix.confstr("CS_PATH")) > 0, True) + with self.assertRaisesRegex( + ValueError, "unrecognized configuration name" + ): + posix.confstr("CS_garbage") + + with self.assertRaisesRegex( + TypeError, "configuration names must be strings or integers" + ): + posix.confstr(1.23) + + path = posix.confstr("CS_PATH") + self.assertGreater(len(path), 0) + self.assertEqual(posix.confstr(posix.confstr_names["CS_PATH"]), path) + + @unittest.skipUnless(hasattr(posix, 'sysconf'), + 'test needs posix.sysconf()') + def test_sysconf(self): + with self.assertRaisesRegex( + ValueError, "unrecognized configuration name" + ): + posix.sysconf("SC_garbage") + + with self.assertRaisesRegex( + TypeError, "configuration names must be strings or integers" + ): + posix.sysconf(1.23) + + arg_max = posix.sysconf("SC_ARG_MAX") + self.assertGreater(arg_max, 0) + self.assertEqual( + posix.sysconf(posix.sysconf_names["SC_ARG_MAX"]), arg_max) @unittest.skipUnless(hasattr(posix, 'dup2'), 'test needs posix.dup2()') diff --git a/Misc/NEWS.d/next/Library/2024-10-28-19-49-18.gh-issue-118201.v41XXh.rst b/Misc/NEWS.d/next/Library/2024-10-28-19-49-18.gh-issue-118201.v41XXh.rst new file mode 100644 index 00000000000..bed4b3b5956 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-28-19-49-18.gh-issue-118201.v41XXh.rst @@ -0,0 +1,2 @@ +Fixed intermittent failures of :any:`os.confstr`, :any:`os.pathconf` and +:any:`os.sysconf` on iOS and Android. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index dce0ea100ec..cd0c4faeac8 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10128,7 +10128,7 @@ os_fpathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (fd < 0) { goto exit; } - if (!conv_path_confname(args[1], &name)) { + if (!conv_confname(module, args[1], &name, "pathconf_names")) { goto exit; } _return_value = os_fpathconf_impl(module, fd, name); @@ -10203,7 +10203,7 @@ os_pathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!path_converter(args[0], &path)) { goto exit; } - if (!conv_path_confname(args[1], &name)) { + if (!conv_confname(module, args[1], &name, "pathconf_names")) { goto exit; } _return_value = os_pathconf_impl(module, &path, name); @@ -10241,7 +10241,7 @@ os_confstr(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int name; - if (!conv_confstr_confname(arg, &name)) { + if (!conv_confname(module, arg, &name, "confstr_names")) { goto exit; } return_value = os_confstr_impl(module, name); @@ -10273,7 +10273,7 @@ os_sysconf(PyObject *module, PyObject *arg) int name; long _return_value; - if (!conv_sysconf_confname(arg, &name)) { + if (!conv_confname(module, arg, &name, "sysconf_names")) { goto exit; } _return_value = os_sysconf_impl(module, name); @@ -13114,4 +13114,4 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=5358a13b4ce6148b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7ee14f5e880092f5 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index da7399de86f..6eb7054b566 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3113,18 +3113,22 @@ class Py_off_t_return_converter(long_return_converter): type = 'Py_off_t' conversion_fn = 'PyLong_FromPy_off_t' -class path_confname_converter(CConverter): +class confname_converter(CConverter): type="int" - converter="conv_path_confname" + converter="conv_confname" -class confstr_confname_converter(path_confname_converter): - converter='conv_confstr_confname' + def converter_init(self, *, table): + self.table = table -class sysconf_confname_converter(path_confname_converter): - converter="conv_sysconf_confname" + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" + if (!{converter}(module, {argname}, &{paramname}, "{table}")) {{{{ + goto exit; + }}}} + """, argname=argname, converter=self.converter, table=self.table) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=1860d32584c2a539]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=8189d5ae78244626]*/ /*[clinic input] @@ -13547,46 +13551,38 @@ struct constdef { }; static int -conv_confname(PyObject *arg, int *valuep, struct constdef *table, - size_t tablesize) +conv_confname(PyObject *module, PyObject *arg, int *valuep, const char *tablename) { - if (PyLong_Check(arg)) { + if (PyUnicode_Check(arg)) { + PyObject *table = PyObject_GetAttrString(module, tablename); + if (table == NULL) { + return 0; + } + + arg = PyObject_GetItem(table, arg); + Py_DECREF(table); + if (arg == NULL) { + PyErr_SetString( + PyExc_ValueError, "unrecognized configuration name"); + return 0; + } + } else { + Py_INCREF(arg); // Match the Py_DECREF below. + } + + int success = 0; + if (!PyLong_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "configuration names must be strings or integers"); + } else { int value = PyLong_AsInt(arg); - if (value == -1 && PyErr_Occurred()) - return 0; - *valuep = value; - return 1; - } - else { - /* look up the value in the table using a binary search */ - size_t lo = 0; - size_t mid; - size_t hi = tablesize; - int cmp; - const char *confname; - if (!PyUnicode_Check(arg)) { - PyErr_SetString(PyExc_TypeError, - "configuration names must be strings or integers"); - return 0; + if (!(value == -1 && PyErr_Occurred())) { + *valuep = value; + success = 1; } - confname = PyUnicode_AsUTF8(arg); - if (confname == NULL) - return 0; - while (lo < hi) { - mid = (lo + hi) / 2; - cmp = strcmp(confname, table[mid].name); - if (cmp < 0) - hi = mid; - else if (cmp > 0) - lo = mid + 1; - else { - *valuep = table[mid].value; - return 1; - } - } - PyErr_SetString(PyExc_ValueError, "unrecognized configuration name"); - return 0; } + Py_DECREF(arg); + return success; } @@ -13677,14 +13673,6 @@ static struct constdef posix_constants_pathconf[] = { {"PC_TIMESTAMP_RESOLUTION", _PC_TIMESTAMP_RESOLUTION}, #endif }; - -static int -conv_path_confname(PyObject *arg, int *valuep) -{ - return conv_confname(arg, valuep, posix_constants_pathconf, - sizeof(posix_constants_pathconf) - / sizeof(struct constdef)); -} #endif @@ -13693,7 +13681,7 @@ conv_path_confname(PyObject *arg, int *valuep) os.fpathconf -> long fd: fildes - name: path_confname + name: confname(table="pathconf_names") / Return the configuration limit name for the file descriptor fd. @@ -13703,7 +13691,7 @@ If there is no limit, return -1. static long os_fpathconf_impl(PyObject *module, int fd, int name) -/*[clinic end generated code: output=d5b7042425fc3e21 input=5b8d2471cfaae186]*/ +/*[clinic end generated code: output=d5b7042425fc3e21 input=023d44589c9ed6aa]*/ { long limit; @@ -13721,7 +13709,7 @@ os_fpathconf_impl(PyObject *module, int fd, int name) /*[clinic input] os.pathconf -> long path: path_t(allow_fd='PATH_HAVE_FPATHCONF') - name: path_confname + name: confname(table="pathconf_names") Return the configuration limit name for the file or directory path. @@ -13732,7 +13720,7 @@ On some platforms, path may also be specified as an open file descriptor. static long os_pathconf_impl(PyObject *module, path_t *path, int name) -/*[clinic end generated code: output=5bedee35b293a089 input=bc3e2a985af27e5e]*/ +/*[clinic end generated code: output=5bedee35b293a089 input=6f6072f57b10c787]*/ { long limit; @@ -13909,19 +13897,11 @@ static struct constdef posix_constants_confstr[] = { #endif }; -static int -conv_confstr_confname(PyObject *arg, int *valuep) -{ - return conv_confname(arg, valuep, posix_constants_confstr, - sizeof(posix_constants_confstr) - / sizeof(struct constdef)); -} - /*[clinic input] os.confstr - name: confstr_confname + name: confname(table="confstr_names") / Return a string-valued system configuration variable. @@ -13929,7 +13909,7 @@ Return a string-valued system configuration variable. static PyObject * os_confstr_impl(PyObject *module, int name) -/*[clinic end generated code: output=bfb0b1b1e49b9383 input=18fb4d0567242e65]*/ +/*[clinic end generated code: output=bfb0b1b1e49b9383 input=4c6ffca2837ec959]*/ { PyObject *result = NULL; char buffer[255]; @@ -14466,18 +14446,10 @@ static struct constdef posix_constants_sysconf[] = { #endif }; -static int -conv_sysconf_confname(PyObject *arg, int *valuep) -{ - return conv_confname(arg, valuep, posix_constants_sysconf, - sizeof(posix_constants_sysconf) - / sizeof(struct constdef)); -} - /*[clinic input] os.sysconf -> long - name: sysconf_confname + name: confname(table="sysconf_names") / Return an integer-valued system configuration variable. @@ -14485,7 +14457,7 @@ Return an integer-valued system configuration variable. static long os_sysconf_impl(PyObject *module, int name) -/*[clinic end generated code: output=3662f945fc0cc756 input=279e3430a33f29e4]*/ +/*[clinic end generated code: output=3662f945fc0cc756 input=930b8f23b5d15086]*/ { long value; @@ -14498,40 +14470,15 @@ os_sysconf_impl(PyObject *module, int name) #endif /* HAVE_SYSCONF */ -/* This code is used to ensure that the tables of configuration value names - * are in sorted order as required by conv_confname(), and also to build - * the exported dictionaries that are used to publish information about the - * names available on the host platform. - * - * Sorting the table at runtime ensures that the table is properly ordered - * when used, even for platforms we're not able to test on. It also makes - * it easier to add additional entries to the tables. - */ - -static int -cmp_constdefs(const void *v1, const void *v2) -{ - const struct constdef *c1 = - (const struct constdef *) v1; - const struct constdef *c2 = - (const struct constdef *) v2; - - return strcmp(c1->name, c2->name); -} - static int setup_confname_table(struct constdef *table, size_t tablesize, const char *tablename, PyObject *module) { - PyObject *d = NULL; - size_t i; - - qsort(table, tablesize, sizeof(struct constdef), cmp_constdefs); - d = PyDict_New(); + PyObject *d = PyDict_New(); if (d == NULL) return -1; - for (i=0; i < tablesize; ++i) { + for (size_t i=0; i < tablesize; ++i) { PyObject *o = PyLong_FromLong(table[i].value); if (o == NULL || PyDict_SetItemString(d, table[i].name, o) == -1) { Py_XDECREF(o); From 29cbcbd73bbfd8c953c0b213fb33682c289934ff Mon Sep 17 00:00:00 2001 From: Justin Applegate <70449145+Legoclones@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:00:35 -0500 Subject: [PATCH 4/8] gh-126991: Fix reference leak in loading pickle's opcode BUILD (GH-126990) If PyObject_SetItem() fails in the `load_build()` function of _pickle.c, no DECREF for the `dict` variable. --- Modules/_pickle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 5837cd41a40..2696f380461 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -6730,6 +6730,7 @@ load_build(PickleState *st, UnpicklerObject *self) } if (PyObject_SetItem(dict, d_key, d_value) < 0) { Py_DECREF(d_key); + Py_DECREF(dict); goto error; } Py_DECREF(d_key); From 88dc84bcf9fef32afa9af0ab41fa467c9733483f Mon Sep 17 00:00:00 2001 From: CoderTCY <73740771+CoderTCY@users.noreply.github.com> Date: Wed, 20 Nov 2024 01:19:33 +0800 Subject: [PATCH 5/8] gh-125729: Makes the installation of the turtle module dependent on the Tcl/Tk install option (GH-126176) --- .../Windows/2024-10-31-09-46-53.gh-issue-125729.KdKVLa.rst | 1 + Tools/msi/bundle/Default.wxl | 4 ++-- Tools/msi/lib/lib.wixproj | 3 ++- Tools/msi/tcltk/tcltk.wixproj | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-10-31-09-46-53.gh-issue-125729.KdKVLa.rst diff --git a/Misc/NEWS.d/next/Windows/2024-10-31-09-46-53.gh-issue-125729.KdKVLa.rst b/Misc/NEWS.d/next/Windows/2024-10-31-09-46-53.gh-issue-125729.KdKVLa.rst new file mode 100644 index 00000000000..fbf4ab1cd1a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-10-31-09-46-53.gh-issue-125729.KdKVLa.rst @@ -0,0 +1 @@ +Makes the presence of the :mod:`turtle` module dependent on the Tcl/Tk installer option. Previously, the module was always installed but would be unusable without Tcl/Tk. diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 49f681d3e11..7208d83ddae 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -70,8 +70,8 @@ Select Customize to review current options. Installs the Python documentation files. &pip Installs pip, which can download and install other Python packages. - tcl/tk and &IDLE - Installs tkinter and the IDLE development environment. + Tcl/Tk, turtle and &IDLE + Installs tkinter, turtle and the IDLE development environment. Python &test suite Installs the standard library test suite. py &launcher diff --git a/Tools/msi/lib/lib.wixproj b/Tools/msi/lib/lib.wixproj index 26311ea3272..02078e503d7 100644 --- a/Tools/msi/lib/lib.wixproj +++ b/Tools/msi/lib/lib.wixproj @@ -19,6 +19,7 @@ @@ -32,4 +33,4 @@ - \ No newline at end of file + diff --git a/Tools/msi/tcltk/tcltk.wixproj b/Tools/msi/tcltk/tcltk.wixproj index 218f3d15ec8..c8b7ab77c4d 100644 --- a/Tools/msi/tcltk/tcltk.wixproj +++ b/Tools/msi/tcltk/tcltk.wixproj @@ -28,7 +28,7 @@ tcltk_lib - $(PySourcePath) !(bindpath.src) @@ -39,4 +39,4 @@ - \ No newline at end of file + From 8da9920a80c60fb3fc326c623e0f217c84011c1d Mon Sep 17 00:00:00 2001 From: Beomsoo Kim Date: Wed, 20 Nov 2024 04:40:52 +0900 Subject: [PATCH 6/8] gh-126947: Typechecking for _pydatetime.timedelta.__new__ arguments (#126949) Co-authored-by: sobolevn Co-authored-by: Peter Bierma --- Lib/_pydatetime.py | 14 +++++++++++++- Lib/test/datetimetester.py | 10 ++++++++++ .../2024-11-18-19-03-46.gh-issue-126947.NiDYUe.rst | 2 ++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-18-19-03-46.gh-issue-126947.NiDYUe.rst diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index 78e03e32896..ed01670cfec 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -651,7 +651,19 @@ def __new__(cls, days=0, seconds=0, microseconds=0, # guide the C implementation; it's way more convoluted than speed- # ignoring auto-overflow-to-long idiomatic Python could be. - # XXX Check that all inputs are ints or floats. + for name, value in ( + ("days", days), + ("seconds", seconds), + ("microseconds", microseconds), + ("milliseconds", milliseconds), + ("minutes", minutes), + ("hours", hours), + ("weeks", weeks) + ): + if not isinstance(value, (int, float)): + raise TypeError( + f"unsupported type for timedelta {name} component: {type(value).__name__}" + ) # Final values, all integer. # s and us fit in 32-bit signed ints; d isn't bounded. diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index dbe25ef57de..25a3015c4e1 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -510,6 +510,7 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): def test_constructor(self): eq = self.assertEqual + ra = self.assertRaises td = timedelta # Check keyword args to constructor @@ -533,6 +534,15 @@ def test_constructor(self): eq(td(seconds=0.001), td(milliseconds=1)) eq(td(milliseconds=0.001), td(microseconds=1)) + # Check type of args to constructor + ra(TypeError, lambda: td(weeks='1')) + ra(TypeError, lambda: td(days='1')) + ra(TypeError, lambda: td(hours='1')) + ra(TypeError, lambda: td(minutes='1')) + ra(TypeError, lambda: td(seconds='1')) + ra(TypeError, lambda: td(milliseconds='1')) + ra(TypeError, lambda: td(microseconds='1')) + def test_computations(self): eq = self.assertEqual td = timedelta diff --git a/Misc/NEWS.d/next/Library/2024-11-18-19-03-46.gh-issue-126947.NiDYUe.rst b/Misc/NEWS.d/next/Library/2024-11-18-19-03-46.gh-issue-126947.NiDYUe.rst new file mode 100644 index 00000000000..29ba4f21454 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-18-19-03-46.gh-issue-126947.NiDYUe.rst @@ -0,0 +1,2 @@ +Raise :exc:`TypeError` in :meth:`!_pydatetime.timedelta.__new__` if the passed arguments are not :class:`int` or :class:`float`, so that the Python +implementation is in line with the C implementation. From 824afbf548e7128ca57c6faf45cfd6b066a6ee45 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 19 Nov 2024 22:41:59 +0300 Subject: [PATCH 7/8] gh-109413: Enable mypy's `disallow_any_generics` setting when checking `libregrtest` (#127033) --- Lib/test/libregrtest/mypy.ini | 1 - Lib/test/libregrtest/results.py | 6 +++++- Lib/test/libregrtest/runtests.py | 2 +- Lib/test/libregrtest/worker.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/mypy.ini b/Lib/test/libregrtest/mypy.ini index 905341cc04b..3fa9afcb7a4 100644 --- a/Lib/test/libregrtest/mypy.ini +++ b/Lib/test/libregrtest/mypy.ini @@ -15,7 +15,6 @@ strict = True # Various stricter settings that we can't yet enable # Try to enable these in the following order: -disallow_any_generics = False disallow_incomplete_defs = False disallow_untyped_calls = False disallow_untyped_defs = False diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 4f3e84282dc..9eda926966d 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,5 +1,6 @@ import sys import trace +from typing import TYPE_CHECKING from .runtests import RunTests from .result import State, TestResult, TestStats, Location @@ -7,6 +8,9 @@ StrPath, TestName, TestTuple, TestList, FilterDict, printlist, count, format_duration) +if TYPE_CHECKING: + from xml.etree.ElementTree import Element + # Python uses exit code 1 when an exception is not caught # argparse.ArgumentParser.error() uses exit code 2 @@ -34,7 +38,7 @@ def __init__(self) -> None: self.test_times: list[tuple[float, TestName]] = [] self.stats = TestStats() # used by --junit-xml - self.testsuite_xml: list = [] + self.testsuite_xml: list['Element'] = [] # used by -T with -j self.covered_lines: set[Location] = set() diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index cd1ce8080a0..130c036a62e 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -28,7 +28,7 @@ class JsonFile: file: int | None file_type: str - def configure_subprocess(self, popen_kwargs: dict) -> None: + def configure_subprocess(self, popen_kwargs: dict[str, Any]) -> None: match self.file_type: case JsonFileType.UNIX_FD: # Unix file descriptor diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index da24760a82c..0c9f5bd6e42 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -20,7 +20,7 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, - tmp_dir: StrPath | None = None) -> subprocess.Popen: + tmp_dir: StrPath | None = None) -> subprocess.Popen[str]: worker_json = runtests.as_json() cmd = runtests.create_python_cmd() From 1c0a104eca189a932e0b44ca9bef46cce3d0b801 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 19 Nov 2024 12:59:19 -0700 Subject: [PATCH 8/8] gh-126914: Store the Preallocated Thread State's Pointer in a PyInterpreterState Field (gh-126989) This approach eliminates the originally reported race. It also gets rid of the deadlock reported in gh-96071, so we can remove the workaround added then. --- Include/internal/pycore_interp.h | 4 +- Include/internal/pycore_runtime_init.h | 3 + Lib/test/test_interpreters/test_stress.py | 30 +++++++ Python/pystate.c | 97 +++++++++++------------ 4 files changed, 84 insertions(+), 50 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 824b865eda6..5e4bcbf835a 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -130,6 +130,7 @@ struct _is { uint64_t next_unique_id; /* The linked list of threads, newest first. */ PyThreadState *head; + _PyThreadStateImpl *preallocated; /* The thread currently executing in the __main__ module, if any. */ PyThreadState *main; /* Used in Modules/_threadmodule.c. */ @@ -278,9 +279,10 @@ struct _is { struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; + Py_ssize_t _interactive_src_count; + /* the initial PyInterpreterState.threads.head */ _PyThreadStateImpl _initial_thread; - Py_ssize_t _interactive_src_count; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 8a8f47695fb..9f6748945ba 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -118,6 +118,9 @@ extern PyTypeObject _PyExc_MemoryError; { \ .id_refcount = -1, \ ._whence = _PyInterpreterState_WHENCE_NOTSET, \ + .threads = { \ + .preallocated = &(INTERP)._initial_thread, \ + }, \ .imports = IMPORTS_INIT, \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index e400535b2a0..56bfc172199 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -23,6 +23,7 @@ def test_create_many_sequential(self): alive.append(interp) @support.requires_resource('cpu') + @threading_helper.requires_working_threading() def test_create_many_threaded(self): alive = [] def task(): @@ -32,6 +33,35 @@ def task(): with threading_helper.start_threads(threads): pass + @support.requires_resource('cpu') + @threading_helper.requires_working_threading() + def test_many_threads_running_interp_in_other_interp(self): + interp = interpreters.create() + + script = f"""if True: + import _interpreters + _interpreters.run_string({interp.id}, '1') + """ + + def run(): + interp = interpreters.create() + alreadyrunning = (f'{interpreters.InterpreterError}: ' + 'interpreter already running') + success = False + while not success: + try: + interp.exec(script) + except interpreters.ExecutionFailed as exc: + if exc.excinfo.msg != 'interpreter already running': + raise # re-raise + assert exc.excinfo.type.__name__ == 'InterpreterError' + else: + success = True + + threads = (threading.Thread(target=run) for _ in range(200)) + with threading_helper.start_threads(threads): + pass + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Python/pystate.c b/Python/pystate.c index a209a26f16f..01e54fc745d 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -629,6 +629,8 @@ init_interpreter(PyInterpreterState *interp, assert(next != NULL || (interp == runtime->interpreters.main)); interp->next = next; + interp->threads.preallocated = &interp->_initial_thread; + // We would call _PyObject_InitState() at this point // if interp->feature_flags were alredy set. @@ -766,7 +768,6 @@ PyInterpreterState_New(void) return interp; } - static void interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) { @@ -910,6 +911,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. + + // We could clear interp->threads.freelist here + // if it held more than just the initial thread state. } @@ -1386,22 +1390,45 @@ allocate_chunk(int size_in_bytes, _PyStackChunk* previous) return res; } -static _PyThreadStateImpl * -alloc_threadstate(void) +static void +reset_threadstate(_PyThreadStateImpl *tstate) { - return PyMem_RawCalloc(1, sizeof(_PyThreadStateImpl)); + // Set to _PyThreadState_INIT directly? + memcpy(tstate, + &initial._main_interpreter._initial_thread, + sizeof(*tstate)); +} + +static _PyThreadStateImpl * +alloc_threadstate(PyInterpreterState *interp) +{ + _PyThreadStateImpl *tstate; + + // Try the preallocated tstate first. + tstate = _Py_atomic_exchange_ptr(&interp->threads.preallocated, NULL); + + // Fall back to the allocator. + if (tstate == NULL) { + tstate = PyMem_RawCalloc(1, sizeof(_PyThreadStateImpl)); + if (tstate == NULL) { + return NULL; + } + reset_threadstate(tstate); + } + return tstate; } static void free_threadstate(_PyThreadStateImpl *tstate) { + PyInterpreterState *interp = tstate->base.interp; // The initial thread state of the interpreter is allocated // as part of the interpreter state so should not be freed. - if (tstate == &tstate->base.interp->_initial_thread) { - // Restore to _PyThreadState_INIT. - memcpy(tstate, - &initial._main_interpreter._initial_thread, - sizeof(*tstate)); + if (tstate == &interp->_initial_thread) { + // Make it available again. + reset_threadstate(tstate); + assert(interp->threads.preallocated == NULL); + _Py_atomic_store_ptr(&interp->threads.preallocated, tstate); } else { PyMem_RawFree(tstate); @@ -1492,66 +1519,38 @@ add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, static PyThreadState * new_threadstate(PyInterpreterState *interp, int whence) { - _PyThreadStateImpl *tstate; - _PyRuntimeState *runtime = interp->runtime; - // We don't need to allocate a thread state for the main interpreter - // (the common case), but doing it later for the other case revealed a - // reentrancy problem (deadlock). So for now we always allocate before - // taking the interpreters lock. See GH-96071. - _PyThreadStateImpl *new_tstate = alloc_threadstate(); - int used_newtstate; - if (new_tstate == NULL) { + // Allocate the thread state. + _PyThreadStateImpl *tstate = alloc_threadstate(interp); + if (tstate == NULL) { return NULL; } + #ifdef Py_GIL_DISABLED Py_ssize_t qsbr_idx = _Py_qsbr_reserve(interp); if (qsbr_idx < 0) { - PyMem_RawFree(new_tstate); + free_threadstate(tstate); return NULL; } int32_t tlbc_idx = _Py_ReserveTLBCIndex(interp); if (tlbc_idx < 0) { - PyMem_RawFree(new_tstate); + free_threadstate(tstate); return NULL; } #endif /* We serialize concurrent creation to protect global state. */ - HEAD_LOCK(runtime); + HEAD_LOCK(interp->runtime); + // Initialize the new thread state. interp->threads.next_unique_id += 1; uint64_t id = interp->threads.next_unique_id; - - // Allocate the thread state and add it to the interpreter. - PyThreadState *old_head = interp->threads.head; - if (old_head == NULL) { - // It's the interpreter's initial thread state. - used_newtstate = 0; - tstate = &interp->_initial_thread; - } - // XXX Re-use interp->_initial_thread if not in use? - else { - // Every valid interpreter must have at least one thread. - assert(id > 1); - assert(old_head->prev == NULL); - used_newtstate = 1; - tstate = new_tstate; - // Set to _PyThreadState_INIT. - memcpy(tstate, - &initial._main_interpreter._initial_thread, - sizeof(*tstate)); - } - init_threadstate(tstate, interp, id, whence); + + // Add the new thread state to the interpreter. + PyThreadState *old_head = interp->threads.head; add_threadstate(interp, (PyThreadState *)tstate, old_head); - HEAD_UNLOCK(runtime); - if (!used_newtstate) { - // Must be called with lock unlocked to avoid re-entrancy deadlock. - PyMem_RawFree(new_tstate); - } - else { - } + HEAD_UNLOCK(interp->runtime); #ifdef Py_GIL_DISABLED // Must be called with lock unlocked to avoid lock ordering deadlocks.