pythongh-113964: Don't prevent new threads until all non-daemon threa…

…ds exit

Starting in Python 3.12, we started preventing fork() and starting new
threads during interpreter finalization (shutdown). This has led to a
number of regressions and flaky tests. We should not prevent starting
new threads (or fork()) until all non-daemon threads exit and
finalization starts in earnest.

This changes the checks to use
`_PyInterpreterState_GetFinalizing(interp)`, which is set immediately
before terminating non-daemon threads.

Lib/test/test_threading.py

@@ -1340,6 +1340,30 @@ def main():
         rc, out, err = assert_python_ok('-c', script)
         self.assertFalse(err)
 
+    def test_thread_from_thread(self):
+        script = """if True:
+            import threading
+            import time
+
+            def thread2():
+                time.sleep(0.05)
+                print("OK")
+
+            def thread1():
+                time.sleep(0.05)
+                t2 = threading.Thread(target=thread2)
+                t2.start()
+
+            t = threading.Thread(target=thread1)
+            t.start()
+            # do not join() -- the interpreter waits for non-daemon threads to
+            # finish.
+            """
+        rc, out, err = assert_python_ok('-c', script)
+        self.assertEqual(err, b"")
+        self.assertEqual(out, b"OK\n")
+        self.assertEqual(rc, 0)
+
     @skip_unless_reliable_fork
     def test_reinit_tls_after_fork(self):
         # Issue #13817: fork() would deadlock in a multithreaded program with

Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst

@@ -0,0 +1,2 @@
+Starting new threads and process creation through :func:`os.fork` are now
+only prevented once all non-daemon threads exit.

Modules/_posixsubprocess.c

@@ -1031,7 +1031,9 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args,
     Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep);
 
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    if ((preexec_fn != Py_None) && interp->finalizing) {
+    if ((preexec_fn != Py_None) &&
+        _PyInterpreterState_GetFinalizing(interp) != NULL)
+    {
         PyErr_SetString(PyExc_PythonFinalizationError,
                         "preexec_fn not supported at interpreter shutdown");
         return NULL;

Modules/_threadmodule.c

@@ -1395,7 +1395,7 @@ do_start_new_thread(thread_module_state* state,
                         "thread is not supported for isolated subinterpreters");
         return -1;
     }
-    if (interp->finalizing) {
+    if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
         PyErr_SetString(PyExc_PythonFinalizationError,
                         "can't create new thread at interpreter shutdown");
         return -1;

Modules/posixmodule.c

@@ -7842,7 +7842,7 @@ os_fork1_impl(PyObject *module)
     pid_t pid;
 
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    if (interp->finalizing) {
+    if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
         PyErr_SetString(PyExc_PythonFinalizationError,
                         "can't fork at interpreter shutdown");
         return NULL;
@@ -7886,7 +7886,7 @@ os_fork_impl(PyObject *module)
 {
     pid_t pid;
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    if (interp->finalizing) {
+    if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
         PyErr_SetString(PyExc_PythonFinalizationError,
                         "can't fork at interpreter shutdown");
         return NULL;
@@ -8719,7 +8719,7 @@ os_forkpty_impl(PyObject *module)
     pid_t pid;
 
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    if (interp->finalizing) {
+    if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
         PyErr_SetString(PyExc_PythonFinalizationError,
                         "can't fork at interpreter shutdown");
         return NULL;

Objects/unicodeobject.c

@@ -505,7 +505,7 @@ unicode_check_encoding_errors(const char *encoding, const char *errors)
 
     /* Disable checks during Python finalization. For example, it allows to
        call _PyObject_Dump() during finalization for debugging purpose. */
-    if (interp->finalizing) {
+    if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
         return 0;
     }