Merge remote-tracking branch 'upstream/main' into pythongh-111650

.github/CODEOWNERS

@@ -190,3 +190,7 @@ Doc/howto/clinic.rst          @erlend-aasland
 
 # WebAssembly
 /Tools/wasm/                  @brettcannon
+
+# SBOM
+/Misc/sbom.spdx.json          @sethmlarson
+/Tools/build/generate_sbom.py @sethmlarson

.github/workflows/mypy.yml

@@ -9,6 +9,7 @@ on:
     paths:
       - ".github/workflows/mypy.yml"
       - "Lib/test/libregrtest/**"
+      - "Tools/build/generate_sbom.py"
       - "Tools/cases_generator/**"
       - "Tools/clinic/**"
       - "Tools/peg_generator/**"
@@ -34,6 +35,7 @@ jobs:
       matrix:
         target: [
           "Lib/test/libregrtest",
+          "Tools/build/",
           "Tools/cases_generator",
           "Tools/clinic",
           "Tools/peg_generator",

Doc/library/pdb.rst

@@ -570,10 +570,27 @@ can be overridden by the local file.
 
    Start an interactive interpreter (using the :mod:`code` module) whose global
    namespace contains all the (global and local) names found in the current
-   scope.
+   scope. Use ``exit()`` or ``quit()`` to exit the interpreter and return to
+   the debugger.
+
+   .. note::
+
+      Because interact creates a new global namespace with the current global
+      and local namespace for execution, assignment to variables will not
+      affect the original namespaces.
+      However, modification to the mutable objects will be reflected in the
+      original namespaces.
 
    .. versionadded:: 3.2
 
+   .. versionadded:: 3.13
+      ``exit()`` and ``quit()`` can be used to exit :pdbcmd:`interact`
+      command.
+
+   .. versionchanged:: 3.13
+      :pdbcmd:`interact` directs its output to the debugger's
+      output channel rather than :data:`sys.stderr`.
+
 .. _debugger-aliases:
 
 .. pdbcommand:: alias [name [command]]

Doc/using/cmdline.rst

@@ -612,6 +612,27 @@ Miscellaneous options
    .. versionadded:: 3.13
       The ``-X presite`` option.
 
+Controlling Color
+~~~~~~~~~~~~~~~~~
+
+The Python interpreter is configured by default to use colors to highlight
+output in certain situations such as when displaying tracebacks. This
+behavior can be controlled by setting different environment variables.
+
+Setting the environment variable ``TERM`` to ``dumb`` will disable color.
+
+If the environment variable ``FORCE_COLOR`` is set, then color will be
+enabled regardless of the value of TERM. This is useful on CI systems which
+aren’t terminals but can none-the-less display ANSI escape sequences.
+
+If the environment variable ``NO_COLOR`` is set, Python will disable all color
+in the output. This takes precedence over ``FORCE_COLOR``.
+
+All these environment variables are used also by other tools to control color
+output. To control the color output only in the Python interpreter, the
+:envvar:`PYTHON_COLORS` environment variable can be used. This variable takes
+precedence over ``NO_COLOR``, which in turn takes precedence over
+``FORCE_COLOR``.
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1110,6 +1131,12 @@ conflict.
 
    .. versionadded:: 3.13
 
+.. envvar:: PYTHON_COLORS
+
+   If this variable is set to ``1``, the interpreter will colorize various kinds
+   of output. Setting it to ``0`` deactivates this behavior.
+
+   .. versionadded:: 3.13
 
 Debug-mode variables
 ~~~~~~~~~~~~~~~~~~~~

Doc/whatsnew/3.13.rst

@@ -85,7 +85,13 @@ Important deprecations, removals or restrictions:
 New Features
 ============
 
+Improved Error Messages
+-----------------------
 
+* The interpreter now colorizes error messages when displaying tracebacks by default.
+  This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment
+  variable as well as the canonical ``NO_COLOR`` and ``FORCE_COLOR`` environment
+  variables. (Contributed by Pablo Galindo Salgado in :gh:`112730`.)
 
 Other Language Changes
 ======================

Include/internal/pycore_dtoa.h

@@ -35,16 +35,20 @@ struct _dtoa_state {
 /* The size of the Bigint freelist */
 #define Bigint_Kmax 7
 
+/* The size of the cached powers of 5 array */
+#define Bigint_Pow5size 8
+
 #ifndef PRIVATE_MEM
 #define PRIVATE_MEM 2304
 #endif
 #define Bigint_PREALLOC_SIZE \
     ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double))
 
 struct _dtoa_state {
-    /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */
+    // p5s is an array of powers of 5 of the form:
+    // 5**(2**(i+2)) for 0 <= i < Bigint_Pow5size
+    struct Bigint *p5s[Bigint_Pow5size];
     // XXX This should be freed during runtime fini.
-    struct Bigint *p5s;
     struct Bigint *freelist[Bigint_Kmax+1];
     double preallocated[Bigint_PREALLOC_SIZE];
     double *preallocated_next;
@@ -57,16 +61,18 @@ struct _dtoa_state {
 #endif  // !Py_USING_MEMORY_DEBUGGER
 
 
-/* These functions are used by modules compiled as C extension like math:
-   they must be exported. */
-
 extern double _Py_dg_strtod(const char *str, char **ptr);
 extern char* _Py_dg_dtoa(double d, int mode, int ndigits,
                          int *decpt, int *sign, char **rve);
 extern void _Py_dg_freedtoa(char *s);
 
 #endif // _PY_SHORT_FLOAT_REPR == 1
 
+
+extern PyStatus _PyDtoa_Init(PyInterpreterState *interp);
+extern void _PyDtoa_Fini(PyInterpreterState *interp);
+
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_typeobject.h

@@ -135,6 +135,8 @@ extern PyObject* _Py_type_getattro_impl(PyTypeObject *type, PyObject *name,
                                         int *suppress_missing_attribute);
 extern PyObject* _Py_type_getattro(PyTypeObject *type, PyObject *name);
 
+extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op);
+
 extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name);
 extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name);
 

Include/objimpl.h

@@ -35,7 +35,7 @@ Functions and macros for modules that implement new object types.
    fields, this also fills in the ob_size field.
 
  - PyObject_Free(op) releases the memory allocated for an object.  It does not
-   run a destructor -- it only frees the memory.  PyObject_Free is identical.
+   run a destructor -- it only frees the memory.
 
  - PyObject_Init(op, typeobj) and PyObject_InitVar(op, typeobj, n) don't
    allocate memory.  Instead of a 'type' parameter, they take a pointer to a

Lib/pdb.py

@@ -207,6 +207,15 @@ def namespace(self):
         )
 
 
+class _PdbInteractiveConsole(code.InteractiveConsole):
+    def __init__(self, ns, message):
+        self._message = message
+        super().__init__(locals=ns, local_exit=True)
+
+    def write(self, data):
+        self._message(data, end='')
+
+
 # Interaction prompt line will separate file and call info from code
 # text using value of line_prefix string.  A newline and arrow may
 # be to your liking.  You can set it once pdb is imported using the
@@ -672,8 +681,8 @@ def handle_command_def(self, line):
 
     # interface abstraction functions
 
-    def message(self, msg):
-        print(msg, file=self.stdout)
+    def message(self, msg, end='\n'):
+        print(msg, end=end, file=self.stdout)
 
     def error(self, msg):
         print('***', msg, file=self.stdout)
@@ -1786,7 +1795,9 @@ def do_interact(self, arg):
         contains all the (global and local) names found in the current scope.
         """
         ns = {**self.curframe.f_globals, **self.curframe_locals}
-        code.interact("*interactive*", local=ns, local_exit=True)
+        console = _PdbInteractiveConsole(ns, message=self.message)
+        console.interact(banner="*pdb interact start*",
+                         exitmsg="*exit from pdb interact command*")
 
     def do_alias(self, arg):
         """alias [name [command]]

Lib/tempfile.py

@@ -270,6 +270,22 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type):
     raise FileExistsError(_errno.EEXIST,
                           "No usable temporary file name found")
 
+def _dont_follow_symlinks(func, path, *args):
+    # Pass follow_symlinks=False, unless not supported on this platform.
+    if func in _os.supports_follow_symlinks:
+        func(path, *args, follow_symlinks=False)
+    elif _os.name == 'nt' or not _os.path.islink(path):
+        func(path, *args)
+
+def _resetperms(path):
+    try:
+        chflags = _os.chflags
+    except AttributeError:
+        pass
+    else:
+        _dont_follow_symlinks(chflags, path, 0)
+    _dont_follow_symlinks(_os.chmod, path, 0o700)
+
 
 # User visible interfaces.
 
@@ -876,17 +892,10 @@ def __init__(self, suffix=None, prefix=None, dir=None,
     def _rmtree(cls, name, ignore_errors=False):
         def onexc(func, path, exc):
             if isinstance(exc, PermissionError):
-                def resetperms(path):
-                    try:
-                        _os.chflags(path, 0)
-                    except AttributeError:
-                        pass
-                    _os.chmod(path, 0o700)
-
                 try:
                     if path != name:
-                        resetperms(_os.path.dirname(path))
-                    resetperms(path)
+                        _resetperms(_os.path.dirname(path))
+                    _resetperms(path)
 
                     try:
                         _os.unlink(path)

Lib/test/support/os_helper.py

@@ -592,10 +592,17 @@ def fd_count():
     """Count the number of open file descriptors.
     """
     if sys.platform.startswith(('linux', 'freebsd', 'emscripten')):
+        fd_path = "/proc/self/fd"
+    elif sys.platform == "darwin":
+        fd_path = "/dev/fd"
+    else:
+        fd_path = None
+
+    if fd_path is not None:
         try:
-            names = os.listdir("/proc/self/fd")
+            names = os.listdir(fd_path)
             # Subtract one because listdir() internally opens a file
-            # descriptor to list the content of the /proc/self/fd/ directory.
+            # descriptor to list the content of the directory.
             return len(names) - 1
         except FileNotFoundError:
             pass

Lib/test/test_builtin.py

@@ -627,6 +627,11 @@ def __dir__(self):
         # test that object has a __dir__()
         self.assertEqual(sorted([].__dir__()), dir([]))
 
+    def test___ne__(self):
+        self.assertFalse(None.__ne__(None))
+        self.assertTrue(None.__ne__(0))
+        self.assertTrue(None.__ne__("abc"))
+
     def test_divmod(self):
         self.assertEqual(divmod(12, 7), (1, 5))
         self.assertEqual(divmod(-12, 7), (-2, 2))
@@ -2269,30 +2274,37 @@ def _run_child(self, child, terminal_input):
 
         return lines
 
-    def check_input_tty(self, prompt, terminal_input, stdio_encoding=None):
+    def check_input_tty(self, prompt, terminal_input, stdio_encoding=None, *,
+                        expected=None,
+                        stdin_errors='surrogateescape',
+                        stdout_errors='replace'):
         if not sys.stdin.isatty() or not sys.stdout.isatty():
             self.skipTest("stdin and stdout must be ttys")
         def child(wpipe):
             # Check the error handlers are accounted for
             if stdio_encoding:
                 sys.stdin = io.TextIOWrapper(sys.stdin.detach(),
                                              encoding=stdio_encoding,
-                                             errors='surrogateescape')
+                                             errors=stdin_errors)
                 sys.stdout = io.TextIOWrapper(sys.stdout.detach(),
                                               encoding=stdio_encoding,
-                                              errors='replace')
+                                              errors=stdout_errors)
             print("tty =", sys.stdin.isatty() and sys.stdout.isatty(), file=wpipe)
-            print(ascii(input(prompt)), file=wpipe)
+            try:
+                print(ascii(input(prompt)), file=wpipe)
+            except BaseException as e:
+                print(ascii(f'{e.__class__.__name__}: {e!s}'), file=wpipe)
         lines = self.run_child(child, terminal_input + b"\r\n")
         # Check we did exercise the GNU readline path
         self.assertIn(lines[0], {'tty = True', 'tty = False'})
         if lines[0] != 'tty = True':
             self.skipTest("standard IO in should have been a tty")
         input_result = eval(lines[1])   # ascii() -> eval() roundtrip
-        if stdio_encoding:
-            expected = terminal_input.decode(stdio_encoding, 'surrogateescape')
-        else:
-            expected = terminal_input.decode(sys.stdin.encoding)  # what else?
+        if expected is None:
+            if stdio_encoding:
+                expected = terminal_input.decode(stdio_encoding, 'surrogateescape')
+            else:
+                expected = terminal_input.decode(sys.stdin.encoding)  # what else?
         self.assertEqual(input_result, expected)
 
     def test_input_tty(self):
@@ -2313,13 +2325,32 @@ def skip_if_readline(self):
     def test_input_tty_non_ascii(self):
         self.skip_if_readline()
         # Check stdin/stdout encoding is used when invoking PyOS_Readline()
-        self.check_input_tty("prompté", b"quux\xe9", "utf-8")
+        self.check_input_tty("prompté", b"quux\xc3\xa9", "utf-8")
 
     def test_input_tty_non_ascii_unicode_errors(self):
         self.skip_if_readline()
         # Check stdin/stdout error handler is used when invoking PyOS_Readline()
         self.check_input_tty("prompté", b"quux\xe9", "ascii")
 
+    def test_input_tty_null_in_prompt(self):
+        self.check_input_tty("prompt\0", b"",
+                expected='ValueError: input: prompt string cannot contain '
+                         'null characters')
+
+    def test_input_tty_nonencodable_prompt(self):
+        self.skip_if_readline()
+        self.check_input_tty("prompté", b"quux", "ascii", stdout_errors='strict',
+                expected="UnicodeEncodeError: 'ascii' codec can't encode "
+                         "character '\\xe9' in position 6: ordinal not in "
+                         "range(128)")
+
+    def test_input_tty_nondecodable_input(self):
+        self.skip_if_readline()
+        self.check_input_tty("prompt", b"quux\xe9", "ascii", stdin_errors='strict',
+                expected="UnicodeDecodeError: 'ascii' codec can't decode "
+                         "byte 0xe9 in position 4: ordinal not in "
+                         "range(128)")
+
     def test_input_no_stdout_fileno(self):
         # Issue #24402: If stdin is the original terminal but stdout.fileno()
         # fails, do not use the original stdout file descriptor