bpo-46841: Quicken code in-place

Include/cpython/code.h

@@ -26,91 +26,80 @@ typedef uint16_t _Py_CODEUNIT;
 // Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing:
 #define _Py_SET_OPCODE(word, opcode) (((unsigned char *)&(word))[0] = (opcode))
 
+// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
+// defined in this macro:
+#define _PyCode_DEF(SIZE) {                                                    \
+    PyObject_VAR_HEAD                                                          \
+                                                                               \
+    /* Note only the following fields are used in hash and/or comparisons      \
+     *                                                                         \
+     * - co_name                                                               \
+     * - co_argcount                                                           \
+     * - co_posonlyargcount                                                    \
+     * - co_kwonlyargcount                                                     \
+     * - co_nlocals                                                            \
+     * - co_stacksize                                                          \
+     * - co_flags                                                              \
+     * - co_firstlineno                                                        \
+     * - co_consts                                                             \
+     * - co_names                                                              \
+     * - co_localsplusnames                                                    \
+     * This is done to preserve the name and line number for tracebacks        \
+     * and debuggers; otherwise, constant de-duplication would collapse        \
+     * identical functions/lambdas defined on different lines.                 \
+     */                                                                        \
+                                                                               \
+    /* These fields are set with provided values on new code objects. */       \
+                                                                               \
+    /* The hottest fields (in the eval loop) are grouped here at the top. */   \
+    PyObject *co_consts;           /* list (constants used) */                 \
+    PyObject *co_names;            /* list of strings (names used) */          \
+    PyObject *co_exceptiontable;   /* Byte string encoding exception handling  \
+                                      table */                                 \
+    int co_flags;                  /* CO_..., see below */                     \
+    int co_warmup;                 /* Warmup counter for quickening */         \
+                                                                               \
+    /* The rest are not so impactful on performance. */                        \
+    int co_argcount;              /* #arguments, except *args */               \
+    int co_posonlyargcount;       /* #positional only arguments */             \
+    int co_kwonlyargcount;        /* #keyword only arguments */                \
+    int co_stacksize;             /* #entries needed for evaluation stack */   \
+    int co_firstlineno;           /* first source line number */               \
+                                                                               \
+    /* redundant values (derived from co_localsplusnames and                   \
+       co_localspluskinds) */                                                  \
+    int co_nlocalsplus;           /* number of local + cell + free variables   \
+                                  */                                           \
+    int co_nlocals;               /* number of local variables */              \
+    int co_nplaincellvars;        /* number of non-arg cell variables */       \
+    int co_ncellvars;             /* total number of cell variables */         \
+    int co_nfreevars;             /* number of free variables */               \
+                                                                               \
+    PyObject *co_localsplusnames; /* tuple mapping offsets to names */         \
+    PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte    \
+                                     per variable) */                          \
+    PyObject *co_filename;        /* unicode (where it was loaded from) */     \
+    PyObject *co_name;            /* unicode (name, for reference) */          \
+    PyObject *co_qualname;        /* unicode (qualname, for reference) */      \
+    PyObject *co_linetable;       /* bytes (encoding addr<->lineno mapping)    \
+                                     See Objects/lnotab_notes.txt for details. \
+                                  */                                           \
+    PyObject *co_endlinetable;    /* bytes object that holds end lineno for    \
+                                     instructions separated across different   \
+                                     lines */                                  \
+    PyObject *co_columntable;     /* bytes object that holds start/end column  \
+                                     offset each instruction */                \
+                                                                               \
+    PyObject *co_weakreflist;     /* to support weakrefs to code objects */    \
+    /* Scratch space for extra data relating to the code object.               \
+       Type is a void* to keep the format private in codeobject.c to force     \
+       people to go through the proper APIs. */                                \
+    void *co_extra;                                                            \
+    char co_code_adaptive[(SIZE)];                                             \
+}
 
 /* Bytecode object */
-struct PyCodeObject {
-    PyObject_HEAD
-
-    /* Note only the following fields are used in hash and/or comparisons
-     *
-     * - co_name
-     * - co_argcount
-     * - co_posonlyargcount
-     * - co_kwonlyargcount
-     * - co_nlocals
-     * - co_stacksize
-     * - co_flags
-     * - co_firstlineno
-     * - co_code
-     * - co_consts
-     * - co_names
-     * - co_varnames
-     * - co_freevars
-     * - co_cellvars
-     *
-     * This is done to preserve the name and line number for tracebacks
-     * and debuggers; otherwise, constant de-duplication would collapse
-     * identical functions/lambdas defined on different lines.
-     */
-
-    /* These fields are set with provided values on new code objects. */
-
-    // The hottest fields (in the eval loop) are grouped here at the top.
-    PyObject *co_consts;        /* list (constants used) */
-    PyObject *co_names;         /* list of strings (names used) */
-    _Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening.
-                                    Unlike the other "hot" fields, this one is
-                                    actually derived from co_code. */
-    PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
-    int co_flags;               /* CO_..., see below */
-    int co_warmup;              /* Warmup counter for quickening */
-
-    // The rest are not so impactful on performance.
-    int co_argcount;            /* #arguments, except *args */
-    int co_posonlyargcount;     /* #positional only arguments */
-    int co_kwonlyargcount;      /* #keyword only arguments */
-    int co_stacksize;           /* #entries needed for evaluation stack */
-    int co_firstlineno;         /* first source line number */
-    PyObject *co_code;          /* instruction opcodes */
-    PyObject *co_localsplusnames;  /* tuple mapping offsets to names */
-    PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */
-    PyObject *co_filename;      /* unicode (where it was loaded from) */
-    PyObject *co_name;          /* unicode (name, for reference) */
-    PyObject *co_qualname;      /* unicode (qualname, for reference) */
-    PyObject *co_linetable;     /* bytes (encoding addr<->lineno mapping) See
-                                   Objects/lnotab_notes.txt for details. */
-    PyObject *co_endlinetable;  /* bytes object that holds end lineno for
-                                   instructions separated across different
-                                   lines */
-    PyObject *co_columntable;   /* bytes object that holds start/end column
-                                   offset each instruction */
-
-    /* These fields are set with computed values on new code objects. */
-
-    // redundant values (derived from co_localsplusnames and co_localspluskinds)
-    int co_nlocalsplus;         /* number of local + cell + free variables */
-    int co_nlocals;             /* number of local variables */
-    int co_nplaincellvars;      /* number of non-arg cell variables */
-    int co_ncellvars;           /* total number of cell variables */
-    int co_nfreevars;           /* number of free variables */
-    // lazily-computed values
-    PyObject *co_varnames;      /* tuple of strings (local variable names) */
-    PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
-    PyObject *co_freevars;      /* tuple of strings (free variable names) */
-
-    /* The remaining fields are zeroed out on new code objects. */
-
-    PyObject *co_weakreflist;   /* to support weakrefs to code objects */
-    /* Scratch space for extra data relating to the code object.
-       Type is a void* to keep the format private in codeobject.c to force
-       people to go through the proper APIs. */
-    void *co_extra;
-    /* Quickened instructions and cache, or NULL
-     This should be treated as opaque by all code except the specializer and
-     interpreter. */
-    _Py_CODEUNIT *co_quickened;
-
-};
+struct PyCodeObject _PyCode_DEF(1);
 
 /* Masks for co_flags above */
 #define CO_OPTIMIZED    0x0001
@@ -151,6 +140,8 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;
 
 #define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type)
 #define PyCode_GetNumFree(op) ((op)->co_nfreevars)
+#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive)
+#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))
 
 /* Public interface */
 PyAPI_FUNC(PyCodeObject *) PyCode_New(

Include/internal/pycore_code.h

@@ -92,30 +92,22 @@ typedef struct {
 
 #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
 
-/* Maximum size of code to quicken, in code units. */
-#define MAX_SIZE_TO_QUICKEN 10000
-
 #define QUICKENING_WARMUP_DELAY 8
 
 /* We want to compare to zero for efficiency, so we offset values accordingly */
 #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
-#define QUICKENING_WARMUP_COLDEST 1
 
-int _Py_Quicken(PyCodeObject *code);
+void _PyCode_Quicken(PyCodeObject *code);
 
-/* Returns 1 if quickening occurs.
- * -1 if an error occurs
- * 0 otherwise */
-static inline int
-_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code)
+static inline void
+_PyCode_Warmup(PyCodeObject *code)
 {
     if (code->co_warmup != 0) {
         code->co_warmup++;
         if (code->co_warmup == 0) {
-            return _Py_Quicken(code) ? -1 : 1;
+            _PyCode_Quicken(code);
         }
     }
-    return 0;
 }
 
 extern Py_ssize_t _Py_QuickenedCount;
@@ -225,6 +217,7 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
 extern PyObject* _PyCode_GetVarnames(PyCodeObject *);
 extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
 extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
+extern PyObject* _PyCode_GetCode(PyCodeObject *);
 
 /* Return the ending source code line number from a bytecode index. */
 extern int _PyCode_Addr2EndLine(PyCodeObject *, int);