From 1e197e63e21f77b102ff2601a549dda4b6439455 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Nov 2022 04:34:57 -0800 Subject: [PATCH] GH-96421: Insert shim frame on entry to interpreter (GH-96319) * Adds EXIT_INTERPRETER instruction to exit PyEval_EvalDefault() * Simplifies RETURN_VALUE, YIELD_VALUE and RETURN_GENERATOR instructions as they no longer need to check for entry frames. --- Include/internal/pycore_code.h | 10 ++ Include/internal/pycore_frame.h | 17 +-- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_interp.h | 1 + Include/internal/pycore_opcode.h | 44 +++--- .../internal/pycore_runtime_init_generated.h | 1 + Include/opcode.h | 127 ++++++++-------- Lib/opcode.py | 1 + ...2-10-19-15-59-08.gh-issue-96421.e22y3r.rst | 13 ++ Objects/codeobject.c | 75 ++++++++++ Objects/frame_layout.md | 22 ++- Objects/frameobject.c | 10 +- Objects/genobject.c | 15 +- Python/bytecodes.c | 141 ++++++++---------- Python/ceval.c | 80 +++++----- Python/compile.c | 2 + Python/frame.c | 4 +- Python/generated_cases.c.h | 136 ++++++++--------- Python/opcode_targets.h | 42 +++--- Python/pylifecycle.c | 21 ++- Python/pystate.c | 1 + Python/traceback.c | 9 ++ Tools/gdb/libpython.py | 18 ++- 24 files changed, 448 insertions(+), 344 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-19-15-59-08.gh-issue-96421.e22y3r.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index b0703073f1d14b..0af240ca362103 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -456,6 +456,16 @@ _PyCode_LineNumberFromArray(PyCodeObject *co, int index) } } +typedef struct _PyShimCodeDef { + const uint8_t *code; + int codelen; + int stacksize; + const char *cname; +} _PyShimCodeDef; + +extern PyCodeObject * +_Py_MakeShimCode(const _PyShimCodeDef *code); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index feee692d0f169e..7fa410d288c33a 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -42,17 +42,18 @@ typedef enum _framestate { enum _frameowner { FRAME_OWNED_BY_THREAD = 0, FRAME_OWNED_BY_GENERATOR = 1, - FRAME_OWNED_BY_FRAME_OBJECT = 2 + FRAME_OWNED_BY_FRAME_OBJECT = 2, + FRAME_OWNED_BY_CSTACK = 3, }; typedef struct _PyInterpreterFrame { /* "Specials" section */ - PyObject *f_funcobj; /* Strong reference */ - PyObject *f_globals; /* Borrowed reference */ - PyObject *f_builtins; /* Borrowed reference */ - PyObject *f_locals; /* Strong reference, may be NULL */ + PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ + PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ + PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ + PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyCodeObject *f_code; /* Strong reference */ - PyFrameObject *frame_obj; /* Strong reference, may be NULL */ + PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ /* Linkage section */ struct _PyInterpreterFrame *previous; // NOTE: This is not necessarily the last instruction started in the given @@ -60,9 +61,8 @@ typedef struct _PyInterpreterFrame { // example, it may be an inline CACHE entry, an instruction we just jumped // over, or (in the case of a newly-created frame) a totally invalid value: _Py_CODEUNIT *prev_instr; - int stacktop; /* Offset of TOS from localsplus */ + int stacktop; /* Offset of TOS from localsplus */ uint16_t yield_offset; - bool is_entry; // Whether this is the "root" frame for the current _PyCFrame. char owner; /* Locals and stack */ PyObject *localsplus[1]; @@ -110,7 +110,6 @@ _PyFrame_InitializeSpecials( frame->stacktop = code->co_nlocalsplus; frame->frame_obj = NULL; frame->prev_instr = _PyCode_CODE(code) - 1; - frame->is_entry = false; frame->yield_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; } diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index f734e4b48a34ae..3d8bc7284f13b9 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -564,6 +564,7 @@ _PyStaticObjects_CheckRefcnt(void) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(newline)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(open_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(percent)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(shim_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(FINISHED)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8588aaeb930004..bf83bc45a51f8e 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -48,6 +48,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(newline, "\n") STRUCT_FOR_STR(open_br, "{") STRUCT_FOR_STR(percent, "%") + STRUCT_FOR_STR(shim_name, "") STRUCT_FOR_STR(utf_8, "utf-8") } literals; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 6f8fbeb1ba3e31..ae2a3d3b13cfa9 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -186,6 +186,7 @@ struct _is { struct ast_state ast; struct types_state types; struct callable_cache callable_cache; + PyCodeObject *interpreter_trampoline; /* The following fields are here to avoid allocation during init. The data is exposed through PyInterpreterState pointer fields. diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 949d2c161d6dfb..3f44511240aa14 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -136,6 +136,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, [IMPORT_STAR] = IMPORT_STAR, + [INTERPRETER_EXIT] = INTERPRETER_EXIT, [IS_OP] = IS_OP, [JUMP_BACKWARD] = JUMP_BACKWARD, [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, @@ -235,19 +236,20 @@ static const char *const _PyOpcode_OpName[263] = { [CACHE] = "CACHE", [POP_TOP] = "POP_TOP", [PUSH_NULL] = "PUSH_NULL", - [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", + [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [END_FOR] = "END_FOR", + [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", [BINARY_OP_GENERIC] = "BINARY_OP_GENERIC", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [NOP] = "NOP", [UNARY_POSITIVE] = "UNARY_POSITIVE", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", - [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", [UNARY_INVERT] = "UNARY_INVERT", + [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", @@ -256,20 +258,20 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", @@ -280,7 +282,6 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -288,37 +289,37 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", [COMPARE_OP_GENERIC] = "COMPARE_OP_GENERIC", [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", - [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", - [FOR_ITER_LIST] = "FOR_ITER_LIST", + [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [STOPITERATION_ERROR] = "STOPITERATION_ERROR", + [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -345,7 +346,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -353,7 +354,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -373,9 +374,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -385,27 +386,27 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", - [173] = "<173>", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [174] = "<174>", [175] = "<175>", [176] = "<176>", @@ -499,7 +500,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 173: \ case 174: \ case 175: \ case 176: \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 0dcb26d46f9cf6..a2b529b3977727 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -558,6 +558,7 @@ extern "C" { INIT_STR(newline, "\n"), \ INIT_STR(open_br, "{"), \ INIT_STR(percent, "%"), \ + INIT_STR(shim_name, ""), \ INIT_STR(utf_8, "utf-8"), \ }, \ .identifiers = { \ diff --git a/Include/opcode.h b/Include/opcode.h index 9fbdbe5c6a6c6a..c18d9c057b849b 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -11,6 +11,7 @@ extern "C" { #define CACHE 0 #define POP_TOP 1 #define PUSH_NULL 2 +#define INTERPRETER_EXIT 3 #define END_FOR 4 #define NOP 9 #define UNARY_POSITIVE 10 @@ -128,69 +129,69 @@ extern "C" { #define JUMP_NO_INTERRUPT 261 #define LOAD_METHOD 262 #define MAX_PSEUDO_OPCODE 262 -#define BINARY_OP_ADD_FLOAT 3 -#define BINARY_OP_ADD_INT 5 -#define BINARY_OP_ADD_UNICODE 6 -#define BINARY_OP_GENERIC 7 -#define BINARY_OP_INPLACE_ADD_UNICODE 8 -#define BINARY_OP_MULTIPLY_FLOAT 13 -#define BINARY_OP_MULTIPLY_INT 14 -#define BINARY_OP_SUBTRACT_FLOAT 16 -#define BINARY_OP_SUBTRACT_INT 17 -#define BINARY_SUBSCR_DICT 18 -#define BINARY_SUBSCR_GETITEM 19 -#define BINARY_SUBSCR_LIST_INT 20 -#define BINARY_SUBSCR_TUPLE_INT 21 -#define CALL_PY_EXACT_ARGS 22 -#define CALL_PY_WITH_DEFAULTS 23 -#define CALL_BOUND_METHOD_EXACT_ARGS 24 -#define CALL_BUILTIN_CLASS 28 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 29 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 34 -#define CALL_NO_KW_BUILTIN_FAST 38 -#define CALL_NO_KW_BUILTIN_O 39 -#define CALL_NO_KW_ISINSTANCE 40 -#define CALL_NO_KW_LEN 41 -#define CALL_NO_KW_LIST_APPEND 42 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45 -#define CALL_NO_KW_STR_1 46 -#define CALL_NO_KW_TUPLE_1 47 -#define CALL_NO_KW_TYPE_1 48 -#define COMPARE_OP_FLOAT_JUMP 56 -#define COMPARE_OP_GENERIC 57 -#define COMPARE_OP_INT_JUMP 58 -#define COMPARE_OP_STR_JUMP 59 -#define FOR_ITER_LIST 62 -#define FOR_ITER_RANGE 64 -#define FOR_ITER_GEN 65 -#define LOAD_ATTR_CLASS 66 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 67 -#define LOAD_ATTR_INSTANCE_VALUE 72 -#define LOAD_ATTR_MODULE 73 -#define LOAD_ATTR_PROPERTY 76 -#define LOAD_ATTR_SLOT 77 -#define LOAD_ATTR_WITH_HINT 78 -#define LOAD_ATTR_METHOD_LAZY_DICT 79 -#define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_WITH_DICT 81 -#define LOAD_ATTR_METHOD_WITH_VALUES 86 -#define LOAD_CONST__LOAD_FAST 113 -#define LOAD_FAST__LOAD_CONST 121 -#define LOAD_FAST__LOAD_FAST 141 -#define LOAD_GLOBAL_BUILTIN 143 -#define LOAD_GLOBAL_MODULE 153 -#define STORE_ATTR_INSTANCE_VALUE 154 -#define STORE_ATTR_SLOT 158 -#define STORE_ATTR_WITH_HINT 159 -#define STORE_FAST__LOAD_FAST 160 -#define STORE_FAST__STORE_FAST 161 -#define STORE_SUBSCR_DICT 166 -#define STORE_SUBSCR_LIST_INT 167 -#define UNPACK_SEQUENCE_LIST 168 -#define UNPACK_SEQUENCE_TUPLE 169 -#define UNPACK_SEQUENCE_TWO_TUPLE 170 +#define BINARY_OP_ADD_FLOAT 5 +#define BINARY_OP_ADD_INT 6 +#define BINARY_OP_ADD_UNICODE 7 +#define BINARY_OP_GENERIC 8 +#define BINARY_OP_INPLACE_ADD_UNICODE 13 +#define BINARY_OP_MULTIPLY_FLOAT 14 +#define BINARY_OP_MULTIPLY_INT 16 +#define BINARY_OP_SUBTRACT_FLOAT 17 +#define BINARY_OP_SUBTRACT_INT 18 +#define BINARY_SUBSCR_DICT 19 +#define BINARY_SUBSCR_GETITEM 20 +#define BINARY_SUBSCR_LIST_INT 21 +#define BINARY_SUBSCR_TUPLE_INT 22 +#define CALL_PY_EXACT_ARGS 23 +#define CALL_PY_WITH_DEFAULTS 24 +#define CALL_BOUND_METHOD_EXACT_ARGS 28 +#define CALL_BUILTIN_CLASS 29 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 34 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 38 +#define CALL_NO_KW_BUILTIN_FAST 39 +#define CALL_NO_KW_BUILTIN_O 40 +#define CALL_NO_KW_ISINSTANCE 41 +#define CALL_NO_KW_LEN 42 +#define CALL_NO_KW_LIST_APPEND 43 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 46 +#define CALL_NO_KW_STR_1 47 +#define CALL_NO_KW_TUPLE_1 48 +#define CALL_NO_KW_TYPE_1 56 +#define COMPARE_OP_FLOAT_JUMP 57 +#define COMPARE_OP_GENERIC 58 +#define COMPARE_OP_INT_JUMP 59 +#define COMPARE_OP_STR_JUMP 62 +#define FOR_ITER_LIST 64 +#define FOR_ITER_RANGE 65 +#define FOR_ITER_GEN 66 +#define LOAD_ATTR_CLASS 67 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 72 +#define LOAD_ATTR_INSTANCE_VALUE 73 +#define LOAD_ATTR_MODULE 76 +#define LOAD_ATTR_PROPERTY 77 +#define LOAD_ATTR_SLOT 78 +#define LOAD_ATTR_WITH_HINT 79 +#define LOAD_ATTR_METHOD_LAZY_DICT 80 +#define LOAD_ATTR_METHOD_NO_DICT 81 +#define LOAD_ATTR_METHOD_WITH_DICT 86 +#define LOAD_ATTR_METHOD_WITH_VALUES 113 +#define LOAD_CONST__LOAD_FAST 121 +#define LOAD_FAST__LOAD_CONST 141 +#define LOAD_FAST__LOAD_FAST 143 +#define LOAD_GLOBAL_BUILTIN 153 +#define LOAD_GLOBAL_MODULE 154 +#define STORE_ATTR_INSTANCE_VALUE 158 +#define STORE_ATTR_SLOT 159 +#define STORE_ATTR_WITH_HINT 160 +#define STORE_FAST__LOAD_FAST 161 +#define STORE_FAST__STORE_FAST 166 +#define STORE_SUBSCR_DICT 167 +#define STORE_SUBSCR_LIST_INT 168 +#define UNPACK_SEQUENCE_LIST 169 +#define UNPACK_SEQUENCE_TUPLE 170 +#define UNPACK_SEQUENCE_TWO_TUPLE 173 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 00ef78aa0baebb..0ee75958508ac7 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -77,6 +77,7 @@ def pseudo_op(name, op, real_ops): def_op('CACHE', 0) def_op('POP_TOP', 1) def_op('PUSH_NULL', 2) +def_op('INTERPRETER_EXIT', 3) def_op('END_FOR', 4) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-15-59-08.gh-issue-96421.e22y3r.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-15-59-08.gh-issue-96421.e22y3r.rst new file mode 100644 index 00000000000000..857ce7decea285 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-15-59-08.gh-issue-96421.e22y3r.rst @@ -0,0 +1,13 @@ +When calling into Python code from C code, through :c:func:`PyEval_EvalFrameEx` or +a related C-API function, a shim frame in inserted into the call stack. +This occurs in the ``_PyEval_EvalFrameDefault()`` function. +The extra frame should be invisible to all Python and most C extensions, +but out-of-process profilers and debuggers need to be aware of it. +These shim frames can be detected by checking +``frame->owner == FRAME_OWNED_BY_CSTACK``. + +Extensions implementing their own interpreters using PEP 523 need to be +aware of this shim frame and the changes to the semantics of +:opcode:`RETURN_VALUE`, :opcode:`YIELD_VALUE`, and :opcode:`RETURN_GENERATOR`, +which now clear the frame. + diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 854611ff858319..3824fc0f5a4e10 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2264,3 +2264,78 @@ _PyStaticCode_Init(PyCodeObject *co) _PyCode_Quicken(co); return 0; } + +#define MAX_CODE_UNITS_PER_LOC_ENTRY 8 + +PyCodeObject * +_Py_MakeShimCode(const _PyShimCodeDef *codedef) +{ + PyObject *name = NULL; + PyObject *co_code = NULL; + PyObject *lines = NULL; + PyCodeObject *codeobj = NULL; + uint8_t *loc_table = NULL; + + name = _PyUnicode_FromASCII(codedef->cname, strlen(codedef->cname)); + if (name == NULL) { + goto cleanup; + } + co_code = PyBytes_FromStringAndSize( + (const char *)codedef->code, codedef->codelen); + if (co_code == NULL) { + goto cleanup; + } + int code_units = codedef->codelen / sizeof(_Py_CODEUNIT); + int loc_entries = (code_units + MAX_CODE_UNITS_PER_LOC_ENTRY - 1) / + MAX_CODE_UNITS_PER_LOC_ENTRY; + loc_table = PyMem_Malloc(loc_entries); + if (loc_table == NULL) { + PyErr_NoMemory(); + goto cleanup; + } + for (int i = 0; i < loc_entries-1; i++) { + loc_table[i] = 0x80 | (PY_CODE_LOCATION_INFO_NONE << 3) | 7; + code_units -= MAX_CODE_UNITS_PER_LOC_ENTRY; + } + assert(loc_entries > 0); + assert(code_units > 0 && code_units <= MAX_CODE_UNITS_PER_LOC_ENTRY); + loc_table[loc_entries-1] = 0x80 | + (PY_CODE_LOCATION_INFO_NONE << 3) | (code_units-1); + lines = PyBytes_FromStringAndSize((const char *)loc_table, loc_entries); + PyMem_Free(loc_table); + if (lines == NULL) { + goto cleanup; + } + _Py_DECLARE_STR(shim_name, ""); + struct _PyCodeConstructor con = { + .filename = &_Py_STR(shim_name), + .name = name, + .qualname = name, + .flags = CO_NEWLOCALS | CO_OPTIMIZED, + + .code = co_code, + .firstlineno = 1, + .linetable = lines, + + .consts = (PyObject *)&_Py_SINGLETON(tuple_empty), + .names = (PyObject *)&_Py_SINGLETON(tuple_empty), + + .localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty), + .localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty), + + .argcount = 0, + .posonlyargcount = 0, + .kwonlyargcount = 0, + + .stacksize = codedef->stacksize, + + .exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), + }; + + codeobj = _PyCode_New(&con); +cleanup: + Py_XDECREF(name); + Py_XDECREF(co_code); + Py_XDECREF(lines); + return codeobj; +} diff --git a/Objects/frame_layout.md b/Objects/frame_layout.md index 11688f68e42e3d..2f95214db56498 100644 --- a/Objects/frame_layout.md +++ b/Objects/frame_layout.md @@ -9,10 +9,10 @@ results in poor locality of reference. In 3.11, rather than have these frames scattered about memory, as happens for heap-allocated objects, frames are allocated -contiguously in a per-thread stack. +contiguously in a per-thread stack. This improves performance significantly for two reasons: * It reduces allocation overhead to a pointer comparison and increment. -* Stack allocated data has the best possible locality and will always be in +* Stack allocated data has the best possible locality and will always be in CPU cache. Generator and coroutines still need heap allocated activation records, but @@ -63,7 +63,7 @@ We may implement this in the future. > In a contiguous stack, we would need to save one fewer registers, as the > top of the caller's activation record would be the same at the base of the -> callee's. However, since some activation records are kept on the heap we +> callee's. However, since some activation records are kept on the heap we > cannot do this. ### Generators and Coroutines @@ -85,7 +85,7 @@ and builtins, than strong references to both globals and builtins. ### Frame objects When creating a backtrace or when calling `sys._getframe()` the frame becomes -visible to Python code. When this happens a new `PyFrameObject` is created +visible to Python code. When this happens a new `PyFrameObject` is created and a strong reference to it placed in the `frame_obj` field of the specials section. The `frame_obj` field is initially `NULL`. @@ -104,7 +104,7 @@ Generator objects have a `_PyInterpreterFrame` embedded in them. This means that creating a generator requires only a single allocation, reducing allocation overhead and improving locality of reference. The embedded frame is linked into the per-thread frame when iterated or -awaited. +awaited. If a frame object associated with a generator outlives the generator, then the embedded `_PyInterpreterFrame` is copied into the frame object. @@ -119,4 +119,14 @@ Thus, some of the field names may be a bit misleading. For example the `f_globals` field has a `f_` prefix implying it belongs to the `PyFrameObject` struct, although it belongs to the `_PyInterpreterFrame` struct. -We may rationalize this naming scheme for 3.12. \ No newline at end of file +We may rationalize this naming scheme for 3.12. + + +### Shim frames + +On entry to `_PyEval_EvalFrameDefault()` a shim `_PyInterpreterFrame` is pushed. +This frame is stored on the C stack, and popped when `_PyEval_EvalFrameDefault()` +returns. This extra frame is inserted so that `RETURN_VALUE`, `YIELD_VALUE`, and +`RETURN_GENERATOR` do not need to check whether the current frame is the entry frame. +The shim frame points to a special code object containing the `INTERPRETER_EXIT` +instruction which cleans up the shim frame and returns. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 6337501cfca884..9e2f1f474891b0 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1329,15 +1329,15 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) } } - -int _PyFrame_IsEntryFrame(PyFrameObject *frame) +int +_PyFrame_IsEntryFrame(PyFrameObject *frame) { assert(frame != NULL); - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - return frame->f_frame->is_entry; + _PyInterpreterFrame *f = frame->f_frame; + assert(!_PyFrame_IsIncomplete(f)); + return f->previous && f->previous->owner == FRAME_OWNED_BY_CSTACK; } - PyCodeObject * PyFrame_GetCode(PyFrameObject *frame) { diff --git a/Objects/genobject.c b/Objects/genobject.c index 44d9820ce665d0..3886e72add2dd2 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -207,8 +207,6 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, Py_INCREF(result); _PyFrame_StackPush(frame, result); - frame->previous = tstate->cframe->current_frame; - _PyErr_StackItem *prev_exc_info = tstate->exc_info; gen->gi_exc_state.previous_item = prev_exc_info; tstate->exc_info = &gen->gi_exc_state; @@ -223,14 +221,8 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, result = _PyEval_EvalFrame(tstate, frame, exc); assert(tstate->exc_info == prev_exc_info); assert(gen->gi_exc_state.previous_item == NULL); - if (gen->gi_frame_state == FRAME_EXECUTING) { - gen->gi_frame_state = FRAME_COMPLETED; - } - assert(tstate->cframe->current_frame == frame->previous); - /* Don't keep the reference to previous any longer than necessary. It - * may keep a chain of frames alive or it could create a reference - * cycle. */ - frame->previous = NULL; + assert(gen->gi_frame_state != FRAME_EXECUTING); + assert(frame->previous == NULL); /* If the generator just returned (as opposed to yielding), signal * that the generator is exhausted. */ @@ -255,8 +247,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, /* first clean reference cycle through stored exception traceback */ _PyErr_ClearExcState(&gen->gi_exc_state); - gen->gi_frame_state = FRAME_CLEARED; - _PyFrame_Clear(frame); + assert(gen->gi_frame_state == FRAME_CLEARED); *presult = result; return result ? PYGEN_RETURN : PYGEN_ERROR; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c1f416e6bb8df5..ed12b94e3142c1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -97,8 +97,7 @@ dummy_func( PyObject *consts, _Py_CODEUNIT *next_instr, PyObject **stack_pointer, - CallShape call_shape, - _Py_CODEUNIT *first_instr, + PyObject *kwnames, int throwflag, binaryfunc binary_ops[] ) @@ -618,32 +617,34 @@ dummy_func( } // stack effect: (__0 -- ) - inst(RETURN_VALUE) { + inst(INTERPRETER_EXIT) { + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); PyObject *retval = POP(); assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); - _Py_LeaveRecursiveCallPy(tstate); - if (!frame->is_entry) { - frame = cframe.current_frame = pop_frame(tstate, frame); - _PyFrame_StackPush(frame, retval); - goto resume_frame; - } - _Py_LeaveRecursiveCallTstate(tstate); - if (frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - } /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallTstate(tstate); return retval; } + // stack effect: (__0 -- ) + inst(RETURN_VALUE) { + PyObject *retval = POP(); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + TRACE_FUNCTION_EXIT(); + DTRACE_FUNCTION_EXIT(); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + frame = cframe.current_frame = pop_frame(tstate, frame); + _PyFrame_StackPush(frame, retval); + goto resume_frame; + } + // stack effect: ( -- ) inst(GET_AITER) { unaryfunc getter = NULL; @@ -775,6 +776,7 @@ dummy_func( // error: SEND stack effect depends on jump flag inst(SEND) { + assert(frame != &entry_frame); assert(STACK_LEVEL() >= 2); PyObject *v = POP(); PyObject *receiver = TOP(); @@ -839,6 +841,7 @@ dummy_func( // The compiler treats any exception raised here as a failed close() // or throw() call. assert(oparg == STACK_LEVEL()); + assert(frame != &entry_frame); PyObject *retval = POP(); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -848,19 +851,12 @@ dummy_func( tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); - if (!frame->is_entry) { - frame = cframe.current_frame = frame->previous; - frame->prev_instr -= frame->yield_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - } - _Py_LeaveRecursiveCallTstate(tstate); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; - assert(tstate->cframe->current_frame == frame->previous); - assert(!_PyErr_Occurred(tstate)); - return retval; + _PyInterpreterFrame *gen_frame = frame; + frame = cframe.current_frame = frame->previous; + gen_frame->previous = NULL; + frame->prev_instr -= frame->yield_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; } // stack effect: (__0 -- ) @@ -876,7 +872,7 @@ dummy_func( if (oparg) { PyObject *lasti = PEEK(oparg + 1); if (PyLong_Check(lasti)) { - frame->prev_instr = first_instr + PyLong_AsLong(lasti); + frame->prev_instr = _PyCode_CODE(frame->f_code) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); } else { @@ -2696,12 +2692,10 @@ dummy_func( gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; gen_frame->previous = frame; - gen_frame->is_entry = false; frame = cframe.current_frame = gen_frame; goto start_frame; } - // stack effect: ( -- __0) inst(BEFORE_ASYNC_WITH) { PyObject *mgr = TOP(); @@ -2929,9 +2923,9 @@ dummy_func( // stack effect: ( -- ) inst(KW_NAMES) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(consts)); - call_shape.kwnames = GETITEM(consts, oparg); + kwnames = GETITEM(consts, oparg); } // stack effect: (__0, __array[oparg] -- ) @@ -2943,7 +2937,7 @@ dummy_func( int nargs = oparg + is_meth; PyObject *callable = PEEK(nargs + 1); next_instr--; - _Py_Specialize_Call(callable, next_instr, nargs, call_shape.kwnames); + _Py_Specialize_Call(callable, next_instr, nargs, kwnames); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); @@ -2972,9 +2966,9 @@ dummy_func( STACK_SHRINK(total_args); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)function, locals, - stack_pointer, positional_args, call_shape.kwnames + stack_pointer, positional_args, kwnames ); - call_shape.kwnames = NULL; + kwnames = NULL; STACK_SHRINK(2-is_meth); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -2994,15 +2988,15 @@ dummy_func( if (cframe.use_tracing) { res = trace_call_function( tstate, function, stack_pointer-total_args, - positional_args, call_shape.kwnames); + positional_args, kwnames); } else { res = PyObject_Vectorcall( function, stack_pointer-total_args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - call_shape.kwnames); + kwnames); } - call_shape.kwnames = NULL; + kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(function); /* Clear the stack */ @@ -3021,7 +3015,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_PY_EXACT_ARGS) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); _PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = is_method(stack_pointer, oparg); @@ -3054,7 +3048,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_PY_WITH_DEFAULTS) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); _PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = is_method(stack_pointer, oparg); @@ -3094,7 +3088,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_TYPE_1) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); @@ -3112,7 +3106,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_STR_1) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); @@ -3134,7 +3128,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_TUPLE_1) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *callable = PEEK(2); @@ -3166,8 +3160,8 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_CALL); STACK_SHRINK(total_args); PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, - total_args-kwnames_len, call_shape.kwnames); - call_shape.kwnames = NULL; + total_args-kwnames_len, kwnames); + kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(stack_pointer[i]); @@ -3185,7 +3179,7 @@ dummy_func( inst(CALL_NO_KW_BUILTIN_O) { assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; DEOPT_IF(total_args != 1, CALL); @@ -3219,7 +3213,7 @@ dummy_func( inst(CALL_NO_KW_BUILTIN_FAST) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyObject *callable = PEEK(total_args + 1); @@ -3276,10 +3270,10 @@ dummy_func( PyCFunction_GET_SELF(callable), stack_pointer, total_args - KWNAMES_LEN(), - call_shape.kwnames + kwnames ); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - call_shape.kwnames = NULL; + kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < total_args; i++) { @@ -3297,7 +3291,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_LEN) { assert(cframe.use_tracing == 0); - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); /* len(o) */ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3327,7 +3321,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_ISINSTANCE) { assert(cframe.use_tracing == 0); - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3360,7 +3354,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_LIST_APPEND) { assert(cframe.use_tracing == 0); - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(oparg == 1); PyObject *callable = PEEK(3); PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -3382,7 +3376,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_METHOD_DESCRIPTOR_O) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyMethodDescrObject *callable = @@ -3435,9 +3429,9 @@ dummy_func( _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), - call_shape.kwnames); + kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - call_shape.kwnames = NULL; + kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < nargs; i++) { @@ -3455,7 +3449,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3489,7 +3483,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyMethodDescrObject *callable = @@ -3606,25 +3600,12 @@ dummy_func( gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - if (!frame->is_entry) { - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = cframe.current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - goto resume_frame; - } - _Py_LeaveRecursiveCallTstate(tstate); - /* Make sure that frame is in a valid state */ - frame->stacktop = 0; - frame->f_locals = NULL; - Py_INCREF(frame->f_funcobj); - Py_INCREF(frame->f_code); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; - assert(tstate->cframe->current_frame == frame->previous); - assert(!_PyErr_Occurred(tstate)); - return (PyObject *)gen; + assert(frame != &entry_frame); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = cframe.current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + goto resume_frame; } // error: BUILD_SLICE has irregular stack effect diff --git a/Python/ceval.c b/Python/ceval.c index 8462cb1ba9195b..bff8b5c4d488cb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -155,7 +155,10 @@ static void lltrace_resume_frame(_PyInterpreterFrame *frame) { PyObject *fobj = frame->f_funcobj; - if (fobj == NULL || !PyFunction_Check(fobj)) { + if (frame->owner == FRAME_OWNED_BY_CSTACK || + fobj == NULL || + !PyFunction_Check(fobj) + ) { printf("\nResuming frame."); return; } @@ -733,13 +736,13 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* Code access macros */ /* The integer overflow is checked by an assertion below. */ -#define INSTR_OFFSET() ((int)(next_instr - first_instr)) +#define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(frame->f_code))) #define NEXTOPARG() do { \ _Py_CODEUNIT word = *next_instr; \ opcode = _Py_OPCODE(word); \ oparg = _Py_OPARG(word); \ } while (0) -#define JUMPTO(x) (next_instr = first_instr + (x)) +#define JUMPTO(x) (next_instr = _PyCode_CODE(frame->f_code) + (x)) #define JUMPBY(x) (next_instr += (x)) /* OpCode prediction macros @@ -1037,18 +1040,11 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { } -/* It is only between the KW_NAMES instruction and the following CALL, - * that this has any meaning. - */ -typedef struct { - PyObject *kwnames; -} CallShape; - // GH-89279: Must be a macro to be sure it's inlined by MSVC. #define is_method(stack_pointer, args) (PEEK((args)+2) != NULL) #define KWNAMES_LEN() \ - (call_shape.kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(call_shape.kwnames))) + (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames))) PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) @@ -1074,8 +1070,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif _PyCFrame cframe; - CallShape call_shape; - call_shape.kwnames = NULL; // Borrowed reference. Reset by CALL instructions. + _PyInterpreterFrame entry_frame; + PyObject *kwnames = NULL; // Borrowed reference. Reset by CALL instructions. /* WARNING: Because the _PyCFrame lives on the C stack, * but can be accessed from a heap allocated object (tstate) @@ -1086,9 +1082,24 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int cframe.previous = prev_cframe; tstate->cframe = &cframe; - frame->is_entry = true; + assert(tstate->interp->interpreter_trampoline != NULL); +#ifdef Py_DEBUG + /* Set these to invalid but identifiable values for debugging. */ + entry_frame.f_funcobj = (PyObject*)0xaaa0; + entry_frame.f_locals = (PyObject*)0xaaa1; + entry_frame.frame_obj = (PyFrameObject*)0xaaa2; + entry_frame.f_globals = (PyObject*)0xaaa3; + entry_frame.f_builtins = (PyObject*)0xaaa4; +#endif + entry_frame.f_code = tstate->interp->interpreter_trampoline; + entry_frame.prev_instr = + _PyCode_CODE(tstate->interp->interpreter_trampoline); + entry_frame.stacktop = 0; + entry_frame.owner = FRAME_OWNED_BY_CSTACK; + entry_frame.yield_offset = 0; /* Push frame */ - frame->previous = prev_cframe->current_frame; + entry_frame.previous = prev_cframe->current_frame; + frame->previous = &entry_frame; cframe.current_frame = frame; if (_Py_EnterRecursiveCallTstate(tstate, "")) { @@ -1112,7 +1123,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject *names; PyObject *consts; - _Py_CODEUNIT *first_instr; _Py_CODEUNIT *next_instr; PyObject **stack_pointer; @@ -1122,7 +1132,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyCodeObject *co = frame->f_code; \ names = co->co_names; \ consts = co->co_consts; \ - first_instr = _PyCode_CODE(co); \ } \ assert(_PyInterpreterFrame_LASTI(frame) >= -1); \ /* Jump back to the last instruction executed... */ \ @@ -1147,14 +1156,16 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef LLTRACE { - int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__)); - if (r < 0) { - goto exit_unwind; + if (frame != &entry_frame) { + int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__)); + if (r < 0) { + goto exit_unwind; + } + lltrace = r; + } + if (lltrace) { + lltrace_resume_frame(frame); } - lltrace = r; - } - if (lltrace) { - lltrace_resume_frame(frame); } #endif @@ -1313,7 +1324,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int pop_1_error: STACK_SHRINK(1); error: - call_shape.kwnames = NULL; + kwnames = NULL; /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -1325,6 +1336,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif /* Log traceback info. */ + assert(frame != &entry_frame); if (!_PyFrame_IsIncomplete(frame)) { PyFrameObject *f = _PyFrame_GetFrameObject(frame); if (f != NULL) { @@ -1397,12 +1409,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int exit_unwind: assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - if (frame->is_entry) { - if (frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - } + assert(frame != &entry_frame); + frame = cframe.current_frame = pop_frame(tstate, frame); + if (frame == &entry_frame) { /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; tstate->cframe->use_tracing = cframe.use_tracing; @@ -1410,7 +1419,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_LeaveRecursiveCallTstate(tstate); return NULL; } - frame = cframe.current_frame = pop_frame(tstate, frame); resume_with_error: SET_LOCALS_FROM_FRAME(); @@ -2038,13 +2046,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, return NULL; } EVAL_CALL_STAT_INC(EVAL_CALL_VECTOR); - PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0); - assert( - _PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame) || - _PyFrame_GetStackPointer(frame) == frame->localsplus - ); - _PyEvalFrameClearAndPop(tstate, frame); - return retval; + return _PyEval_EvalFrame(tstate, frame, 0); } /* Legacy API */ diff --git a/Python/compile.c b/Python/compile.c index 177409adeb6193..c71563f81609a2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1256,6 +1256,8 @@ stack_effect(int opcode, int oparg, int jump) return 1; case BINARY_OP: return -1; + case INTERPRETER_EXIT: + return -1; default: return PY_INVALID_STACK_EFFECT; } diff --git a/Python/frame.c b/Python/frame.c index 89f084b110cb5a..52f6ef428291c5 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -80,6 +80,7 @@ _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) static void take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) { + assert(frame->owner != FRAME_OWNED_BY_CSTACK); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); assert(frame->owner != FRAME_CLEARED); Py_ssize_t size = ((char*)&frame->localsplus[frame->stacktop]) - (char *)frame; @@ -99,7 +100,9 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) while (prev && _PyFrame_IsIncomplete(prev)) { prev = prev->previous; } + frame->previous = NULL; if (prev) { + assert(prev->owner != FRAME_OWNED_BY_CSTACK); /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrame.previous */ PyFrameObject *back = _PyFrame_GetFrameObject(prev); if (back == NULL) { @@ -111,7 +114,6 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) else { f->f_back = (PyFrameObject *)Py_NewRef(back); } - frame->previous = NULL; } if (!_PyObject_GC_IS_TRACKED((PyObject *)f)) { _PyObject_GC_TRACK((PyObject *)f); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d09384e7433819..0487851db6106e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -611,32 +611,33 @@ goto error; } - TARGET(RETURN_VALUE) { + TARGET(INTERPRETER_EXIT) { + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); PyObject *retval = POP(); assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); - _Py_LeaveRecursiveCallPy(tstate); - if (!frame->is_entry) { - frame = cframe.current_frame = pop_frame(tstate, frame); - _PyFrame_StackPush(frame, retval); - goto resume_frame; - } - _Py_LeaveRecursiveCallTstate(tstate); - if (frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - } /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallTstate(tstate); return retval; } + TARGET(RETURN_VALUE) { + PyObject *retval = POP(); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + TRACE_FUNCTION_EXIT(); + DTRACE_FUNCTION_EXIT(); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + frame = cframe.current_frame = pop_frame(tstate, frame); + _PyFrame_StackPush(frame, retval); + goto resume_frame; + } + TARGET(GET_AITER) { unaryfunc getter = NULL; PyObject *iter = NULL; @@ -768,6 +769,7 @@ } TARGET(SEND) { + assert(frame != &entry_frame); assert(STACK_LEVEL() >= 2); PyObject *v = POP(); PyObject *receiver = TOP(); @@ -832,6 +834,7 @@ // The compiler treats any exception raised here as a failed close() // or throw() call. assert(oparg == STACK_LEVEL()); + assert(frame != &entry_frame); PyObject *retval = POP(); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -841,19 +844,12 @@ tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); - if (!frame->is_entry) { - frame = cframe.current_frame = frame->previous; - frame->prev_instr -= frame->yield_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - } - _Py_LeaveRecursiveCallTstate(tstate); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; - assert(tstate->cframe->current_frame == frame->previous); - assert(!_PyErr_Occurred(tstate)); - return retval; + _PyInterpreterFrame *gen_frame = frame; + frame = cframe.current_frame = frame->previous; + gen_frame->previous = NULL; + frame->prev_instr -= frame->yield_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; } TARGET(POP_EXCEPT) { @@ -868,7 +864,7 @@ if (oparg) { PyObject *lasti = PEEK(oparg + 1); if (PyLong_Check(lasti)) { - frame->prev_instr = first_instr + PyLong_AsLong(lasti); + frame->prev_instr = _PyCode_CODE(frame->f_code) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); } else { @@ -2690,7 +2686,6 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; gen_frame->previous = frame; - gen_frame->is_entry = false; frame = cframe.current_frame = gen_frame; goto start_frame; } @@ -2919,9 +2914,9 @@ } TARGET(KW_NAMES) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(consts)); - call_shape.kwnames = GETITEM(consts, oparg); + kwnames = GETITEM(consts, oparg); DISPATCH(); } @@ -2934,7 +2929,7 @@ int nargs = oparg + is_meth; PyObject *callable = PEEK(nargs + 1); next_instr--; - _Py_Specialize_Call(callable, next_instr, nargs, call_shape.kwnames); + _Py_Specialize_Call(callable, next_instr, nargs, kwnames); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); @@ -2963,9 +2958,9 @@ STACK_SHRINK(total_args); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)function, locals, - stack_pointer, positional_args, call_shape.kwnames + stack_pointer, positional_args, kwnames ); - call_shape.kwnames = NULL; + kwnames = NULL; STACK_SHRINK(2-is_meth); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -2985,15 +2980,15 @@ if (cframe.use_tracing) { res = trace_call_function( tstate, function, stack_pointer-total_args, - positional_args, call_shape.kwnames); + positional_args, kwnames); } else { res = PyObject_Vectorcall( function, stack_pointer-total_args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - call_shape.kwnames); + kwnames); } - call_shape.kwnames = NULL; + kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(function); /* Clear the stack */ @@ -3013,7 +3008,7 @@ TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); _PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = is_method(stack_pointer, oparg); @@ -3045,7 +3040,7 @@ } TARGET(CALL_PY_WITH_DEFAULTS) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); _PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = is_method(stack_pointer, oparg); @@ -3084,7 +3079,7 @@ } TARGET(CALL_NO_KW_TYPE_1) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); @@ -3102,7 +3097,7 @@ } TARGET(CALL_NO_KW_STR_1) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); @@ -3124,7 +3119,7 @@ } TARGET(CALL_NO_KW_TUPLE_1) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *callable = PEEK(2); @@ -3156,8 +3151,8 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); STACK_SHRINK(total_args); PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, - total_args-kwnames_len, call_shape.kwnames); - call_shape.kwnames = NULL; + total_args-kwnames_len, kwnames); + kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(stack_pointer[i]); @@ -3175,7 +3170,7 @@ TARGET(CALL_NO_KW_BUILTIN_O) { assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; DEOPT_IF(total_args != 1, CALL); @@ -3209,7 +3204,7 @@ TARGET(CALL_NO_KW_BUILTIN_FAST) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyObject *callable = PEEK(total_args + 1); @@ -3266,10 +3261,10 @@ PyCFunction_GET_SELF(callable), stack_pointer, total_args - KWNAMES_LEN(), - call_shape.kwnames + kwnames ); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - call_shape.kwnames = NULL; + kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < total_args; i++) { @@ -3287,7 +3282,7 @@ TARGET(CALL_NO_KW_LEN) { assert(cframe.use_tracing == 0); - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); /* len(o) */ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3317,7 +3312,7 @@ TARGET(CALL_NO_KW_ISINSTANCE) { assert(cframe.use_tracing == 0); - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3350,7 +3345,7 @@ TARGET(CALL_NO_KW_LIST_APPEND) { assert(cframe.use_tracing == 0); - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(oparg == 1); PyObject *callable = PEEK(3); PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -3372,7 +3367,7 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyMethodDescrObject *callable = @@ -3425,9 +3420,9 @@ _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), - call_shape.kwnames); + kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - call_shape.kwnames = NULL; + kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < nargs; i++) { @@ -3445,7 +3440,7 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3479,7 +3474,7 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { - assert(call_shape.kwnames == NULL); + assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyMethodDescrObject *callable = @@ -3597,25 +3592,12 @@ gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - if (!frame->is_entry) { - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = cframe.current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - goto resume_frame; - } - _Py_LeaveRecursiveCallTstate(tstate); - /* Make sure that frame is in a valid state */ - frame->stacktop = 0; - frame->f_locals = NULL; - Py_INCREF(frame->f_funcobj); - Py_INCREF(frame->f_code); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; - assert(tstate->cframe->current_frame == frame->previous); - assert(!_PyErr_Occurred(tstate)); - return (PyObject *)gen; + assert(frame != &entry_frame); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = cframe.current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + goto resume_frame; } TARGET(BUILD_SLICE) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 0603b44184b4c4..237d3b946b1066 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -2,19 +2,20 @@ static void *opcode_targets[256] = { &&TARGET_CACHE, &&TARGET_POP_TOP, &&TARGET_PUSH_NULL, - &&TARGET_BINARY_OP_ADD_FLOAT, + &&TARGET_INTERPRETER_EXIT, &&TARGET_END_FOR, + &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, &&TARGET_BINARY_OP_GENERIC, - &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_NOP, &&TARGET_UNARY_POSITIVE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_BINARY_OP_MULTIPLY_FLOAT, - &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_SUBSCR_DICT, @@ -23,20 +24,20 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_CALL_BUILTIN_CLASS, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, @@ -47,7 +48,6 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,37 +55,37 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_COMPARE_OP_GENERIC, &&TARGET_COMPARE_OP_INT_JUMP, - &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_FOR_ITER_LIST, + &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_STOPITERATION_ERROR, + &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_ATTR_CLASS, - &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,27 +152,27 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&_unknown_opcode, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 51192bd5bb9b88..13519762fa87bf 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -29,6 +29,7 @@ #include "pycore_tuple.h" // _PyTuple_InitTypes() #include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() +#include "opcode.h" extern void _PyIO_Fini(void); @@ -779,6 +780,21 @@ pycore_init_types(PyInterpreterState *interp) return _PyStatus_OK(); } +static const uint8_t INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { + /* Put a NOP at the start, so that the IP points into + * the code, rather than before it */ + NOP, 0, + INTERPRETER_EXIT, 0, + /* RESUME at end makes sure that the frame appears incomplete */ + RESUME, 0 +}; + +static const _PyShimCodeDef INTERPRETER_TRAMPOLINE_CODEDEF = { + INTERPRETER_TRAMPOLINE_INSTRUCTIONS, + sizeof(INTERPRETER_TRAMPOLINE_INSTRUCTIONS), + 1, + "" +}; static PyStatus pycore_init_builtins(PyThreadState *tstate) @@ -812,7 +828,10 @@ pycore_init_builtins(PyThreadState *tstate) PyObject *object__getattribute__ = _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__getattribute__)); assert(object__getattribute__); interp->callable_cache.object__getattribute__ = object__getattribute__; - + interp->interpreter_trampoline = _Py_MakeShimCode(&INTERPRETER_TRAMPOLINE_CODEDEF); + if (interp->interpreter_trampoline == NULL) { + return _PyStatus_ERR("failed to create interpreter trampoline."); + } if (_PyBuiltins_AddExceptions(bimod) < 0) { return _PyStatus_ERR("failed to add exceptions to builtins"); } diff --git a/Python/pystate.c b/Python/pystate.c index b5fd8c2b7ae297..04db1fb419af62 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -454,6 +454,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) PyDict_Clear(interp->builtins); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + Py_CLEAR(interp->interpreter_trampoline); for (int i=0; i < DICT_MAX_WATCHERS; i++) { interp->dict_watchers[i] = NULL; diff --git a/Python/traceback.c b/Python/traceback.c index 2d9da0e692f10f..aacdb33d39b8a2 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1224,6 +1224,15 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) if (frame == NULL) { break; } + if (frame->owner == FRAME_OWNED_BY_CSTACK) { + /* Trampoline frame */ + frame = frame->previous; + } + if (frame == NULL) { + break; + } + /* Can't have more than one shim frame in a row */ + assert(frame->owner != FRAME_OWNED_BY_CSTACK); depth++; } } diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 303409cb0077d1..c003c1ab4a23bd 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -82,6 +82,8 @@ def _sizeof_void_p(): Py_TPFLAGS_BASE_EXC_SUBCLASS = (1 << 30) Py_TPFLAGS_TYPE_SUBCLASS = (1 << 31) +#From pycore_frame.h +FRAME_OWNED_BY_CSTACK = 3 MAX_OUTPUT_LEN=1024 @@ -1077,8 +1079,8 @@ def _f_lasti(self): first_instr = self._f_code().field("co_code_adaptive").cast(codeunit_p) return int(prev_instr - first_instr) - def is_entry(self): - return self._f_special("is_entry", bool) + def is_shim(self): + return self._f_special("owner", int) == FRAME_OWNED_BY_CSTACK def previous(self): return self._f_special("previous", PyFramePtr) @@ -1821,14 +1823,14 @@ def print_summary(self): interp_frame = self.get_pyop() while True: if interp_frame: + if interp_frame.is_shim(): + break line = interp_frame.get_truncated_repr(MAX_OUTPUT_LEN) sys.stdout.write('#%i %s\n' % (self.get_index(), line)) if not interp_frame.is_optimized_out(): line = interp_frame.current_line() if line is not None: sys.stdout.write(' %s\n' % line.strip()) - if interp_frame.is_entry(): - break else: sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index()) break @@ -1845,13 +1847,13 @@ def print_traceback(self): interp_frame = self.get_pyop() while True: if interp_frame: + if interp_frame.is_shim(): + break interp_frame.print_traceback() if not interp_frame.is_optimized_out(): line = interp_frame.current_line() if line is not None: sys.stdout.write(' %s\n' % line.strip()) - if interp_frame.is_entry(): - break else: sys.stdout.write(' (unable to read python frame information)\n') break @@ -2106,6 +2108,8 @@ def invoke(self, args, from_tty): while True: if not pyop_frame: print(UNABLE_READ_INFO_PYTHON_FRAME) + if pyop_frame.is_shim(): + break sys.stdout.write('Locals for %s\n' % (pyop_frame.co_name.proxyval(set()))) @@ -2114,8 +2118,6 @@ def invoke(self, args, from_tty): % (pyop_name.proxyval(set()), pyop_value.get_truncated_repr(MAX_OUTPUT_LEN))) - if pyop_frame.is_entry(): - break pyop_frame = pyop_frame.previous()