diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index de7db70275441a..f64f88dcd2c6f3 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2355,8 +2355,15 @@ def adjust_int_max_str_digits(max_digits): #For recursion tests, easily exceeds default recursion limit EXCEEDS_RECURSION_LIMIT = 5000 -# The default C recursion limit (from Include/cpython/pystate.h). -Py_C_RECURSION_LIMIT = 1500 +def _get_c_recursion_limit(): + try: + import _testcapi + return _testcapi.Py_C_RECURSION_LIMIT + except (ImportError, AttributeError): + return 1500 # (from Include/cpython/pystate.h) + +# The default C recursion limit. +Py_C_RECURSION_LIMIT = _get_c_recursion_limit() #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 475d7492040b7b..ea0c11621a2ca9 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -118,17 +118,19 @@ def run_cases_test(self, input: str, expected: str): with open(self.temp_output_filename) as temp_output: lines = temp_output.readlines() - while lines and lines[0].startswith("// "): + while lines and lines[0].startswith(("// ", "#", " #", "\n")): lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) actual = "".join(lines) - # if actual.rstrip() != expected.rstrip(): + # if actual.strip() != expected.strip(): # print("Actual:") # print(actual) # print("Expected:") # print(expected) # print("End") - self.assertEqual(actual.rstrip(), expected.rstrip()) + self.assertEqual(actual.strip(), expected.strip()) def test_inst_no_args(self): input = """ diff --git a/Makefile.pre.in b/Makefile.pre.in index f2b252cce9775c..2950d87700ed5e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -412,7 +412,6 @@ PYTHON_OBJS= \ Python/crossinterp.o \ Python/dynamic_annotations.o \ Python/errors.o \ - Python/executor.o \ Python/flowgraph.o \ Python/frame.o \ Python/frozenmain.o \ @@ -1603,12 +1602,6 @@ Python/ceval.o: \ $(srcdir)/Python/generated_cases.c.h \ $(srcdir)/Python/opcode_targets.h -Python/executor.o: \ - $(srcdir)/Include/internal/pycore_opcode_metadata.h \ - $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Python/ceval_macros.h \ - $(srcdir)/Python/executor_cases.c.h - Python/flowgraph.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-31-21-33-35.gh-issue-111520.vw-rxJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-31-21-33-35.gh-issue-111520.vw-rxJ.rst new file mode 100644 index 00000000000000..67ca9c6671f99b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-31-21-33-35.gh-issue-111520.vw-rxJ.rst @@ -0,0 +1,9 @@ +Merge the Tier 1 (bytecode) and Tier 2 (micro-ops) interpreters together, +moving the Tier 2 interpreter loop and switch into +``_PyEval_EvalFrameDefault()`` in ``Python/ceval.c``. +The ``Python/executor.c`` file is gone. +Also the ``TIER_ONE`` and ``TIER_TWO`` macros are now handled +by the code generator. + +**Beware!** This changes the environment variables to enable micro-ops and +their debugging to ``PYTHON_UOPS`` and ``PYTHON_LLTRACE``. diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 20d800a6959552..e4ce3fefc91008 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -200,7 +200,6 @@ - diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 40256a22fb5562..8e987cdf2b86e0 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -127,9 +127,6 @@ Source Files - - Source Files - Source Files diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index f4640454a73070..8b733865962373 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -95,7 +95,7 @@ Console 2000000 - 8000000 + 12000000 diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 4d3621a6146807..eca2671b0cc090 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -552,7 +552,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 69d8e0312e0169..447bb266d7587e 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1253,9 +1253,6 @@ Python - - Python - Python diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4c9ce7daa9daa1..98af114a72fa60 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -63,6 +63,7 @@ static size_t jump; static uint16_t invert, counter, index, hint; #define unused 0 // Used in a macro def, can't be static static uint32_t type_version; +static _PyUOpExecutorObject *current_executor; static PyObject * dummy_func( @@ -171,7 +172,7 @@ dummy_func( uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - goto error; + GOTO_ERROR(error); } next_instr = this_instr; } @@ -268,7 +269,7 @@ dummy_func( if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - goto error; + GOTO_ERROR(error); } PyErr_SetRaisedException(NULL); } @@ -284,7 +285,7 @@ dummy_func( if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - goto error; + GOTO_ERROR(error); } PyErr_SetRaisedException(NULL); } @@ -826,7 +827,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) goto error; + if (err) GOTO_ERROR(error); STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -850,7 +851,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) goto error; + if (err) GOTO_ERROR(error); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -906,7 +907,7 @@ dummy_func( if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - goto error; + GOTO_ERROR(error); } } else { if (type->tp_as_async != NULL){ @@ -916,7 +917,7 @@ dummy_func( if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - goto error; + GOTO_ERROR(error); } } else { @@ -924,7 +925,7 @@ dummy_func( "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - goto error; + GOTO_ERROR(error); } awaitable = _PyCoro_GetAwaitableIter(next_iter); @@ -936,7 +937,7 @@ dummy_func( Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - goto error; + GOTO_ERROR(error); } else { Py_DECREF(next_iter); } @@ -1019,7 +1020,7 @@ dummy_func( JUMPBY(oparg); } else { - goto error; + GOTO_ERROR(error); } } Py_DECREF(v); @@ -1054,7 +1055,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, retval); - if (err) goto error; + if (err) GOTO_ERROR(error); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1108,7 +1109,7 @@ dummy_func( else { assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - goto error; + GOTO_ERROR(error); } } assert(exc && PyExceptionInstance_Check(exc)); @@ -1187,7 +1188,7 @@ dummy_func( if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - goto error; + GOTO_ERROR(error); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -1195,7 +1196,7 @@ dummy_func( _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + GOTO_ERROR(error); } } @@ -1319,7 +1320,7 @@ dummy_func( _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } - goto error; + GOTO_ERROR(error); } } @@ -1336,7 +1337,7 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); @@ -1344,17 +1345,17 @@ dummy_func( Py_INCREF(v); } else if (_PyErr_Occurred(tstate)) { - goto error; + GOTO_ERROR(error); } else { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + GOTO_ERROR(error); } } } @@ -1370,7 +1371,7 @@ dummy_func( } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); @@ -1378,17 +1379,17 @@ dummy_func( Py_INCREF(v); } else if (_PyErr_Occurred(tstate)) { - goto error; + GOTO_ERROR(error); } else { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + GOTO_ERROR(error); } } } @@ -1510,7 +1511,7 @@ dummy_func( PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - goto error; + GOTO_ERROR(error); } SETLOCAL(oparg, cell); } @@ -1522,7 +1523,7 @@ dummy_func( // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + GOTO_ERROR(error); } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -1535,7 +1536,7 @@ dummy_func( name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { Py_DECREF(class_dict); - goto error; + GOTO_ERROR(error); } Py_DECREF(class_dict); if (!value) { @@ -1543,7 +1544,7 @@ dummy_func( value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + GOTO_ERROR(error); } Py_INCREF(value); } @@ -1622,7 +1623,7 @@ dummy_func( inst(BUILD_SET, (values[oparg] -- set)) { set = PySet_New(NULL); if (set == NULL) - goto error; + GOTO_ERROR(error); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; @@ -1690,7 +1691,7 @@ dummy_func( PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, "bad BUILD_CONST_KEY_MAP keys argument"); - goto error; // Pop the keys and values. + GOTO_ERROR(error); // Pop the keys and values. } map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, @@ -2363,13 +2364,16 @@ dummy_func( JUMPBY(1-original_oparg); frame->instr_ptr = next_instr; Py_INCREF(executor); + if (executor->execute == _PyUopExecute) { + current_executor = (_PyUOpExecutorObject *)executor; + GOTO_TIER_TWO(); + } frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { frame = tstate->current_frame; goto resume_with_error; } - next_instr = frame->instr_ptr; - goto resume_frame; + goto enter_tier_one; } inst(POP_JUMP_IF_FALSE, (unused/1, cond -- )) { @@ -2469,7 +2473,7 @@ dummy_func( _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - goto error; + GOTO_ERROR(error); } iter = iterable; } @@ -2480,7 +2484,7 @@ dummy_func( /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - goto error; + GOTO_ERROR(error); } DECREF_INPUTS(); } @@ -2518,7 +2522,7 @@ dummy_func( if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; + GOTO_ERROR(error); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2548,7 +2552,7 @@ dummy_func( else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; + GOTO_ERROR(error); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2743,7 +2747,7 @@ dummy_func( "asynchronous context manager protocol", Py_TYPE(mgr)->tp_name); } - goto error; + GOTO_ERROR(error); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); if (exit == NULL) { @@ -2755,7 +2759,7 @@ dummy_func( Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - goto error; + GOTO_ERROR(error); } DECREF_INPUTS(); res = _PyObject_CallNoArgs(enter); @@ -2779,7 +2783,7 @@ dummy_func( "context manager protocol", Py_TYPE(mgr)->tp_name); } - goto error; + GOTO_ERROR(error); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); if (exit == NULL) { @@ -2791,7 +2795,7 @@ dummy_func( Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - goto error; + GOTO_ERROR(error); } DECREF_INPUTS(); res = _PyObject_CallNoArgs(enter); @@ -3041,7 +3045,7 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + GOTO_ERROR(error); } frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -3258,7 +3262,7 @@ dummy_func( STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { - goto error; + GOTO_ERROR(error); } Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -3295,7 +3299,7 @@ dummy_func( PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - goto error; + GOTO_ERROR(error); } } @@ -3334,7 +3338,7 @@ dummy_func( // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + GOTO_ERROR(error); } PyObject *arg = args[0]; res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); @@ -3419,7 +3423,7 @@ dummy_func( PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - goto error; + GOTO_ERROR(error); } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3444,7 +3448,7 @@ dummy_func( PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - goto error; + GOTO_ERROR(error); } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3494,7 +3498,7 @@ dummy_func( // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); @@ -3554,7 +3558,7 @@ dummy_func( // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); @@ -3641,7 +3645,7 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + GOTO_ERROR(error); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -3689,11 +3693,11 @@ dummy_func( assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { - goto error; + GOTO_ERROR(error); } PyObject *tuple = PySequence_Tuple(callargs); if (tuple == NULL) { - goto error; + GOTO_ERROR(error); } Py_SETREF(callargs, tuple); } @@ -3707,7 +3711,7 @@ dummy_func( int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) goto error; + if (err) GOTO_ERROR(error); result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( @@ -3738,7 +3742,7 @@ dummy_func( // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { - goto error; + GOTO_ERROR(error); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -3759,7 +3763,7 @@ dummy_func( Py_DECREF(codeobj); if (func_obj == NULL) { - goto error; + GOTO_ERROR(error); } _PyFunction_SetVersion( @@ -3799,7 +3803,7 @@ dummy_func( PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { - goto error; + GOTO_ERROR(error); } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3988,24 +3992,25 @@ dummy_func( op(_POP_JUMP_IF_FALSE, (flag -- )) { if (Py_IsFalse(flag)) { - pc = oparg; + next_uop = current_executor->trace + oparg; } } op(_POP_JUMP_IF_TRUE, (flag -- )) { if (Py_IsTrue(flag)) { - pc = oparg; + next_uop = current_executor->trace + oparg; } } op(_JUMP_TO_TOP, (--)) { - pc = 0; + next_uop = current_executor->trace; CHECK_EVAL_BREAKER(); } op(_SET_IP, (--)) { TIER_TWO_ONLY - frame->instr_ptr = ip_offset + oparg; + // TODO: Put the code pointer in `operand` to avoid indirection via `frame` + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + oparg; } op(_SAVE_RETURN_OFFSET, (--)) { @@ -4019,10 +4024,7 @@ dummy_func( op(_EXIT_TRACE, (--)) { TIER_TWO_ONLY - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - return frame; + GOTO_TIER_ONE(); } op(_INSERT, (unused[oparg], top -- top, unused[oparg])) { diff --git a/Python/ceval.c b/Python/ceval.c index bada365011226c..6e156ab2009468 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -202,15 +202,15 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, _PyInterpreterFrame *skip if (r < 0) { return -1; } - int lltrace = r; + int lltrace = r * 5; // Levels 1-4 only trace uops if (!lltrace) { - // When tracing executed uops, also trace bytecode - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = (*uop_debug - '0') >= 5; // TODO: Parse an int and all that + // Can also be controlled by environment variable + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } } - if (lltrace) { + if (lltrace >= 5) { lltrace_resume_frame(frame); } return lltrace; @@ -611,10 +611,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) return _PyEval_EvalFrame(tstate, f->f_frame, throwflag); } -#define TIER_ONE 1 #include "ceval_macros.h" - int _Py_CheckRecursiveCallPy( PyThreadState *tstate) { @@ -680,9 +678,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_STATS int lastopcode = 0; #endif - // opcode is an 8-bit value to improve the code generated by MSVC - // for the big switch below (in combination with the EXTRA_CASES macro). - uint8_t opcode; /* Current opcode */ + int opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ #ifdef LLTRACE int lltrace = 0; @@ -730,6 +726,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto resume_with_error; } + /* State shared between Tier 1 and Tier 2 interpreter */ + _PyUOpExecutorObject *current_executor = NULL; + /* Local "register" variables. * These are cached values from the frame and code object. */ @@ -766,7 +765,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Start instructions */ #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + // Cast to an 8-bit value to improve the code generated by MSVC + // (in combination with the EXTRA_CASES macro). + switch ((uint8_t)opcode) #endif { @@ -914,7 +915,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } /* Resume normal execution */ #ifdef LLTRACE - if (lltrace) { + if (lltrace >= 5) { lltrace_resume_frame(frame); } #endif @@ -943,6 +944,135 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int stack_pointer = _PyFrame_GetStackPointer(frame); goto error; + + +// The Tier 2 interpreter is also here! +enter_tier_two: + +#undef LOAD_IP +#define LOAD_IP(UNUSED) (void)0 + +#undef GOTO_ERROR +#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two + +#undef DEOPT_IF +#define DEOPT_IF(COND, INSTNAME) \ + if ((COND)) { \ + goto deoptimize;\ + } + +#ifdef Py_STATS +// Disable these macros that apply to Tier 1 stats when we are in Tier 2 +#undef STAT_INC +#define STAT_INC(opname, name) ((void)0) +#undef STAT_DEC +#define STAT_DEC(opname, name) ((void)0) +#undef CALL_STAT_INC +#define CALL_STAT_INC(name) ((void)0) +#endif + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION 0 + +#ifdef Py_DEBUG + #define DPRINTF(level, ...) \ + if (lltrace >= (level)) { printf(__VA_ARGS__); } +#else + #define DPRINTF(level, ...) +#endif + + OPT_STAT_INC(traces_executed); + _PyUOpInstruction *next_uop = current_executor->trace; +#ifdef Py_DEBUG + uint64_t operand; // Used by several DPRINTF() calls +#endif +#ifdef Py_STATS + uint64_t trace_uop_execution_counter = 0; +#endif + + for (;;) { + opcode = next_uop->opcode; + oparg = next_uop->oparg; +#ifdef Py_DEBUG + operand = next_uop->operand; +#endif + DPRINTF(3, + "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n", + (int)(next_uop - current_executor->trace), + opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode], + oparg, + operand, + (int)(stack_pointer - _PyFrame_Stackbase(frame))); + next_uop++; + OPT_STAT_INC(uops_executed); +#ifdef Py_STATS + trace_uop_execution_counter++; +#endif + + switch (opcode) { + +#include "executor_cases.c.h" + + default: +#ifdef Py_DEBUG + { + fprintf(stderr, "Unknown uop %d, oparg %d, operand %" PRIu64 "\n", + opcode, oparg, operand); + Py_FatalError("Unknown uop"); + } +#else + Py_UNREACHABLE(); +#endif + + } + } + +// Jump here from ERROR_IF(..., unbound_local_error) +unbound_local_error_tier_two: + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + goto error_tier_two; + +// JUMP to any of these from ERROR_IF(..., error) +pop_4_error_tier_two: + STACK_SHRINK(1); +pop_3_error_tier_two: + STACK_SHRINK(1); +pop_2_error_tier_two: + STACK_SHRINK(1); +pop_1_error_tier_two: + STACK_SHRINK(1); +error_tier_two: + DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + frame->return_offset = 0; // Don't leave this random + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(current_executor); + goto resume_with_error; + +// Jump here from DEOPT_IF() +deoptimize: + // On DEOPT_IF we just repeat the last instruction. + // This presumes nothing was popped from the stack (nor pushed). + DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + frame->return_offset = 0; // Dispatch to frame->instr_ptr + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(current_executor); + // Fall through +// Jump here from ENTER_EXECUTOR +enter_tier_one: + next_instr = frame->instr_ptr; + goto resume_frame; + +// Jump here from _EXIT_TRACE +exit_trace: + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(current_executor); + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + goto enter_tier_one; } #if defined(__GNUC__) # pragma GCC diagnostic pop diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b11240b1faed49..546adbe5f438d1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -1,4 +1,4 @@ -// Macros and other things needed by ceval.c, executor.c, and bytecodes.c +// Macros and other things needed by ceval.c, and bytecodes.c /* Computed GOTOs, or the-optimization-commonly-but-improperly-known-as-"threaded code" @@ -80,7 +80,7 @@ /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef LLTRACE -#define PRE_DISPATCH_GOTO() if (lltrace) { \ +#define PRE_DISPATCH_GOTO() if (lltrace >= 5) { \ lltrace_instruction(frame, stack_pointer, next_instr); } #else #define PRE_DISPATCH_GOTO() ((void)0) @@ -112,11 +112,14 @@ goto start_frame; \ } while (0) +// Use this instead of 'goto error' so Tier 2 can go to a different label +#define GOTO_ERROR(LABEL) goto LABEL + #define CHECK_EVAL_BREAKER() \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ if (_Py_HandlePending(tstate) != 0) { \ - goto error; \ + GOTO_ERROR(error); \ } \ } @@ -322,7 +325,7 @@ do { \ }\ else { \ result = PyFloat_FromDouble(dval); \ - if ((result) == NULL) goto error; \ + if ((result) == NULL) GOTO_ERROR(error); \ _Py_DECREF_NO_DEALLOC(left); \ _Py_DECREF_NO_DEALLOC(right); \ } \ @@ -374,27 +377,14 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { /* Implementation of "macros" that modify the instruction pointer, * stack pointer, or frame pointer. - * These need to treated differently by tier 1 and 2. */ - -#if TIER_ONE + * These need to treated differently by tier 1 and 2. + * The Tier 1 version is here; Tier 2 is inlined in ceval.c. */ #define LOAD_IP(OFFSET) do { \ next_instr = frame->instr_ptr + (OFFSET); \ } while (0) -#define STORE_SP() \ -_PyFrame_SetStackPointer(frame, stack_pointer) - -#define LOAD_SP() \ -stack_pointer = _PyFrame_GetStackPointer(frame); - -#endif - - -#if TIER_TWO - -#define LOAD_IP(UNUSED) \ -do { ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; } while (0) +/* There's no STORE_IP(), it's inlined by the code generator. */ #define STORE_SP() \ _PyFrame_SetStackPointer(frame, stack_pointer) @@ -402,8 +392,8 @@ _PyFrame_SetStackPointer(frame, stack_pointer) #define LOAD_SP() \ stack_pointer = _PyFrame_GetStackPointer(frame); -#endif - - +/* Tier-switching macros. */ +#define GOTO_TIER_TWO() goto enter_tier_two; +#define GOTO_TIER_ONE() goto exit_trace; diff --git a/Python/executor.c b/Python/executor.c deleted file mode 100644 index 2884565689171c..00000000000000 --- a/Python/executor.c +++ /dev/null @@ -1,149 +0,0 @@ -#include "Python.h" - -#include "opcode.h" - -#include "pycore_bitutils.h" -#include "pycore_call.h" -#include "pycore_ceval.h" -#include "pycore_dict.h" -#include "pycore_emscripten_signal.h" -#include "pycore_intrinsics.h" -#include "pycore_long.h" -#include "pycore_object.h" -#include "pycore_opcode_metadata.h" -#include "pycore_opcode_utils.h" -#include "pycore_pyerrors.h" -#include "pycore_range.h" -#include "pycore_setobject.h" // _PySet_Update() -#include "pycore_sliceobject.h" -#include "pycore_uops.h" - -#define TIER_TWO 2 -#include "ceval_macros.h" - - -#undef DEOPT_IF -#define DEOPT_IF(COND, INSTNAME) \ - if ((COND)) { \ - UOP_STAT_INC(INSTNAME, miss); \ - goto deoptimize; \ - } - -#ifdef Py_STATS -// Disable these macros that apply to Tier 1 stats when we are in Tier 2 -#undef STAT_INC -#define STAT_INC(opname, name) ((void)0) -#undef STAT_DEC -#define STAT_DEC(opname, name) ((void)0) -#undef CALL_STAT_INC -#define CALL_STAT_INC(name) ((void)0) -#endif - -#undef ENABLE_SPECIALIZATION -#define ENABLE_SPECIALIZATION 0 - - -_PyInterpreterFrame * -_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) -{ -#ifdef Py_DEBUG - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); - int lltrace = 0; - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = *uop_debug - '0'; // TODO: Parse an int and all that - } - #define DPRINTF(level, ...) \ - if (lltrace >= (level)) { printf(__VA_ARGS__); } -#else - #define DPRINTF(level, ...) -#endif - - DPRINTF(3, - "Entering _PyUopExecute for %s (%s:%d) at byte offset %ld\n", - PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname), - PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename), - _PyFrame_GetCode(frame)->co_firstlineno, - 2 * (long)(frame->instr_ptr - - (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive)); - - PyThreadState *tstate = _PyThreadState_GET(); - _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor; - - CHECK_EVAL_BREAKER(); - - OPT_STAT_INC(traces_executed); - _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; - int pc = 0; - int opcode; - int oparg; - uint64_t operand; -#ifdef Py_STATS - uint64_t trace_uop_execution_counter = 0; -#endif - - for (;;) { - opcode = self->trace[pc].opcode; - oparg = self->trace[pc].oparg; - operand = self->trace[pc].operand; - DPRINTF(3, - "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n", - pc, - opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode], - oparg, - operand, - (int)(stack_pointer - _PyFrame_Stackbase(frame))); - pc++; - OPT_STAT_INC(uops_executed); - UOP_STAT_INC(opcode, execution_count); -#ifdef Py_STATS - trace_uop_execution_counter++; -#endif - - switch (opcode) { - -#include "executor_cases.c.h" - - default: - { - fprintf(stderr, "Unknown uop %d, operand %" PRIu64 "\n", opcode, operand); - Py_FatalError("Unknown uop"); - } - - } - } - -unbound_local_error: - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - goto error; - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - // On ERROR_IF we return NULL as the frame. - // The caller recovers the frame from tstate->current_frame. - DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - frame->return_offset = 0; // Don't leave this random - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return NULL; - -deoptimize: - // On DEOPT_IF we just repeat the last instruction. - // This presumes nothing was popped from the stack (nor pushed). - DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - frame->return_offset = 0; // Dispatch to frame->instr_ptr - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return frame; -} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3477733c8c4402..fdf3e4b01a88b3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3,6 +3,11 @@ // Python/bytecodes.c // Do not edit! +#ifdef TIER_ONE + #error "This file is for Tier 2 only" +#endif +#define TIER_TWO 2 + case NOP: { break; } @@ -22,7 +27,7 @@ case LOAD_FAST_CHECK: { PyObject *value; value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; + if (value == NULL) goto unbound_local_error_tier_two; Py_INCREF(value); STACK_GROW(1); stack_pointer[-1] = value; @@ -99,7 +104,7 @@ value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); - if (res == NULL) goto pop_1_error; + if (res == NULL) goto pop_1_error_tier_two; stack_pointer[-1] = res; break; } @@ -120,7 +125,7 @@ value = stack_pointer[-1]; int err = PyObject_IsTrue(value); Py_DECREF(value); - if (err < 0) goto pop_1_error; + if (err < 0) goto pop_1_error_tier_two; res = err ? Py_True : Py_False; stack_pointer[-1] = res; break; @@ -199,7 +204,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - uint32_t version = (uint32_t)operand; + uint32_t version = (uint32_t)next_uop[-1].operand; // This one is a bit weird, because we expect *some* failures: assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); @@ -216,7 +221,7 @@ value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); - if (res == NULL) goto pop_1_error; + if (res == NULL) goto pop_1_error_tier_two; stack_pointer[-1] = res; break; } @@ -241,7 +246,7 @@ res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + if (res == NULL) goto pop_2_error_tier_two; STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -257,7 +262,7 @@ res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + if (res == NULL) goto pop_2_error_tier_two; STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -273,7 +278,7 @@ res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + if (res == NULL) goto pop_2_error_tier_two; STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -357,7 +362,7 @@ res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; + if (res == NULL) goto pop_2_error_tier_two; STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -372,7 +377,7 @@ res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (res == NULL) goto pop_2_error; + if (res == NULL) goto pop_2_error_tier_two; STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -397,7 +402,7 @@ Py_DECREF(slice); } Py_DECREF(container); - if (res == NULL) goto pop_3_error; + if (res == NULL) goto pop_3_error_tier_two; STACK_SHRINK(2); stack_pointer[-1] = res; break; @@ -423,7 +428,7 @@ } Py_DECREF(v); Py_DECREF(container); - if (err) goto pop_4_error; + if (err) goto pop_4_error_tier_two; STACK_SHRINK(4); break; } @@ -514,7 +519,7 @@ } Py_DECREF(dict); Py_DECREF(sub); - if (true) goto pop_2_error; + if (true) goto pop_2_error_tier_two; } Py_INCREF(res); // Do this before DECREF'ing dict, sub Py_DECREF(dict); @@ -529,7 +534,7 @@ PyObject *list; v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error_tier_two; STACK_SHRINK(1); break; } @@ -541,7 +546,7 @@ set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); - if (err) goto pop_1_error; + if (err) goto pop_1_error_tier_two; STACK_SHRINK(1); break; } @@ -558,7 +563,7 @@ Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - if (err) goto pop_3_error; + if (err) goto pop_3_error_tier_two; STACK_SHRINK(3); break; } @@ -601,7 +606,7 @@ STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); - if (err) goto pop_3_error; + if (err) goto pop_3_error_tier_two; STACK_SHRINK(3); break; } @@ -615,7 +620,7 @@ int err = PyObject_DelItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (err) goto pop_2_error; + if (err) goto pop_2_error_tier_two; STACK_SHRINK(2); break; } @@ -627,7 +632,7 @@ assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); - if (res == NULL) goto pop_1_error; + if (res == NULL) goto pop_1_error_tier_two; stack_pointer[-1] = res; break; } @@ -642,7 +647,7 @@ res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); Py_DECREF(value1); - if (res == NULL) goto pop_2_error; + if (res == NULL) goto pop_2_error_tier_two; STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -691,12 +696,12 @@ "__aiter__ method, got %.100s", type->tp_name); Py_DECREF(obj); - if (true) goto pop_1_error; + if (true) goto pop_1_error_tier_two; } iter = (*getter)(obj); Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; + if (iter == NULL) goto pop_1_error_tier_two; if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { @@ -706,7 +711,7 @@ "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); Py_DECREF(iter); - if (true) goto pop_1_error; + if (true) goto pop_1_error_tier_two; } stack_pointer[-1] = iter; break; @@ -723,7 +728,7 @@ if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - goto error; + GOTO_ERROR(error); } } else { if (type->tp_as_async != NULL){ @@ -733,7 +738,7 @@ if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - goto error; + GOTO_ERROR(error); } } else { @@ -741,7 +746,7 @@ "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - goto error; + GOTO_ERROR(error); } awaitable = _PyCoro_GetAwaitableIter(next_iter); @@ -753,7 +758,7 @@ Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - goto error; + GOTO_ERROR(error); } else { Py_DECREF(next_iter); } @@ -789,7 +794,7 @@ } } - if (iter == NULL) goto pop_1_error; + if (iter == NULL) goto pop_1_error_tier_two; stack_pointer[-1] = iter; break; } @@ -813,11 +818,11 @@ case LOAD_BUILD_CLASS: { PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error_tier_two; if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - if (true) goto error; + if (true) goto error_tier_two; } STACK_GROW(1); stack_pointer[-1] = bc; @@ -834,14 +839,14 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); Py_DECREF(v); - if (true) goto pop_1_error; + if (true) goto pop_1_error_tier_two; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); Py_DECREF(v); - if (err) goto pop_1_error; + if (err) goto pop_1_error_tier_two; STACK_SHRINK(1); break; } @@ -853,7 +858,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - goto error; + GOTO_ERROR(error); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -861,7 +866,7 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + GOTO_ERROR(error); } break; } @@ -869,7 +874,7 @@ case _SPECIALIZE_UNPACK_SEQUENCE: { PyObject *seq; seq = stack_pointer[-1]; - uint16_t counter = (uint16_t)operand; + uint16_t counter = (uint16_t)next_uop[-1].operand; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -888,7 +893,7 @@ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); - if (res == 0) goto pop_1_error; + if (res == 0) goto pop_1_error_tier_two; STACK_SHRINK(1); STACK_GROW(oparg); break; @@ -954,7 +959,7 @@ PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); - if (res == 0) goto pop_1_error; + if (res == 0) goto pop_1_error_tier_two; STACK_GROW((oparg & 0xFF) + (oparg >> 8)); break; } @@ -968,7 +973,7 @@ int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); - if (err) goto pop_2_error; + if (err) goto pop_2_error_tier_two; STACK_SHRINK(2); break; } @@ -979,7 +984,7 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); - if (err) goto pop_1_error; + if (err) goto pop_1_error_tier_two; STACK_SHRINK(1); break; } @@ -990,7 +995,7 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); - if (err) goto pop_1_error; + if (err) goto pop_1_error_tier_two; STACK_SHRINK(1); break; } @@ -1005,7 +1010,7 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } - goto error; + GOTO_ERROR(error); } break; } @@ -1016,7 +1021,7 @@ if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error; + if (true) goto error_tier_two; } Py_INCREF(locals); STACK_GROW(1); @@ -1030,7 +1035,7 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); @@ -1038,17 +1043,17 @@ Py_INCREF(v); } else if (_PyErr_Occurred(tstate)) { - goto error; + GOTO_ERROR(error); } else { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + GOTO_ERROR(error); } } } @@ -1063,11 +1068,11 @@ if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error; + if (true) goto error_tier_two; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); @@ -1075,17 +1080,17 @@ Py_INCREF(v); } else if (_PyErr_Occurred(tstate)) { - goto error; + GOTO_ERROR(error); } else { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + GOTO_ERROR(error); } } } @@ -1111,22 +1116,22 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } - if (true) goto error; + if (true) goto error_tier_two; } Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error_tier_two; if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error_tier_two; if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - if (true) goto error; + if (true) goto error_tier_two; } } } @@ -1139,7 +1144,7 @@ } case _GUARD_GLOBALS_VERSION: { - uint16_t version = (uint16_t)operand; + uint16_t version = (uint16_t)next_uop[-1].operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); @@ -1148,7 +1153,7 @@ } case _GUARD_BUILTINS_VERSION: { - uint16_t version = (uint16_t)operand; + uint16_t version = (uint16_t)next_uop[-1].operand; PyDictObject *dict = (PyDictObject *)BUILTINS(); DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); @@ -1159,7 +1164,7 @@ case _LOAD_GLOBAL_MODULE: { PyObject *res; PyObject *null = NULL; - uint16_t index = (uint16_t)operand; + uint16_t index = (uint16_t)next_uop[-1].operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; @@ -1177,7 +1182,7 @@ case _LOAD_GLOBAL_BUILTINS: { PyObject *res; PyObject *null = NULL; - uint16_t index = (uint16_t)operand; + uint16_t index = (uint16_t)next_uop[-1].operand; PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; @@ -1194,7 +1199,7 @@ case DELETE_FAST: { PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; + if (v == NULL) goto unbound_local_error_tier_two; SETLOCAL(oparg, NULL); break; } @@ -1205,7 +1210,7 @@ PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - goto error; + GOTO_ERROR(error); } SETLOCAL(oparg, cell); break; @@ -1218,7 +1223,7 @@ // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + GOTO_ERROR(error); } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -1235,7 +1240,7 @@ name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { Py_DECREF(class_dict); - goto error; + GOTO_ERROR(error); } Py_DECREF(class_dict); if (!value) { @@ -1243,7 +1248,7 @@ value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + GOTO_ERROR(error); } Py_INCREF(value); } @@ -1257,7 +1262,7 @@ value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error; + if (true) goto error_tier_two; } Py_INCREF(value); STACK_GROW(1); @@ -1298,7 +1303,7 @@ for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) { STACK_SHRINK(oparg); goto error; } + if (str == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1310,7 +1315,7 @@ PyObject *tup; values = stack_pointer - oparg; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error; } + if (tup == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1322,7 +1327,7 @@ PyObject *list; values = stack_pointer - oparg; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error; } + if (list == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1345,7 +1350,7 @@ Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); - if (true) goto pop_1_error; + if (true) goto pop_1_error_tier_two; } assert(Py_IsNone(none_val)); Py_DECREF(iterable); @@ -1360,7 +1365,7 @@ set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); - if (err < 0) goto pop_1_error; + if (err < 0) goto pop_1_error_tier_two; STACK_SHRINK(1); break; } @@ -1371,7 +1376,7 @@ values = stack_pointer - oparg; set = PySet_New(NULL); if (set == NULL) - goto error; + GOTO_ERROR(error); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; @@ -1381,7 +1386,7 @@ } if (err != 0) { Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error; } + if (true) { STACK_SHRINK(oparg); goto error_tier_two; } } STACK_SHRINK(oparg); STACK_GROW(1); @@ -1400,7 +1405,7 @@ for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } + if (map == NULL) { STACK_SHRINK(oparg*2); goto error_tier_two; } STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -1413,33 +1418,33 @@ if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - if (true) goto error; + if (true) goto error_tier_two; } /* check if __annotations__ in locals()... */ if (PyDict_CheckExact(LOCALS())) { ann_dict = _PyDict_GetItemWithError(LOCALS(), &_Py_ID(__annotations__)); if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) goto error; + if (_PyErr_Occurred(tstate)) goto error_tier_two; /* ...if not, create a new one */ ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; + if (ann_dict == NULL) goto error_tier_two; err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err) goto error; + if (err) goto error_tier_two; } } else { /* do the same if locals() is not a dict */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error_tier_two; if (ann_dict == NULL) { ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; + if (ann_dict == NULL) goto error_tier_two; err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err) goto error; + if (err) goto error_tier_two; } else { Py_DECREF(ann_dict); @@ -1458,7 +1463,7 @@ PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, "bad BUILD_CONST_KEY_MAP keys argument"); - goto error; // Pop the keys and values. + GOTO_ERROR(error); // Pop the keys and values. } map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, @@ -1467,7 +1472,7 @@ Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } + if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error_tier_two; } STACK_SHRINK(oparg); stack_pointer[-1] = map; break; @@ -1485,7 +1490,7 @@ Py_TYPE(update)->tp_name); } Py_DECREF(update); - if (true) goto pop_1_error; + if (true) goto pop_1_error_tier_two; } Py_DECREF(update); STACK_SHRINK(1); @@ -1502,7 +1507,7 @@ if (_PyDict_MergeEx(dict, update, 2) < 0) { _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); - if (true) goto pop_1_error; + if (true) goto pop_1_error_tier_two; } Py_DECREF(update); STACK_SHRINK(1); @@ -1519,7 +1524,7 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error_tier_two; STACK_SHRINK(2); break; } @@ -1541,7 +1546,7 @@ Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - if (attr == NULL) goto pop_3_error; + if (attr == NULL) goto pop_3_error_tier_two; STACK_SHRINK(2); stack_pointer[-1] = attr; break; @@ -1569,7 +1574,7 @@ Py_DECREF(class); if (attr == NULL) { Py_DECREF(self); - if (true) goto pop_3_error; + if (true) goto pop_3_error_tier_two; } if (method_found) { self_or_null = self; // transfer ownership @@ -1608,7 +1613,7 @@ NULL | meth | arg1 | ... | argN */ Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; + if (attr == NULL) goto pop_1_error_tier_two; self_or_null = NULL; } } @@ -1616,7 +1621,7 @@ /* Classic, pushes one value. */ attr = PyObject_GetAttr(owner, name); Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; + if (attr == NULL) goto pop_1_error_tier_two; } STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; @@ -1627,7 +1632,7 @@ case _GUARD_TYPE_VERSION: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; + uint32_t type_version = (uint32_t)next_uop[-1].operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); @@ -1649,7 +1654,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)operand; + uint16_t index = (uint16_t)next_uop[-1].operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); @@ -1666,7 +1671,7 @@ case _CHECK_ATTR_MODULE: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; + uint32_t type_version = (uint32_t)next_uop[-1].operand; DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -1679,7 +1684,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)operand; + uint16_t index = (uint16_t)next_uop[-1].operand; PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); @@ -1713,7 +1718,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)operand; + uint16_t hint = (uint16_t)next_uop[-1].operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT); @@ -1744,7 +1749,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)operand; + uint16_t index = (uint16_t)next_uop[-1].operand; char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); @@ -1761,7 +1766,7 @@ case _CHECK_ATTR_CLASS: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; + uint32_t type_version = (uint32_t)next_uop[-1].operand; DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS); assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS); @@ -1773,7 +1778,7 @@ PyObject *attr; PyObject *null = NULL; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)next_uop[-1].operand; STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -1799,7 +1804,7 @@ PyObject *value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)operand; + uint16_t index = (uint16_t)next_uop[-1].operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); PyDictValues *values = _PyDictOrValues_GetValues(dorv); @@ -1821,7 +1826,7 @@ PyObject *value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)operand; + uint16_t index = (uint16_t)next_uop[-1].operand; char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -1842,11 +1847,11 @@ res = PyObject_RichCompare(left, right, oparg >> 5); Py_DECREF(left); Py_DECREF(right); - if (res == NULL) goto pop_2_error; + if (res == NULL) goto pop_2_error_tier_two; if (oparg & 16) { int res_bool = PyObject_IsTrue(res); Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; + if (res_bool < 0) goto pop_2_error_tier_two; res = res_bool ? Py_True : Py_False; } STACK_SHRINK(1); @@ -1949,7 +1954,7 @@ int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) goto pop_2_error; + if (res < 0) goto pop_2_error_tier_two; b = (res ^ oparg) ? Py_True : Py_False; STACK_SHRINK(1); stack_pointer[-1] = b; @@ -1966,7 +1971,7 @@ if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); - if (true) goto pop_2_error; + if (true) goto pop_2_error_tier_two; } match = NULL; @@ -1975,10 +1980,10 @@ &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); - if (res < 0) goto pop_2_error; + if (res < 0) goto pop_2_error_tier_two; assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; + if (match == NULL) goto pop_2_error_tier_two; if (!Py_IsNone(match)) { PyErr_SetHandledException(match); @@ -1997,7 +2002,7 @@ assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); - if (true) goto pop_1_error; + if (true) goto pop_1_error_tier_two; } int res = PyErr_GivenExceptionMatches(left, right); @@ -2028,9 +2033,9 @@ obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error; + if (len_i < 0) goto error_tier_two; len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; + if (len_o == NULL) goto error_tier_two; STACK_GROW(1); stack_pointer[-1] = len_o; break; @@ -2055,7 +2060,7 @@ assert(PyTuple_CheckExact(attrs)); // Success! } else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; + if (_PyErr_Occurred(tstate)) goto pop_3_error_tier_two; attrs = Py_None; // Failure! } STACK_SHRINK(2); @@ -2093,7 +2098,7 @@ subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; + if (values_or_none == NULL) goto error_tier_two; STACK_GROW(1); stack_pointer[-1] = values_or_none; break; @@ -2106,7 +2111,7 @@ /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; + if (iter == NULL) goto pop_1_error_tier_two; stack_pointer[-1] = iter; break; } @@ -2124,7 +2129,7 @@ _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - goto error; + GOTO_ERROR(error); } iter = iterable; } @@ -2135,7 +2140,7 @@ /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - goto error; + GOTO_ERROR(error); } Py_DECREF(iterable); } @@ -2264,7 +2269,7 @@ r->start = value + r->step; r->len--; next = PyLong_FromLong(value); - if (next == NULL) goto error; + if (next == NULL) goto error_tier_two; STACK_GROW(1); stack_pointer[-1] = next; break; @@ -2283,7 +2288,7 @@ "asynchronous context manager protocol", Py_TYPE(mgr)->tp_name); } - goto error; + GOTO_ERROR(error); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); if (exit == NULL) { @@ -2295,14 +2300,14 @@ Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - goto error; + GOTO_ERROR(error); } Py_DECREF(mgr); res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); - if (true) goto pop_1_error; + if (true) goto pop_1_error_tier_two; } STACK_GROW(1); stack_pointer[-2] = exit; @@ -2342,7 +2347,7 @@ PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; + if (res == NULL) goto error_tier_two; STACK_GROW(1); stack_pointer[-1] = res; break; @@ -2379,7 +2384,7 @@ case _GUARD_KEYS_VERSION: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)operand; + uint32_t keys_version = (uint32_t)next_uop[-1].operand; PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); @@ -2391,7 +2396,7 @@ PyObject *attr; PyObject *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)next_uop[-1].operand; assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -2410,7 +2415,7 @@ PyObject *attr; PyObject *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)next_uop[-1].operand; assert(oparg & 1); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -2428,7 +2433,7 @@ PyObject *owner; PyObject *attr; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)next_uop[-1].operand; assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -2442,7 +2447,7 @@ PyObject *owner; PyObject *attr; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)next_uop[-1].operand; assert((oparg & 1) == 0); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -2469,7 +2474,7 @@ PyObject *attr; PyObject *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; + PyObject *descr = (PyObject *)next_uop[-1].operand; assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -2518,7 +2523,7 @@ PyObject *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)operand; + uint32_t func_version = (uint32_t)next_uop[-1].operand; DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); @@ -2623,7 +2628,7 @@ res = PyObject_Str(arg); Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -2647,7 +2652,7 @@ res = PySequence_Tuple(arg); Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -2663,7 +2668,7 @@ PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - goto error; + GOTO_ERROR(error); } STACK_SHRINK(1); break; @@ -2692,7 +2697,7 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -2722,7 +2727,7 @@ // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + GOTO_ERROR(error); } PyObject *arg = args[0]; res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); @@ -2731,7 +2736,7 @@ Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -2769,7 +2774,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } /* Not deopting because this doesn't mean our optimization was wrong. `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid'). In those cases an exception is set, so we must @@ -2811,7 +2816,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -2840,14 +2845,14 @@ PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - goto error; + GOTO_ERROR(error); } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -2876,7 +2881,7 @@ PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - goto error; + GOTO_ERROR(error); } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -2884,7 +2889,7 @@ Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -2917,7 +2922,7 @@ // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); @@ -2925,7 +2930,7 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -2965,7 +2970,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -2999,14 +3004,14 @@ // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3045,7 +3050,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3063,7 +3068,7 @@ Py_DECREF(codeobj); if (func_obj == NULL) { - goto error; + GOTO_ERROR(error); } _PyFunction_SetVersion( @@ -3119,7 +3124,7 @@ Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } + if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error_tier_two; } STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -3135,7 +3140,7 @@ conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); Py_DECREF(value); - if (result == NULL) goto pop_1_error; + if (result == NULL) goto pop_1_error_tier_two; stack_pointer[-1] = result; break; } @@ -3149,7 +3154,7 @@ if (!PyUnicode_CheckExact(value)) { res = PyObject_Format(value, NULL); Py_DECREF(value); - if (res == NULL) goto pop_1_error; + if (res == NULL) goto pop_1_error_tier_two; } else { res = value; @@ -3167,7 +3172,7 @@ res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; + if (res == NULL) goto pop_2_error_tier_two; STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -3194,7 +3199,7 @@ res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; + if (res == NULL) goto pop_2_error_tier_two; STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -3215,7 +3220,7 @@ PyObject *flag; flag = stack_pointer[-1]; if (Py_IsFalse(flag)) { - pc = oparg; + next_uop = current_executor->trace + oparg; } STACK_SHRINK(1); break; @@ -3225,21 +3230,22 @@ PyObject *flag; flag = stack_pointer[-1]; if (Py_IsTrue(flag)) { - pc = oparg; + next_uop = current_executor->trace + oparg; } STACK_SHRINK(1); break; } case _JUMP_TO_TOP: { - pc = 0; + next_uop = current_executor->trace; CHECK_EVAL_BREAKER(); break; } case _SET_IP: { TIER_TWO_ONLY - frame->instr_ptr = ip_offset + oparg; + // TODO: Put the code pointer in `operand` to avoid indirection via `frame` + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + oparg; break; } @@ -3255,10 +3261,7 @@ case _EXIT_TRACE: { TIER_TWO_ONLY - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - return frame; + GOTO_TIER_ONE(); break; } @@ -3270,3 +3273,5 @@ stack_pointer[-1 - oparg] = top; break; } + +#undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cb773f302710e7..e4b328fee99738 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3,6 +3,11 @@ // Python/bytecodes.c // Do not edit! +#ifdef TIER_TWO + #error "This file is for Tier 1 only" +#endif +#define TIER_ONE 1 + TARGET(NOP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -61,7 +66,7 @@ uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - goto error; + GOTO_ERROR(error); } next_instr = this_instr; } @@ -250,7 +255,7 @@ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - goto error; + GOTO_ERROR(error); } PyErr_SetRaisedException(NULL); } @@ -286,7 +291,7 @@ if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - goto error; + GOTO_ERROR(error); } PyErr_SetRaisedException(NULL); } @@ -1190,7 +1195,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) goto error; + if (err) GOTO_ERROR(error); STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1250,7 +1255,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) goto error; + if (err) GOTO_ERROR(error); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1320,7 +1325,7 @@ if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - goto error; + GOTO_ERROR(error); } } else { if (type->tp_as_async != NULL){ @@ -1330,7 +1335,7 @@ if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - goto error; + GOTO_ERROR(error); } } else { @@ -1338,7 +1343,7 @@ "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - goto error; + GOTO_ERROR(error); } awaitable = _PyCoro_GetAwaitableIter(next_iter); @@ -1350,7 +1355,7 @@ Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - goto error; + GOTO_ERROR(error); } else { Py_DECREF(next_iter); } @@ -1454,7 +1459,7 @@ JUMPBY(oparg); } else { - goto error; + GOTO_ERROR(error); } } Py_DECREF(v); @@ -1502,7 +1507,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, retval); - if (err) goto error; + if (err) GOTO_ERROR(error); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1575,7 +1580,7 @@ else { assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - goto error; + GOTO_ERROR(error); } } assert(exc && PyExceptionInstance_Check(exc)); @@ -1704,7 +1709,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - goto error; + GOTO_ERROR(error); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -1712,7 +1717,7 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + GOTO_ERROR(error); } DISPATCH(); } @@ -1907,7 +1912,7 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } - goto error; + GOTO_ERROR(error); } DISPATCH(); } @@ -1938,7 +1943,7 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); @@ -1946,17 +1951,17 @@ Py_INCREF(v); } else if (_PyErr_Occurred(tstate)) { - goto error; + GOTO_ERROR(error); } else { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + GOTO_ERROR(error); } } } @@ -1978,7 +1983,7 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { v = PyDict_GetItemWithError(GLOBALS(), name); @@ -1986,17 +1991,17 @@ Py_INCREF(v); } else if (_PyErr_Occurred(tstate)) { - goto error; + GOTO_ERROR(error); } else { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; + GOTO_ERROR(error); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + GOTO_ERROR(error); } } } @@ -2164,7 +2169,7 @@ PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - goto error; + GOTO_ERROR(error); } SETLOCAL(oparg, cell); DISPATCH(); @@ -2180,7 +2185,7 @@ // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + GOTO_ERROR(error); } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -2200,7 +2205,7 @@ name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { Py_DECREF(class_dict); - goto error; + GOTO_ERROR(error); } Py_DECREF(class_dict); if (!value) { @@ -2208,7 +2213,7 @@ value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - goto error; + GOTO_ERROR(error); } Py_INCREF(value); } @@ -2363,7 +2368,7 @@ values = stack_pointer - oparg; set = PySet_New(NULL); if (set == NULL) - goto error; + GOTO_ERROR(error); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; @@ -2459,7 +2464,7 @@ PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, "bad BUILD_CONST_KEY_MAP keys argument"); - goto error; // Pop the keys and values. + GOTO_ERROR(error); // Pop the keys and values. } map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, @@ -3432,13 +3437,16 @@ JUMPBY(1-original_oparg); frame->instr_ptr = next_instr; Py_INCREF(executor); + if (executor->execute == _PyUopExecute) { + current_executor = (_PyUOpExecutorObject *)executor; + GOTO_TIER_TWO(); + } frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { frame = tstate->current_frame; goto resume_with_error; } - next_instr = frame->instr_ptr; - goto resume_frame; + goto enter_tier_one; } TARGET(POP_JUMP_IF_FALSE) { @@ -3673,7 +3681,7 @@ _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - goto error; + GOTO_ERROR(error); } iter = iterable; } @@ -3684,7 +3692,7 @@ /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - goto error; + GOTO_ERROR(error); } Py_DECREF(iterable); } @@ -3723,7 +3731,7 @@ if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; + GOTO_ERROR(error); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -3758,7 +3766,7 @@ else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; + GOTO_ERROR(error); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -3941,7 +3949,7 @@ "asynchronous context manager protocol", Py_TYPE(mgr)->tp_name); } - goto error; + GOTO_ERROR(error); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); if (exit == NULL) { @@ -3953,7 +3961,7 @@ Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - goto error; + GOTO_ERROR(error); } Py_DECREF(mgr); res = _PyObject_CallNoArgs(enter); @@ -3988,7 +3996,7 @@ "context manager protocol", Py_TYPE(mgr)->tp_name); } - goto error; + GOTO_ERROR(error); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); if (exit == NULL) { @@ -4000,7 +4008,7 @@ Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - goto error; + GOTO_ERROR(error); } Py_DECREF(mgr); res = _PyObject_CallNoArgs(enter); @@ -4341,7 +4349,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + GOTO_ERROR(error); } frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -4708,7 +4716,7 @@ STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { - goto error; + GOTO_ERROR(error); } Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -4750,7 +4758,7 @@ PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - goto error; + GOTO_ERROR(error); } STACK_SHRINK(1); DISPATCH(); @@ -4815,7 +4823,7 @@ // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + GOTO_ERROR(error); } PyObject *arg = args[0]; res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); @@ -4942,7 +4950,7 @@ PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - goto error; + GOTO_ERROR(error); } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -4981,7 +4989,7 @@ PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - goto error; + GOTO_ERROR(error); } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -5053,7 +5061,7 @@ // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); @@ -5141,7 +5149,7 @@ // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; + GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); @@ -5263,7 +5271,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + GOTO_ERROR(error); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -5330,11 +5338,11 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { - goto error; + GOTO_ERROR(error); } PyObject *tuple = PySequence_Tuple(callargs); if (tuple == NULL) { - goto error; + GOTO_ERROR(error); } Py_SETREF(callargs, tuple); } @@ -5348,7 +5356,7 @@ int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) goto error; + if (err) GOTO_ERROR(error); result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( @@ -5379,7 +5387,7 @@ // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { - goto error; + GOTO_ERROR(error); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -5412,7 +5420,7 @@ Py_DECREF(codeobj); if (func_obj == NULL) { - goto error; + GOTO_ERROR(error); } _PyFunction_SetVersion( @@ -5467,7 +5475,7 @@ PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { - goto error; + GOTO_ERROR(error); } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5765,3 +5773,5 @@ assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } + +#undef TIER_ONE diff --git a/Python/optimizer.c b/Python/optimizer.c index c239b03053a896..0e5b4370ccb946 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -433,10 +433,10 @@ translate_bytecode_to_trace( int trace_stack_depth = 0; #ifdef Py_DEBUG - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); int lltrace = 0; - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif @@ -881,10 +881,10 @@ remove_unneeded_uops(_PyUOpInstruction *trace, int trace_length) if (dest < last_instr) { int new_trace_length = move_stubs(trace, dest, last_instr, trace_length); #ifdef Py_DEBUG - char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); int lltrace = 0; - if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } if (lltrace >= 2) { printf("Optimized trace (length %d+%d = %d, saved %d):\n", @@ -939,6 +939,15 @@ uop_optimize( return 1; } +/* Dummy execute() function for Uop Executor. + * The actual implementation is inlined in ceval.c, + * in _PyEval_EvalFrameDefault(). */ +_PyInterpreterFrame * +_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) +{ + Py_FatalError("Tier 2 is now inlined into Tier 1"); +} + static void uop_opt_dealloc(PyObject *self) { PyObject_Free(self); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 7b56034541756a..3c57056fb81e81 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1229,7 +1229,7 @@ init_interp_main(PyThreadState *tstate) // Turn on experimental tier 2 (uops-based) optimizer if (is_main_interp) { - char *envvar = Py_GETENV("PYTHONUOPS"); + char *envvar = Py_GETENV("PYTHON_UOPS"); int enabled = envvar != NULL && *envvar > '0'; if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) { enabled = 1; diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index f5853703481d8a..54bd51885d9785 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -758,6 +758,12 @@ def write_instructions( self.write_provenance_header() + self.out.write_raw("\n") + self.out.write_raw("#ifdef TIER_TWO\n") + self.out.write_raw(" #error \"This file is for Tier 1 only\"\n") + self.out.write_raw("#endif\n") + self.out.write_raw("#define TIER_ONE 1\n") + # Write and count instructions of all kinds n_macros = 0 for thing in self.everything: @@ -773,6 +779,9 @@ def write_instructions( case _: assert_never(thing) + self.out.write_raw("\n") + self.out.write_raw("#undef TIER_ONE\n") + print( f"Wrote {n_macros} cases to {output_filename}", file=sys.stderr, @@ -786,6 +795,13 @@ def write_executor_instructions( with open(executor_filename, "w") as f: self.out = Formatter(f, 8, emit_line_directives) self.write_provenance_header() + + self.out.write_raw("\n") + self.out.write_raw("#ifdef TIER_ONE\n") + self.out.write_raw(" #error \"This file is for Tier 2 only\"\n") + self.out.write_raw("#endif\n") + self.out.write_raw("#define TIER_TWO 2\n") + for instr in self.instrs.values(): if instr.is_viable_uop(): n_uops += 1 @@ -795,6 +811,10 @@ def write_executor_instructions( if instr.check_eval_breaker: self.out.emit("CHECK_EVAL_BREAKER();") self.out.emit("break;") + + self.out.write_raw("\n") + self.out.write_raw("#undef TIER_TWO\n") + print( f"Wrote {n_uops} cases to {executor_filename}", file=sys.stderr, diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 4ecb73921952ad..f3e4f76697853b 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -162,7 +162,7 @@ def write_body( f"{func}(&this_instr[{active.offset + 1}].cache);" ) else: - out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;") + out.emit(f"{typ}{ceffect.name} = ({typ.strip()})next_uop[-1].operand;") # Write the body, substituting a goto for ERROR_IF() and other stuff assert dedent <= 0 @@ -197,6 +197,8 @@ def write_body( ninputs, symbolic = list_effect_size(ieffs) if ninputs: label = f"pop_{ninputs}_{label}" + if tier == TIER_TWO: + label = label + "_tier_two" if symbolic: out.write_raw( f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n"