From 66ff374d4f353ae427c148d2a1d141d223303a82 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 15:56:36 -0700 Subject: [PATCH 01/50] gh-100227: Move func_state.next_version to PyInterpreterState (gh-102334) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_function.h | 2 +- Include/internal/pycore_interp.h | 1 + Include/internal/pycore_runtime.h | 2 -- Include/internal/pycore_runtime_init.h | 6 +++--- Objects/funcobject.c | 5 +++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 5cedb33d7e3afd..11988149843fef 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -10,7 +10,7 @@ extern "C" { #define FUNC_MAX_WATCHERS 8 -struct _py_func_runtime_state { +struct _py_func_state { uint32_t next_version; }; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 7ef9c40153e421..9efed0a1cf90c2 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -141,6 +141,7 @@ struct _is { struct _Py_float_state float_state; struct _Py_long_state long_state; struct _dtoa_state dtoa; + struct _py_func_state func_state; /* Using a cache is very effective since typically only a single slice is created and then deleted again. */ PySliceObject *slice_cache; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 2350eaab5976ca..e0e3d4ace0cfde 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -13,7 +13,6 @@ extern "C" { #include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state -#include "pycore_function.h" // struct _func_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState @@ -155,7 +154,6 @@ typedef struct pyruntimestate { struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; struct _Py_dict_runtime_state dict_state; - struct _py_func_runtime_state func_state; struct { /* Used to set PyTypeObject.tp_version_tag */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index a2cc7c87c2f3e2..aeabcfd2b9056c 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -68,9 +68,6 @@ extern PyTypeObject _PyExc_MemoryError; .dict_state = { \ .next_keys_version = 2, \ }, \ - .func_state = { \ - .next_version = 1, \ - }, \ .types = { \ .next_version_tag = 1, \ }, \ @@ -116,6 +113,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ + .func_state = { \ + .next_version = 1, \ + }, \ .static_objects = { \ .singletons = { \ ._not_used = 1, \ diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 99048ea41c6c80..ce5d7bda32c032 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -227,10 +227,11 @@ uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) if (func->vectorcall != _PyFunction_Vectorcall) { return 0; } - if (_PyRuntime.func_state.next_version == 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->func_state.next_version == 0) { return 0; } - uint32_t v = _PyRuntime.func_state.next_version++; + uint32_t v = interp->func_state.next_version++; func->func_version = v; return v; } From 5e5acd291f4387876afc641163e9f8ae5c65086c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 18:04:16 -0700 Subject: [PATCH 02/50] gh-100227: Move next_keys_version to PyInterpreterState (gh-102335) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_dict.h | 3 ++- Include/internal/pycore_dict_state.h | 4 +++- Include/internal/pycore_runtime_init.h | 6 +++--- Objects/dictobject.c | 7 ++++--- Python/specialize.c | 16 +++++++++++----- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 1af5e59a677a9a..12c3c708e29b95 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -37,7 +37,8 @@ extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); /* Gets a version number unique to the current state of the keys of dict, if possible. * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys); +extern uint32_t _PyDictKeys_GetVersionForCurrentState( + PyInterpreterState *interp, PyDictKeysObject *dictkeys); extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 77375ea8beb877..c5142ee6763a86 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -14,7 +14,6 @@ struct _Py_dict_runtime_state { * It is incremented each time that a dictionary is created and each * time that a dictionary is modified. */ uint64_t global_version; - uint32_t next_keys_version; }; @@ -30,6 +29,8 @@ struct _Py_dict_runtime_state { #define DICT_MAX_WATCHERS 8 struct _Py_dict_state { + uint32_t next_keys_version; + #if PyDict_MAXFREELIST > 0 /* Dictionary reuse scheme to save calls to malloc and free */ PyDictObject *free_list[PyDict_MAXFREELIST]; @@ -37,6 +38,7 @@ struct _Py_dict_state { int numfree; int keys_numfree; #endif + PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index aeabcfd2b9056c..efc82b43a61dab 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -65,9 +65,6 @@ extern PyTypeObject _PyExc_MemoryError; .float_format = _py_float_format_unknown, \ .double_format = _py_float_format_unknown, \ }, \ - .dict_state = { \ - .next_keys_version = 2, \ - }, \ .types = { \ .next_version_tag = 1, \ }, \ @@ -113,6 +110,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ + .dict_state = { \ + .next_keys_version = 2, \ + }, \ .func_state = { \ .next_version = 1, \ }, \ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a60f275742a534..b58e93f4320d42 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5655,15 +5655,16 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys) dictkeys_decref(keys); } -uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys) +uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, + PyDictKeysObject *dictkeys) { if (dictkeys->dk_version != 0) { return dictkeys->dk_version; } - if (_PyRuntime.dict_state.next_keys_version == 0) { + if (interp->dict_state.next_keys_version == 0) { return 0; } - uint32_t v = _PyRuntime.dict_state.next_keys_version++; + uint32_t v = interp->dict_state.next_keys_version++; dictkeys->dk_version = v; return v; } diff --git a/Python/specialize.c b/Python/specialize.c index 3405d2b0ab0680..0a7af8991eec83 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -512,7 +512,8 @@ specialize_module_load_attr( SPEC_FAIL_OUT_OF_RANGE); return -1; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + _PyInterpreterState_GET(), dict->ma_keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; @@ -1063,7 +1064,8 @@ PyObject *descr, DescriptorClassification kind) SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); return 0; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + _PyInterpreterState_GET(), keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; @@ -1134,12 +1136,14 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); goto fail; } + PyInterpreterState *interp = _PyInterpreterState_GET(); if (index != DKIX_EMPTY) { if (index != (uint16_t)index) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + interp, globals_keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; @@ -1167,12 +1171,14 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); + uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState( + interp, globals_keys); if (globals_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } - uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(builtin_keys); + uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState( + interp, builtin_keys); if (builtins_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; From 58d761e5b5f253892a697941c71bac0159a1d74e Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Wed, 8 Mar 2023 18:39:33 -0800 Subject: [PATCH 03/50] GH-84783: Document GH-101264 (Make the slice object hashable) in What's New. (#102548) --- Doc/whatsnew/3.12.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index c0c021c679147f..d982cb62ec2f4e 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -189,6 +189,8 @@ Other Language Changes part of comprehensions (like ``a``) is still disallowed, as per :pep:`572`. (Contributed by Nikita Sobolev in :gh:`100581`.) +* :class:`slice` objects are now hashable, allowing them to be used as dict keys and + set items. (Contributed by Furkan Onder in :gh:`101264`.) New Modules =========== From b45d14b88611fefc6f054226d3e1117082d322c8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 08:16:30 -0700 Subject: [PATCH 04/50] gh-100227: Move dict_state.global_version to PyInterpreterState (gh-102338) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_dict.h | 11 +- Include/internal/pycore_dict_state.h | 12 +- Include/internal/pycore_runtime.h | 2 - Objects/dictobject.c | 236 +++++++++++++++++---------- Python/bytecodes.c | 4 +- Python/generated_cases.c.h | 4 +- 6 files changed, 160 insertions(+), 109 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 12c3c708e29b95..6253e0841ad349 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -149,8 +149,8 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) #define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1) -#define DICT_NEXT_VERSION() \ - (_PyRuntime.dict_state.global_version += DICT_VERSION_INCREMENT) +#define DICT_NEXT_VERSION(INTERP) \ + ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) void _PyDict_SendEvent(int watcher_bits, @@ -160,7 +160,8 @@ _PyDict_SendEvent(int watcher_bits, PyObject *value); static inline uint64_t -_PyDict_NotifyEvent(PyDict_WatchEvent event, +_PyDict_NotifyEvent(PyInterpreterState *interp, + PyDict_WatchEvent event, PyDictObject *mp, PyObject *key, PyObject *value) @@ -169,9 +170,9 @@ _PyDict_NotifyEvent(PyDict_WatchEvent event, int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; if (watcher_bits) { _PyDict_SendEvent(watcher_bits, event, mp, key, value); - return DICT_NEXT_VERSION() | watcher_bits; + return DICT_NEXT_VERSION(interp) | watcher_bits; } - return DICT_NEXT_VERSION(); + return DICT_NEXT_VERSION(interp); } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index c5142ee6763a86..d608088ed2d7ce 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -9,14 +9,6 @@ extern "C" { #endif -struct _Py_dict_runtime_state { - /*Global counter used to set ma_version_tag field of dictionary. - * It is incremented each time that a dictionary is created and each - * time that a dictionary is modified. */ - uint64_t global_version; -}; - - #ifndef WITH_FREELISTS // without freelists # define PyDict_MAXFREELIST 0 @@ -29,6 +21,10 @@ struct _Py_dict_runtime_state { #define DICT_MAX_WATCHERS 8 struct _Py_dict_state { + /*Global counter used to set ma_version_tag field of dictionary. + * It is incremented each time that a dictionary is created and each + * time that a dictionary is modified. */ + uint64_t global_version; uint32_t next_keys_version; #if PyDict_MAXFREELIST > 0 diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index e0e3d4ace0cfde..520109ca440444 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -10,7 +10,6 @@ extern "C" { #include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects @@ -153,7 +152,6 @@ typedef struct pyruntimestate { struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; - struct _Py_dict_runtime_state dict_state; struct { /* Used to set PyTypeObject.tp_version_tag */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b58e93f4320d42..227e438a8dfffc 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -232,7 +232,8 @@ equally good collision statistics, needed less code & used less memory. */ -static int dictresize(PyDictObject *mp, uint8_t log_newsize, int unicode); +static int dictresize(PyInterpreterState *interp, PyDictObject *mp, + uint8_t log_newsize, int unicode); static PyObject* dict_iter(PyDictObject *dict); @@ -241,9 +242,8 @@ static PyObject* dict_iter(PyDictObject *dict); #if PyDict_MAXFREELIST > 0 static struct _Py_dict_state * -get_dict_state(void) +get_dict_state(PyInterpreterState *interp) { - PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->dict_state; } #endif @@ -289,7 +289,8 @@ void _PyDict_DebugMallocStats(FILE *out) { #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _Py_dict_state *state = get_dict_state(interp); _PyDebugAllocatorStats(out, "free PyDictObject", state->numfree, sizeof(PyDictObject)); #endif @@ -297,7 +298,7 @@ _PyDict_DebugMallocStats(FILE *out) #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyDictKeysObject *keys); +static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys); static inline void dictkeys_incref(PyDictKeysObject *dk) @@ -309,14 +310,14 @@ dictkeys_incref(PyDictKeysObject *dk) } static inline void -dictkeys_decref(PyDictKeysObject *dk) +dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) { assert(dk->dk_refcnt > 0); #ifdef Py_REF_DEBUG _Py_DecRefTotal(); #endif if (--dk->dk_refcnt == 0) { - free_keys_object(dk); + free_keys_object(interp, dk); } } @@ -586,7 +587,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* -new_keys_object(uint8_t log2_size, bool unicode) +new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) { PyDictKeysObject *dk; Py_ssize_t usable; @@ -612,7 +613,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_keys_object() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -648,7 +649,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } static void -free_keys_object(PyDictKeysObject *keys) +free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) { assert(keys != Py_EMPTY_KEYS); if (DK_IS_UNICODE(keys)) { @@ -668,7 +669,7 @@ free_keys_object(PyDictKeysObject *keys) } } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // free_keys_object() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -709,12 +710,14 @@ free_values(PyDictValues *values) /* Consumes a reference to the keys object */ static PyObject * -new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) +new_dict(PyInterpreterState *interp, + PyDictKeysObject *keys, PyDictValues *values, + Py_ssize_t used, int free_values_on_failure) { PyDictObject *mp; assert(keys != NULL); #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() assert(state->numfree != -1); @@ -731,7 +734,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(keys); + dictkeys_decref(interp, keys); if (free_values_on_failure) { free_values(values); } @@ -741,7 +744,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = DICT_NEXT_VERSION(interp); ASSERT_CONSISTENT(mp); return (PyObject *)mp; } @@ -754,19 +757,19 @@ shared_keys_usable_size(PyDictKeysObject *keys) /* Consumes a reference to the keys object */ static PyObject * -new_dict_with_shared_keys(PyDictKeysObject *keys) +new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(keys); + dictkeys_decref(interp, keys); return PyErr_NoMemory(); } ((char *)values)[-2] = 0; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - return new_dict(keys, values, 0, 1); + return new_dict(interp, keys, values, 0, 1); } @@ -829,8 +832,9 @@ clone_combined_dict_keys(PyDictObject *orig) PyObject * PyDict_New(void) { + PyInterpreterState *interp = _PyInterpreterState_GET(); dictkeys_incref(Py_EMPTY_KEYS); - return new_dict(Py_EMPTY_KEYS, NULL, 0, 0); + return new_dict(interp, Py_EMPTY_KEYS, NULL, 0, 0); } /* Search index of hash table from offset of entry table */ @@ -1170,9 +1174,9 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) } static int -insertion_resize(PyDictObject *mp, int unicode) +insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) { - return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); + return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); } static Py_ssize_t @@ -1214,12 +1218,13 @@ Returns -1 if an error occurred, or 0 on success. Consumes key and value references. */ static int -insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) +insertdict(PyInterpreterState *interp, PyDictObject *mp, + PyObject *key, Py_hash_t hash, PyObject *value) { PyObject *old_value; if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(mp, 0) < 0) + if (insertion_resize(interp, mp, 0) < 0) goto Fail; assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); } @@ -1231,13 +1236,14 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) MAINTAIN_TRACKING(mp, key, value); if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); /* Insert into new slot. */ mp->ma_keys->dk_version = 0; assert(old_value == NULL); if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(mp, 1) < 0) + if (insertion_resize(interp, mp, 1) < 0) goto Fail; } @@ -1275,7 +1281,8 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) } if (old_value != value) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_MODIFIED, mp, key, value); if (_PyDict_HasSplitTable(mp)) { mp->ma_values->values[ix] = value; if (old_value == NULL) { @@ -1308,21 +1315,23 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) // Same to insertdict but specialized for ma_keys = Py_EMPTY_KEYS. // Consumes key and value references. static int -insert_to_emptydict(PyDictObject *mp, PyObject *key, Py_hash_t hash, - PyObject *value) +insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, + PyObject *key, Py_hash_t hash, PyObject *value) { assert(mp->ma_keys == Py_EMPTY_KEYS); - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); int unicode = PyUnicode_CheckExact(key); - PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode); + PyDictKeysObject *newkeys = new_keys_object( + interp, PyDict_LOG_MINSIZE, unicode); if (newkeys == NULL) { Py_DECREF(key); Py_DECREF(value); return -1; } - dictkeys_decref(Py_EMPTY_KEYS); + dictkeys_decref(interp, Py_EMPTY_KEYS); mp->ma_keys = newkeys; mp->ma_values = NULL; @@ -1397,7 +1406,8 @@ This function supports: - Generic -> Generic */ static int -dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) +dictresize(PyInterpreterState *interp, PyDictObject *mp, + uint8_t log2_newsize, int unicode) { PyDictKeysObject *oldkeys; PyDictValues *oldvalues; @@ -1421,7 +1431,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(log2_newsize, unicode); + mp->ma_keys = new_keys_object(interp, log2_newsize, unicode); if (mp->ma_keys == NULL) { mp->ma_keys = oldkeys; return -1; @@ -1462,7 +1472,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) } build_indices_unicode(mp->ma_keys, newentries, numentries); } - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); mp->ma_values = NULL; free_values(oldvalues); } @@ -1530,7 +1540,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // dictresize() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -1557,7 +1567,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) } static PyObject * -dict_new_presized(Py_ssize_t minused, bool unicode) +dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) { const uint8_t log2_max_presize = 17; const Py_ssize_t max_presize = ((Py_ssize_t)1) << log2_max_presize; @@ -1578,16 +1588,17 @@ dict_new_presized(Py_ssize_t minused, bool unicode) log2_newsize = estimate_log2_keysize(minused); } - new_keys = new_keys_object(log2_newsize, unicode); + new_keys = new_keys_object(interp, log2_newsize, unicode); if (new_keys == NULL) return NULL; - return new_dict(new_keys, NULL, 0, 0); + return new_dict(interp, new_keys, NULL, 0, 0); } PyObject * _PyDict_NewPresized(Py_ssize_t minused) { - return dict_new_presized(minused, false); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_new_presized(interp, minused, false); } PyObject * @@ -1597,6 +1608,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, { bool unicode = true; PyObject *const *ks = keys; + PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = 0; i < length; i++) { if (!PyUnicode_CheckExact(*ks)) { @@ -1606,7 +1618,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, ks += keys_offset; } - PyObject *dict = dict_new_presized(length, unicode); + PyObject *dict = dict_new_presized(interp, length, unicode); if (dict == NULL) { return NULL; } @@ -1834,11 +1846,12 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return -1; } } + PyInterpreterState *interp = _PyInterpreterState_GET(); if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(mp, key, hash, value); + return insert_to_emptydict(interp, mp, key, hash, value); } /* insertdict() handles any resizing that might be necessary */ - return insertdict(mp, key, hash, value); + return insertdict(interp, mp, key, hash, value); } /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the @@ -1875,11 +1888,12 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(hash != -1); mp = (PyDictObject *)op; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } /* insertdict() handles any resizing that might be necessary */ - return insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } static void @@ -1977,7 +1991,9 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return -1; } - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); return delitem_common(mp, hash, ix, old_value, new_version); } @@ -2020,7 +2036,9 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, assert(hashpos >= 0); if (res > 0) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); return delitem_common(mp, hashpos, ix, old_value, new_version); } else { return 0; @@ -2045,7 +2063,9 @@ PyDict_Clear(PyObject *op) return; } /* Empty the dict... */ - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLEARED, mp, NULL, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); dictkeys_incref(Py_EMPTY_KEYS); mp->ma_keys = Py_EMPTY_KEYS; mp->ma_values = NULL; @@ -2057,11 +2077,11 @@ PyDict_Clear(PyObject *op) for (i = 0; i < n; i++) Py_CLEAR(oldvalues->values[i]); free_values(oldvalues); - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); } else { assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); } ASSERT_CONSISTENT(mp); } @@ -2165,6 +2185,7 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d Py_ssize_t ix; PyObject *old_value; PyDictObject *mp; + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(PyDict_Check(dict)); mp = (PyDictObject *)dict; @@ -2187,7 +2208,8 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d return NULL; } assert(old_value != NULL); - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); ASSERT_CONSISTENT(mp); @@ -2222,6 +2244,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; PyObject *d; int status; + PyInterpreterState *interp = _PyInterpreterState_GET(); d = _PyObject_CallNoArgs(cls); if (d == NULL) @@ -2236,13 +2259,16 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_hash_t hash; int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); - if (dictresize(mp, estimate_log2_keysize(PyDict_GET_SIZE(iterable)), unicode)) { + if (dictresize(interp, mp, + estimate_log2_keysize(PyDict_GET_SIZE(iterable)), + unicode)) { Py_DECREF(d); return NULL; } while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { - if (insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value))) { + if (insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value))) { Py_DECREF(d); return NULL; } @@ -2255,13 +2281,14 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + if (dictresize(interp, mp, + estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { Py_DECREF(d); return NULL; } while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value))) { + if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { Py_DECREF(d); return NULL; } @@ -2308,9 +2335,10 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(Py_REFCNT(mp) == 0); Py_SET_REFCNT(mp, 1); - _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); if (Py_REFCNT(mp) > 1) { Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); return; @@ -2328,14 +2356,14 @@ dict_dealloc(PyDictObject *mp) Py_XDECREF(values->values[i]); } free_values(values); - dictkeys_decref(keys); + dictkeys_decref(interp, keys); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(keys); + dictkeys_decref(interp, keys); } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() assert(state->numfree != -1); @@ -2765,7 +2793,7 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) } static int -dict_merge(PyObject *a, PyObject *b, int override) +dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) { PyDictObject *mp, *other; @@ -2799,13 +2827,14 @@ dict_merge(PyObject *a, PyObject *b, int override) other->ma_used == okeys->dk_nentries && (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLONED, mp, b, NULL); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLONED, mp, b, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) { return -1; } - dictkeys_decref(mp->ma_keys); + dictkeys_decref(interp, mp->ma_keys); mp->ma_keys = keys; if (mp->ma_values != NULL) { free_values(mp->ma_values); @@ -2830,7 +2859,9 @@ dict_merge(PyObject *a, PyObject *b, int override) */ if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(mp, estimate_log2_keysize(mp->ma_used + other->ma_used), unicode)) { + if (dictresize(interp, mp, + estimate_log2_keysize(mp->ma_used + other->ma_used), + unicode)) { return -1; } } @@ -2845,12 +2876,14 @@ dict_merge(PyObject *a, PyObject *b, int override) Py_INCREF(key); Py_INCREF(value); if (override == 1) { - err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); } else { err = _PyDict_Contains_KnownHash(a, key, hash); if (err == 0) { - err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); } else if (err > 0) { if (override != 0) { @@ -2936,20 +2969,23 @@ dict_merge(PyObject *a, PyObject *b, int override) int PyDict_Update(PyObject *a, PyObject *b) { - return dict_merge(a, b, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_merge(interp, a, b, 1); } int PyDict_Merge(PyObject *a, PyObject *b, int override) { + PyInterpreterState *interp = _PyInterpreterState_GET(); /* XXX Deprecate override not in (0, 1). */ - return dict_merge(a, b, override != 0); + return dict_merge(interp, a, b, override != 0); } int _PyDict_MergeEx(PyObject *a, PyObject *b, int override) { - return dict_merge(a, b, override); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_merge(interp, a, b, override); } static PyObject * @@ -2963,6 +2999,7 @@ PyDict_Copy(PyObject *o) { PyObject *copy; PyDictObject *mp; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (o == NULL || !PyDict_Check(o)) { PyErr_BadInternalCall(); @@ -2991,7 +3028,7 @@ PyDict_Copy(PyObject *o) split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; - split_copy->ma_version_tag = DICT_NEXT_VERSION(); + split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); dictkeys_incref(mp->ma_keys); for (size_t i = 0; i < size; i++) { PyObject *value = mp->ma_values->values[i]; @@ -3024,7 +3061,7 @@ PyDict_Copy(PyObject *o) if (keys == NULL) { return NULL; } - PyDictObject *new = (PyDictObject *)new_dict(keys, NULL, 0, 0); + PyDictObject *new = (PyDictObject *)new_dict(interp, keys, NULL, 0, 0); if (new == NULL) { /* In case of an error, `new_dict()` takes care of cleaning up `keys`. */ @@ -3044,7 +3081,7 @@ PyDict_Copy(PyObject *o) copy = PyDict_New(); if (copy == NULL) return NULL; - if (dict_merge(copy, o, 1) == 0) + if (dict_merge(interp, copy, o, 1) == 0) return copy; Py_DECREF(copy); return NULL; @@ -3244,6 +3281,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyDict_Check(d)) { PyErr_BadInternalCall(); @@ -3257,7 +3295,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) } if (mp->ma_keys == Py_EMPTY_KEYS) { - if (insert_to_emptydict(mp, Py_NewRef(key), hash, + if (insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(defaultobj)) < 0) { return NULL; } @@ -3265,7 +3303,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(mp, 0) < 0) { + if (insertion_resize(interp, mp, 0) < 0) { return NULL; } } @@ -3275,11 +3313,12 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) return NULL; if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, defaultobj); mp->ma_keys->dk_version = 0; value = defaultobj; if (mp->ma_keys->dk_usable <= 0) { - if (insertion_resize(mp, 1) < 0) { + if (insertion_resize(interp, mp, 1) < 0) { return NULL; } } @@ -3314,7 +3353,8 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) assert(mp->ma_keys->dk_usable >= 0); } else if (value == NULL) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, defaultobj); value = defaultobj; assert(_PyDict_HasSplitTable(mp)); assert(mp->ma_values->values[ix] == NULL); @@ -3395,6 +3435,7 @@ dict_popitem_impl(PyDictObject *self) Py_ssize_t i, j; PyObject *res; uint64_t new_version; + PyInterpreterState *interp = _PyInterpreterState_GET(); /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which @@ -3415,7 +3456,7 @@ dict_popitem_impl(PyDictObject *self) } /* Convert split table to combined table */ if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { - if (dictresize(self, DK_LOG_SIZE(self->ma_keys), 1)) { + if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1)) { Py_DECREF(res); return NULL; } @@ -3434,7 +3475,8 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); + new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3449,7 +3491,8 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); + new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3708,7 +3751,8 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDictObject *d = (PyDictObject *)self; d->ma_used = 0; - d->ma_version_tag = DICT_NEXT_VERSION(); + d->ma_version_tag = DICT_NEXT_VERSION( + _PyInterpreterState_GET()); dictkeys_incref(Py_EMPTY_KEYS); d->ma_keys = Py_EMPTY_KEYS; d->ma_values = NULL; @@ -5262,7 +5306,9 @@ dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(void) { - PyDictKeysObject *keys = new_keys_object(NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *keys = new_keys_object( + interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { PyErr_Clear(); } @@ -5306,6 +5352,7 @@ init_inline_values(PyObject *obj, PyTypeObject *tp) int _PyObject_InitializeDict(PyObject *obj) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (tp->tp_dictoffset == 0) { return 0; @@ -5317,7 +5364,7 @@ _PyObject_InitializeDict(PyObject *obj) PyObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); } else { dict = PyDict_New(); @@ -5331,7 +5378,8 @@ _PyObject_InitializeDict(PyObject *obj) } static PyObject * -make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) +make_dict_from_instance_attributes(PyInterpreterState *interp, + PyDictKeysObject *keys, PyDictValues *values) { dictkeys_incref(keys); Py_ssize_t used = 0; @@ -5344,7 +5392,7 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(keys, values, used, 0); + PyObject *res = new_dict(interp, keys, values, used, 0); if (track && res) { _PyObject_GC_TRACK(res); } @@ -5354,15 +5402,17 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) PyObject * _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); - return make_dict_from_instance_attributes(keys, values); + return make_dict_from_instance_attributes(interp, keys, values); } int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); @@ -5385,7 +5435,8 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, OBJECT_STAT_INC(dict_materialized_str_subclass); } #endif - PyObject *dict = make_dict_from_instance_attributes(keys, values); + PyObject *dict = make_dict_from_instance_attributes( + interp, keys, values); if (dict == NULL) { return -1; } @@ -5564,13 +5615,15 @@ PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { PyObject *dict; + PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); if (_PyDictOrValues_IsValues(*dorv_ptr)) { PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); OBJECT_STAT_INC(dict_materialized_on_request); - dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), values); + dict = make_dict_from_instance_attributes( + interp, CACHED_KEYS(tp), values); if (dict != NULL) { dorv_ptr->dict = dict; } @@ -5579,7 +5632,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = _PyDictOrValues_GetDict(*dorv_ptr); if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); dorv_ptr->dict = dict; } } @@ -5596,7 +5649,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context) PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + *dictptr = dict = new_dict_with_shared_keys( + interp, CACHED_KEYS(tp)); } else { *dictptr = dict = PyDict_New(); @@ -5613,6 +5667,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *dict; int res; PyDictKeysObject *cached; + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(dictptr != NULL); if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { @@ -5620,7 +5675,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, dict = *dictptr; if (dict == NULL) { dictkeys_incref(cached); - dict = new_dict_with_shared_keys(cached); + dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) return -1; *dictptr = dict; @@ -5652,7 +5707,8 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, void _PyDictKeys_DecRef(PyDictKeysObject *keys) { - dictkeys_decref(keys); + PyInterpreterState *interp = _PyInterpreterState_GET(); + dictkeys_decref(interp, keys); } uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 63dbecad3b45fc..45d50726674321 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1679,7 +1679,7 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { @@ -1687,7 +1687,7 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } Py_DECREF(old_value); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 82e18505b0d430..51357cda4c8949 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2114,7 +2114,7 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { @@ -2122,7 +2122,7 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } Py_DECREF(old_value); From cf6e7c5e551b3513817d6a77ba88253dc8473298 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 09:46:21 -0700 Subject: [PATCH 05/50] gh-100227: Isolate the Import State to Each Interpreter (gh-101941) Specific changes: * move the import lock to PyInterpreterState * move the "find_and_load" diagnostic state to PyInterpreterState Note that the import lock exists to keep multiple imports of the same module in the same interpreter (but in different threads) from stomping on each other. Independently, we use a distinct global lock to protect globally shared import state, especially related to loaded extension modules. For now we can rely on the GIL as that lock but with a per-interpreter GIL we'll need a new global lock. The remaining state in _PyRuntimeState.imports will (probably) continue being global. https://github.com/python/cpython/issues/100227 --- Include/cpython/import.h | 4 +- Include/internal/pycore_import.h | 34 +++++---- Include/internal/pycore_runtime_init.h | 10 --- Modules/posixmodule.c | 13 ++-- Python/import.c | 98 ++++++++++++++------------ 5 files changed, 85 insertions(+), 74 deletions(-) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index a58801b47f1bec..2bca4ade4c4f2c 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,8 +10,8 @@ PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(_Py_Identifier *name); PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); -PyAPI_FUNC(void) _PyImport_AcquireLock(void); -PyAPI_FUNC(int) _PyImport_ReleaseLock(void); +PyAPI_FUNC(void) _PyImport_AcquireLock(PyInterpreterState *interp); +PyAPI_FUNC(int) _PyImport_ReleaseLock(PyInterpreterState *interp); PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index b7ffe01c0c0e20..69ed6273b7e609 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -21,17 +21,6 @@ struct _import_runtime_state { This is initialized lazily in _PyImport_FixupExtensionObject(). Modules are added there and looked up in _imp.find_extension(). */ PyObject *extensions; - /* The global import lock. */ - struct { - PyThread_type_lock mutex; - unsigned long thread; - int level; - } lock; - struct { - int import_level; - _PyTime_t accumulated; - int header; - } find_and_load; /* Package context -- the full module name for package imports */ const char * pkgcontext; }; @@ -69,6 +58,18 @@ struct _import_state { int dlopenflags; #endif PyObject *import_func; + /* The global import lock. */ + struct { + PyThread_type_lock mutex; + unsigned long thread; + int level; + } lock; + /* diagnostic info in PyImport_ImportModuleLevelObject() */ + struct { + int import_level; + _PyTime_t accumulated; + int header; + } find_and_load; }; #ifdef HAVE_DLOPEN @@ -86,8 +87,15 @@ struct _import_state { #define IMPORTS_INIT \ { \ - .override_frozen_modules = 0, \ DLOPENFLAGS_INIT \ + .lock = { \ + .mutex = NULL, \ + .thread = PYTHREAD_INVALID_THREAD_ID, \ + .level = 0, \ + }, \ + .find_and_load = { \ + .header = 1, \ + }, \ } extern void _PyImport_ClearCore(PyInterpreterState *interp); @@ -138,7 +146,7 @@ extern void _PyImport_FiniExternal(PyInterpreterState *interp); #ifdef HAVE_FORK -extern PyStatus _PyImport_ReInitLock(void); +extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp); #endif diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index efc82b43a61dab..bdecac944dfd3a 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -40,16 +40,6 @@ extern PyTypeObject _PyExc_MemoryError; in accordance with the specification. */ \ .autoTSSkey = Py_tss_NEEDS_INIT, \ .parser = _parser_runtime_state_INIT, \ - .imports = { \ - .lock = { \ - .mutex = NULL, \ - .thread = PYTHREAD_INVALID_THREAD_ID, \ - .level = 0, \ - }, \ - .find_and_load = { \ - .header = 1, \ - }, \ - }, \ .ceval = { \ .perf = _PyEval_RUNTIME_PERF_INIT, \ }, \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7beb2cee64a05c..a3d86cbe7a57fb 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -567,18 +567,21 @@ run_at_forkers(PyObject *lst, int reverse) void PyOS_BeforeFork(void) { - run_at_forkers(_PyInterpreterState_GET()->before_forkers, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + run_at_forkers(interp->before_forkers, 1); - _PyImport_AcquireLock(); + _PyImport_AcquireLock(interp); } void PyOS_AfterFork_Parent(void) { - if (_PyImport_ReleaseLock() <= 0) + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyImport_ReleaseLock(interp) <= 0) { Py_FatalError("failed releasing import lock after fork"); + } - run_at_forkers(_PyInterpreterState_GET()->after_forkers_parent, 0); + run_at_forkers(interp->after_forkers_parent, 0); } void @@ -604,7 +607,7 @@ PyOS_AfterFork_Child(void) goto fatal_error; } - status = _PyImport_ReInitLock(); + status = _PyImport_ReInitLock(tstate->interp); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; } diff --git a/Python/import.c b/Python/import.c index 57d4eea148810f..1bf4199e125aa5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -56,11 +56,6 @@ static struct _inittab *inittab_copy = NULL; #define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index #define EXTENSIONS _PyRuntime.imports.extensions -#define import_lock _PyRuntime.imports.lock.mutex -#define import_lock_thread _PyRuntime.imports.lock.thread -#define import_lock_level _PyRuntime.imports.lock.level - -#define FIND_AND_LOAD _PyRuntime.imports.find_and_load #define PKGCONTEXT (_PyRuntime.imports.pkgcontext) @@ -85,6 +80,16 @@ static struct _inittab *inittab_copy = NULL; #define IMPORT_FUNC(interp) \ (interp)->imports.import_func +#define IMPORT_LOCK(interp) \ + (interp)->imports.lock.mutex +#define IMPORT_LOCK_THREAD(interp) \ + (interp)->imports.lock.thread +#define IMPORT_LOCK_LEVEL(interp) \ + (interp)->imports.lock.level + +#define FIND_AND_LOAD(interp) \ + (interp)->imports.find_and_load + /*******************/ /* the import lock */ @@ -95,45 +100,45 @@ static struct _inittab *inittab_copy = NULL; These calls are serialized by the global interpreter lock. */ void -_PyImport_AcquireLock(void) +_PyImport_AcquireLock(PyInterpreterState *interp) { unsigned long me = PyThread_get_thread_ident(); if (me == PYTHREAD_INVALID_THREAD_ID) return; /* Too bad */ - if (import_lock == NULL) { - import_lock = PyThread_allocate_lock(); - if (import_lock == NULL) + if (IMPORT_LOCK(interp) == NULL) { + IMPORT_LOCK(interp) = PyThread_allocate_lock(); + if (IMPORT_LOCK(interp) == NULL) return; /* Nothing much we can do. */ } - if (import_lock_thread == me) { - import_lock_level++; + if (IMPORT_LOCK_THREAD(interp) == me) { + IMPORT_LOCK_LEVEL(interp)++; return; } - if (import_lock_thread != PYTHREAD_INVALID_THREAD_ID || - !PyThread_acquire_lock(import_lock, 0)) + if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID || + !PyThread_acquire_lock(IMPORT_LOCK(interp), 0)) { PyThreadState *tstate = PyEval_SaveThread(); - PyThread_acquire_lock(import_lock, WAIT_LOCK); + PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); PyEval_RestoreThread(tstate); } - assert(import_lock_level == 0); - import_lock_thread = me; - import_lock_level = 1; + assert(IMPORT_LOCK_LEVEL(interp) == 0); + IMPORT_LOCK_THREAD(interp) = me; + IMPORT_LOCK_LEVEL(interp) = 1; } int -_PyImport_ReleaseLock(void) +_PyImport_ReleaseLock(PyInterpreterState *interp) { unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID || import_lock == NULL) + if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL) return 0; /* Too bad */ - if (import_lock_thread != me) + if (IMPORT_LOCK_THREAD(interp) != me) return -1; - import_lock_level--; - assert(import_lock_level >= 0); - if (import_lock_level == 0) { - import_lock_thread = PYTHREAD_INVALID_THREAD_ID; - PyThread_release_lock(import_lock); + IMPORT_LOCK_LEVEL(interp)--; + assert(IMPORT_LOCK_LEVEL(interp) >= 0); + if (IMPORT_LOCK_LEVEL(interp) == 0) { + IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; + PyThread_release_lock(IMPORT_LOCK(interp)); } return 1; } @@ -144,23 +149,23 @@ _PyImport_ReleaseLock(void) We now acquire the import lock around fork() calls but on some platforms (Solaris 9 and earlier? see isue7242) that still left us with problems. */ PyStatus -_PyImport_ReInitLock(void) +_PyImport_ReInitLock(PyInterpreterState *interp) { - if (import_lock != NULL) { - if (_PyThread_at_fork_reinit(&import_lock) < 0) { + if (IMPORT_LOCK(interp) != NULL) { + if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) { return _PyStatus_ERR("failed to create a new lock"); } } - if (import_lock_level > 1) { + if (IMPORT_LOCK_LEVEL(interp) > 1) { /* Forked as a side effect of import */ unsigned long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(import_lock, WAIT_LOCK); - import_lock_thread = me; - import_lock_level--; + PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); + IMPORT_LOCK_THREAD(interp) = me; + IMPORT_LOCK_LEVEL(interp)--; } else { - import_lock_thread = PYTHREAD_INVALID_THREAD_ID; - import_lock_level = 0; + IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; + IMPORT_LOCK_LEVEL(interp) = 0; } return _PyStatus_OK(); } @@ -2506,8 +2511,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; int import_time = _PyInterpreterState_GetConfig(interp)->import_time; -#define import_level FIND_AND_LOAD.import_level -#define accumulated FIND_AND_LOAD.accumulated +#define import_level FIND_AND_LOAD(interp).import_level +#define accumulated FIND_AND_LOAD(interp).accumulated _PyTime_t t1 = 0, accumulated_copy = accumulated; @@ -2528,7 +2533,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) * _PyDict_GetItemIdWithError(). */ if (import_time) { -#define header FIND_AND_LOAD.header +#define header FIND_AND_LOAD(interp).header if (header) { fputs("import time: self [us] | cumulative | imported package\n", stderr); @@ -2867,10 +2872,6 @@ _PyImport_Fini(void) { /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ _extensions_cache_clear_all(); - if (import_lock != NULL) { - PyThread_free_lock(import_lock); - import_lock = NULL; - } /* Use the same memory allocator as _PyImport_Init(). */ PyMemAllocatorEx old_alloc; @@ -2959,6 +2960,11 @@ _PyImport_FiniCore(PyInterpreterState *interp) PyErr_WriteUnraisable(NULL); } + if (IMPORT_LOCK(interp) != NULL) { + PyThread_free_lock(IMPORT_LOCK(interp)); + IMPORT_LOCK(interp) = NULL; + } + _PyImport_ClearCore(interp); } @@ -3090,7 +3096,9 @@ static PyObject * _imp_lock_held_impl(PyObject *module) /*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ { - return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return PyBool_FromLong( + IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID); } /*[clinic input] @@ -3106,7 +3114,8 @@ static PyObject * _imp_acquire_lock_impl(PyObject *module) /*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ { - _PyImport_AcquireLock(); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyImport_AcquireLock(interp); Py_RETURN_NONE; } @@ -3122,7 +3131,8 @@ static PyObject * _imp_release_lock_impl(PyObject *module) /*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ { - if (_PyImport_ReleaseLock() < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyImport_ReleaseLock(interp) < 0) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; From ca066bdbed85094a9c4d9930823ce3587807db48 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 09:50:33 -0700 Subject: [PATCH 06/50] gh-81057: Vendor a Subset of distutils for the c-analyzer Tool (gh-102505) distutils was removed in November. However, the c-analyzer relies on it. To solve that here, we vendor the parts the tool needs so it can be run against 3.12+. (Also see gh-92584.) Note that we may end up removing this code later in favor of a solution in common with the peg_generator tool (which also relies on distutils). At the least, the copy here makes sure the c-analyzer tool works on 3.12+ in the meantime. --- Tools/c-analyzer/distutils/README | 2 + Tools/c-analyzer/distutils/__init__.py | 0 Tools/c-analyzer/distutils/_msvccompiler.py | 203 ++++++++ Tools/c-analyzer/distutils/bcppcompiler.py | 109 ++++ Tools/c-analyzer/distutils/ccompiler.py | 470 ++++++++++++++++++ Tools/c-analyzer/distutils/cygwinccompiler.py | 286 +++++++++++ Tools/c-analyzer/distutils/debug.py | 5 + Tools/c-analyzer/distutils/dep_util.py | 29 ++ Tools/c-analyzer/distutils/errors.py | 48 ++ Tools/c-analyzer/distutils/log.py | 63 +++ Tools/c-analyzer/distutils/msvc9compiler.py | 438 ++++++++++++++++ Tools/c-analyzer/distutils/msvccompiler.py | 327 ++++++++++++ Tools/c-analyzer/distutils/spawn.py | 48 ++ Tools/c-analyzer/distutils/unixccompiler.py | 102 ++++ Tools/c-analyzer/distutils/util.py | 171 +++++++ 15 files changed, 2301 insertions(+) create mode 100644 Tools/c-analyzer/distutils/README create mode 100644 Tools/c-analyzer/distutils/__init__.py create mode 100644 Tools/c-analyzer/distutils/_msvccompiler.py create mode 100644 Tools/c-analyzer/distutils/bcppcompiler.py create mode 100644 Tools/c-analyzer/distutils/ccompiler.py create mode 100644 Tools/c-analyzer/distutils/cygwinccompiler.py create mode 100644 Tools/c-analyzer/distutils/debug.py create mode 100644 Tools/c-analyzer/distutils/dep_util.py create mode 100644 Tools/c-analyzer/distutils/errors.py create mode 100644 Tools/c-analyzer/distutils/log.py create mode 100644 Tools/c-analyzer/distutils/msvc9compiler.py create mode 100644 Tools/c-analyzer/distutils/msvccompiler.py create mode 100644 Tools/c-analyzer/distutils/spawn.py create mode 100644 Tools/c-analyzer/distutils/unixccompiler.py create mode 100644 Tools/c-analyzer/distutils/util.py diff --git a/Tools/c-analyzer/distutils/README b/Tools/c-analyzer/distutils/README new file mode 100644 index 00000000000000..b260b8e06fac06 --- /dev/null +++ b/Tools/c-analyzer/distutils/README @@ -0,0 +1,2 @@ +This is a partial copy of distutils as it was removed in 0faa0ba240e. +It only includes the parts needed by the C parser. diff --git a/Tools/c-analyzer/distutils/__init__.py b/Tools/c-analyzer/distutils/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Tools/c-analyzer/distutils/_msvccompiler.py b/Tools/c-analyzer/distutils/_msvccompiler.py new file mode 100644 index 00000000000000..1e67870d13323d --- /dev/null +++ b/Tools/c-analyzer/distutils/_msvccompiler.py @@ -0,0 +1,203 @@ +"""distutils._msvccompiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for Microsoft Visual Studio 2015. + +The module is compatible with VS 2015 and later. You can find legacy support +for older versions in distutils.msvc9compiler and distutils.msvccompiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS 2005 and VS 2008 by Christian Heimes +# ported to VS 2015 by Steve Dower + +import os +import subprocess +import winreg + +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +from itertools import count + +def _find_vc2015(): + try: + key = winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"Software\Microsoft\VisualStudio\SxS\VC7", + access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY + ) + except OSError: + log.debug("Visual C++ is not registered") + return None, None + + best_version = 0 + best_dir = None + with key: + for i in count(): + try: + v, vc_dir, vt = winreg.EnumValue(key, i) + except OSError: + break + if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir): + try: + version = int(float(v)) + except (ValueError, TypeError): + continue + if version >= 14 and version > best_version: + best_version, best_dir = version, vc_dir + return best_version, best_dir + +def _find_vc2017(): + """Returns "15, path" based on the result of invoking vswhere.exe + If no install is found, returns "None, None" + + The version is returned to avoid unnecessarily changing the function + result. It may be ignored when the path is not None. + + If vswhere.exe is not available, by definition, VS 2017 is not + installed. + """ + root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") + if not root: + return None, None + + try: + path = subprocess.check_output([ + os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), + "-latest", + "-prerelease", + "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", "installationPath", + "-products", "*", + ], encoding="mbcs", errors="strict").strip() + except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): + return None, None + + path = os.path.join(path, "VC", "Auxiliary", "Build") + if os.path.isdir(path): + return 15, path + + return None, None + +PLAT_SPEC_TO_RUNTIME = { + 'x86' : 'x86', + 'x86_amd64' : 'x64', + 'x86_arm' : 'arm', + 'x86_arm64' : 'arm64' +} + +def _find_vcvarsall(plat_spec): + # bpo-38597: Removed vcruntime return value + _, best_dir = _find_vc2017() + + if not best_dir: + best_version, best_dir = _find_vc2015() + + if not best_dir: + log.debug("No suitable Visual C++ version found") + return None, None + + vcvarsall = os.path.join(best_dir, "vcvarsall.bat") + if not os.path.isfile(vcvarsall): + log.debug("%s cannot be found", vcvarsall) + return None, None + + return vcvarsall, None + +def _get_vc_env(plat_spec): + if os.getenv("DISTUTILS_USE_SDK"): + return { + key.lower(): value + for key, value in os.environ.items() + } + + vcvarsall, _ = _find_vcvarsall(plat_spec) + if not vcvarsall: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + + try: + out = subprocess.check_output( + 'cmd /u /c "{}" {} && set'.format(vcvarsall, plat_spec), + stderr=subprocess.STDOUT, + ).decode('utf-16le', errors='replace') + except subprocess.CalledProcessError as exc: + log.error(exc.output) + raise DistutilsPlatformError("Error executing {}" + .format(exc.cmd)) + + env = { + key.lower(): value + for key, _, value in + (line.partition('=') for line in out.splitlines()) + if key and value + } + + return env + +def _find_exe(exe, paths=None): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + if not paths: + paths = os.getenv('path').split(os.pathsep) + for p in paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + return exe + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Always cross-compile from x86 to work with the +# lighter-weight MSVC installs that do not include native 64-bit tools. +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'x86_amd64', + 'win-arm32' : 'x86_arm', + 'win-arm64' : 'x86_arm64' +} + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.initialized = False diff --git a/Tools/c-analyzer/distutils/bcppcompiler.py b/Tools/c-analyzer/distutils/bcppcompiler.py new file mode 100644 index 00000000000000..4575b665c10c0e --- /dev/null +++ b/Tools/c-analyzer/distutils/bcppcompiler.py @@ -0,0 +1,109 @@ +"""distutils.bcppcompiler + +Contains BorlandCCompiler, an implementation of the abstract CCompiler class +for the Borland C++ compiler. +""" + +# This implementation by Lyle Johnson, based on the original msvccompiler.py +# module and using the directions originally published by Gordon Williams. + +# XXX looks like there's a LOT of overlap between these two classes: +# someone should sit down and factor out the common code as +# WindowsCCompiler! --GPW + + +import os +from distutils.errors import DistutilsExecError, CompileError +from distutils.ccompiler import \ + CCompiler, gen_preprocess_options +from distutils.dep_util import newer + +class BCPPCompiler(CCompiler) : + """Concrete class that implements an interface to the Borland C/C++ + compiler, as defined by the CCompiler abstract class. + """ + + compiler_type = 'bcpp' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = _c_extensions + _cpp_extensions + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__ (self, + verbose=0, + dry_run=0, + force=0): + + CCompiler.__init__ (self, verbose, dry_run, force) + + # These executables are assumed to all be in the path. + # Borland doesn't seem to use any special registry settings to + # indicate their installation locations. + + self.cc = "bcc32.exe" + self.linker = "ilink32.exe" + self.lib = "tlib.exe" + + self.preprocess_options = None + self.compile_options = ['/tWM', '/O2', '/q', '/g0'] + self.compile_options_debug = ['/tWM', '/Od', '/q', '/g0'] + + self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_static = [] + self.ldflags_exe = ['/Gn', '/q', '/x'] + self.ldflags_exe_debug = ['/Gn', '/q', '/x','/r'] + + + # -- Worker methods ------------------------------------------------ + + def preprocess (self, + source, + output_file=None, + macros=None, + include_dirs=None, + extra_preargs=None, + extra_postargs=None): + + (_, macros, include_dirs) = \ + self._fix_compile_args(None, macros, include_dirs) + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = ['cpp32.exe'] + pp_opts + if output_file is not None: + pp_args.append('-o' + output_file) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or the + # source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + print(msg) + raise CompileError(msg) + + # preprocess() diff --git a/Tools/c-analyzer/distutils/ccompiler.py b/Tools/c-analyzer/distutils/ccompiler.py new file mode 100644 index 00000000000000..13e43103b94f5e --- /dev/null +++ b/Tools/c-analyzer/distutils/ccompiler.py @@ -0,0 +1,470 @@ +"""distutils.ccompiler + +Contains CCompiler, an abstract base class that defines the interface +for the Distutils compiler abstraction model.""" + +import sys, os, re +from distutils.errors import ( + DistutilsModuleError, DistutilsPlatformError, +) +from distutils.util import split_quoted + +class CCompiler: + """Abstract base class to define the interface that must be implemented + by real compiler classes. Also has some utility methods used by + several compiler classes. + + The basic idea behind a compiler abstraction class is that each + instance can be used for all the compile/link steps in building a + single project. Thus, attributes common to all of those compile and + link steps -- include directories, macros to define, libraries to link + against, etc. -- are attributes of the compiler instance. To allow for + variability in how individual files are treated, most of those + attributes may be varied on a per-compilation or per-link basis. + """ + + # 'compiler_type' is a class attribute that identifies this class. It + # keeps code that wants to know what kind of compiler it's dealing with + # from having to import all possible compiler classes just to do an + # 'isinstance'. In concrete CCompiler subclasses, 'compiler_type' + # should really, really be one of the keys of the 'compiler_class' + # dictionary (see below -- used by the 'new_compiler()' factory + # function) -- authors of new compiler interface classes are + # responsible for updating 'compiler_class'! + compiler_type = None + + # XXX things not handled by this compiler abstraction model: + # * client can't provide additional options for a compiler, + # e.g. warning, optimization, debugging flags. Perhaps this + # should be the domain of concrete compiler abstraction classes + # (UnixCCompiler, MSVCCompiler, etc.) -- or perhaps the base + # class should have methods for the common ones. + # * can't completely override the include or library searchg + # path, ie. no "cc -I -Idir1 -Idir2" or "cc -L -Ldir1 -Ldir2". + # I'm not sure how widely supported this is even by Unix + # compilers, much less on other platforms. And I'm even less + # sure how useful it is; maybe for cross-compiling, but + # support for that is a ways off. (And anyways, cross + # compilers probably have a dedicated binary with the + # right paths compiled in. I hope.) + # * can't do really freaky things with the library list/library + # dirs, e.g. "-Ldir1 -lfoo -Ldir2 -lfoo" to link against + # different versions of libfoo.a in different locations. I + # think this is useless without the ability to null out the + # library search path anyways. + + + # Subclasses that rely on the standard filename generation methods + # implemented below should override these; see the comment near + # those methods ('object_filenames()' et. al.) for details: + src_extensions = None # list of strings + obj_extension = None # string + static_lib_extension = None + shared_lib_extension = None # string + static_lib_format = None # format string + shared_lib_format = None # prob. same as static_lib_format + exe_extension = None # string + + # Default language settings. language_map is used to detect a source + # file or Extension target language, checking source filenames. + # language_order is used to detect the language precedence, when deciding + # what language to use when mixing source types. For example, if some + # extension has two files with ".c" extension, and one with ".cpp", it + # is still linked as c++. + language_map = {".c" : "c", + ".cc" : "c++", + ".cpp" : "c++", + ".cxx" : "c++", + ".m" : "objc", + } + language_order = ["c++", "objc", "c"] + + def __init__(self, verbose=0, dry_run=0, force=0): + self.dry_run = dry_run + self.force = force + self.verbose = verbose + + # 'output_dir': a common output directory for object, library, + # shared object, and shared library files + self.output_dir = None + + # 'macros': a list of macro definitions (or undefinitions). A + # macro definition is a 2-tuple (name, value), where the value is + # either a string or None (no explicit value). A macro + # undefinition is a 1-tuple (name,). + self.macros = [] + + # 'include_dirs': a list of directories to search for include files + self.include_dirs = [] + + # 'libraries': a list of libraries to include in any link + # (library names, not filenames: eg. "foo" not "libfoo.a") + self.libraries = [] + + # 'library_dirs': a list of directories to search for libraries + self.library_dirs = [] + + # 'runtime_library_dirs': a list of directories to search for + # shared libraries/objects at runtime + self.runtime_library_dirs = [] + + # 'objects': a list of object files (or similar, such as explicitly + # named library files) to include on any link + self.objects = [] + + for key in self.executables.keys(): + self.set_executable(key, self.executables[key]) + + def set_executables(self, **kwargs): + """Define the executables (and options for them) that will be run + to perform the various stages of compilation. The exact set of + executables that may be specified here depends on the compiler + class (via the 'executables' class attribute), but most will have: + compiler the C/C++ compiler + linker_so linker used to create shared objects and libraries + linker_exe linker used to create binary executables + archiver static library creator + + On platforms with a command-line (Unix, DOS/Windows), each of these + is a string that will be split into executable name and (optional) + list of arguments. (Splitting the string is done similarly to how + Unix shells operate: words are delimited by spaces, but quotes and + backslashes can override this. See + 'distutils.util.split_quoted()'.) + """ + + # Note that some CCompiler implementation classes will define class + # attributes 'cpp', 'cc', etc. with hard-coded executable names; + # this is appropriate when a compiler class is for exactly one + # compiler/OS combination (eg. MSVCCompiler). Other compiler + # classes (UnixCCompiler, in particular) are driven by information + # discovered at run-time, since there are many different ways to do + # basically the same things with Unix C compilers. + + for key in kwargs: + if key not in self.executables: + raise ValueError("unknown executable '%s' for class %s" % + (key, self.__class__.__name__)) + self.set_executable(key, kwargs[key]) + + def set_executable(self, key, value): + if isinstance(value, str): + setattr(self, key, split_quoted(value)) + else: + setattr(self, key, value) + + def _find_macro(self, name): + i = 0 + for defn in self.macros: + if defn[0] == name: + return i + i += 1 + return None + + def _check_macro_definitions(self, definitions): + """Ensures that every element of 'definitions' is a valid macro + definition, ie. either (name,value) 2-tuple or a (name,) tuple. Do + nothing if all definitions are OK, raise TypeError otherwise. + """ + for defn in definitions: + if not (isinstance(defn, tuple) and + (len(defn) in (1, 2) and + (isinstance (defn[1], str) or defn[1] is None)) and + isinstance (defn[0], str)): + raise TypeError(("invalid macro definition '%s': " % defn) + \ + "must be tuple (string,), (string, string), or " + \ + "(string, None)") + + + # -- Bookkeeping methods ------------------------------------------- + + def define_macro(self, name, value=None): + """Define a preprocessor macro for all compilations driven by this + compiler object. The optional parameter 'value' should be a + string; if it is not supplied, then the macro will be defined + without an explicit value and the exact outcome depends on the + compiler used (XXX true? does ANSI say anything about this?) + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro (name) + if i is not None: + del self.macros[i] + + self.macros.append((name, value)) + + def undefine_macro(self, name): + """Undefine a preprocessor macro for all compilations driven by + this compiler object. If the same macro is defined by + 'define_macro()' and undefined by 'undefine_macro()' the last call + takes precedence (including multiple redefinitions or + undefinitions). If the macro is redefined/undefined on a + per-compilation basis (ie. in the call to 'compile()'), then that + takes precedence. + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro (name) + if i is not None: + del self.macros[i] + + undefn = (name,) + self.macros.append(undefn) + + def add_include_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + header files. The compiler is instructed to search directories in + the order in which they are supplied by successive calls to + 'add_include_dir()'. + """ + self.include_dirs.append(dir) + + def set_include_dirs(self, dirs): + """Set the list of directories that will be searched to 'dirs' (a + list of strings). Overrides any preceding calls to + 'add_include_dir()'; subsequence calls to 'add_include_dir()' add + to the list passed to 'set_include_dirs()'. This does not affect + any list of standard include directories that the compiler may + search by default. + """ + self.include_dirs = dirs[:] + + + # -- Private utility methods -------------------------------------- + # (here for the convenience of subclasses) + + # Helper method to prep compiler in subclass compile() methods + + def _fix_compile_args(self, output_dir, macros, include_dirs): + """Typecheck and fix-up some of the arguments to the 'compile()' + method, and return fixed-up values. Specifically: if 'output_dir' + is None, replaces it with 'self.output_dir'; ensures that 'macros' + is a list, and augments it with 'self.macros'; ensures that + 'include_dirs' is a list, and augments it with 'self.include_dirs'. + Guarantees that the returned values are of the correct type, + i.e. for 'output_dir' either string or None, and for 'macros' and + 'include_dirs' either list or None. + """ + if output_dir is None: + output_dir = self.output_dir + elif not isinstance(output_dir, str): + raise TypeError("'output_dir' must be a string or None") + + if macros is None: + macros = self.macros + elif isinstance(macros, list): + macros = macros + (self.macros or []) + else: + raise TypeError("'macros' (if supplied) must be a list of tuples") + + if include_dirs is None: + include_dirs = self.include_dirs + elif isinstance(include_dirs, (list, tuple)): + include_dirs = list(include_dirs) + (self.include_dirs or []) + else: + raise TypeError( + "'include_dirs' (if supplied) must be a list of strings") + + return output_dir, macros, include_dirs + + + # -- Worker methods ------------------------------------------------ + # (must be implemented by subclasses) + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + """Preprocess a single C/C++ source file, named in 'source'. + Output will be written to file named 'output_file', or stdout if + 'output_file' not supplied. 'macros' is a list of macro + definitions as for 'compile()', which will augment the macros set + with 'define_macro()' and 'undefine_macro()'. 'include_dirs' is a + list of directory names that will be added to the default list. + + Raises PreprocessError on failure. + """ + pass + + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function; there is + # no appropriate default implementation so subclasses should + # implement all of these. + +# def library_dir_option(self, dir): +# """Return the compiler option to add 'dir' to the list of +# directories searched for libraries. +# """ +# raise NotImplementedError +# +# def runtime_library_dir_option(self, dir): +# """Return the compiler option to add 'dir' to the list of +# directories searched for runtime libraries. +# """ +# raise NotImplementedError +# +# def library_option(self, lib): +# """Return the compiler option to add 'lib' to the list of libraries +# linked into the shared library or executable. +# """ +# raise NotImplementedError +# +# def find_library_file (self, dirs, lib, debug=0): +# """Search the specified list of directories for a static or shared +# library file 'lib' and return the full path to that file. If +# 'debug' true, look for a debugging version (if that makes sense on +# the current platform). Return None if 'lib' wasn't found in any of +# the specified directories. +# """ +# raise NotImplementedError + + + # -- Utility methods ----------------------------------------------- + + def spawn(self, cmd): + raise NotImplementedError + + +# Map a sys.platform/os.name ('posix', 'nt') to the default compiler +# type for that platform. Keys are interpreted as re match +# patterns. Order is important; platform mappings are preferred over +# OS names. +_default_compilers = ( + + # Platform string mappings + + # on a cygwin built python we can use gcc like an ordinary UNIXish + # compiler + ('cygwin.*', 'unix'), + + # OS name mappings + ('posix', 'unix'), + ('nt', 'msvc'), + + ) + +def get_default_compiler(osname=None, platform=None): + """Determine the default compiler to use for the given platform. + + osname should be one of the standard Python OS names (i.e. the + ones returned by os.name) and platform the common value + returned by sys.platform for the platform in question. + + The default values are os.name and sys.platform in case the + parameters are not given. + """ + if osname is None: + osname = os.name + if platform is None: + platform = sys.platform + for pattern, compiler in _default_compilers: + if re.match(pattern, platform) is not None or \ + re.match(pattern, osname) is not None: + return compiler + # Default to Unix compiler + return 'unix' + +# Map compiler types to (module_name, class_name) pairs -- ie. where to +# find the code that implements an interface to this compiler. (The module +# is assumed to be in the 'distutils' package.) +compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', + "standard UNIX-style compiler"), + 'msvc': ('_msvccompiler', 'MSVCCompiler', + "Microsoft Visual C++"), + 'cygwin': ('cygwinccompiler', 'CygwinCCompiler', + "Cygwin port of GNU C Compiler for Win32"), + 'mingw32': ('cygwinccompiler', 'Mingw32CCompiler', + "Mingw32 port of GNU C Compiler for Win32"), + 'bcpp': ('bcppcompiler', 'BCPPCompiler', + "Borland C++ Compiler"), + } + + +def new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0): + """Generate an instance of some CCompiler subclass for the supplied + platform/compiler combination. 'plat' defaults to 'os.name' + (eg. 'posix', 'nt'), and 'compiler' defaults to the default compiler + for that platform. Currently only 'posix' and 'nt' are supported, and + the default compilers are "traditional Unix interface" (UnixCCompiler + class) and Visual C++ (MSVCCompiler class). Note that it's perfectly + possible to ask for a Unix compiler object under Windows, and a + Microsoft compiler object under Unix -- if you supply a value for + 'compiler', 'plat' is ignored. + """ + if plat is None: + plat = os.name + + try: + if compiler is None: + compiler = get_default_compiler(plat) + + (module_name, class_name, long_description) = compiler_class[compiler] + except KeyError: + msg = "don't know how to compile C/C++ code on platform '%s'" % plat + if compiler is not None: + msg = msg + " with '%s' compiler" % compiler + raise DistutilsPlatformError(msg) + + try: + module_name = "distutils." + module_name + __import__ (module_name) + module = sys.modules[module_name] + klass = vars(module)[class_name] + except ImportError: + raise + raise DistutilsModuleError( + "can't compile C/C++ code: unable to load module '%s'" % \ + module_name) + except KeyError: + raise DistutilsModuleError( + "can't compile C/C++ code: unable to find class '%s' " + "in module '%s'" % (class_name, module_name)) + + # XXX The None is necessary to preserve backwards compatibility + # with classes that expect verbose to be the first positional + # argument. + return klass(None, dry_run, force) + + +def gen_preprocess_options(macros, include_dirs): + """Generate C pre-processor options (-D, -U, -I) as used by at least + two types of compilers: the typical Unix compiler and Visual C++. + 'macros' is the usual thing, a list of 1- or 2-tuples, where (name,) + means undefine (-U) macro 'name', and (name,value) means define (-D) + macro 'name' to 'value'. 'include_dirs' is just a list of directory + names to be added to the header file search path (-I). Returns a list + of command-line options suitable for either Unix compilers or Visual + C++. + """ + # XXX it would be nice (mainly aesthetic, and so we don't generate + # stupid-looking command lines) to go over 'macros' and eliminate + # redundant definitions/undefinitions (ie. ensure that only the + # latest mention of a particular macro winds up on the command + # line). I don't think it's essential, though, since most (all?) + # Unix C compilers only pay attention to the latest -D or -U + # mention of a macro on their command line. Similar situation for + # 'include_dirs'. I'm punting on both for now. Anyways, weeding out + # redundancies like this should probably be the province of + # CCompiler, since the data structures used are inherited from it + # and therefore common to all CCompiler classes. + pp_opts = [] + for macro in macros: + if not (isinstance(macro, tuple) and 1 <= len(macro) <= 2): + raise TypeError( + "bad macro definition '%s': " + "each element of 'macros' list must be a 1- or 2-tuple" + % macro) + + if len(macro) == 1: # undefine this macro + pp_opts.append("-U%s" % macro[0]) + elif len(macro) == 2: + if macro[1] is None: # define with no explicit value + pp_opts.append("-D%s" % macro[0]) + else: + # XXX *don't* need to be clever about quoting the + # macro value here, because we're going to avoid the + # shell at all costs when we spawn the command! + pp_opts.append("-D%s=%s" % macro) + + for dir in include_dirs: + pp_opts.append("-I%s" % dir) + return pp_opts diff --git a/Tools/c-analyzer/distutils/cygwinccompiler.py b/Tools/c-analyzer/distutils/cygwinccompiler.py new file mode 100644 index 00000000000000..a3505f31f1f0a2 --- /dev/null +++ b/Tools/c-analyzer/distutils/cygwinccompiler.py @@ -0,0 +1,286 @@ +"""distutils.cygwinccompiler + +Provides the CygwinCCompiler class, a subclass of UnixCCompiler that +handles the Cygwin port of the GNU C compiler to Windows. It also contains +the Mingw32CCompiler class which handles the mingw32 port of GCC (same as +cygwin in no-cygwin mode). +""" + +# problems: +# +# * if you use a msvc compiled python version (1.5.2) +# 1. you have to insert a __GNUC__ section in its config.h +# 2. you have to generate an import library for its dll +# - create a def-file for python??.dll +# - create an import library using +# dlltool --dllname python15.dll --def python15.def \ +# --output-lib libpython15.a +# +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# +# * We put export_symbols in a def-file, and don't use +# --export-all-symbols because it doesn't worked reliable in some +# tested configurations. And because other windows compilers also +# need their symbols specified this no serious problem. +# +# tested configurations: +# +# * cygwin gcc 2.91.57/ld 2.9.4/dllwrap 0.2.4 works +# (after patching python's config.h and for C++ some other include files) +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# * mingw32 gcc 2.95.2/ld 2.9.4/dllwrap 0.2.4 works +# (ld doesn't support -shared, so we use dllwrap) +# * cygwin gcc 2.95.2/ld 2.10.90/dllwrap 2.10.90 works now +# - its dllwrap doesn't work, there is a bug in binutils 2.10.90 +# see also http://sources.redhat.com/ml/cygwin/2000-06/msg01274.html +# - using gcc -mdll instead dllwrap doesn't work without -static because +# it tries to link against dlls instead their import libraries. (If +# it finds the dll first.) +# By specifying -static we force ld to link against the import libraries, +# this is windows standard and there are normally not the necessary symbols +# in the dlls. +# *** only the version of June 2000 shows these problems +# * cygwin gcc 3.2/ld 2.13.90 works +# (ld supports -shared) +# * mingw gcc 3.2/ld 2.13 works +# (ld supports -shared) + +import os +import sys +from subprocess import Popen, PIPE, check_output +import re + +from distutils.unixccompiler import UnixCCompiler +from distutils.errors import CCompilerError +from distutils.version import LooseVersion +from distutils.spawn import find_executable + +def get_msvcr(): + """Include the appropriate MSVC runtime library if Python was built + with MSVC 7.0 or later. + """ + msc_pos = sys.version.find('MSC v.') + if msc_pos != -1: + msc_ver = sys.version[msc_pos+6:msc_pos+10] + if msc_ver == '1300': + # MSVC 7.0 + return ['msvcr70'] + elif msc_ver == '1310': + # MSVC 7.1 + return ['msvcr71'] + elif msc_ver == '1400': + # VS2005 / MSVC 8.0 + return ['msvcr80'] + elif msc_ver == '1500': + # VS2008 / MSVC 9.0 + return ['msvcr90'] + elif msc_ver == '1600': + # VS2010 / MSVC 10.0 + return ['msvcr100'] + else: + raise ValueError("Unknown MS Compiler version %s " % msc_ver) + + +class CygwinCCompiler(UnixCCompiler): + """ Handles the Cygwin port of the GNU C compiler to Windows. + """ + compiler_type = 'cygwin' + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".dll" + static_lib_format = "lib%s%s" + shared_lib_format = "%s%s" + exe_extension = ".exe" + + def __init__(self, verbose=0, dry_run=0, force=0): + + UnixCCompiler.__init__(self, verbose, dry_run, force) + + status, details = check_config_h() + self.debug_print("Python's GCC status: %s (details: %s)" % + (status, details)) + if status is not CONFIG_H_OK: + self.warn( + "Python's pyconfig.h doesn't seem to support your compiler. " + "Reason: %s. " + "Compiling may fail because of undefined preprocessor macros." + % details) + + self.gcc_version, self.ld_version, self.dllwrap_version = \ + get_versions() + self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % + (self.gcc_version, + self.ld_version, + self.dllwrap_version) ) + + # ld_version >= "2.10.90" and < "2.13" should also be able to use + # gcc -mdll instead of dllwrap + # Older dllwraps had own version numbers, newer ones use the + # same as the rest of binutils ( also ld ) + # dllwrap 2.10.90 is buggy + if self.ld_version >= "2.10.90": + self.linker_dll = "gcc" + else: + self.linker_dll = "dllwrap" + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # Hard-code GCC because that's what this is all about. + # XXX optimization, warnings etc. should be customizable. + self.set_executables(compiler='gcc -mcygwin -O -Wall', + compiler_so='gcc -mcygwin -mdll -O -Wall', + compiler_cxx='g++ -mcygwin -O -Wall', + linker_exe='gcc -mcygwin', + linker_so=('%s -mcygwin %s' % + (self.linker_dll, shared_option))) + + # cygwin and mingw32 need different sets of libraries + if self.gcc_version == "2.91.57": + # cygwin shouldn't need msvcrt, but without the dlls will crash + # (gcc version 2.91.57) -- perhaps something about initialization + self.dll_libraries=["msvcrt"] + self.warn( + "Consider upgrading to a newer version of gcc") + else: + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + + +# the same as cygwin plus some additional parameters +class Mingw32CCompiler(CygwinCCompiler): + """ Handles the Mingw32 port of the GNU C compiler to Windows. + """ + compiler_type = 'mingw32' + + def __init__(self, verbose=0, dry_run=0, force=0): + + CygwinCCompiler.__init__ (self, verbose, dry_run, force) + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # A real mingw32 doesn't need to specify a different entry point, + # but cygwin 2.91.57 in no-cygwin-mode needs it. + if self.gcc_version <= "2.91.57": + entry_point = '--entry _DllMain@12' + else: + entry_point = '' + + if is_cygwingcc(): + raise CCompilerError( + 'Cygwin gcc cannot be used with --compiler=mingw32') + + self.set_executables(compiler='gcc -O -Wall', + compiler_so='gcc -mdll -O -Wall', + compiler_cxx='g++ -O -Wall', + linker_exe='gcc', + linker_so='%s %s %s' + % (self.linker_dll, shared_option, + entry_point)) + # Maybe we should also append -mthreads, but then the finished + # dlls need another dll (mingwm10.dll see Mingw32 docs) + # (-mthreads: Support thread-safe exception handling on `Mingw32') + + # no additional libraries needed + self.dll_libraries=[] + + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + +# Because these compilers aren't configured in Python's pyconfig.h file by +# default, we should at least warn the user if he is using an unmodified +# version. + +CONFIG_H_OK = "ok" +CONFIG_H_NOTOK = "not ok" +CONFIG_H_UNCERTAIN = "uncertain" + +def check_config_h(): + """Check if the current Python installation appears amenable to building + extensions with GCC. + + Returns a tuple (status, details), where 'status' is one of the following + constants: + + - CONFIG_H_OK: all is well, go ahead and compile + - CONFIG_H_NOTOK: doesn't look good + - CONFIG_H_UNCERTAIN: not sure -- unable to read pyconfig.h + + 'details' is a human-readable string explaining the situation. + + Note there are two ways to conclude "OK": either 'sys.version' contains + the string "GCC" (implying that this Python was built with GCC), or the + installed "pyconfig.h" contains the string "__GNUC__". + """ + + # XXX since this function also checks sys.version, it's not strictly a + # "pyconfig.h" check -- should probably be renamed... + + import sysconfig + + # if sys.version contains GCC then python was compiled with GCC, and the + # pyconfig.h file should be OK + if "GCC" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'GCC'" + + # let's see if __GNUC__ is mentioned in python.h + fn = sysconfig.get_config_h_filename() + try: + config_h = open(fn) + try: + if "__GNUC__" in config_h.read(): + return CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn + else: + return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn + finally: + config_h.close() + except OSError as exc: + return (CONFIG_H_UNCERTAIN, + "couldn't read '%s': %s" % (fn, exc.strerror)) + +RE_VERSION = re.compile(br'(\d+\.\d+(\.\d+)*)') + +def _find_exe_version(cmd): + """Find the version of an executable by running `cmd` in the shell. + + If the command is not found, or the output does not match + `RE_VERSION`, returns None. + """ + executable = cmd.split()[0] + if find_executable(executable) is None: + return None + out = Popen(cmd, shell=True, stdout=PIPE).stdout + try: + out_string = out.read() + finally: + out.close() + result = RE_VERSION.search(out_string) + if result is None: + return None + # LooseVersion works with strings + # so we need to decode our bytes + return LooseVersion(result.group(1).decode()) + +def get_versions(): + """ Try to find out the versions of gcc, ld and dllwrap. + + If not possible it returns None for it. + """ + commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] + return tuple([_find_exe_version(cmd) for cmd in commands]) + +def is_cygwingcc(): + '''Try to determine if the gcc that would be used is from cygwin.''' + out_string = check_output(['gcc', '-dumpmachine']) + return out_string.strip().endswith(b'cygwin') diff --git a/Tools/c-analyzer/distutils/debug.py b/Tools/c-analyzer/distutils/debug.py new file mode 100644 index 00000000000000..daf1660f0d8211 --- /dev/null +++ b/Tools/c-analyzer/distutils/debug.py @@ -0,0 +1,5 @@ +import os + +# If DISTUTILS_DEBUG is anything other than the empty string, we run in +# debug mode. +DEBUG = os.environ.get('DISTUTILS_DEBUG') diff --git a/Tools/c-analyzer/distutils/dep_util.py b/Tools/c-analyzer/distutils/dep_util.py new file mode 100644 index 00000000000000..318c830f2eab3e --- /dev/null +++ b/Tools/c-analyzer/distutils/dep_util.py @@ -0,0 +1,29 @@ +"""distutils.dep_util + +Utility functions for simple, timestamp-based dependency of files +and groups of files; also, function based entirely on such +timestamp dependency analysis.""" + +import os +from distutils.errors import DistutilsFileError + + +def newer (source, target): + """Return true if 'source' exists and is more recently modified than + 'target', or if 'source' exists and 'target' doesn't. Return false if + both exist and 'target' is the same age or younger than 'source'. + Raise DistutilsFileError if 'source' does not exist. + """ + if not os.path.exists(source): + raise DistutilsFileError("file '%s' does not exist" % + os.path.abspath(source)) + if not os.path.exists(target): + return 1 + + from stat import ST_MTIME + mtime1 = os.stat(source)[ST_MTIME] + mtime2 = os.stat(target)[ST_MTIME] + + return mtime1 > mtime2 + +# newer () diff --git a/Tools/c-analyzer/distutils/errors.py b/Tools/c-analyzer/distutils/errors.py new file mode 100644 index 00000000000000..10f8e316f675f0 --- /dev/null +++ b/Tools/c-analyzer/distutils/errors.py @@ -0,0 +1,48 @@ +"""distutils.errors + +Provides exceptions used by the Distutils modules. Note that Distutils +modules may raise standard exceptions; in particular, SystemExit is +usually raised for errors that are obviously the end-user's fault +(eg. bad command-line arguments). + +This module is safe to use in "from ... import *" mode; it only exports +symbols whose names start with "Distutils" and end with "Error".""" + +class DistutilsError (Exception): + """The root of all Distutils evil.""" + pass + +class DistutilsModuleError (DistutilsError): + """Unable to load an expected module, or to find an expected class + within some module (in particular, command modules and classes).""" + pass + +class DistutilsFileError (DistutilsError): + """Any problems in the filesystem: expected file not found, etc. + Typically this is for problems that we detect before OSError + could be raised.""" + pass + +class DistutilsPlatformError (DistutilsError): + """We don't know how to do something on the current platform (but + we do know how to do it on some platform) -- eg. trying to compile + C files on a platform not supported by a CCompiler subclass.""" + pass + +class DistutilsExecError (DistutilsError): + """Any problems executing an external program (such as the C + compiler, when compiling C files).""" + pass + +# Exception classes used by the CCompiler implementation classes +class CCompilerError (Exception): + """Some compile/link operation failed.""" + +class PreprocessError (CCompilerError): + """Failure to preprocess one or more C/C++ files.""" + +class CompileError (CCompilerError): + """Failure to compile one or more C/C++ source files.""" + +class UnknownFileError (CCompilerError): + """Attempt to process an unknown file type.""" diff --git a/Tools/c-analyzer/distutils/log.py b/Tools/c-analyzer/distutils/log.py new file mode 100644 index 00000000000000..26ecf22ae19bef --- /dev/null +++ b/Tools/c-analyzer/distutils/log.py @@ -0,0 +1,63 @@ +"""A simple log mechanism styled after PEP 282.""" + +# The class here is styled after PEP 282 so that it could later be +# replaced with a standard Python logging implementation. + +DEBUG = 1 +INFO = 2 +WARN = 3 +ERROR = 4 +FATAL = 5 + +import sys + +class Log: + + def __init__(self, threshold=WARN): + self.threshold = threshold + + def _log(self, level, msg, args): + if level not in (DEBUG, INFO, WARN, ERROR, FATAL): + raise ValueError('%s wrong log level' % str(level)) + + if level >= self.threshold: + if args: + msg = msg % args + if level in (WARN, ERROR, FATAL): + stream = sys.stderr + else: + stream = sys.stdout + try: + stream.write('%s\n' % msg) + except UnicodeEncodeError: + # emulate backslashreplace error handler + encoding = stream.encoding + msg = msg.encode(encoding, "backslashreplace").decode(encoding) + stream.write('%s\n' % msg) + stream.flush() + + def log(self, level, msg, *args): + self._log(level, msg, args) + + def debug(self, msg, *args): + self._log(DEBUG, msg, args) + + def info(self, msg, *args): + self._log(INFO, msg, args) + + def warn(self, msg, *args): + self._log(WARN, msg, args) + + def error(self, msg, *args): + self._log(ERROR, msg, args) + + def fatal(self, msg, *args): + self._log(FATAL, msg, args) + +_global_log = Log() +log = _global_log.log +debug = _global_log.debug +info = _global_log.info +warn = _global_log.warn +error = _global_log.error +fatal = _global_log.fatal diff --git a/Tools/c-analyzer/distutils/msvc9compiler.py b/Tools/c-analyzer/distutils/msvc9compiler.py new file mode 100644 index 00000000000000..38fff9b2d53120 --- /dev/null +++ b/Tools/c-analyzer/distutils/msvc9compiler.py @@ -0,0 +1,438 @@ +"""distutils.msvc9compiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for the Microsoft Visual Studio 2008. + +The module is compatible with VS 2005 and VS 2008. You can find legacy support +for older versions of VS in distutils.msvccompiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS2005 and VS 2008 by Christian Heimes + +import os +import subprocess +import sys +import re + +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +import winreg + +RegOpenKeyEx = winreg.OpenKeyEx +RegEnumKey = winreg.EnumKey +RegEnumValue = winreg.EnumValue +RegError = winreg.error + +HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + +NATIVE_WIN64 = (sys.platform == 'win32' and sys.maxsize > 2**32) +if NATIVE_WIN64: + # Visual C++ is a 32-bit application, so we need to look in + # the corresponding registry branch, if we're running a + # 64-bit Python on Win64 + VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f" + WINSDK_BASE = r"Software\Wow6432Node\Microsoft\Microsoft SDKs\Windows" + NET_BASE = r"Software\Wow6432Node\Microsoft\.NETFramework" +else: + VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" + WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" + NET_BASE = r"Software\Microsoft\.NETFramework" + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is +# the param to cross-compile on x86 targeting amd64.) +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'amd64', +} + +class Reg: + """Helper class to read values from the registry + """ + + def get_value(cls, path, key): + for base in HKEYS: + d = cls.read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + get_value = classmethod(get_value) + + def read_keys(cls, base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + read_keys = classmethod(read_keys) + + def read_values(cls, base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) + i += 1 + return d + read_values = classmethod(read_values) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + convert_mbcs = staticmethod(convert_mbcs) + +class MacroExpander: + + def __init__(self, version): + self.macros = {} + self.vsbase = VS_BASE % version + self.load_macros(version) + + def set_macro(self, macro, path, key): + self.macros["$(%s)" % macro] = Reg.get_value(path, key) + + def load_macros(self, version): + self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") + self.set_macro("FrameworkDir", NET_BASE, "installroot") + try: + if version >= 8.0: + self.set_macro("FrameworkSDKDir", NET_BASE, + "sdkinstallrootv2.0") + else: + raise KeyError("sdkinstallrootv2.0") + except KeyError: + raise DistutilsPlatformError( + """Python was built with Visual Studio 2008; +extensions must be built with a compiler than can generate compatible binaries. +Visual Studio 2008 was not found on this system. If you have Cygwin installed, +you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") + + if version >= 9.0: + self.set_macro("FrameworkVersion", self.vsbase, "clr version") + self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") + else: + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = Reg.get_value(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + if majorVersion >= 13: + # v13 was skipped and should be v14 + majorVersion += 1 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + +def removeDuplicates(variable): + """Remove duplicate values of an environment variable. + """ + oldList = variable.split(os.pathsep) + newList = [] + for i in oldList: + if i not in newList: + newList.append(i) + newVariable = os.pathsep.join(newList) + return newVariable + +def find_vcvarsall(version): + """Find the vcvarsall.bat file + + At first it tries to find the productdir of VS 2008 in the registry. If + that fails it falls back to the VS90COMNTOOLS env var. + """ + vsbase = VS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + log.debug("Unable to find productdir in registry") + productdir = None + + if not productdir or not os.path.isdir(productdir): + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + + if toolsdir and os.path.isdir(toolsdir): + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) + return None + else: + log.debug("Env var %s is not set or invalid" % toolskey) + if not productdir: + log.debug("No productdir found") + return None + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + log.debug("Unable to find vcvarsall.bat") + return None + +def query_vcvarsall(version, arch="x86"): + """Launch vcvarsall.bat and read the settings from its environment + """ + vcvarsall = find_vcvarsall(version) + interesting = {"include", "lib", "libpath", "path"} + result = {} + + if vcvarsall is None: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) + popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise DistutilsPlatformError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.split("\n"): + line = Reg.convert_mbcs(line) + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key in interesting: + if value.endswith(os.pathsep): + value = value[:-1] + result[key] = removeDuplicates(value) + + finally: + popen.stdout.close() + popen.stderr.close() + + if len(result) != len(interesting): + raise ValueError(str(list(result.keys()))) + + return result + +# More globals +VERSION = get_build_version() +if VERSION < 8.0: + raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) +# MACROS = MacroExpander(VERSION) + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + self.__version = VERSION + self.__root = r"Software\Microsoft\VisualStudio" + # self.__macros = MACROS + self.__paths = [] + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.__arch = None # deprecated name + self.initialized = False + + # -- Worker methods ------------------------------------------------ + + def manifest_setup_ldargs(self, output_filename, build_temp, ld_args): + # If we need a manifest at all, an embedded manifest is recommended. + # See MSDN article titled + # "How to: Embed a Manifest Inside a C/C++ Application" + # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) + # Ask the linker to generate the manifest in the temp dir, so + # we can check it, and possibly embed it, later. + temp_manifest = os.path.join( + build_temp, + os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFESTFILE:' + temp_manifest) + + def manifest_get_embed_info(self, target_desc, ld_args): + # If a manifest should be embedded, return a tuple of + # (manifest_filename, resource_id). Returns None if no manifest + # should be embedded. See http://bugs.python.org/issue7833 for why + # we want to avoid any manifest for extension modules if we can. + for arg in ld_args: + if arg.startswith("/MANIFESTFILE:"): + temp_manifest = arg.split(":", 1)[1] + break + else: + # no /MANIFESTFILE so nothing to do. + return None + if target_desc == CCompiler.EXECUTABLE: + # by default, executables always get the manifest with the + # CRT referenced. + mfid = 1 + else: + # Extension modules try and avoid any manifest if possible. + mfid = 2 + temp_manifest = self._remove_visual_c_ref(temp_manifest) + if temp_manifest is None: + return None + return temp_manifest, mfid + + def _remove_visual_c_ref(self, manifest_file): + try: + # Remove references to the Visual C runtime, so they will + # fall through to the Visual C dependency of Python.exe. + # This way, when installed for a restricted user (e.g. + # runtimes are not in WinSxS folder, but in Python's own + # folder), the runtimes do not need to be in every folder + # with .pyd's. + # Returns either the filename of the modified manifest or + # None if no manifest should be embedded. + manifest_f = open(manifest_file) + try: + manifest_buf = manifest_f.read() + finally: + manifest_f.close() + pattern = re.compile( + r"""|)""", + re.DOTALL) + manifest_buf = re.sub(pattern, "", manifest_buf) + pattern = r"\s*" + manifest_buf = re.sub(pattern, "", manifest_buf) + # Now see if any other assemblies are referenced - if not, we + # don't want a manifest embedded. + pattern = re.compile( + r"""|)""", re.DOTALL) + if re.search(pattern, manifest_buf) is None: + return None + + manifest_f = open(manifest_file, 'w') + try: + manifest_f.write(manifest_buf) + return manifest_file + finally: + manifest_f.close() + except OSError: + pass + + # -- Miscellaneous methods ----------------------------------------- + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe diff --git a/Tools/c-analyzer/distutils/msvccompiler.py b/Tools/c-analyzer/distutils/msvccompiler.py new file mode 100644 index 00000000000000..c0864b1f95d4e2 --- /dev/null +++ b/Tools/c-analyzer/distutils/msvccompiler.py @@ -0,0 +1,327 @@ +"""distutils.msvccompiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for the Microsoft Visual Studio. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) + +import sys, os +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +_can_read_reg = False +try: + import winreg + + _can_read_reg = True + hkey_mod = winreg + + RegOpenKeyEx = winreg.OpenKeyEx + RegEnumKey = winreg.EnumKey + RegEnumValue = winreg.EnumValue + RegError = winreg.error + +except ImportError: + try: + import win32api + import win32con + _can_read_reg = True + hkey_mod = win32con + + RegOpenKeyEx = win32api.RegOpenKeyEx + RegEnumKey = win32api.RegEnumKey + RegEnumValue = win32api.RegEnumValue + RegError = win32api.error + except ImportError: + log.info("Warning: Can't read registry to find the " + "necessary compiler setting\n" + "Make sure that Python modules winreg, " + "win32api or win32con are installed.") + +if _can_read_reg: + HKEYS = (hkey_mod.HKEY_USERS, + hkey_mod.HKEY_CURRENT_USER, + hkey_mod.HKEY_LOCAL_MACHINE, + hkey_mod.HKEY_CLASSES_ROOT) + +def read_keys(base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + +def read_values(base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[convert_mbcs(name)] = convert_mbcs(value) + i += 1 + return d + +def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + +class MacroExpander: + def __init__(self, version): + self.macros = {} + self.load_macros(version) + + def set_macro(self, macro, path, key): + for base in HKEYS: + d = read_values(base, path) + if d: + self.macros["$(%s)" % macro] = d[key] + break + + def load_macros(self, version): + vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version + self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") + net = r"Software\Microsoft\.NETFramework" + self.set_macro("FrameworkDir", net, "installroot") + try: + if version > 7.0: + self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") + else: + self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") + except KeyError as exc: # + raise DistutilsPlatformError( + """Python was built with Visual Studio 2003; +extensions must be built with a compiler than can generate compatible binaries. +Visual Studio 2003 was not found on this system. If you have Cygwin installed, +you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") + + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = read_values(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + if majorVersion >= 13: + # v13 was skipped and should be v14 + majorVersion += 1 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def get_build_architecture(): + """Return the processor architecture. + + Possible results are "Intel" or "AMD64". + """ + + prefix = " bit (" + i = sys.version.find(prefix) + if i == -1: + return "Intel" + j = sys.version.find(")", i) + return sys.version[i+len(prefix):j] + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + self.__version = get_build_version() + self.__arch = get_build_architecture() + if self.__arch == "Intel": + # x86 + if self.__version >= 7: + self.__root = r"Software\Microsoft\VisualStudio" + self.__macros = MacroExpander(self.__version) + else: + self.__root = r"Software\Microsoft\Devstudio" + self.__product = "Visual Studio version %s" % self.__version + else: + # Win64. Assume this was built with the platform SDK + self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) + + self.initialized = False + + + # -- Miscellaneous methods ----------------------------------------- + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe + + def get_msvc_paths(self, path, platform='x86'): + """Get a list of devstudio directories (include, lib or path). + + Return a list of strings. The list will be empty if unable to + access the registry or appropriate registry keys not found. + """ + if not _can_read_reg: + return [] + + path = path + " dirs" + if self.__version >= 7: + key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" + % (self.__root, self.__version)) + else: + key = (r"%s\6.0\Build System\Components\Platforms" + r"\Win32 (%s)\Directories" % (self.__root, platform)) + + for base in HKEYS: + d = read_values(base, key) + if d: + if self.__version >= 7: + return self.__macros.sub(d[path]).split(";") + else: + return d[path].split(";") + # MSVC 6 seems to create the registry entries we need only when + # the GUI is run. + if self.__version == 6: + for base in HKEYS: + if read_values(base, r"%s\6.0" % self.__root) is not None: + self.warn("It seems you have Visual Studio 6 installed, " + "but the expected registry settings are not present.\n" + "You must at least run the Visual Studio GUI once " + "so that these entries are created.") + break + return [] + + def set_path_env_var(self, name): + """Set environment variable 'name' to an MSVC path type value. + + This is equivalent to a SET command prior to execution of spawned + commands. + """ + + if name == "lib": + p = self.get_msvc_paths("library") + else: + p = self.get_msvc_paths(name) + if p: + os.environ[name] = ';'.join(p) + + +if get_build_version() >= 8.0: + log.debug("Importing new compiler from distutils.msvc9compiler") + OldMSVCCompiler = MSVCCompiler + from distutils.msvc9compiler import MSVCCompiler + # get_build_architecture not really relevant now we support cross-compile + from distutils.msvc9compiler import MacroExpander diff --git a/Tools/c-analyzer/distutils/spawn.py b/Tools/c-analyzer/distutils/spawn.py new file mode 100644 index 00000000000000..eb9794964d7acb --- /dev/null +++ b/Tools/c-analyzer/distutils/spawn.py @@ -0,0 +1,48 @@ +"""distutils.spawn + +Provides the 'spawn()' function, a front-end to various platform- +specific functions for launching another program in a sub-process. +Also provides the 'find_executable()' to search the path for a given +executable name. +""" + +import sys +import os +import os.path + + +def find_executable(executable, path=None): + """Tries to find 'executable' in the directories listed in 'path'. + + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + """ + _, ext = os.path.splitext(executable) + if (sys.platform == 'win32') and (ext != '.exe'): + executable = executable + '.exe' + + if os.path.isfile(executable): + return executable + + if path is None: + path = os.environ.get('PATH', None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable is + # set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current directory + if not path: + return None + + paths = path.split(os.pathsep) + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None diff --git a/Tools/c-analyzer/distutils/unixccompiler.py b/Tools/c-analyzer/distutils/unixccompiler.py new file mode 100644 index 00000000000000..1a3a8e5255070d --- /dev/null +++ b/Tools/c-analyzer/distutils/unixccompiler.py @@ -0,0 +1,102 @@ +"""distutils.unixccompiler + +Contains the UnixCCompiler class, a subclass of CCompiler that handles +the "typical" Unix-style command-line C compiler: + * macros defined with -Dname[=value] + * macros undefined with -Uname + * include search directories specified with -Idir + * libraries specified with -lllib + * library search directories specified with -Ldir + * compile handled by 'cc' (or similar) executable with -c option: + compiles .c to .o + * link static library handled by 'ar' command (possibly with 'ranlib') + * link shared library handled by 'cc -shared' +""" + +import os, sys, re + +from distutils.dep_util import newer +from distutils.ccompiler import CCompiler, gen_preprocess_options +from distutils.errors import DistutilsExecError, CompileError + +# XXX Things not currently handled: +# * optimization/debug/warning flags; we just use whatever's in Python's +# Makefile and live with it. Is this adequate? If not, we might +# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, +# SunCCompiler, and I suspect down that road lies madness. +# * even if we don't know a warning flag from an optimization flag, +# we need some way for outsiders to feed preprocessor/compiler/linker +# flags in to us -- eg. a sysadmin might want to mandate certain flags +# via a site config file, or a user might want to set something for +# compiling this module distribution only via the setup.py command +# line, whatever. As long as these options come from something on the +# current system, they can be as system-dependent as they like, and we +# should just happily stuff them into the preprocessor/compiler/linker +# options and carry on. + + +class UnixCCompiler(CCompiler): + + compiler_type = 'unix' + + # These are used by CCompiler in two places: the constructor sets + # instance attributes 'preprocessor', 'compiler', etc. from them, and + # 'set_executable()' allows any of these to be set. The defaults here + # are pretty generic; they will probably have to be set by an outsider + # (eg. using information discovered by the sysconfig about building + # Python extensions). + executables = {'preprocessor' : None, + 'compiler' : ["cc"], + 'compiler_so' : ["cc"], + 'compiler_cxx' : ["cc"], + 'linker_so' : ["cc", "-shared"], + 'linker_exe' : ["cc"], + 'archiver' : ["ar", "-cr"], + 'ranlib' : None, + } + + if sys.platform[:6] == "darwin": + executables['ranlib'] = ["ranlib"] + + # Needed for the filename generation methods provided by the base + # class, CCompiler. NB. whoever instantiates/uses a particular + # UnixCCompiler instance should set 'shared_lib_ext' -- we set a + # reasonable common default here, but it's not necessarily used on all + # Unices! + + src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".so" + dylib_lib_extension = ".dylib" + xcode_stub_lib_extension = ".tbd" + static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + xcode_stub_lib_format = dylib_lib_format + if sys.platform == "cygwin": + exe_extension = ".exe" + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + fixed_args = self._fix_compile_args(None, macros, include_dirs) + ignore, macros, include_dirs = fixed_args + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = self.preprocessor + pp_opts + if output_file: + pp_args.extend(['-o', output_file]) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or we're + # generating output to stdout, or there's a target output file and + # the source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + raise CompileError(msg) diff --git a/Tools/c-analyzer/distutils/util.py b/Tools/c-analyzer/distutils/util.py new file mode 100644 index 00000000000000..89ca094336fdb8 --- /dev/null +++ b/Tools/c-analyzer/distutils/util.py @@ -0,0 +1,171 @@ +"""distutils.util + +Miscellaneous utility functions -- anything that doesn't fit into +one of the other *util.py modules. +""" + +import os +import re +import string +import sys +from distutils.errors import DistutilsPlatformError + +def get_host_platform(): + """Return a string that identifies the current platform. This is used mainly to + distinguish platform-specific build directories and platform-specific built + distributions. Typically includes the OS name and version and the + architecture (as supplied by 'os.uname()'), although the exact information + included depends on the OS; eg. on Linux, the kernel version isn't + particularly important. + + Examples of returned values: + linux-i586 + linux-alpha (?) + solaris-2.6-sun4u + + Windows will return one of: + win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win32 (all others - specifically, sys.platform is returned) + + For other non-POSIX platforms, currently just returns 'sys.platform'. + + """ + if os.name == 'nt': + if 'amd64' in sys.version.lower(): + return 'win-amd64' + if '(arm)' in sys.version.lower(): + return 'win-arm32' + if '(arm64)' in sys.version.lower(): + return 'win-arm64' + return sys.platform + + # Set for cross builds explicitly + if "_PYTHON_HOST_PLATFORM" in os.environ: + return os.environ["_PYTHON_HOST_PLATFORM"] + + if os.name != "posix" or not hasattr(os, 'uname'): + # XXX what about the architecture? NT is Intel or Alpha, + # Mac OS is M68k or PPC, etc. + return sys.platform + + # Try to distinguish various flavours of Unix + + (osname, host, release, version, machine) = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace('/', '') + machine = machine.replace(' ', '_') + machine = machine.replace('/', '-') + + if osname[:5] == "linux": + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return "%s-%s" % (osname, machine) + elif osname[:5] == "sunos": + if release[0] >= "5": # SunOS 5 == Solaris 2 + osname = "solaris" + release = "%d.%s" % (int(release[0]) - 3, release[2:]) + # We can't use "platform.architecture()[0]" because a + # bootstrap problem. We use a dict to get an error + # if some suspicious happens. + bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} + machine += ".%s" % bitness[sys.maxsize] + # fall through to standard osname-release-machine representation + elif osname[:3] == "aix": + from _aix_support import aix_platform + return aix_platform() + elif osname[:6] == "cygwin": + osname = "cygwin" + rel_re = re.compile (r'[\d.]+', re.ASCII) + m = rel_re.match(release) + if m: + release = m.group() + elif osname[:6] == "darwin": + import _osx_support, sysconfig + osname, release, machine = _osx_support.get_platform_osx( + sysconfig.get_config_vars(), + osname, release, machine) + + return "%s-%s-%s" % (osname, release, machine) + +def get_platform(): + if os.name == 'nt': + TARGET_TO_PLAT = { + 'x86' : 'win32', + 'x64' : 'win-amd64', + 'arm' : 'win-arm32', + } + return TARGET_TO_PLAT.get(os.environ.get('VSCMD_ARG_TGT_ARCH')) or get_host_platform() + else: + return get_host_platform() + + +# Needed by 'split_quoted()' +_wordchars_re = _squote_re = _dquote_re = None +def _init_regex(): + global _wordchars_re, _squote_re, _dquote_re + _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) + _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") + _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') + +def split_quoted (s): + """Split a string up according to Unix shell-like rules for quotes and + backslashes. In short: words are delimited by spaces, as long as those + spaces are not escaped by a backslash, or inside a quoted string. + Single and double quotes are equivalent, and the quote characters can + be backslash-escaped. The backslash is stripped from any two-character + escape sequence, leaving only the escaped character. The quote + characters are stripped from any quoted string. Returns a list of + words. + """ + + # This is a nice algorithm for splitting up a single string, since it + # doesn't require character-by-character examination. It was a little + # bit of a brain-bender to get it working right, though... + if _wordchars_re is None: _init_regex() + + s = s.strip() + words = [] + pos = 0 + + while s: + m = _wordchars_re.match(s, pos) + end = m.end() + if end == len(s): + words.append(s[:end]) + break + + if s[end] in string.whitespace: # unescaped, unquoted whitespace: now + words.append(s[:end]) # we definitely have a word delimiter + s = s[end:].lstrip() + pos = 0 + + elif s[end] == '\\': # preserve whatever is being escaped; + # will become part of the current word + s = s[:end] + s[end+1:] + pos = end+1 + + else: + if s[end] == "'": # slurp singly-quoted string + m = _squote_re.match(s, end) + elif s[end] == '"': # slurp doubly-quoted string + m = _dquote_re.match(s, end) + else: + raise RuntimeError("this can't happen (bad char '%c')" % s[end]) + + if m is None: + raise ValueError("bad string (mismatched %s quotes?)" % s[end]) + + (beg, end) = m.span() + s = s[:beg] + s[beg+1:end-1] + s[end:] + pos = m.end() - 2 + + if pos >= len(s): + words.append(s) + break + + return words + +# split_quoted () From c6858d1e7f4cd3184d5ddea4025ad5dfc7596546 Mon Sep 17 00:00:00 2001 From: Max Bachmann Date: Thu, 9 Mar 2023 22:09:12 +0100 Subject: [PATCH 07/50] gh-102255: Improve build support for Windows API partitions (GH-102256) Add `MS_WINDOWS_DESKTOP`, `MS_WINDOWS_APPS`, `MS_WINDOWS_SYSTEM` and `MS_WINDOWS_GAMES` preprocessor definitions to allow switching off functionality missing from particular API partitions ("partitions" are used in Windows to identify overlapping subsets of APIs). CPython only officially supports `MS_WINDOWS_DESKTOP` and `MS_WINDOWS_SYSTEM` (APPS is included by normal desktop builds, but APPS without DESKTOP is not covered). Other configurations are a convenience for people building their own runtimes. `MS_WINDOWS_GAMES` is for the Xbox subset of the Windows API, which is also available on client OS, but is restricted compared to `MS_WINDOWS_DESKTOP`. These restrictions may change over time, as they relate to the build headers rather than the OS support, and so we assume that Xbox builds will use the latest available version of the GDK. --- Include/internal/pycore_fileutils.h | 8 + Lib/test/test_os.py | 8 +- ...-02-26-11-43-56.gh-issue-102255.cRnI5x.rst | 1 + Modules/_io/_iomodule.c | 6 +- Modules/_io/_iomodule.h | 6 +- Modules/_io/clinic/winconsoleio.c.h | 42 ++-- Modules/_io/fileio.c | 2 + Modules/_io/winconsoleio.c | 6 +- Modules/_localemodule.c | 8 +- Modules/_multiprocessing/multiprocessing.h | 4 +- Modules/_pickle.c | 6 + Modules/_randommodule.c | 6 +- Modules/_ssl.c | 4 + Modules/_winapi.c | 13 ++ Modules/clinic/posixmodule.c.h | 10 +- Modules/errnomodule.c | 2 + Modules/faulthandler.c | 2 +- Modules/getpath.c | 2 + Modules/mmapmodule.c | 8 +- Modules/posixmodule.c | 58 +++-- Modules/selectmodule.c | 6 +- Modules/socketmodule.c | 60 +++-- Modules/timemodule.c | 8 +- Objects/obmalloc.c | 1 - PC/clinic/msvcrtmodule.c.h | 42 +++- PC/clinic/winreg.c.h | 218 +++++++++++++++++- PC/config.c | 8 + PC/config_minimal.c | 6 + PC/msvcrtmodule.c | 22 ++ PC/pyconfig.h | 24 +- PC/winreg.c | 25 ++ Parser/myreadline.c | 8 +- Python/dynload_win.c | 8 + Python/fileutils.c | 89 ++++++- Python/pylifecycle.c | 2 +- Python/sysmodule.c | 4 + 36 files changed, 633 insertions(+), 100 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index f8e2bf22590888..445ac0a3d955d9 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -251,6 +251,14 @@ extern int _Py_add_relfile(wchar_t *dirname, extern size_t _Py_find_basename(const wchar_t *filename); PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); +// The Windows Games API family does not provide these functions +// so provide our own implementations. Remove them in case they get added +// to the Games API family +#if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +#include + +extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); +#endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 792794ca109489..ba6feb69ea1712 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3084,11 +3084,13 @@ def test_device_encoding(self): class PidTests(unittest.TestCase): @unittest.skipUnless(hasattr(os, 'getppid'), "test needs os.getppid") def test_getppid(self): - p = subprocess.Popen([sys.executable, '-c', + p = subprocess.Popen([sys._base_executable, '-c', 'import os; print(os.getppid())'], - stdout=subprocess.PIPE) - stdout, _ = p.communicate() + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, error = p.communicate() # We are the parent of our subprocess + self.assertEqual(error, b'') self.assertEqual(int(stdout), os.getpid()) def check_waitpid(self, code, exitcode, callback=None): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst new file mode 100644 index 00000000000000..daabc3c15f6ee2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst @@ -0,0 +1 @@ +Improve build support for the Xbox. Patch by Max Bachmann. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index d8d836b8382eb1..1506755427fc0d 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -317,7 +317,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, _PyIO_State *state = get_io_state(module); { PyObject *RawIO_class = (PyObject *)state->PyFileIO_Type; -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _Py_GetConfig(); if (!config->legacy_windows_stdio && _PyIO_get_console_type(path_or_fd) != '\0') { RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type; @@ -660,7 +660,7 @@ static PyTypeObject* static_types[] = { // PyRawIOBase_Type(PyIOBase_Type) subclasses &_PyBytesIOBuffer_Type, -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO &PyWindowsConsoleIO_Type, #endif }; @@ -718,7 +718,7 @@ PyInit__io(void) } // Set type base classes -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; #endif diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 02daef9e85677e..d7224e56f9a722 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -26,9 +26,9 @@ extern PyType_Spec fileio_spec; extern PyType_Spec stringio_spec; extern PyType_Spec textiowrapper_spec; -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO extern PyTypeObject PyWindowsConsoleIO_Type; -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* These functions are used as METH_NOARGS methods, are normally called * with args=NULL, and return a new reference. @@ -178,7 +178,7 @@ find_io_state_by_def(PyTypeObject *type) extern _PyIO_State *_PyIO_get_module_state(void); -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO extern char _PyIO_get_console_type(PyObject *); #endif diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index df834dbde40f5b..4c5cd0892c4a6d 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -8,7 +8,7 @@ preserve #endif -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_close__doc__, "close($self, /)\n" @@ -31,9 +31,9 @@ _io__WindowsConsoleIO_close(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_close_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__, "_WindowsConsoleIO(file, mode=\'r\', closefd=True, opener=None)\n" @@ -131,9 +131,9 @@ _io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_fileno__doc__, "fileno($self, /)\n" @@ -153,9 +153,9 @@ _io__WindowsConsoleIO_fileno(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_fileno_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readable__doc__, "readable($self, /)\n" @@ -175,9 +175,9 @@ _io__WindowsConsoleIO_readable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_readable_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_writable__doc__, "writable($self, /)\n" @@ -197,9 +197,9 @@ _io__WindowsConsoleIO_writable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_writable_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readinto__doc__, "readinto($self, buffer, /)\n" @@ -239,9 +239,9 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyObject *arg) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readall__doc__, "readall($self, /)\n" @@ -263,9 +263,9 @@ _io__WindowsConsoleIO_readall(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_readall_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_read__doc__, "read($self, size=-1, /)\n" @@ -305,9 +305,9 @@ _io__WindowsConsoleIO_read(winconsoleio *self, PyObject *const *args, Py_ssize_t return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_write__doc__, "write($self, b, /)\n" @@ -348,9 +348,9 @@ _io__WindowsConsoleIO_write(winconsoleio *self, PyObject *arg) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_isatty__doc__, "isatty($self, /)\n" @@ -370,7 +370,7 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_isatty_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ #ifndef _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF #define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF @@ -407,4 +407,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=4920e9068e0cf08a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=163e934aa9b0ef16 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 35a498ce5a8354..1118d86e6c9a10 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -37,7 +37,9 @@ #ifdef MS_WINDOWS /* can simulate truncate with Win32 API functions; see file_truncate */ #define HAVE_FTRUNCATE +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #endif diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index de07b50f5ce4cb..f836e230243020 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -11,7 +11,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO #include "structmember.h" // PyMemberDef #ifdef HAVE_SYS_TYPES_H @@ -22,7 +22,9 @@ #endif #include /* For offsetof */ +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #include @@ -1177,4 +1179,4 @@ PyTypeObject PyWindowsConsoleIO_Type = { 0, /* tp_finalize */ }; -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 23c38e14d997d1..96675cdfb661ad 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -35,7 +35,9 @@ This software comes with no warranty. Use at your own risk. #endif #if defined(MS_WINDOWS) +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #endif @@ -457,12 +459,12 @@ _locale__getdefaultlocale_impl(PyObject *module) PyOS_snprintf(encoding, sizeof(encoding), "cp%u", GetACP()); - if (GetLocaleInfo(LOCALE_USER_DEFAULT, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_SISO639LANGNAME, locale, sizeof(locale))) { Py_ssize_t i = strlen(locale); locale[i++] = '_'; - if (GetLocaleInfo(LOCALE_USER_DEFAULT, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_SISO3166CTRYNAME, locale+i, (int)(sizeof(locale)-i))) return Py_BuildValue("ss", locale, encoding); @@ -474,7 +476,7 @@ _locale__getdefaultlocale_impl(PyObject *module) locale[0] = '0'; locale[1] = 'x'; - if (GetLocaleInfo(LOCALE_USER_DEFAULT, LOCALE_IDEFAULTLANGUAGE, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_IDEFAULTLANGUAGE, locale+2, sizeof(locale)-2)) { return Py_BuildValue("ss", locale, encoding); } diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index b595e5a8dd18de..dfc2a8e0799a60 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -12,7 +12,9 @@ */ #ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif # include # include # include /* getpid() */ diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 1b34977806b661..a26732af8ba2a1 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -42,6 +42,12 @@ enum { #define FLOAT FLOAT_ #define INT INT_ #define LONG LONG_ + +/* This can already be defined on Windows to set the character set + the Windows header files treat as default */ +#ifdef UNICODE +#undef UNICODE +#endif #endif /* Pickle opcodes. These must be kept updated with pickle.py. diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 68060c07033d34..6e22053239305a 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -77,6 +77,10 @@ # include // getpid() #endif +#ifdef MS_WINDOWS +# include +#endif + /* Period parameters -- These are all magic. Don't change. */ #define N 624 #define M 397 @@ -259,7 +263,7 @@ random_seed_time_pid(RandomObject *self) key[0] = (uint32_t)(now & 0xffffffffU); key[1] = (uint32_t)(now >> 32); -#ifdef MS_WINDOWS_NON_DESKTOP +#if defined(MS_WINDOWS) && !defined(MS_WINDOWS_DESKTOP) && !defined(MS_WINDOWS_SYSTEM) key[2] = (uint32_t)GetCurrentProcessId(); #elif defined(HAVE_GETPID) key[2] = (uint32_t)getpid(); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 8f03a846aed089..28112317bc289e 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -28,6 +28,10 @@ /* Include symbols from _socket module */ #include "socketmodule.h" +#ifdef MS_WINDOWS +# include +#endif + #include "_ssl.h" /* Redefined below for Windows debug builds after important #includes */ diff --git a/Modules/_winapi.c b/Modules/_winapi.c index eefc2571ee8287..83cde7501176b6 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -39,8 +39,11 @@ #include "structmember.h" // PyMemberDef +#ifndef WINDOWS_LEAN_AND_MEAN #define WINDOWS_LEAN_AND_MEAN +#endif #include "windows.h" +#include #include #include "winreparse.h" @@ -63,6 +66,14 @@ #define T_HANDLE T_POINTER +// winbase.h limits the STARTF_* flags to the desktop API as of 10.0.19041. +#ifndef STARTF_USESHOWWINDOW +#define STARTF_USESHOWWINDOW 0x00000001 +#endif +#ifndef STARTF_USESTDHANDLES +#define STARTF_USESTDHANDLES 0x00000100 +#endif + typedef struct { PyTypeObject *overlapped_type; } WinApiState; @@ -1201,8 +1212,10 @@ _winapi_ExitProcess_impl(PyObject *module, UINT ExitCode) /*[clinic end generated code: output=a387deb651175301 input=4f05466a9406c558]*/ { #if defined(Py_DEBUG) +#ifdef MS_WINDOWS_DESKTOP SetErrorMode(SEM_FAILCRITICALERRORS|SEM_NOALIGNMENTFAULTEXCEPT| SEM_NOGPFAULTERRORBOX|SEM_NOOPENFILEERRORBOX); +#endif _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_DEBUG); #endif diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index dcd25c28370c93..6565f8df935cb8 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10988,7 +10988,7 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #endif /* defined(HAVE_GETRANDOM_SYSCALL) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os__add_dll_directory__doc__, "_add_dll_directory($module, /, path)\n" @@ -11057,9 +11057,9 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os__remove_dll_directory__doc__, "_remove_dll_directory($module, /, cookie)\n" @@ -11120,7 +11120,7 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ #if (defined(WIFEXITED) || defined(MS_WINDOWS)) @@ -11796,4 +11796,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=1b0eb6a76b1a0e28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9495478e51701b8a input=a9049054013a1b77]*/ diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 4de4144520aa48..df4e494ba8a973 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -5,7 +5,9 @@ /* Windows socket errors (WSA*) */ #ifdef MS_WINDOWS +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include /* The following constants were added to errno.h in VS2010 but have preferred WSA equivalents. */ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 5309a3728c5e07..bfe35fed7a450a 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -953,7 +953,7 @@ faulthandler_unregister_py(PyObject *self, PyObject *args) static void faulthandler_suppress_crash_report(void) { -#ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP UINT mode; /* Configure Windows to not display the Windows Error Reporting dialog */ diff --git a/Modules/getpath.c b/Modules/getpath.c index c807a3c8e9a2b9..2f20521592ce2e 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -745,10 +745,12 @@ static int library_to_dict(PyObject *dict, const char *key) { #ifdef MS_WINDOWS +#ifdef Py_ENABLE_SHARED extern HMODULE PyWin_DLLhModule; if (PyWin_DLLhModule) { return winmodule_to_dict(dict, key, PyWin_DLLhModule); } +#endif #elif defined(WITH_NEXT_FRAMEWORK) static char modPath[MAXPATHLEN + 1]; static int modPathInitialized = -1; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6054c2853d7a78..fe76ca6eafaa88 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -29,6 +29,10 @@ #include "structmember.h" // PyMemberDef #include // offsetof() +// to support MS_WINDOWS_SYSTEM OpenFileMappingA / CreateFileMappingA +// need to be replaced with OpenFileMappingW / CreateFileMappingW +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) + #ifndef MS_WINDOWS #define UNIX # ifdef HAVE_FCNTL_H @@ -647,7 +651,7 @@ mmap_flush_method(mmap_object *self, PyObject *args) if (self->access == ACCESS_READ || self->access == ACCESS_COPY) Py_RETURN_NONE; -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) if (!FlushViewOfFile(self->data+offset, size)) { PyErr_SetFromWindowsErr(GetLastError()); return NULL; @@ -1724,3 +1728,5 @@ PyInit_mmap(void) { return PyModuleDef_Init(&mmapmodule); } + +#endif /* !MS_WINDOWS || MS_WINDOWS_DESKTOP || MS_WINDOWS_GAMES */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a3d86cbe7a57fb..0d534f35f6a420 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -26,11 +26,15 @@ #ifdef MS_WINDOWS # include -# include +# if !defined(MS_WINDOWS_GAMES) || defined(MS_WINDOWS_DESKTOP) +# include +# endif # include # include // UNLEN # include "osdefs.h" // SEP -# define HAVE_SYMLINK +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) +# define HAVE_SYMLINK +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ #endif #include "structmember.h" // PyMemberDef @@ -311,7 +315,7 @@ corresponding Unix manual entries for more information on calls."); # include #endif -#if defined(MS_WINDOWS) +#ifdef HAVE_WINDOWS_CONSOLE_IO # define TERMSIZE_USE_CONIO #elif defined(HAVE_SYS_IOCTL_H) # include @@ -321,7 +325,7 @@ corresponding Unix manual entries for more information on calls."); # if defined(TIOCGWINSZ) # define TERMSIZE_USE_IOCTL # endif -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* Various compilers have only certain posix functions */ /* XXX Gosh I wish these were all moved into pyconfig.h */ @@ -329,21 +333,25 @@ corresponding Unix manual entries for more information on calls."); # define HAVE_OPENDIR 1 # define HAVE_SYSTEM 1 # include -#else -# ifdef _MSC_VER - /* Microsoft compiler */ +#elif defined( _MSC_VER) + /* Microsoft compiler */ +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_GETPPID 1 +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_APP | MS_WINDOWS_SYSTEM */ +# if defined(MS_WINDOWS_DESKTOP) # define HAVE_GETLOGIN 1 +# endif /* MS_WINDOWS_DESKTOP */ +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SPAWNV 1 # define HAVE_EXECV 1 # define HAVE_WSPAWNV 1 # define HAVE_WEXECV 1 -# define HAVE_PIPE 1 # define HAVE_SYSTEM 1 # define HAVE_CWAIT 1 -# define HAVE_FSYNC 1 -# define fsync _commit -# endif /* _MSC_VER */ +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ +# define HAVE_PIPE 1 +# define HAVE_FSYNC 1 +# define fsync _commit #endif /* ! __WATCOMC__ || __QNX__ */ /*[clinic input] @@ -1536,7 +1544,7 @@ convertenviron(void) #ifdef MS_WINDOWS /* _wenviron must be initialized in this way if the program is started through main() instead of wmain(). */ - _wgetenv(L""); + (void)_wgetenv(L""); e = _wenviron; #elif defined(WITH_NEXT_FRAMEWORK) || (defined(__APPLE__) && defined(Py_ENABLE_SHARED)) /* environ is not accessible as an extern in a shared object on OSX; use @@ -1785,6 +1793,10 @@ attributes_from_dir(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *re if (n && (pszFile[n - 1] == L'\\' || pszFile[n - 1] == L'/')) { // cannot use PyMem_Malloc here because we do not hold the GIL filename = (LPCWSTR)malloc((n + 1) * sizeof(filename[0])); + if(!filename) { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + return FALSE; + } wcsncpy_s((LPWSTR)filename, n + 1, pszFile, n); while (--n > 0 && (filename[n] == L'\\' || filename[n] == L'/')) { ((LPWSTR)filename)[n] = L'\0'; @@ -7933,10 +7945,10 @@ static PyObject * os_getpid_impl(PyObject *module) /*[clinic end generated code: output=9ea6fdac01ed2b3c input=5a9a00f0ab68aa00]*/ { -#ifdef MS_WINDOWS_NON_DESKTOP - return PyLong_FromUnsignedLong(GetCurrentProcessId()); -#else +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) return PyLong_FromPid(getpid()); +#else + return PyLong_FromUnsignedLong(GetCurrentProcessId()); #endif } #endif /* defined(HAVE_GETPID) */ @@ -8392,6 +8404,7 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) DWORD err; HANDLE handle; +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Console processes which share a common console can be sent CTRL+C or CTRL+BREAK events, provided they handle said events. */ if (sig == CTRL_C_EVENT || sig == CTRL_BREAK_EVENT) { @@ -8399,9 +8412,11 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) err = GetLastError(); PyErr_SetFromWindowsErr(err); } - else + else { Py_RETURN_NONE; + } } +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* If the signal is outside of what GenerateConsoleCtrlEvent can use, attempt to open and terminate the process. */ @@ -13776,7 +13791,9 @@ os_cpu_count_impl(PyObject *module) { int ncpu = 0; #ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); +#endif #elif defined(__hpux) ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL); #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) @@ -13848,6 +13865,10 @@ os_set_inheritable_impl(PyObject *module, int fd, int inheritable) #ifdef MS_WINDOWS +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif + /*[clinic input] os.get_handle_inheritable -> bool handle: intptr_t @@ -15023,7 +15044,8 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) } #endif /* HAVE_GETRANDOM_SYSCALL */ -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) + /* bpo-36085: Helper functions for managing DLL search directories * on win32 */ @@ -15114,7 +15136,7 @@ os__remove_dll_directory_impl(PyObject *module, PyObject *cookie) Py_RETURN_NONE; } -#endif +#endif /* MS_WINDOWS_APP || MS_WINDOWS_SYSTEM */ /* Only check if WIFEXITED is available: expect that it comes diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index df4043de08dacc..5a1e40d0b4a482 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -57,8 +57,10 @@ extern void bzero(void *, int); #endif #ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN -# include +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include #else # define SOCKET int #endif diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 43a0cc0f963f9d..b7927750e334b7 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -270,7 +270,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include -#else +#else /* MS_WINDOWS */ /* MS_WINDOWS includes */ # ifdef HAVE_FCNTL_H @@ -281,7 +281,6 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include /* Macros based on the IPPROTO enum, see: https://bugs.python.org/issue29515 */ -#ifdef MS_WINDOWS #define IPPROTO_ICMP IPPROTO_ICMP #define IPPROTO_IGMP IPPROTO_IGMP #define IPPROTO_GGP IPPROTO_GGP @@ -312,7 +311,6 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #define IPPROTO_PGM IPPROTO_PGM // WinSock2 only #define IPPROTO_L2TP IPPROTO_L2TP // WinSock2 only #define IPPROTO_SCTP IPPROTO_SCTP // WinSock2 only -#endif /* MS_WINDOWS */ /* Provides the IsWindows7SP1OrGreater() function */ #include @@ -348,13 +346,18 @@ remove_unusable_flags(PyObject *m) { PyObject *dict; OSVERSIONINFOEX info; - DWORDLONG dwlConditionMask; dict = PyModule_GetDict(m); if (dict == NULL) { return -1; } - +#ifndef MS_WINDOWS_DESKTOP + info.dwOSVersionInfoSize = sizeof(info); + if (!GetVersionExW((OSVERSIONINFOW*) &info)) { + PyErr_SetFromWindowsErr(0); + return -1; + } +#else /* set to Windows 10, except BuildNumber. */ memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof(info); @@ -362,19 +365,30 @@ remove_unusable_flags(PyObject *m) info.dwMinorVersion = 0; /* set Condition Mask */ - dwlConditionMask = 0; + DWORDLONG dwlConditionMask = 0; VER_SET_CONDITION(dwlConditionMask, VER_MAJORVERSION, VER_GREATER_EQUAL); VER_SET_CONDITION(dwlConditionMask, VER_MINORVERSION, VER_GREATER_EQUAL); VER_SET_CONDITION(dwlConditionMask, VER_BUILDNUMBER, VER_GREATER_EQUAL); +#endif for (int i=0; i 10 || + (info.dwMajorVersion == 10 && info.dwMinorVersion > 0) || + (info.dwMajorVersion == 10 && info.dwMinorVersion == 0 && + info.dwBuildNumber >= win_runtime_flags[i].build_number); +#endif + if (isSupported) { break; } else { @@ -497,14 +511,14 @@ remove_unusable_flags(PyObject *m) #endif #endif -#ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP #define sockaddr_rc SOCKADDR_BTH_REDEF #define USE_BLUETOOTH 1 #define AF_BLUETOOTH AF_BTH #define BTPROTO_RFCOMM BTHPROTO_RFCOMM #define _BT_RC_MEMB(sa, memb) ((sa)->memb) -#endif +#endif /* MS_WINDOWS_DESKTOP */ /* Convert "sock_addr_t *" to "struct sockaddr *". */ #define SAS2SA(x) (&((x)->sa)) @@ -2869,11 +2883,16 @@ sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) newfd = ctx.result; #ifdef MS_WINDOWS +#if defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { PyErr_SetFromWindowsErr(0); SOCKETCLOSE(newfd); goto finally; } +#endif #else #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) @@ -5434,11 +5453,6 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, proto = 0; } #ifdef MS_WINDOWS - /* Windows implementation */ -#ifndef WSA_FLAG_NO_HANDLE_INHERIT -#define WSA_FLAG_NO_HANDLE_INHERIT 0x80 -#endif - Py_BEGIN_ALLOW_THREADS fd = WSASocketW(family, type, proto, NULL, 0, @@ -6150,8 +6164,9 @@ socket_dup(PyObject *self, PyObject *fdobj) #endif fd = PyLong_AsSocket_t(fdobj); - if (fd == (SOCKET_T)(-1) && PyErr_Occurred()) + if (fd == (SOCKET_T)(-1) && PyErr_Occurred()) { return NULL; + } #ifdef MS_WINDOWS if (WSADuplicateSocketW(fd, GetCurrentProcessId(), &info)) @@ -6160,8 +6175,9 @@ socket_dup(PyObject *self, PyObject *fdobj) newfd = WSASocketW(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, &info, 0, WSA_FLAG_OVERLAPPED); - if (newfd == INVALID_SOCKET) + if (newfd == INVALID_SOCKET) { return set_error(); + } if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { closesocket(newfd); @@ -6171,13 +6187,15 @@ socket_dup(PyObject *self, PyObject *fdobj) #else /* On UNIX, dup can be used to duplicate the file descriptor of a socket */ newfd = _Py_dup(fd); - if (newfd == INVALID_SOCKET) + if (newfd == INVALID_SOCKET) { return NULL; + } #endif newfdobj = PyLong_FromSocket_t(newfd); - if (newfdobj == NULL) + if (newfdobj == NULL) { SOCKETCLOSE(newfd); + } return newfdobj; } diff --git a/Modules/timemodule.c b/Modules/timemodule.c index c2bacaae0c0339..c50e689bb6986c 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -30,7 +30,9 @@ # include #else # ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif # include # endif /* MS_WINDOWS */ #endif /* !__WATCOMC__ || __QNX__ */ @@ -1135,7 +1137,9 @@ time_tzset(PyObject *self, PyObject *unused) return NULL; } +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); +#endif /* Reset timezone, altzone, daylight and tzname */ if (init_timezone(m) < 0) { @@ -1753,7 +1757,9 @@ init_timezone(PyObject *m) */ #ifdef HAVE_DECL_TZNAME PyObject *otz0, *otz1; +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); +#endif PyModule_AddIntConstant(m, "timezone", _Py_timezone); #ifdef HAVE_ALTZONE PyModule_AddIntConstant(m, "altzone", altzone); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 276c5a276c06e6..5e1bcda1d976bb 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -8,7 +8,6 @@ #include // malloc() #include - #undef uint #define uint pymem_uint diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index d808ef0bbd0ffe..b708c6cdde757c 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -261,6 +261,8 @@ msvcrt_getch(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_getwch__doc__, "getwch($module, /)\n" "--\n" @@ -285,6 +287,8 @@ msvcrt_getwch(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_getche__doc__, "getche($module, /)\n" "--\n" @@ -309,6 +313,8 @@ msvcrt_getche(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_getwche__doc__, "getwche($module, /)\n" "--\n" @@ -333,6 +339,8 @@ msvcrt_getwche(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_putch__doc__, "putch($module, char, /)\n" "--\n" @@ -367,6 +375,8 @@ msvcrt_putch(PyObject *module, PyObject *arg) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_putwch__doc__, "putwch($module, unicode_char, /)\n" "--\n" @@ -403,6 +413,8 @@ msvcrt_putwch(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_ungetch__doc__, "ungetch($module, char, /)\n" "--\n" @@ -441,6 +453,8 @@ msvcrt_ungetch(PyObject *module, PyObject *arg) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_ungetwch__doc__, "ungetwch($module, unicode_char, /)\n" "--\n" @@ -477,6 +491,8 @@ msvcrt_ungetwch(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + #if defined(_DEBUG) PyDoc_STRVAR(msvcrt_CrtSetReportFile__doc__, @@ -610,6 +626,8 @@ msvcrt_set_error_mode(PyObject *module, PyObject *arg) #endif /* defined(_DEBUG) */ +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(msvcrt_GetErrorMode__doc__, "GetErrorMode($module, /)\n" "--\n" @@ -628,6 +646,8 @@ msvcrt_GetErrorMode(PyObject *module, PyObject *Py_UNUSED(ignored)) return msvcrt_GetErrorMode_impl(module); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ + PyDoc_STRVAR(msvcrt_SetErrorMode__doc__, "SetErrorMode($module, mode, /)\n" "--\n" @@ -656,6 +676,22 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) return return_value; } +#ifndef MSVCRT_GETWCH_METHODDEF + #define MSVCRT_GETWCH_METHODDEF +#endif /* !defined(MSVCRT_GETWCH_METHODDEF) */ + +#ifndef MSVCRT_GETWCHE_METHODDEF + #define MSVCRT_GETWCHE_METHODDEF +#endif /* !defined(MSVCRT_GETWCHE_METHODDEF) */ + +#ifndef MSVCRT_PUTWCH_METHODDEF + #define MSVCRT_PUTWCH_METHODDEF +#endif /* !defined(MSVCRT_PUTWCH_METHODDEF) */ + +#ifndef MSVCRT_UNGETWCH_METHODDEF + #define MSVCRT_UNGETWCH_METHODDEF +#endif /* !defined(MSVCRT_UNGETWCH_METHODDEF) */ + #ifndef MSVCRT_CRTSETREPORTFILE_METHODDEF #define MSVCRT_CRTSETREPORTFILE_METHODDEF #endif /* !defined(MSVCRT_CRTSETREPORTFILE_METHODDEF) */ @@ -667,4 +703,8 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_SET_ERROR_MODE_METHODDEF #define MSVCRT_SET_ERROR_MODE_METHODDEF #endif /* !defined(MSVCRT_SET_ERROR_MODE_METHODDEF) */ -/*[clinic end generated code: output=204bae9fee7f6124 input=a9049054013a1b77]*/ + +#ifndef MSVCRT_GETERRORMODE_METHODDEF + #define MSVCRT_GETERRORMODE_METHODDEF +#endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ +/*[clinic end generated code: output=2db6197608a6aab3 input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 2834d9967a7726..7a9474301da8a1 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -8,6 +8,8 @@ preserve #endif +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType_Close__doc__, "Close($self, /)\n" "--\n" @@ -28,6 +30,10 @@ winreg_HKEYType_Close(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return winreg_HKEYType_Close_impl(self); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType_Detach__doc__, "Detach($self, /)\n" "--\n" @@ -54,6 +60,10 @@ winreg_HKEYType_Detach(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return winreg_HKEYType_Detach_impl(self); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType___enter____doc__, "__enter__($self, /)\n" "--\n" @@ -77,6 +87,10 @@ winreg_HKEYType___enter__(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType___exit____doc__, "__exit__($self, /, exc_type, exc_value, traceback)\n" "--\n" @@ -136,6 +150,10 @@ winreg_HKEYType___exit__(PyHKEYObject *self, PyObject *const *args, Py_ssize_t n return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CloseKey__doc__, "CloseKey($module, hkey, /)\n" "--\n" @@ -151,6 +169,10 @@ PyDoc_STRVAR(winreg_CloseKey__doc__, #define WINREG_CLOSEKEY_METHODDEF \ {"CloseKey", (PyCFunction)winreg_CloseKey, METH_O, winreg_CloseKey__doc__}, +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_ConnectRegistry__doc__, "ConnectRegistry($module, computer_name, key, /)\n" "--\n" @@ -213,6 +235,10 @@ winreg_ConnectRegistry(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CreateKey__doc__, "CreateKey($module, key, sub_key, /)\n" "--\n" @@ -278,6 +304,10 @@ winreg_CreateKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CreateKeyEx__doc__, "CreateKeyEx($module, /, key, sub_key, reserved=0,\n" " access=winreg.KEY_WRITE)\n" @@ -398,6 +428,10 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteKey__doc__, "DeleteKey($module, key, sub_key, /)\n" "--\n" @@ -452,6 +486,10 @@ winreg_DeleteKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteKeyEx__doc__, "DeleteKeyEx($module, /, key, sub_key, access=winreg.KEY_WOW64_64KEY,\n" " reserved=0)\n" @@ -565,6 +603,10 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteValue__doc__, "DeleteValue($module, key, value, /)\n" "--\n" @@ -617,6 +659,10 @@ winreg_DeleteValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_EnumKey__doc__, "EnumKey($module, key, index, /)\n" "--\n" @@ -661,6 +707,10 @@ winreg_EnumKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_EnumValue__doc__, "EnumValue($module, key, index, /)\n" "--\n" @@ -714,6 +764,10 @@ winreg_EnumValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_ExpandEnvironmentStrings__doc__, "ExpandEnvironmentStrings($module, string, /)\n" "--\n" @@ -750,6 +804,10 @@ winreg_ExpandEnvironmentStrings(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_FlushKey__doc__, "FlushKey($module, key, /)\n" "--\n" @@ -790,6 +848,10 @@ winreg_FlushKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_LoadKey__doc__, "LoadKey($module, key, sub_key, file_name, /)\n" "--\n" @@ -866,6 +928,10 @@ winreg_LoadKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_OpenKey__doc__, "OpenKey($module, /, key, sub_key, reserved=0, access=winreg.KEY_READ)\n" "--\n" @@ -979,6 +1045,10 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_OpenKeyEx__doc__, "OpenKeyEx($module, /, key, sub_key, reserved=0, access=winreg.KEY_READ)\n" "--\n" @@ -1092,6 +1162,10 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryInfoKey__doc__, "QueryInfoKey($module, key, /)\n" "--\n" @@ -1128,6 +1202,10 @@ winreg_QueryInfoKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryValue__doc__, "QueryValue($module, key, sub_key, /)\n" "--\n" @@ -1189,6 +1267,10 @@ winreg_QueryValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryValueEx__doc__, "QueryValueEx($module, key, name, /)\n" "--\n" @@ -1246,6 +1328,10 @@ winreg_QueryValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_SaveKey__doc__, "SaveKey($module, key, file_name, /)\n" "--\n" @@ -1303,6 +1389,10 @@ winreg_SaveKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_SetValue__doc__, "SetValue($module, key, sub_key, type, value, /)\n" "--\n" @@ -1384,6 +1474,10 @@ winreg_SetValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_SetValueEx__doc__, "SetValueEx($module, key, value_name, reserved, type, value, /)\n" "--\n" @@ -1478,6 +1572,10 @@ winreg_SetValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_DisableReflectionKey__doc__, "DisableReflectionKey($module, key, /)\n" "--\n" @@ -1514,6 +1612,10 @@ winreg_DisableReflectionKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_EnableReflectionKey__doc__, "EnableReflectionKey($module, key, /)\n" "--\n" @@ -1548,6 +1650,10 @@ winreg_EnableReflectionKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_QueryReflectionKey__doc__, "QueryReflectionKey($module, key, /)\n" "--\n" @@ -1579,4 +1685,114 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7e817dc5edc914d3 input=a9049054013a1b77]*/ + +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#ifndef WINREG_HKEYTYPE_CLOSE_METHODDEF + #define WINREG_HKEYTYPE_CLOSE_METHODDEF +#endif /* !defined(WINREG_HKEYTYPE_CLOSE_METHODDEF) */ + +#ifndef WINREG_HKEYTYPE_DETACH_METHODDEF + #define WINREG_HKEYTYPE_DETACH_METHODDEF +#endif /* !defined(WINREG_HKEYTYPE_DETACH_METHODDEF) */ + +#ifndef WINREG_HKEYTYPE___ENTER___METHODDEF + #define WINREG_HKEYTYPE___ENTER___METHODDEF +#endif /* !defined(WINREG_HKEYTYPE___ENTER___METHODDEF) */ + +#ifndef WINREG_HKEYTYPE___EXIT___METHODDEF + #define WINREG_HKEYTYPE___EXIT___METHODDEF +#endif /* !defined(WINREG_HKEYTYPE___EXIT___METHODDEF) */ + +#ifndef WINREG_CLOSEKEY_METHODDEF + #define WINREG_CLOSEKEY_METHODDEF +#endif /* !defined(WINREG_CLOSEKEY_METHODDEF) */ + +#ifndef WINREG_CONNECTREGISTRY_METHODDEF + #define WINREG_CONNECTREGISTRY_METHODDEF +#endif /* !defined(WINREG_CONNECTREGISTRY_METHODDEF) */ + +#ifndef WINREG_CREATEKEY_METHODDEF + #define WINREG_CREATEKEY_METHODDEF +#endif /* !defined(WINREG_CREATEKEY_METHODDEF) */ + +#ifndef WINREG_CREATEKEYEX_METHODDEF + #define WINREG_CREATEKEYEX_METHODDEF +#endif /* !defined(WINREG_CREATEKEYEX_METHODDEF) */ + +#ifndef WINREG_DELETEKEY_METHODDEF + #define WINREG_DELETEKEY_METHODDEF +#endif /* !defined(WINREG_DELETEKEY_METHODDEF) */ + +#ifndef WINREG_DELETEKEYEX_METHODDEF + #define WINREG_DELETEKEYEX_METHODDEF +#endif /* !defined(WINREG_DELETEKEYEX_METHODDEF) */ + +#ifndef WINREG_DELETEVALUE_METHODDEF + #define WINREG_DELETEVALUE_METHODDEF +#endif /* !defined(WINREG_DELETEVALUE_METHODDEF) */ + +#ifndef WINREG_ENUMKEY_METHODDEF + #define WINREG_ENUMKEY_METHODDEF +#endif /* !defined(WINREG_ENUMKEY_METHODDEF) */ + +#ifndef WINREG_ENUMVALUE_METHODDEF + #define WINREG_ENUMVALUE_METHODDEF +#endif /* !defined(WINREG_ENUMVALUE_METHODDEF) */ + +#ifndef WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF + #define WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF +#endif /* !defined(WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF) */ + +#ifndef WINREG_FLUSHKEY_METHODDEF + #define WINREG_FLUSHKEY_METHODDEF +#endif /* !defined(WINREG_FLUSHKEY_METHODDEF) */ + +#ifndef WINREG_LOADKEY_METHODDEF + #define WINREG_LOADKEY_METHODDEF +#endif /* !defined(WINREG_LOADKEY_METHODDEF) */ + +#ifndef WINREG_OPENKEY_METHODDEF + #define WINREG_OPENKEY_METHODDEF +#endif /* !defined(WINREG_OPENKEY_METHODDEF) */ + +#ifndef WINREG_OPENKEYEX_METHODDEF + #define WINREG_OPENKEYEX_METHODDEF +#endif /* !defined(WINREG_OPENKEYEX_METHODDEF) */ + +#ifndef WINREG_QUERYINFOKEY_METHODDEF + #define WINREG_QUERYINFOKEY_METHODDEF +#endif /* !defined(WINREG_QUERYINFOKEY_METHODDEF) */ + +#ifndef WINREG_QUERYVALUE_METHODDEF + #define WINREG_QUERYVALUE_METHODDEF +#endif /* !defined(WINREG_QUERYVALUE_METHODDEF) */ + +#ifndef WINREG_QUERYVALUEEX_METHODDEF + #define WINREG_QUERYVALUEEX_METHODDEF +#endif /* !defined(WINREG_QUERYVALUEEX_METHODDEF) */ + +#ifndef WINREG_SAVEKEY_METHODDEF + #define WINREG_SAVEKEY_METHODDEF +#endif /* !defined(WINREG_SAVEKEY_METHODDEF) */ + +#ifndef WINREG_SETVALUE_METHODDEF + #define WINREG_SETVALUE_METHODDEF +#endif /* !defined(WINREG_SETVALUE_METHODDEF) */ + +#ifndef WINREG_SETVALUEEX_METHODDEF + #define WINREG_SETVALUEEX_METHODDEF +#endif /* !defined(WINREG_SETVALUEEX_METHODDEF) */ + +#ifndef WINREG_DISABLEREFLECTIONKEY_METHODDEF + #define WINREG_DISABLEREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_DISABLEREFLECTIONKEY_METHODDEF) */ + +#ifndef WINREG_ENABLEREFLECTIONKEY_METHODDEF + #define WINREG_ENABLEREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_ENABLEREFLECTIONKEY_METHODDEF) */ + +#ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF + #define WINREG_QUERYREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ +/*[clinic end generated code: output=715db416dc1321ee input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index b1481d79e6508d..9d0fe6f87df69a 100644 --- a/PC/config.c +++ b/PC/config.c @@ -43,10 +43,14 @@ extern PyObject* PyInit__collections(void); extern PyObject* PyInit__heapq(void); extern PyObject* PyInit__bisect(void); extern PyObject* PyInit__symtable(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_mmap(void); +#endif extern PyObject* PyInit__csv(void); extern PyObject* PyInit__sre(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_winreg(void); +#endif extern PyObject* PyInit__struct(void); extern PyObject* PyInit__datetime(void); extern PyObject* PyInit__functools(void); @@ -122,10 +126,14 @@ struct _inittab _PyImport_Inittab[] = { {"itertools", PyInit_itertools}, {"_collections", PyInit__collections}, {"_symtable", PyInit__symtable}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) {"mmap", PyInit_mmap}, +#endif {"_csv", PyInit__csv}, {"_sre", PyInit__sre}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) {"winreg", PyInit_winreg}, +#endif {"_struct", PyInit__struct}, {"_datetime", PyInit__datetime}, {"_functools", PyInit__functools}, diff --git a/PC/config_minimal.c b/PC/config_minimal.c index 928a4efd32e132..9a66ea1d1cd348 100644 --- a/PC/config_minimal.c +++ b/PC/config_minimal.c @@ -5,8 +5,10 @@ #include "Python.h" +#ifdef Py_ENABLE_SHARED /* Define extern variables omitted from minimal builds */ void *PyWin_DLLhModule = NULL; +#endif extern PyObject* PyInit_faulthandler(void); @@ -14,7 +16,9 @@ extern PyObject* PyInit__tracemalloc(void); extern PyObject* PyInit_gc(void); extern PyObject* PyInit_nt(void); extern PyObject* PyInit__signal(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_winreg(void); +#endif extern PyObject* PyInit__ast(void); extern PyObject* PyInit__io(void); @@ -35,7 +39,9 @@ struct _inittab _PyImport_Inittab[] = { {"_tokenize", PyInit__tokenize}, {"_tracemalloc", PyInit__tracemalloc}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) {"winreg", PyInit_winreg}, +#endif /* This module "lives in" with marshal.c */ {"marshal", PyMarshal_Init}, diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 988d9c95aaa22e..face4d03af9d4f 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -253,6 +253,8 @@ msvcrt_getch_impl(PyObject *module) return ch; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.getwch -> wchar_t @@ -271,6 +273,8 @@ msvcrt_getwch_impl(PyObject *module) return ch; } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.getche -> byte_char @@ -289,6 +293,8 @@ msvcrt_getche_impl(PyObject *module) return ch; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.getwche -> wchar_t @@ -307,6 +313,8 @@ msvcrt_getwche_impl(PyObject *module) return ch; } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.putch @@ -326,6 +334,8 @@ msvcrt_putch_impl(PyObject *module, char char_value) Py_RETURN_NONE; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.putwch @@ -346,6 +356,8 @@ msvcrt_putwch_impl(PyObject *module, int unicode_char) } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.ungetch @@ -374,6 +386,8 @@ msvcrt_ungetch_impl(PyObject *module, char char_value) Py_RETURN_NONE; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.ungetwch @@ -398,6 +412,8 @@ msvcrt_ungetwch_impl(PyObject *module, int unicode_char) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP */ + #ifdef _DEBUG /*[clinic input] msvcrt.CrtSetReportFile -> HANDLE @@ -475,6 +491,8 @@ msvcrt_set_error_mode_impl(PyObject *module, int mode) } #endif /* _DEBUG */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] msvcrt.GetErrorMode @@ -494,6 +512,8 @@ msvcrt_GetErrorMode_impl(PyObject *module) return PyLong_FromUnsignedLong(res); } +#endif /* MS_WINDOWS_APP || MS_WINDOWS_SYSTEM */ + /*[clinic input] msvcrt.SetErrorMode @@ -601,10 +621,12 @@ PyInit_msvcrt(void) insertint(d, "LK_NBRLCK", _LK_NBRLCK); insertint(d, "LK_RLCK", _LK_RLCK); insertint(d, "LK_UNLCK", _LK_UNLCK); +#ifdef MS_WINDOWS_DESKTOP insertint(d, "SEM_FAILCRITICALERRORS", SEM_FAILCRITICALERRORS); insertint(d, "SEM_NOALIGNMENTFAULTEXCEPT", SEM_NOALIGNMENTFAULTEXCEPT); insertint(d, "SEM_NOGPFAULTERRORBOX", SEM_NOGPFAULTERRORBOX); insertint(d, "SEM_NOOPENFILEERRORBOX", SEM_NOOPENFILEERRORBOX); +#endif #ifdef _DEBUG insertint(d, "CRT_WARN", _CRT_WARN); insertint(d, "CRT_ERROR", _CRT_ERROR); diff --git a/PC/pyconfig.h b/PC/pyconfig.h index a34d420ab7ecaa..8a3bf8968ce29d 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -72,9 +72,27 @@ WIN32 is still required for the locale module. #define USE_SOCKET #endif -#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP) -#define MS_WINDOWS_NON_DESKTOP +#if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN) || defined(Py_BUILD_CORE_MODULE) +#include + +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#define MS_WINDOWS_DESKTOP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +#define MS_WINDOWS_APP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_SYSTEM) +#define MS_WINDOWS_SYSTEM #endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_GAMES) +#define MS_WINDOWS_GAMES +#endif + +/* Define to 1 if you support windows console io */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) +#define HAVE_WINDOWS_CONSOLE_IO 1 +#endif +#endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ /* Compiler specific defines */ @@ -300,7 +318,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # endif /* Py_BUILD_CORE */ #endif /* MS_COREDLL */ -#if defined(MS_WIN64) +#ifdef MS_WIN64 /* maintain "win32" sys.platform for backward compatibility of Python code, the Win64 API should be close enough to the Win32 API to make this preferable */ diff --git a/PC/winreg.c b/PC/winreg.c index 86efed09855b01..073598a12a68aa 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -18,6 +18,8 @@ #include "structmember.h" // PyMemberDef #include +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) + static BOOL PyHKEY_AsHKEY(PyObject *ob, HKEY *pRes, BOOL bNoneOK); static BOOL clinic_HKEY_converter(PyObject *ob, void *p); static PyObject *PyHKEY_FromHKEY(HKEY h); @@ -829,6 +831,8 @@ winreg_CloseKey(PyObject *module, PyObject *hkey) Py_RETURN_NONE; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.ConnectRegistry -> HKEY @@ -866,6 +870,8 @@ winreg_ConnectRegistry_impl(PyObject *module, return retKey; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.CreateKey -> HKEY @@ -1272,6 +1278,8 @@ winreg_ExpandEnvironmentStrings_impl(PyObject *module, return o; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.FlushKey @@ -1305,6 +1313,9 @@ winreg_FlushKey_impl(PyObject *module, HKEY key) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) /*[clinic input] winreg.LoadKey @@ -1354,6 +1365,8 @@ winreg_LoadKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.OpenKey -> HKEY @@ -1463,6 +1476,7 @@ winreg_QueryInfoKey_impl(PyObject *module, HKEY key) return ret; } + /*[clinic input] winreg.QueryValue @@ -1634,6 +1648,8 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const Py_UNICODE *name) return result; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.SaveKey @@ -1679,6 +1695,8 @@ winreg_SaveKey_impl(PyObject *module, HKEY key, const Py_UNICODE *file_name) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.SetValue @@ -1776,6 +1794,7 @@ winreg_SetValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, return result; } + /*[clinic input] winreg.SetValueEx @@ -1861,6 +1880,8 @@ winreg_SetValueEx_impl(PyObject *module, HKEY key, return result; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.DisableReflectionKey @@ -2009,6 +2030,8 @@ winreg_QueryReflectionKey_impl(PyObject *module, HKEY key) return PyBool_FromLong(result); } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + static struct PyMethodDef winreg_methods[] = { WINREG_CLOSEKEY_METHODDEF WINREG_CONNECTREGISTRY_METHODDEF @@ -2149,3 +2172,5 @@ PyMODINIT_FUNC PyInit_winreg(void) ADD_INT(REG_RESOURCE_REQUIREMENTS_LIST); return m; } + +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM || MS_WINDOWS_GAMES */ diff --git a/Parser/myreadline.c b/Parser/myreadline.c index d55fcefbb6f206..3f0e29f051a438 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -13,7 +13,9 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_pystate.h" // _PyThreadState_GET() #ifdef MS_WINDOWS +# ifndef WIN32_LEAN_AND_MEAN # define WIN32_LEAN_AND_MEAN +# endif # include "windows.h" #endif /* MS_WINDOWS */ @@ -108,7 +110,7 @@ my_fgets(PyThreadState* tstate, char *buf, int len, FILE *fp) /* NOTREACHED */ } -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Readline implementation using ReadConsoleW */ extern char _get_console_type(HANDLE handle); @@ -233,7 +235,7 @@ _PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn) return buf; } -#endif +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* Readline implementation using fgets() */ @@ -246,7 +248,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) PyThreadState *tstate = _PyOS_ReadlineTState; assert(tstate != NULL); -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); if (!config->legacy_windows_stdio && sys_stdin == stdin) { HANDLE hStdIn, hStdErr; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 7bd04d573df4ad..acab05e2c6def3 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -163,6 +163,7 @@ static char *GetPythonImport (HINSTANCE hModule) return NULL; } +#ifdef Py_ENABLE_SHARED /* Load python3.dll before loading any extension module that might refer to it. That way, we can be sure that always the python3.dll corresponding to this python DLL is loaded, not a python3.dll that might be on the path @@ -216,6 +217,7 @@ _Py_CheckPython3(void) return hPython3 != NULL; #undef MAXPATHLEN } +#endif /* Py_ENABLE_SHARED */ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, const char *shortname, @@ -224,7 +226,9 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, dl_funcptr p; char funcname[258], *import_python; +#ifdef Py_ENABLE_SHARED _Py_CheckPython3(); +#endif /* Py_ENABLE_SHARED */ wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL); if (wpathname == NULL) @@ -234,10 +238,12 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, { HINSTANCE hDLL = NULL; +#ifdef MS_WINDOWS_DESKTOP unsigned int old_mode; /* Don't display a message box when Python can't load a DLL */ old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); +#endif /* bpo-36085: We use LoadLibraryEx with restricted search paths to avoid DLL preloading attacks and enable use of the @@ -250,8 +256,10 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, Py_END_ALLOW_THREADS PyMem_Free(wpathname); +#ifdef MS_WINDOWS_DESKTOP /* restore old error mode settings */ SetErrorMode(old_mode); +#endif if (hDLL==NULL){ PyObject *message; diff --git a/Python/fileutils.c b/Python/fileutils.c index 93bee9ee007cd4..4ac759c45a3a1e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -8,7 +8,11 @@ #ifdef MS_WINDOWS # include # include -# include // PathCchCombineEx +# if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +# define PATHCCH_ALLOW_LONG_PATHS 0x01 +# else +# include // PathCchCombineEx +# endif extern int winerror_to_errno(int); #endif @@ -77,7 +81,8 @@ _Py_device_encoding(int fd) if (!valid) Py_RETURN_NONE; -#if defined(MS_WINDOWS) +#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO UINT cp; if (fd == 0) cp = GetConsoleCP(); @@ -92,6 +97,9 @@ _Py_device_encoding(int fd) } return PyUnicode_FromFormat("cp%u", (unsigned int)cp); +#else + Py_RETURN_NONE; +#endif /* HAVE_WINDOWS_CONSOLE_IO */ #else if (_PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); @@ -1270,6 +1278,13 @@ _Py_stat(PyObject *path, struct stat *statbuf) #endif } +#ifdef MS_WINDOWS +// For some Windows API partitions, SetHandleInformation() is declared +// but none of the handle flags are defined. +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif +#endif /* This function MUST be kept async-signal-safe on POSIX when raise=0. */ static int @@ -2096,6 +2111,72 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) #endif } +// The Windows Games API family implements the PathCch* APIs in the Xbox OS, +// but does not expose them yet. Load them dynamically until +// 1) they are officially exposed +// 2) we stop supporting older versions of the GDK which do not expose them +#if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +HRESULT +PathCchSkipRoot(const wchar_t *path, const wchar_t **rootEnd) +{ + static int initialized = 0; + typedef HRESULT(__stdcall *PPathCchSkipRoot) (PCWSTR pszPath, + PCWSTR *ppszRootEnd); + static PPathCchSkipRoot _PathCchSkipRoot; + + if (initialized == 0) { + HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); + if (pathapi) { + _PathCchSkipRoot = (PPathCchSkipRoot)GetProcAddress( + pathapi, "PathCchSkipRoot"); + } + else { + _PathCchSkipRoot = NULL; + } + initialized = 1; + } + + if (!_PathCchSkipRoot) { + return E_NOINTERFACE; + } + + return _PathCchSkipRoot(path, rootEnd); +} + +static HRESULT +PathCchCombineEx(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, + const wchar_t *relfile, unsigned long flags) +{ + static int initialized = 0; + typedef HRESULT(__stdcall *PPathCchCombineEx) (PWSTR pszPathOut, + size_t cchPathOut, + PCWSTR pszPathIn, + PCWSTR pszMore, + unsigned long dwFlags); + static PPathCchCombineEx _PathCchCombineEx; + + if (initialized == 0) { + HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); + if (pathapi) { + _PathCchCombineEx = (PPathCchCombineEx)GetProcAddress( + pathapi, "PathCchCombineEx"); + } + else { + _PathCchCombineEx = NULL; + } + initialized = 1; + } + + if (!_PathCchCombineEx) { + return E_NOINTERFACE; + } + + return _PathCchCombineEx(buffer, bufsize, dirname, relfile, flags); +} + +#endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ // The caller must ensure "buffer" is big enough. static int @@ -2491,12 +2572,12 @@ _Py_get_blocking(int fd) success = GetNamedPipeHandleStateW(handle, &mode, NULL, NULL, NULL, NULL, 0); Py_END_ALLOW_THREADS - + if (!success) { PyErr_SetFromWindowsErr(0); return -1; } - + return !(mode & PIPE_NOWAIT); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e80dd30c89dfd0..82e94090a6027a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2289,7 +2289,7 @@ create_stdio(const PyConfig *config, PyObject* io, raw = Py_NewRef(buf); } -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Windows console IO is always UTF-8 encoded */ PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 207abb964bcac9..764fb70bae6c38 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1490,6 +1490,9 @@ static PyStructSequence_Desc windows_version_desc = { static PyObject * _sys_getwindowsversion_from_kernel32() { +#ifndef MS_WINDOWS_DESKTOP + return NULL; +#else HANDLE hKernel32; wchar_t kernel32_path[MAX_PATH]; LPVOID verblock; @@ -1523,6 +1526,7 @@ _sys_getwindowsversion_from_kernel32() realBuild = HIWORD(ffi->dwProductVersionLS); PyMem_RawFree(verblock); return Py_BuildValue("(kkk)", realMajor, realMinor, realBuild); +#endif /* !MS_WINDOWS_DESKTOP */ } /* Disable deprecation warnings about GetVersionEx as the result is From 71cf7c3dddd9c49ec70c1a95547f2fcd5daa7034 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 9 Mar 2023 23:06:20 +0000 Subject: [PATCH 08/50] gh-102378: don't bother stripping `/` from __text_signature__ (#102379) --- Lib/inspect.py | 57 ++++++------------- Lib/test/test_inspect.py | 23 +++----- ...-03-03-19-53-08.gh-issue-102378.kRdOZc.rst | 1 + 3 files changed, 25 insertions(+), 56 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 166667c62cdccf..edc23b0ffa9201 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2106,26 +2106,21 @@ def _signature_strip_non_python_syntax(signature): Private helper function. Takes a signature in Argument Clinic's extended signature format. - Returns a tuple of three things: - * that signature re-rendered in standard Python syntax, + Returns a tuple of two things: + * that signature re-rendered in standard Python syntax, and * the index of the "self" parameter (generally 0), or None if - the function does not have a "self" parameter, and - * the index of the last "positional only" parameter, - or None if the signature has no positional-only parameters. + the function does not have a "self" parameter. """ if not signature: - return signature, None, None + return signature, None self_parameter = None - last_positional_only = None lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) - delayed_comma = False - skip_next_comma = False text = [] add = text.append @@ -2142,35 +2137,18 @@ def _signature_strip_non_python_syntax(signature): if type == OP: if string == ',': - if skip_next_comma: - skip_next_comma = False - else: - assert not delayed_comma - delayed_comma = True - current_parameter += 1 - continue - - if string == '/': - assert not skip_next_comma - assert last_positional_only is None - skip_next_comma = True - last_positional_only = current_parameter - 1 - continue + current_parameter += 1 if (type == ERRORTOKEN) and (string == '$'): assert self_parameter is None self_parameter = current_parameter continue - if delayed_comma: - delayed_comma = False - if not ((type == OP) and (string == ')')): - add(', ') add(string) if (string == ','): add(' ') clean_signature = ''.join(text) - return clean_signature, self_parameter, last_positional_only + return clean_signature, self_parameter def _signature_fromstr(cls, obj, s, skip_bound_arg=True): @@ -2179,8 +2157,7 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): """ Parameter = cls._parameter_cls - clean_signature, self_parameter, last_positional_only = \ - _signature_strip_non_python_syntax(s) + clean_signature, self_parameter = _signature_strip_non_python_syntax(s) program = "def foo" + clean_signature + ": pass" @@ -2269,17 +2246,17 @@ def p(name_node, default_node, default=empty): parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters - args = reversed(f.args.args) - defaults = reversed(f.args.defaults) - iter = itertools.zip_longest(args, defaults, fillvalue=None) - if last_positional_only is not None: - kind = Parameter.POSITIONAL_ONLY - else: - kind = Parameter.POSITIONAL_OR_KEYWORD - for i, (name, default) in enumerate(reversed(list(iter))): + total_non_kw_args = len(f.args.posonlyargs) + len(f.args.args) + required_non_kw_args = total_non_kw_args - len(f.args.defaults) + defaults = itertools.chain(itertools.repeat(None, required_non_kw_args), f.args.defaults) + + kind = Parameter.POSITIONAL_ONLY + for (name, default) in zip(f.args.posonlyargs, defaults): + p(name, default) + + kind = Parameter.POSITIONAL_OR_KEYWORD + for (name, default) in zip(f.args.args, defaults): p(name, default) - if i == last_positional_only: - kind = Parameter.POSITIONAL_OR_KEYWORD # *args if f.args.vararg: diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 92aba519d28a08..02f8378d0413ea 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -4230,56 +4230,47 @@ def foo(a): pass class TestSignaturePrivateHelpers(unittest.TestCase): def _strip_non_python_syntax(self, input, - clean_signature, self_parameter, last_positional_only): + clean_signature, self_parameter): computed_clean_signature, \ - computed_self_parameter, \ - computed_last_positional_only = \ + computed_self_parameter = \ inspect._signature_strip_non_python_syntax(input) self.assertEqual(computed_clean_signature, clean_signature) self.assertEqual(computed_self_parameter, self_parameter) - self.assertEqual(computed_last_positional_only, last_positional_only) def test_signature_strip_non_python_syntax(self): self._strip_non_python_syntax( "($module, /, path, mode, *, dir_fd=None, " + "effective_ids=False,\n follow_symlinks=True)", - "(module, path, mode, *, dir_fd=None, " + + "(module, /, path, mode, *, dir_fd=None, " + "effective_ids=False, follow_symlinks=True)", - 0, 0) self._strip_non_python_syntax( "($module, word, salt, /)", - "(module, word, salt)", - 0, - 2) + "(module, word, salt, /)", + 0) self._strip_non_python_syntax( "(x, y=None, z=None, /)", - "(x, y=None, z=None)", - None, - 2) + "(x, y=None, z=None, /)", + None) self._strip_non_python_syntax( "(x, y=None, z=None)", "(x, y=None, z=None)", - None, None) self._strip_non_python_syntax( "(x,\n y=None,\n z = None )", "(x, y=None, z=None)", - None, None) self._strip_non_python_syntax( "", "", - None, None) self._strip_non_python_syntax( - None, None, None, None) diff --git a/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst b/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst new file mode 100644 index 00000000000000..d30f65f30d109a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst @@ -0,0 +1 @@ +Private helper method ``inspect._signature_strip_non_python_syntax`` will no longer strip ``/`` from the input string. From 2999e02836f9112de6b17784eaca762fb87e71a9 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 10 Mar 2023 09:02:32 +0000 Subject: [PATCH 09/50] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives in `_ctypes` (#102477) --- Modules/_ctypes/_ctypes.c | 11 ++--------- Modules/_ctypes/callproc.c | 36 +++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8690f2c1b07852..6f92ca08dd537b 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2200,7 +2200,6 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) struct fielddesc *fd; PyObject *as_parameter; int res; - PyObject *exc, *val, *tb; /* If the value is already an instance of the requested type, we can use it as is */ @@ -2234,33 +2233,27 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj) return (PyObject *)parg; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); Py_DECREF(parg); if (_PyObject_LookupAttr(value, &_Py_ID(_as_parameter_), &as_parameter) < 0) { Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return NULL; } if (as_parameter) { if (_Py_EnterRecursiveCall("while processing _as_parameter_")) { Py_DECREF(as_parameter); Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return NULL; } value = PyCSimpleType_from_param(type, as_parameter); _Py_LeaveRecursiveCall(); Py_DECREF(as_parameter); Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return value; } if (exc) { - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); } else { PyErr_SetString(PyExc_TypeError, "wrong type"); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f6d98bbeebc24e..4438727332bc11 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1013,41 +1013,43 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) { va_list vargs; - PyObject *tp, *v, *tb, *s, *cls_str, *msg_str; va_start(vargs, fmt); - s = PyUnicode_FromFormatV(fmt, vargs); + PyObject *s = PyUnicode_FromFormatV(fmt, vargs); va_end(vargs); - if (!s) + if (s == NULL) { return; + } - PyErr_Fetch(&tp, &v, &tb); - PyErr_NormalizeException(&tp, &v, &tb); - if (PyType_Check(tp)) - cls_str = PyType_GetName((PyTypeObject *)tp); - else - cls_str = PyObject_Str(tp); + assert(PyErr_Occurred()); + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + PyObject *cls_str = PyType_GetName(Py_TYPE(exc)); if (cls_str) { PyUnicode_AppendAndDel(&s, cls_str); PyUnicode_AppendAndDel(&s, PyUnicode_FromString(": ")); - if (s == NULL) + if (s == NULL) { goto error; - } else + } + } + else { PyErr_Clear(); - msg_str = PyObject_Str(v); - if (msg_str) + } + + PyObject *msg_str = PyObject_Str(exc); + if (msg_str) { PyUnicode_AppendAndDel(&s, msg_str); + } else { PyErr_Clear(); PyUnicode_AppendAndDel(&s, PyUnicode_FromString("???")); } - if (s == NULL) + if (s == NULL) { goto error; + } PyErr_SetObject(exc_class, s); error: - Py_XDECREF(tp); - Py_XDECREF(v); - Py_XDECREF(tb); + Py_XDECREF(exc); Py_XDECREF(s); } From cb35882773a3ffc7fe0671e64848f4c926a2d52f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Mar 2023 12:21:37 +0000 Subject: [PATCH 10/50] gh-102519: Add os.listdrives, os.listvolumes and os.listmounts on Windows (GH-102544) --- Doc/whatsnew/3.12.rst | 4 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 2 + Lib/test/test_os.py | 43 ++++ ...-03-08-23-08-38.gh-issue-102519.wlcsFI.rst | 2 + Modules/clinic/posixmodule.c.h | 128 +++++++++++- Modules/posixmodule.c | 194 ++++++++++++++++++ 9 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index d982cb62ec2f4e..48b7aab0595ebb 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -294,6 +294,10 @@ os method to check if the entry is a junction. (Contributed by Charles Machalow in :gh:`99547`.) +* Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` + functions on Windows for enumerating drives, volumes and mount points. + (Contributed by Steve Dower in :gh:`102519`.) + os.path ------- diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index dc5cd58d853534..4b12ae523c3260 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1216,6 +1216,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(volume)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnings)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnoptions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8b23aa15479301..17fb9ffbbf9f11 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -702,6 +702,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(value) STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) + STRUCT_FOR_ID(volume) STRUCT_FOR_ID(warnings) STRUCT_FOR_ID(warnoptions) STRUCT_FOR_ID(wbits) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 471efadb13bb4f..b240be57369d9d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1208,6 +1208,7 @@ extern "C" { INIT_ID(value), \ INIT_ID(values), \ INIT_ID(version), \ + INIT_ID(volume), \ INIT_ID(warnings), \ INIT_ID(warnoptions), \ INIT_ID(wbits), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b47d240e492ff9..52af37a8e60aa8 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1310,6 +1310,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(version); PyUnicode_InternInPlace(&string); + string = &_Py_ID(volume); + PyUnicode_InternInPlace(&string); string = &_Py_ID(warnings); PyUnicode_InternInPlace(&string); string = &_Py_ID(warnoptions); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index ba6feb69ea1712..253e2a23238f12 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2648,6 +2648,49 @@ def test_listdir_extended_path(self): [os.fsencode(path) for path in self.created_paths]) +@unittest.skipUnless(os.name == "nt", "NT specific tests") +class Win32ListdriveTests(unittest.TestCase): + """Test listdrive, listmounts and listvolume on Windows.""" + + def setUp(self): + # Get drives and volumes from fsutil + out = subprocess.check_output( + ["fsutil.exe", "volume", "list"], + cwd=os.path.join(os.getenv("SystemRoot", "\\Windows"), "System32"), + encoding="mbcs", + errors="ignore", + ) + lines = out.splitlines() + self.known_volumes = {l for l in lines if l.startswith('\\\\?\\')} + self.known_drives = {l for l in lines if l[1:] == ':\\'} + self.known_mounts = {l for l in lines if l[1:3] == ':\\'} + + def test_listdrives(self): + drives = os.listdrives() + self.assertIsInstance(drives, list) + self.assertSetEqual( + self.known_drives, + self.known_drives & set(drives), + ) + + def test_listvolumes(self): + volumes = os.listvolumes() + self.assertIsInstance(volumes, list) + self.assertSetEqual( + self.known_volumes, + self.known_volumes & set(volumes), + ) + + def test_listmounts(self): + for volume in os.listvolumes(): + mounts = os.listmounts(volume) + self.assertIsInstance(mounts, list) + self.assertSetEqual( + set(mounts), + self.known_mounts & set(mounts), + ) + + @unittest.skipUnless(hasattr(os, 'readlink'), 'needs os.readlink()') class ReadlinkTests(unittest.TestCase): filelink = 'readlinktest' diff --git a/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst b/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst new file mode 100644 index 00000000000000..f47e4f70b1301d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst @@ -0,0 +1,2 @@ +Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` +functions on Windows for enumerating drives, volumes and mount points diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 6565f8df935cb8..8b0550d832fc0a 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1601,6 +1601,120 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #if defined(MS_WINDOWS) +PyDoc_STRVAR(os_listdrives__doc__, +"listdrives($module, /)\n" +"--\n" +"\n" +"Return a list containing the names of drives in the system.\n" +"\n" +"A drive name typically looks like \'C:\\\\\'."); + +#define OS_LISTDRIVES_METHODDEF \ + {"listdrives", (PyCFunction)os_listdrives, METH_NOARGS, os_listdrives__doc__}, + +static PyObject * +os_listdrives_impl(PyObject *module); + +static PyObject * +os_listdrives(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os_listdrives_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os_listvolumes__doc__, +"listvolumes($module, /)\n" +"--\n" +"\n" +"Return a list containing the volumes in the system.\n" +"\n" +"Volumes are typically represented as a GUID path."); + +#define OS_LISTVOLUMES_METHODDEF \ + {"listvolumes", (PyCFunction)os_listvolumes, METH_NOARGS, os_listvolumes__doc__}, + +static PyObject * +os_listvolumes_impl(PyObject *module); + +static PyObject * +os_listvolumes(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os_listvolumes_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os_listmounts__doc__, +"listmounts($module, /, volume)\n" +"--\n" +"\n" +"Return a list containing mount points for a particular volume.\n" +"\n" +"\'volume\' should be a GUID path as returned from os.listvolumes."); + +#define OS_LISTMOUNTS_METHODDEF \ + {"listmounts", _PyCFunction_CAST(os_listmounts), METH_FASTCALL|METH_KEYWORDS, os_listmounts__doc__}, + +static PyObject * +os_listmounts_impl(PyObject *module, path_t *volume); + +static PyObject * +os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(volume), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"volume", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "listmounts", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t volume = PATH_T_INITIALIZE("listmounts", "volume", 0, 0); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &volume)) { + goto exit; + } + return_value = os_listmounts_impl(module, &volume); + +exit: + /* Cleanup for volume */ + path_cleanup(&volume); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + PyDoc_STRVAR(os__getfullpathname__doc__, "_getfullpathname($module, path, /)\n" "--\n" @@ -11253,6 +11367,18 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_LINK_METHODDEF #endif /* !defined(OS_LINK_METHODDEF) */ +#ifndef OS_LISTDRIVES_METHODDEF + #define OS_LISTDRIVES_METHODDEF +#endif /* !defined(OS_LISTDRIVES_METHODDEF) */ + +#ifndef OS_LISTVOLUMES_METHODDEF + #define OS_LISTVOLUMES_METHODDEF +#endif /* !defined(OS_LISTVOLUMES_METHODDEF) */ + +#ifndef OS_LISTMOUNTS_METHODDEF + #define OS_LISTMOUNTS_METHODDEF +#endif /* !defined(OS_LISTMOUNTS_METHODDEF) */ + #ifndef OS__GETFULLPATHNAME_METHODDEF #define OS__GETFULLPATHNAME_METHODDEF #endif /* !defined(OS__GETFULLPATHNAME_METHODDEF) */ @@ -11796,4 +11922,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=9495478e51701b8a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=47750e0e29c8d707 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0d534f35f6a420..7d91f7e4bac76b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4229,7 +4229,198 @@ os_listdir_impl(PyObject *module, path_t *path) #endif } + #ifdef MS_WINDOWS + +/*[clinic input] +os.listdrives + +Return a list containing the names of drives in the system. + +A drive name typically looks like 'C:\\'. + +[clinic start generated code]*/ + +static PyObject * +os_listdrives_impl(PyObject *module) +/*[clinic end generated code: output=aaece9dacdf682b5 input=1af9ccc9e583798e]*/ +{ + /* Number of possible drives is limited, so 256 should always be enough. + On the day when it is not, listmounts() will have to be used. */ + wchar_t buffer[256]; + DWORD buflen = Py_ARRAY_LENGTH(buffer); + PyObject *result = NULL; + if (PySys_Audit("os.listdrives", NULL) < 0) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + buflen = GetLogicalDriveStringsW(buflen, buffer); + Py_END_ALLOW_THREADS; + + if (!buflen) { + PyErr_SetFromWindowsErr(0); + return NULL; + } else if (buflen >= Py_ARRAY_LENGTH(buffer)) { + PyErr_SetFromWindowsErr(ERROR_MORE_DATA); + return NULL; + } + + /* buflen includes a null terminator, so remove it */ + PyObject *str = PyUnicode_FromWideChar(buffer, buflen - 1); + if (str) { + PyObject *nullchar = PyUnicode_FromStringAndSize("\0", 1); + if (nullchar) { + result = PyUnicode_Split(str, nullchar, -1); + Py_DECREF(nullchar); + } + Py_DECREF(str); + } + return result; +} + +/*[clinic input] +os.listvolumes + +Return a list containing the volumes in the system. + +Volumes are typically represented as a GUID path. + +[clinic start generated code]*/ + +static PyObject * +os_listvolumes_impl(PyObject *module) +/*[clinic end generated code: output=534e10ea2bf9d386 input=f6e4e70371f11e99]*/ +{ + PyObject *result = PyList_New(0); + HANDLE find = INVALID_HANDLE_VALUE; + wchar_t buffer[MAX_PATH + 1]; + if (!result) { + return NULL; + } + if (PySys_Audit("os.listvolumes", NULL) < 0) { + Py_DECREF(result); + return NULL; + } + + int err = 0; + Py_BEGIN_ALLOW_THREADS; + find = FindFirstVolumeW(buffer, Py_ARRAY_LENGTH(buffer)); + if (find == INVALID_HANDLE_VALUE) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS; + + while (!err) { + PyObject *s = PyUnicode_FromWideChar(buffer, -1); + if (!s || PyList_Append(result, s) < 0) { + Py_XDECREF(s); + Py_CLEAR(result); + break; + } + Py_DECREF(s); + + Py_BEGIN_ALLOW_THREADS; + if (!FindNextVolumeW(find, buffer, Py_ARRAY_LENGTH(buffer))) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS; + } + + if (find != INVALID_HANDLE_VALUE) { + Py_BEGIN_ALLOW_THREADS; + FindVolumeClose(find); + Py_END_ALLOW_THREADS; + } + if (err && err != ERROR_NO_MORE_FILES) { + PyErr_SetFromWindowsErr(err); + Py_XDECREF(result); + result = NULL; + } + return result; +} + + +/*[clinic input] +os.listmounts + + volume: path_t + +Return a list containing mount points for a particular volume. + +'volume' should be a GUID path as returned from os.listvolumes. + +[clinic start generated code]*/ + +static PyObject * +os_listmounts_impl(PyObject *module, path_t *volume) +/*[clinic end generated code: output=06da49679de4512e input=a8a27178e3f67845]*/ +{ + wchar_t default_buffer[MAX_PATH + 1]; + DWORD buflen = Py_ARRAY_LENGTH(default_buffer); + LPWSTR buffer = default_buffer; + DWORD attributes; + PyObject *str = NULL; + PyObject *nullchar = NULL; + PyObject *result = NULL; + + /* Ensure we have a valid volume path before continuing */ + Py_BEGIN_ALLOW_THREADS + attributes = GetFileAttributesW(volume->wide); + Py_END_ALLOW_THREADS + if (attributes == INVALID_FILE_ATTRIBUTES && + GetLastError() == ERROR_UNRECOGNIZED_VOLUME) + { + return PyErr_SetFromWindowsErr(ERROR_UNRECOGNIZED_VOLUME); + } + + if (PySys_Audit("os.listmounts", "O", volume->object) < 0) { + return NULL; + } + + while (1) { + BOOL success; + Py_BEGIN_ALLOW_THREADS + success = GetVolumePathNamesForVolumeNameW(volume->wide, buffer, + buflen, &buflen); + Py_END_ALLOW_THREADS + if (success) { + break; + } + if (GetLastError() != ERROR_MORE_DATA) { + PyErr_SetFromWindowsErr(0); + goto exit; + } + if (buffer != default_buffer) { + PyMem_Free((void *)buffer); + } + buffer = (wchar_t*)PyMem_Malloc(sizeof(wchar_t) * buflen); + if (!buffer) { + PyErr_NoMemory(); + goto exit; + } + } + if (buflen < 2) { + result = PyList_New(0); + goto exit; + } + // buflen includes two null terminators, one for the last string + // and one for the array of strings. + str = PyUnicode_FromWideChar(buffer, buflen - 2); + nullchar = PyUnicode_FromStringAndSize("\0", 1); + if (str && nullchar) { + result = PyUnicode_Split(str, nullchar, -1); + } +exit: + if (buffer != default_buffer) { + PyMem_Free(buffer); + } + Py_XDECREF(nullchar); + Py_XDECREF(str); + return result; +} + + int _PyOS_getfullpathname(const wchar_t *path, wchar_t **abspath_p) { @@ -15252,6 +15443,9 @@ static PyMethodDef posix_methods[] = { OS_GETCWDB_METHODDEF OS_LINK_METHODDEF OS_LISTDIR_METHODDEF + OS_LISTDRIVES_METHODDEF + OS_LISTMOUNTS_METHODDEF + OS_LISTVOLUMES_METHODDEF OS_LSTAT_METHODDEF OS_MKDIR_METHODDEF OS_NICE_METHODDEF From 53dceb53ade15587b9cfd30c0a0942232517dee9 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Fri, 10 Mar 2023 19:22:02 +0700 Subject: [PATCH 11/50] gh-86509: Add link to Lib/_threading_local.py in threading docs (#101824) --- Doc/library/threading.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index b352125551fa79..83ed48052704fb 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -272,7 +272,7 @@ The instance's values will be different for separate threads. A class that represents thread-local data. For more details and extensive examples, see the documentation string of the - :mod:`_threading_local` module. + :mod:`_threading_local` module: :source:`Lib/_threading_local.py`. .. _thread-objects: From 64bde502cf89963bc7382b03ea9e1c0967d22e35 Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Fri, 10 Mar 2023 10:29:37 -0500 Subject: [PATCH 12/50] GH-102537: Handle check for PYTHONTZPATH failing in zoneinfo test (GH-102538) It is possible but unlikely for the `python_tzpath_context` function to fail between the start of the `try` block and the point where `os.environ.get` succeeds, in which case `old_env` will be undefined. In this case, we want to take no action. Practically speaking this will really only happen in an error condition anyway, so it doesn't really matter, but we should probably do it right anyway. --- Lib/test/test_zoneinfo/test_zoneinfo.py | 9 ++++++++- .../Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 82041a2b488334..ae921f7432c466 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1543,13 +1543,20 @@ class TzPathTest(TzPathUserMixin, ZoneInfoTestBase): @contextlib.contextmanager def python_tzpath_context(value): path_var = "PYTHONTZPATH" + unset_env_sentinel = object() + old_env = unset_env_sentinel try: with OS_ENV_LOCK: old_env = os.environ.get(path_var, None) os.environ[path_var] = value yield finally: - if old_env is None: + if old_env is unset_env_sentinel: + # In this case, `old_env` was never retrieved from the + # environment for whatever reason, so there's no need to + # reset the environment TZPATH. + pass + elif old_env is None: del os.environ[path_var] else: os.environ[path_var] = old_env # pragma: nocover diff --git a/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst b/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst new file mode 100644 index 00000000000000..94d160dd4127a6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst @@ -0,0 +1,2 @@ +Adjust the error handling strategy in +``test_zoneinfo.TzPathTest.python_tzpath_context``. Patch by Paul Ganssle. From 12226bec2588f925f4698e1130ce78e118343934 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Mar 2023 15:41:32 +0000 Subject: [PATCH 13/50] gh-102519: Add doc updates for os.listdrives, listvolumes and listmounts (GH-102585) --- Doc/library/os.rst | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 23ce98785bedfc..5b9f49be1fad55 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2188,6 +2188,69 @@ features: Accepts a :term:`path-like object`. +.. function:: listdrives() + + Return a list containing the names of drives on a Windows system. + + A drive name typically looks like ``'C:\\'``. Not every drive name + will be associated with a volume, and some may be inaccessible for + a variety of reasons, including permissions, network connectivity + or missing media. This function does not test for access. + + May raise :exc:`OSError` if an error occurs collecting the drive + names. + + .. audit-event:: os.listdrives "" os.listdrives + + .. availability:: Windows + + .. versionadded:: 3.12 + + +.. function:: listmounts(volume) + + Return a list containing the mount points for a volume on a Windows + system. + + *volume* must be represented as a GUID path, like those returned by + :func:`os.listvolumes`. Volumes may be mounted in multiple locations + or not at all. In the latter case, the list will be empty. Mount + points that are not associated with a volume will not be returned by + this function. + + The mount points return by this function will be absolute paths, and + may be longer than the drive name. + + Raises :exc:`OSError` if the volume is not recognized or if an error + occurs collecting the paths. + + .. audit-event:: os.listmounts volume os.listmounts + + .. availability:: Windows + + .. versionadded:: 3.12 + + +.. function:: listvolumes() + + Return a list containing the volumes in the system. + + Volumes are typically represented as a GUID path that looks like + ``\\?\Volume{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx}\``. Files can + usually be accessed through a GUID path, permissions allowing. + However, users are generally not familiar with them, and so the + recommended use of this function is to retrieve mount points + using :func:`os.listmounts`. + + May raise :exc:`OSError` if an error occurs collecting the volumes. + + .. audit-event:: os.listvolumes "" os.listvolumes + + .. availability:: Windows + + .. versionadded:: 3.12 + + .. function:: lstat(path, *, dir_fd=None) Perform the equivalent of an :c:func:`lstat` system call on the given path. From 90f1d777177e28b6c7b8d9ba751550e373d61b0a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 10 Mar 2023 17:29:04 +0000 Subject: [PATCH 14/50] GH-80486: Fix handling of NTFS alternate data streams in pathlib (GH-102454) Co-authored-by: Maor Kleinberger --- Lib/pathlib.py | 8 ++++-- Lib/test/test_pathlib.py | 28 ++++++++++++++++++- .../2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst | 2 ++ 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index d375529ff5f767..55c44f12e5a2fb 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -320,8 +320,9 @@ def _from_parsed_parts(cls, drv, root, parts): def _format_parsed_parts(cls, drv, root, parts): if drv or root: return drv + root + cls._flavour.sep.join(parts[1:]) - else: - return cls._flavour.sep.join(parts) + elif parts and cls._flavour.splitdrive(parts[0])[0]: + parts = ['.'] + parts + return cls._flavour.sep.join(parts) def __str__(self): """Return the string representation of the path, suitable for @@ -1188,7 +1189,8 @@ def expanduser(self): homedir = self._flavour.expanduser(self._parts[0]) if homedir[:1] == "~": raise RuntimeError("Could not determine home directory.") - return self._from_parts([homedir] + self._parts[1:]) + drv, root, parts = self._parse_parts((homedir,)) + return self._from_parsed_parts(drv, root, parts + self._parts[1:]) return self diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index f8e2f44d27fc1e..f05dead5886743 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -122,6 +122,13 @@ def test_parse_parts(self): # the second path is relative. check(['c:/a/b', 'c:x/y'], ('c:', '\\', ['c:\\', 'a', 'b', 'x', 'y'])) check(['c:/a/b', 'c:/x/y'], ('c:', '\\', ['c:\\', 'x', 'y'])) + # Paths to files with NTFS alternate data streams + check(['./c:s'], ('', '', ['c:s'])) + check(['cc:s'], ('', '', ['cc:s'])) + check(['C:c:s'], ('C:', '', ['C:', 'c:s'])) + check(['C:/c:s'], ('C:', '\\', ['C:\\', 'c:s'])) + check(['D:a', './c:b'], ('D:', '', ['D:', 'a', 'c:b'])) + check(['D:/a', './c:b'], ('D:', '\\', ['D:\\', 'a', 'c:b'])) # @@ -165,6 +172,7 @@ def test_constructor_common(self): self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + self.assertEqual(P(P('./a:b')), P('./a:b')) def test_bytes(self): P = self.cls @@ -814,7 +822,8 @@ class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase): equivalences = _BasePurePathTest.equivalences.copy() equivalences.update({ - 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('/', 'c:', 'a') ], + './a:b': [ ('./a:b',) ], + 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], 'c:/a': [ ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), @@ -838,6 +847,7 @@ def test_str(self): self.assertEqual(str(p), '\\\\a\\b\\c\\d') def test_str_subclass(self): + self._check_str_subclass('.\\a:b') self._check_str_subclass('c:') self._check_str_subclass('c:a') self._check_str_subclass('c:a\\b.txt') @@ -1005,6 +1015,7 @@ def test_drive(self): self.assertEqual(P('//a/b').drive, '\\\\a\\b') self.assertEqual(P('//a/b/').drive, '\\\\a\\b') self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') def test_root(self): P = self.cls @@ -1341,6 +1352,14 @@ def test_join(self): self.assertEqual(pp, P('C:/a/b/x/y')) pp = p.joinpath('c:/x/y') self.assertEqual(pp, P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath(P('./d:s')) + self.assertEqual(pp, P('C:/a/b/d:s')) + pp = p.joinpath(P('./dd:s')) + self.assertEqual(pp, P('C:/a/b/dd:s')) + pp = p.joinpath(P('E:d:s')) + self.assertEqual(pp, P('E:d:s')) def test_div(self): # Basically the same as joinpath(). @@ -1361,6 +1380,11 @@ def test_div(self): # the second path is relative. self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) self.assertEqual(p / 'c:/x/y', P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) + self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) + self.assertEqual(p / P('E:d:s'), P('E:d:s')) def test_is_reserved(self): P = self.cls @@ -1626,6 +1650,8 @@ def test_expanduser_common(self): self.assertEqual(p.expanduser(), p) p = P(P('').absolute().anchor) / '~' self.assertEqual(p.expanduser(), p) + p = P('~/a:b') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) def test_exists(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst b/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst new file mode 100644 index 00000000000000..d9360496ac24cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst @@ -0,0 +1,2 @@ +Fix handling of Windows filenames that resemble drives, such as ``./a:b``, +in :mod:`pathlib`. From ee6f8413a99d0ee4828e1c81911e203d3fff85d5 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 11 Mar 2023 08:21:22 +0800 Subject: [PATCH 15/50] gh-102578: Optimise setting and deleting mutable attributes on non-dataclass subclasses of frozen dataclasses (gh-102573) --- Lib/dataclasses.py | 10 ++--- Lib/test/test_dataclasses.py | 44 +++++++++++++++++++ ...-03-10-13-21-16.gh-issue-102578.-gujoI.rst | 4 ++ 3 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8bc8594d674bc0..78a126f051e2e7 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -616,21 +616,19 @@ def _repr_fn(fields, globals): def _frozen_get_del_attr(cls, fields, globals): locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} + condition = 'type(self) is cls' if fields: - fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' - else: - # Special case for the zero-length tuple. - fields_str = '()' + condition += ' or name in {' + ', '.join(repr(f.name) for f in fields) + '}' return (_create_fn('__setattr__', ('self', 'name', 'value'), - (f'if type(self) is cls or name in {fields_str}:', + (f'if {condition}:', ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', f'super(cls, self).__setattr__(name, value)'), locals=locals, globals=globals), _create_fn('__delattr__', ('self', 'name'), - (f'if type(self) is cls or name in {fields_str}:', + (f'if {condition}:', ' raise FrozenInstanceError(f"cannot delete field {name!r}")', f'super(cls, self).__delattr__(name)'), locals=locals, diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 589f229f462359..5486b2ef3f47e5 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2767,6 +2767,19 @@ class C: c.i = 5 self.assertEqual(c.i, 10) + def test_frozen_empty(self): + @dataclass(frozen=True) + class C: + pass + + c = C() + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + del c.i + def test_inherit(self): @dataclass(frozen=True) class C: @@ -2890,6 +2903,37 @@ class S(D): self.assertEqual(s.y, 10) self.assertEqual(s.cached, True) + with self.assertRaises(FrozenInstanceError): + del s.x + self.assertEqual(s.x, 3) + with self.assertRaises(FrozenInstanceError): + del s.y + self.assertEqual(s.y, 10) + del s.cached + self.assertFalse(hasattr(s, 'cached')) + with self.assertRaises(AttributeError) as cm: + del s.cached + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + + def test_non_frozen_normal_derived_from_empty_frozen(self): + @dataclass(frozen=True) + class D: + pass + + class S(D): + pass + + s = S() + self.assertFalse(hasattr(s, 'x')) + s.x = 5 + self.assertEqual(s.x, 5) + + del s.x + self.assertFalse(hasattr(s, 'x')) + with self.assertRaises(AttributeError) as cm: + del s.x + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + def test_overwriting_frozen(self): # frozen uses __setattr__ and __delattr__. with self.assertRaisesRegex(TypeError, diff --git a/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst b/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst new file mode 100644 index 00000000000000..7307148d9a81ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst @@ -0,0 +1,4 @@ +Speed up setting or deleting mutable attributes on non-dataclass subclasses of +frozen dataclasses. Due to the implementation of ``__setattr__`` and +``__delattr__`` for frozen dataclasses, this previously had a time complexity +of ``O(n)``. It now has a time complexity of ``O(1)``. From b48be8fa18518583abb21bf6e4f5d7e4b5c9d7b2 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 11 Mar 2023 03:26:46 +0300 Subject: [PATCH 16/50] gh-102103: add `module` argument to `dataclasses.make_dataclass` (#102104) --- Doc/library/dataclasses.rst | 6 ++- Lib/dataclasses.py | 15 ++++++- Lib/test/test_dataclasses.py | 39 +++++++++++++++++++ ...-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst | 2 + 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 82faa7b77450fb..5f4dc25bfd7877 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -389,7 +389,7 @@ Module contents :func:`astuple` raises :exc:`TypeError` if ``obj`` is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None) Creates a new dataclass with name ``cls_name``, fields as defined in ``fields``, base classes as given in ``bases``, and initialized @@ -401,6 +401,10 @@ Module contents ``match_args``, ``kw_only``, ``slots``, and ``weakref_slot`` have the same meaning as they do in :func:`dataclass`. + If ``module`` is defined, the ``__module__`` attribute + of the dataclass is set to that value. + By default, it is set to the module name of the caller. + This function is not strictly required, because any Python mechanism for creating a new class with ``__annotations__`` can then apply the :func:`dataclass` function to convert that class to diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 78a126f051e2e7..24f3779ebb8ec8 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1391,7 +1391,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False): + weakref_slot=False, module=None): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1455,6 +1455,19 @@ def exec_body_callback(ns): # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) + # For pickling to work, the __module__ variable needs to be set to the frame + # where the dataclass is created. + if module is None: + try: + module = sys._getframemodulename(1) or '__main__' + except AttributeError: + try: + module = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + if module is not None: + cls.__module__ = module + # Apply the normal decorator. return dataclass(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 5486b2ef3f47e5..76bed0c3314673 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -3606,6 +3606,15 @@ def test_text_annotations(self): 'return': type(None)}) +ByMakeDataClass = make_dataclass('ByMakeDataClass', [('x', int)]) +ManualModuleMakeDataClass = make_dataclass('ManualModuleMakeDataClass', + [('x', int)], + module='test.test_dataclasses') +WrongNameMakeDataclass = make_dataclass('Wrong', [('x', int)]) +WrongModuleMakeDataclass = make_dataclass('WrongModuleMakeDataclass', + [('x', int)], + module='custom') + class TestMakeDataclass(unittest.TestCase): def test_simple(self): C = make_dataclass('C', @@ -3715,6 +3724,36 @@ def test_no_types(self): 'y': int, 'z': 'typing.Any'}) + def test_module_attr(self): + self.assertEqual(ByMakeDataClass.__module__, __name__) + self.assertEqual(ByMakeDataClass(1).__module__, __name__) + self.assertEqual(WrongModuleMakeDataclass.__module__, "custom") + Nested = make_dataclass('Nested', []) + self.assertEqual(Nested.__module__, __name__) + self.assertEqual(Nested().__module__, __name__) + + def test_pickle_support(self): + for klass in [ByMakeDataClass, ManualModuleMakeDataClass]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(klass, proto)), + klass, + ) + self.assertEqual( + pickle.loads(pickle.dumps(klass(1), proto)), + klass(1), + ) + + def test_cannot_be_pickled(self): + for klass in [WrongNameMakeDataclass, WrongModuleMakeDataclass]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + with self.assertRaises(pickle.PickleError): + pickle.dumps(klass, proto) + with self.assertRaises(pickle.PickleError): + pickle.dumps(klass(1), proto) + def test_invalid_type_specification(self): for bad_field in [(), (1, 2, 3, 4), diff --git a/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst b/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst new file mode 100644 index 00000000000000..feba433f5bee89 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst @@ -0,0 +1,2 @@ +Add ``module`` argument to :func:`dataclasses.make_dataclass` and make +classes produced by it pickleable. From 767d3a8f6f2f94daa15ad3759d0ecdf4c009b7ab Mon Sep 17 00:00:00 2001 From: "Ilya V. Schurov" Date: Sat, 11 Mar 2023 01:27:41 +0100 Subject: [PATCH 17/50] gh-88071: Update docstrings of dataclass' astuple and asdict (#101806) Update dataclasses.astuple and dataclasses.asdict docstrings to reflect that they deep copy objects in the field values. --- Lib/dataclasses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 24f3779ebb8ec8..f4617b1dbdac66 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1281,7 +1281,7 @@ class C: If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): raise TypeError("asdict() should be called on dataclass instances") @@ -1353,7 +1353,7 @@ class C: If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): From 08b67fb34f4519be1b0bb4673643a2c761c7ae92 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 10 Mar 2023 17:01:16 -0800 Subject: [PATCH 18/50] GH-90997: Shrink the LOAD_GLOBAL caches (#102569) --- Doc/library/dis.rst | 6 +- Include/internal/pycore_code.h | 2 +- Include/internal/pycore_opcode.h | 2 +- Lib/importlib/_bootstrap_external.py | 13 +- Lib/opcode.py | 2 +- Lib/test/test_dis.py | 292 +++++++++--------- ...3-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst | 2 + Python/bytecodes.c | 6 +- Python/generated_cases.c.h | 14 +- Python/opcode_metadata.h | 8 +- Python/specialize.c | 12 +- 11 files changed, 187 insertions(+), 172 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a5bc5e7fb6ea71..f4f47b3bf4846d 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -57,9 +57,9 @@ the following command can be used to display the disassembly of 2 0 RESUME 0 3 2 LOAD_GLOBAL 1 (NULL + len) - 14 LOAD_FAST 0 (alist) - 16 CALL 1 - 26 RETURN_VALUE + 12 LOAD_FAST 0 (alist) + 14 CALL 1 + 24 RETURN_VALUE (The "2" is a line number). diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 10f1e320a12ff4..6bd212dd42c6f2 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -20,7 +20,7 @@ extern "C" { typedef struct { uint16_t counter; uint16_t index; - uint16_t module_keys_version[2]; + uint16_t module_keys_version; uint16_t builtin_keys_version; } _PyLoadGlobalCache; diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index f9ab95ca4bb9d3..da94fc81dcbe3e 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -48,7 +48,7 @@ const uint8_t _PyOpcode_Caches[256] = { [STORE_ATTR] = 4, [LOAD_ATTR] = 9, [COMPARE_OP] = 1, - [LOAD_GLOBAL] = 5, + [LOAD_GLOBAL] = 4, [BINARY_OP] = 1, [SEND] = 1, [COMPARE_AND_BRANCH] = 1, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 954401cfa85ed3..a01a0955182de5 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -431,12 +431,17 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) # Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) # Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a5 3518 (Add RETURN_CONST instruction) -# Python 3.12a5 3519 (Modify SEND instruction) -# Python 3.12a5 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) +# Python 3.12a6 3518 (Add RETURN_CONST instruction) +# Python 3.12a6 3519 (Modify SEND instruction) +# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) +# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) # Python 3.13 will start with 3550 +# Please don't copy-paste the same pre-release tag for new entries above!!! +# You should always use the *upcoming* tag. For example, if 3.12a6 came out +# a week ago, I should put "Python 3.12a7" next to my new magic number. + # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). @@ -446,7 +451,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3520).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3521).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 809d24e51676bd..23529d87a09ef9 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -390,7 +390,7 @@ def pseudo_op(name, op, real_ops): "LOAD_GLOBAL": { "counter": 1, "index": 1, - "module_keys_version": 2, + "module_keys_version": 1, "builtin_keys_version": 1, }, "BINARY_OP": { diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 9086824dd6f40c..b77e3b06d0c1f1 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -138,10 +138,10 @@ def bug708901(): %3d CALL 2 GET_ITER - >> FOR_ITER 2 (to 38) + >> FOR_ITER 2 (to 36) STORE_FAST 0 (res) -%3d JUMP_BACKWARD 4 (to 30) +%3d JUMP_BACKWARD 4 (to 28) %3d >> END_FOR RETURN_CONST 0 (None) @@ -354,7 +354,7 @@ def bug42562(): %3d LOAD_GLOBAL 0 (Exception) CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 23 (to 82) + POP_JUMP_IF_FALSE 23 (to 80) STORE_FAST 0 (e) %3d LOAD_FAST 0 (e) @@ -730,14 +730,14 @@ def loop_test(): LOAD_CONST 2 (3) BINARY_OP 5 (*) GET_ITER - >> FOR_ITER_LIST 15 (to 50) + >> FOR_ITER_LIST 14 (to 48) STORE_FAST 0 (i) %3d LOAD_GLOBAL_MODULE 1 (NULL + load_test) LOAD_FAST 0 (i) CALL_PY_WITH_DEFAULTS 1 POP_TOP - JUMP_BACKWARD 17 (to 16) + JUMP_BACKWARD 16 (to 16) %3d >> END_FOR RETURN_CONST 0 (None) @@ -1189,8 +1189,8 @@ def test_show_caches(self): caches = list(self.get_cached_values(quickened, adaptive)) for cache in caches: self.assertRegex(cache, pattern) - total_caches = 23 - empty_caches = 8 + total_caches = 22 + empty_caches = 7 self.assertEqual(caches.count(""), empty_caches) self.assertEqual(len(caches), total_caches) @@ -1483,17 +1483,17 @@ def _prepare_test_cases(): Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=18, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=20, starts_line=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=32, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=34, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=36, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=1, argrepr='1', offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=7, argval=7, argrepr='', offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=58, starts_line=8, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=30, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=32, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=34, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=1, argrepr='1', offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=38, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=7, argval=7, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=56, starts_line=8, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_f = [ @@ -1511,145 +1511,145 @@ def _prepare_test_cases(): Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=22, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=24, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=26, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='d', argrepr='d', offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=4, argval=4, argrepr='', offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=58, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=38, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='d', argrepr='d', offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=4, argval=4, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=56, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_inner = [ Instruction(opname='COPY_FREE_VARS', opcode=149, arg=4, argval=4, argrepr='', offset=0, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=2, starts_line=3, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=4, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=6, argval=6, argrepr='', offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=40, starts_line=None, is_jump_target=False, positions=None) + Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=16, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=18, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=20, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=6, argval=6, argrepr='', offset=26, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=38, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=0, starts_line=1, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='NULL + range', offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='FOR_ITER', opcode=93, arg=28, argval=88, argrepr='to 88', offset=28, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=32, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=34, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=48, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=60, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=62, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=64, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=72, argrepr='to 72', offset=68, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=22, argval=28, argrepr='to 28', offset=70, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=72, starts_line=7, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=74, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=68, argval='>', argrepr='>', offset=76, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=84, argrepr='to 84', offset=80, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=28, argval=28, argrepr='to 28', offset=82, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=84, starts_line=8, is_jump_target=True, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=14, argval=116, argrepr='to 116', offset=86, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=88, starts_line=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=90, starts_line=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=104, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=114, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=116, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=33, argval=186, argrepr='to 186', offset=118, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=120, starts_line=12, is_jump_target=True, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=132, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=134, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=144, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=146, starts_line=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=148, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=150, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=154, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=156, starts_line=14, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=158, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=75, argval='>', argrepr='>', offset=160, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=168, argrepr='to 168', offset=164, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=26, argval=116, argrepr='to 116', offset=166, starts_line=15, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=168, starts_line=16, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=170, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=172, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=180, argrepr='to 180', offset=176, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=16, argval=212, argrepr='to 212', offset=178, starts_line=17, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=180, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=186, argrepr='to 186', offset=182, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=33, argval=120, argrepr='to 120', offset=184, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=186, starts_line=19, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=198, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=200, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=210, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=212, starts_line=20, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=214, starts_line=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=216, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=218, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=222, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=224, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=226, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=228, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=230, starts_line=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=242, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=244, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=254, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=256, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=258, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=260, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=262, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=272, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=274, starts_line=28, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=286, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=288, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=298, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=300, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=302, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=304, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=310, argrepr='to 310', offset=306, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=308, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=310, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=312, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=314, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=316, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=23, argval=274, argrepr='to 274', offset=318, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=320, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=322, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=324, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=326, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=328, starts_line=22, is_jump_target=False, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=340, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=16, argval=376, argrepr='to 376', offset=342, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=344, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=346, starts_line=23, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=360, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=370, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=372, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=51, argval=274, argrepr='to 274', offset=374, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=376, starts_line=22, is_jump_target=True, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=378, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=380, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=382, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=384, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=386, starts_line=28, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=398, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=400, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=410, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=412, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=414, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=416, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=418, starts_line=None, is_jump_target=False, positions=None) + Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=12, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='FOR_ITER', opcode=93, arg=27, argval=84, argrepr='to 84', offset=26, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=30, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=32, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=56, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=58, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=68, argrepr='to 68', offset=64, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=21, argval=26, argrepr='to 26', offset=66, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=68, starts_line=7, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=70, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=68, argval='>', argrepr='>', offset=72, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=80, argrepr='to 80', offset=76, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=27, argval=26, argrepr='to 26', offset=78, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=80, starts_line=8, is_jump_target=True, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=13, argval=110, argrepr='to 110', offset=82, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=84, starts_line=3, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=86, starts_line=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=98, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=108, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=110, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=32, argval=178, argrepr='to 178', offset=112, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=114, starts_line=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=124, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=126, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=136, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=138, starts_line=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=140, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=142, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=146, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=148, starts_line=14, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=150, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=75, argval='>', argrepr='>', offset=152, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=160, argrepr='to 160', offset=156, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=25, argval=110, argrepr='to 110', offset=158, starts_line=15, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=160, starts_line=16, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=162, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=164, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=172, argrepr='to 172', offset=168, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=15, argval=202, argrepr='to 202', offset=170, starts_line=17, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=172, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=178, argrepr='to 178', offset=174, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=32, argval=114, argrepr='to 114', offset=176, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=178, starts_line=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=188, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=190, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=200, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=202, starts_line=20, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=204, starts_line=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=206, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=208, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=214, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=216, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=218, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=220, starts_line=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=230, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=232, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=242, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=244, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=246, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=248, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=250, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=260, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=262, starts_line=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=272, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=274, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=284, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=286, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=288, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=290, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=296, argrepr='to 296', offset=292, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=294, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=296, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=298, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=300, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=302, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=22, argval=262, argrepr='to 262', offset=304, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=306, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=308, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=310, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=312, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=314, starts_line=22, is_jump_target=False, positions=None), + Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=324, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=15, argval=358, argrepr='to 358', offset=326, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=328, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=330, starts_line=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=340, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=342, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=352, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=354, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=48, argval=262, argrepr='to 262', offset=356, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=358, starts_line=22, is_jump_target=True, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=360, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=362, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=364, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=366, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=368, starts_line=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=378, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=380, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=390, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=392, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=394, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=396, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=398, starts_line=None, is_jump_target=False, positions=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst new file mode 100644 index 00000000000000..723a4b9fa777d6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst @@ -0,0 +1,2 @@ +Shrink the number of inline :opcode:`CACHE` entries used by +:opcode:`LOAD_GLOBAL`. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 45d50726674321..dc2a20f22b6038 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1078,7 +1078,7 @@ dummy_func( LOAD_GLOBAL_BUILTIN, }; - inst(LOAD_GLOBAL, (unused/1, unused/1, unused/2, unused/1 -- null if (oparg & 1), v)) { + inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- null if (oparg & 1), v)) { #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1133,7 +1133,7 @@ dummy_func( null = NULL; } - inst(LOAD_GLOBAL_MODULE, (unused/1, index/1, version/2, unused/1 -- null if (oparg & 1), res)) { + inst(LOAD_GLOBAL_MODULE, (unused/1, index/1, version/1, unused/1 -- null if (oparg & 1), res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1147,7 +1147,7 @@ dummy_func( null = NULL; } - inst(LOAD_GLOBAL_BUILTIN, (unused/1, index/1, mod_version/2, bltn_version/1 -- null if (oparg & 1), res)) { + inst(LOAD_GLOBAL_BUILTIN, (unused/1, index/1, mod_version/1, bltn_version/1 -- null if (oparg & 1), res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 51357cda4c8949..3839aeecb912a1 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1349,7 +1349,7 @@ TARGET(LOAD_GLOBAL) { PREDICTED(LOAD_GLOBAL); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 5, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; #if ENABLE_SPECIALIZATION @@ -1408,7 +1408,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 5; + next_instr += 4; DISPATCH(); } @@ -1416,7 +1416,7 @@ PyObject *null = NULL; PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); - uint32_t version = read_u32(&next_instr[2].cache); + uint16_t version = read_u16(&next_instr[2].cache); assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1432,7 +1432,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 5; + next_instr += 4; DISPATCH(); } @@ -1440,8 +1440,8 @@ PyObject *null = NULL; PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); - uint32_t mod_version = read_u32(&next_instr[2].cache); - uint16_t bltn_version = read_u16(&next_instr[4].cache); + uint16_t mod_version = read_u16(&next_instr[2].cache); + uint16_t bltn_version = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); @@ -1460,7 +1460,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 5; + next_instr += 4; DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 67cb0088c3b789..93f3c76c5b8240 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -708,7 +708,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { #endif enum Direction { DIR_NONE, DIR_READ, DIR_WRITE }; -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC0, INSTR_FMT_IBC000, INSTR_FMT_IBC0000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC0, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; struct opcode_metadata { enum Direction dir_op1; enum Direction dir_op2; @@ -790,9 +790,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [STORE_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [DELETE_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_NAME] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0000 }, - [LOAD_GLOBAL_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0000 }, - [LOAD_GLOBAL_BUILTIN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0000 }, + [LOAD_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [LOAD_GLOBAL_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [LOAD_GLOBAL_BUILTIN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [DELETE_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [MAKE_CELL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [DELETE_DEREF] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/specialize.c b/Python/specialize.c index 0a7af8991eec83..719bd5bda329ff 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1148,8 +1148,12 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } + if (keys_version != (uint16_t)keys_version) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } cache->index = (uint16_t)index; - write_u32(cache->module_keys_version, keys_version); + cache->module_keys_version = (uint16_t)keys_version; instr->op.code = LOAD_GLOBAL_MODULE; goto success; } @@ -1177,6 +1181,10 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } + if (globals_version != (uint16_t)globals_version) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState( interp, builtin_keys); if (builtins_version == 0) { @@ -1188,7 +1196,7 @@ _Py_Specialize_LoadGlobal( goto fail; } cache->index = (uint16_t)index; - write_u32(cache->module_keys_version, globals_version); + cache->module_keys_version = (uint16_t)globals_version; cache->builtin_keys_version = (uint16_t)builtins_version; instr->op.code = LOAD_GLOBAL_BUILTIN; goto success; From 5ffdaf748d98da6065158534720f1996a45a0072 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 11 Mar 2023 01:20:20 +0000 Subject: [PATCH 19/50] gh-102433: Add tests for how classes with properties interact with `isinstance()` checks on `typing.runtime_checkable` protocols (#102449) Co-authored-by: Carl Meyer --- Lib/test/test_typing.py | 88 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 0483ca3aa42f94..c17be6cd0bbc4a 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2530,6 +2530,94 @@ def meth(x): ... with self.assertRaises(TypeError): isinstance(C(), BadPG) + def test_protocols_isinstance_properties_and_descriptors(self): + class C: + @property + def attr(self): + return 42 + + class CustomDescriptor: + def __get__(self, obj, objtype=None): + return 42 + + class D: + attr = CustomDescriptor() + + # Check that properties set on superclasses + # are still found by the isinstance() logic + class E(C): ... + class F(D): ... + + class Empty: ... + + T = TypeVar('T') + + @runtime_checkable + class P(Protocol): + @property + def attr(self): ... + + @runtime_checkable + class P1(Protocol): + attr: int + + @runtime_checkable + class PG(Protocol[T]): + @property + def attr(self): ... + + @runtime_checkable + class PG1(Protocol[T]): + attr: T + + for protocol_class in P, P1, PG, PG1: + for klass in C, D, E, F: + with self.subTest( + klass=klass.__name__, + protocol_class=protocol_class.__name__ + ): + self.assertIsInstance(klass(), protocol_class) + + with self.subTest(klass="Empty", protocol_class=protocol_class.__name__): + self.assertNotIsInstance(Empty(), protocol_class) + + class BadP(Protocol): + @property + def attr(self): ... + + class BadP1(Protocol): + attr: int + + class BadPG(Protocol[T]): + @property + def attr(self): ... + + class BadPG1(Protocol[T]): + attr: T + + for obj in PG[T], PG[C], PG1[T], PG1[C], BadP, BadP1, BadPG, BadPG1: + for klass in C, D, E, F, Empty: + with self.subTest(klass=klass.__name__, obj=obj): + with self.assertRaises(TypeError): + isinstance(klass(), obj) + + def test_protocols_isinstance_not_fooled_by_custom_dir(self): + @runtime_checkable + class HasX(Protocol): + x: int + + class CustomDirWithX: + x = 10 + def __dir__(self): + return [] + + class CustomDirWithoutX: + def __dir__(self): + return ["x"] + + self.assertIsInstance(CustomDirWithX(), HasX) + self.assertNotIsInstance(CustomDirWithoutX(), HasX) + def test_protocols_isinstance_py36(self): class APoint: def __init__(self, x, y, label): From aa0a73d1bc53dcb6348a869df1e775138991e561 Mon Sep 17 00:00:00 2001 From: wangxiang-hz <34048878+wangxiang-hz@users.noreply.github.com> Date: Sat, 11 Mar 2023 19:11:37 +0800 Subject: [PATCH 20/50] gh-102213: Optimize the performance of `__getattr__` (GH-102248) When __getattr__ is defined, python with try to find an attribute using _PyObject_GenericGetAttrWithDict find nothing is reasonable so we don't need an exception, it will hurt performance. --- Include/internal/pycore_object.h | 1 + .../2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst | 1 + Objects/object.c | 6 ++++++ Objects/typeobject.c | 9 ++++++--- 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index e15685f174ebcf..318e6f3371c0c3 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -370,6 +370,7 @@ extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); extern int _PyType_HasSubclasses(PyTypeObject *); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); +extern PyObject* _PyObject_GenericTryGetAttr(PyObject *, PyObject *); // Access macro to the members which are floating "behind" the object static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst new file mode 100644 index 00000000000000..997bef226e713f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst @@ -0,0 +1 @@ +Fix performance loss when accessing an object's attributes with ``__getattr__`` defined. diff --git a/Objects/object.c b/Objects/object.c index 38da4d497a96e7..dff5e2afa16ab8 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1405,6 +1405,12 @@ PyObject_GenericGetAttr(PyObject *obj, PyObject *name) return _PyObject_GenericGetAttrWithDict(obj, name, NULL, 0); } +PyObject * +_PyObject_GenericTryGetAttr(PyObject *obj, PyObject *name) +{ + return _PyObject_GenericGetAttrWithDict(obj, name, NULL, 1); +} + int _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, PyObject *value, PyObject *dict) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f486b83fd69e64..f0654c239f6635 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8247,14 +8247,17 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == (void *)PyObject_GenericGetAttr)) - res = PyObject_GenericGetAttr(self, name); + /* finding nothing is reasonable when __getattr__ is defined */ + res = _PyObject_GenericTryGetAttr(self, name); else { Py_INCREF(getattribute); res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); } - if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); + if (res == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } res = call_attribute(self, getattr, name); } Py_DECREF(getattr); From ced13c96a4eb9391a9d27e3e13218f70c579670f Mon Sep 17 00:00:00 2001 From: Thomas Krennwallner Date: Sat, 11 Mar 2023 08:19:40 -0500 Subject: [PATCH 21/50] gh-79940: add introspection API for asynchronous generators to `inspect` module (#11590) --- Doc/library/inspect.rst | 28 ++++- Doc/whatsnew/3.12.rst | 4 + Lib/inspect.py | 50 ++++++++ Lib/test/test_inspect.py | 107 ++++++++++++++++++ ...3-02-26-17-29-57.gh-issue-79940.SAfmAy.rst | 2 + Objects/genobject.c | 10 ++ 6 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 789e9839d22f71..ccf240193d36a9 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1440,8 +1440,8 @@ code execution:: pass -Current State of Generators and Coroutines ------------------------------------------- +Current State of Generators, Coroutines, and Asynchronous Generators +-------------------------------------------------------------------- When implementing coroutine schedulers and for other advanced uses of generators, it is useful to determine whether a generator is currently @@ -1476,6 +1476,22 @@ generator to be determined easily. .. versionadded:: 3.5 +.. function:: getasyncgenstate(agen) + + Get current state of an asynchronous generator object. The function is + intended to be used with asynchronous iterator objects created by + :keyword:`async def` functions which use the :keyword:`yield` statement, + but will accept any asynchronous generator-like object that has + ``ag_running`` and ``ag_frame`` attributes. + + Possible states are: + * AGEN_CREATED: Waiting to start execution. + * AGEN_RUNNING: Currently being executed by the interpreter. + * AGEN_SUSPENDED: Currently suspended at a yield expression. + * AGEN_CLOSED: Execution has completed. + + .. versionadded:: 3.12 + The current internal state of the generator can also be queried. This is mostly useful for testing purposes, to ensure that internal state is being updated as expected: @@ -1507,6 +1523,14 @@ updated as expected: .. versionadded:: 3.5 +.. function:: getasyncgenlocals(agen) + + This function is analogous to :func:`~inspect.getgeneratorlocals`, but + works for asynchronous generator objects created by :keyword:`async def` + functions which use the :keyword:`yield` statement. + + .. versionadded:: 3.12 + .. _inspect-module-co-flags: diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 48b7aab0595ebb..9f33dbc808ddc0 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -244,6 +244,10 @@ inspect a :term:`coroutine` for use with :func:`iscoroutinefunction`. (Contributed Carlton Gibson in :gh:`99247`.) +* Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals` + for determining the current state of asynchronous generators. + (Contributed by Thomas Krennwallner in :issue:`35759`.) + pathlib ------- diff --git a/Lib/inspect.py b/Lib/inspect.py index edc23b0ffa9201..0eceaaf9a24f5d 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -34,6 +34,10 @@ 'Yury Selivanov ') __all__ = [ + "AGEN_CLOSED", + "AGEN_CREATED", + "AGEN_RUNNING", + "AGEN_SUSPENDED", "ArgInfo", "Arguments", "Attribute", @@ -77,6 +81,8 @@ "getabsfile", "getargs", "getargvalues", + "getasyncgenlocals", + "getasyncgenstate", "getattr_static", "getblock", "getcallargs", @@ -1935,6 +1941,50 @@ def getcoroutinelocals(coroutine): return {} +# ----------------------------------- asynchronous generator introspection + +AGEN_CREATED = 'AGEN_CREATED' +AGEN_RUNNING = 'AGEN_RUNNING' +AGEN_SUSPENDED = 'AGEN_SUSPENDED' +AGEN_CLOSED = 'AGEN_CLOSED' + + +def getasyncgenstate(agen): + """Get current state of an asynchronous generator object. + + Possible states are: + AGEN_CREATED: Waiting to start execution. + AGEN_RUNNING: Currently being executed by the interpreter. + AGEN_SUSPENDED: Currently suspended at a yield expression. + AGEN_CLOSED: Execution has completed. + """ + if agen.ag_running: + return AGEN_RUNNING + if agen.ag_suspended: + return AGEN_SUSPENDED + if agen.ag_frame is None: + return AGEN_CLOSED + return AGEN_CREATED + + +def getasyncgenlocals(agen): + """ + Get the mapping of asynchronous generator local variables to their current + values. + + A dict is returned, with the keys the local variable names and values the + bound values.""" + + if not isasyncgen(agen): + raise TypeError(f"{agen!r} is not a Python async generator") + + frame = getattr(agen, "ag_frame", None) + if frame is not None: + return agen.ag_frame.f_locals + else: + return {} + + ############################################################################### ### Function Signature Object (PEP 362) ############################################################################### diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 02f8378d0413ea..410a2e5b5468f0 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1,3 +1,4 @@ +import asyncio import builtins import collections import datetime @@ -65,6 +66,10 @@ def revise(filename, *args): git = mod.StupidGit() +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + def signatures_with_lexicographic_keyword_only_parameters(): """ Yields a whole bunch of functions with only keyword-only parameters, @@ -2321,6 +2326,108 @@ async def func(a=None): {'a': None, 'gencoro': gencoro, 'b': 'spam'}) +class TestGetAsyncGenState(unittest.IsolatedAsyncioTestCase): + + def setUp(self): + async def number_asyncgen(): + for number in range(5): + yield number + self.asyncgen = number_asyncgen() + + async def asyncTearDown(self): + await self.asyncgen.aclose() + + def _asyncgenstate(self): + return inspect.getasyncgenstate(self.asyncgen) + + def test_created(self): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CREATED) + + async def test_suspended(self): + value = await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + self.assertEqual(value, 0) + + async def test_closed_after_exhaustion(self): + countdown = 7 + with self.assertRaises(StopAsyncIteration): + while countdown := countdown - 1: + await anext(self.asyncgen) + self.assertEqual(countdown, 1) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + async def test_closed_after_immediate_exception(self): + with self.assertRaises(RuntimeError): + await self.asyncgen.athrow(RuntimeError) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + async def test_running(self): + async def running_check_asyncgen(): + for number in range(5): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + yield number + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + self.asyncgen = running_check_asyncgen() + # Running up to the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + # Running after the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + + def test_easy_debugging(self): + # repr() and str() of a asyncgen state should contain the state name + names = 'AGEN_CREATED AGEN_RUNNING AGEN_SUSPENDED AGEN_CLOSED'.split() + for name in names: + state = getattr(inspect, name) + self.assertIn(name, repr(state)) + self.assertIn(name, str(state)) + + async def test_getasyncgenlocals(self): + async def each(lst, a=None): + b=(1, 2, 3) + for v in lst: + if v == 3: + c = 12 + yield v + + numbers = each([1, 2, 3]) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3]}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 1, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 2, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 3, + 'b': (1, 2, 3), 'c': 12}) + with self.assertRaises(StopAsyncIteration): + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), {}) + + async def test_getasyncgenlocals_empty(self): + async def yield_one(): + yield 1 + one = yield_one() + self.assertEqual(inspect.getasyncgenlocals(one), {}) + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + with self.assertRaises(StopAsyncIteration): + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + + def test_getasyncgenlocals_error(self): + self.assertRaises(TypeError, inspect.getasyncgenlocals, 1) + self.assertRaises(TypeError, inspect.getasyncgenlocals, lambda x: True) + self.assertRaises(TypeError, inspect.getasyncgenlocals, set) + self.assertRaises(TypeError, inspect.getasyncgenlocals, (2,3)) + + class MySignature(inspect.Signature): # Top-level to make it picklable; # used in test_signature_object_pickle diff --git a/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst b/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst new file mode 100644 index 00000000000000..31b8ead8433279 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst @@ -0,0 +1,2 @@ +Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals`. +Patch by Thomas Krennwallner. diff --git a/Objects/genobject.c b/Objects/genobject.c index 61463774310f88..6316fa9865fe65 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1520,6 +1520,15 @@ ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) return _gen_getcode(gen, "ag_code"); } +static PyObject * +ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) +{ + if (ag->ag_frame_state == FRAME_SUSPENDED) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyGetSetDef async_gen_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, PyDoc_STR("name of the async generator")}, @@ -1529,6 +1538,7 @@ static PyGetSetDef async_gen_getsetlist[] = { PyDoc_STR("object being awaited on, or None")}, {"ag_frame", (getter)ag_getframe, NULL, NULL}, {"ag_code", (getter)ag_getcode, NULL, NULL}, + {"ag_suspended", (getter)ag_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; From 534660f1680955c7a6a47d5c6bd9649704b74a87 Mon Sep 17 00:00:00 2001 From: Thomas Krennwallner Date: Sat, 11 Mar 2023 13:36:49 -0500 Subject: [PATCH 22/50] gh-79940: skip `TestGetAsyncGenState` on wasm as it requires working sockets (GH-102605) Skip `TestGetAsyncGenState` and restoring of the default event loop policy in `test_inspect` if platform lacks working socket support. Fixes #11590 Automerge-Triggered-By: GH:kumaraditya303 --- Lib/test/test_inspect.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 410a2e5b5468f0..803b259d961f54 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -67,7 +67,8 @@ def revise(filename, *args): def tearDownModule(): - asyncio.set_event_loop_policy(None) + if support.has_socket_support: + asyncio.set_event_loop_policy(None) def signatures_with_lexicographic_keyword_only_parameters(): @@ -2326,6 +2327,7 @@ async def func(a=None): {'a': None, 'gencoro': gencoro, 'b': 'spam'}) +@support.requires_working_socket() class TestGetAsyncGenState(unittest.IsolatedAsyncioTestCase): def setUp(self): From bb396eece44036a71427e7766fbb8e0247373102 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 11 Mar 2023 22:10:52 +0300 Subject: [PATCH 23/50] gh-101821: Test coverage for `ast.main` function (#101822) --- Lib/test/test_ast.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 7c9a57c685df75..6c932e1305e1dd 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -11,6 +11,7 @@ from textwrap import dedent from test import support +from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin def to_tuple(t): @@ -2564,6 +2565,25 @@ def test_subinterpreter(self): self.assertEqual(res, 0) +class ASTMainTests(unittest.TestCase): + # Tests `ast.main()` function. + + def test_cli_file_input(self): + code = "print(1, 2, 3)" + expected = ast.dump(ast.parse(code), indent=3) + + with os_helper.temp_dir() as tmp_dir: + filename = os.path.join(tmp_dir, "test_module.py") + with open(filename, 'w', encoding='utf-8') as f: + f.write(code) + res, _ = script_helper.run_python_until_end("-m", "ast", filename) + + self.assertEqual(res.err, b"") + self.assertEqual(expected.splitlines(), + res.out.decode("utf8").splitlines()) + self.assertEqual(res.rc, 0) + + def main(): if __name__ != '__main__': return From e6210621bee4ac10e18b4adc11229b8cc1ee788d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 12 Mar 2023 09:58:02 -0700 Subject: [PATCH 24/50] Fixes duplicated word (#102623) In line 1627, the end of the sentence reads "only that that it may be." but it should read "only that it may be" (or alternatively "only that that may be"). Co-authored-by: Hugo Gabriel Eyherabide --- Doc/library/subprocess.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ccc431b2d92e07..2b5a82e0107fb6 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1622,7 +1622,7 @@ that. It is safe to set these to false on any Python version. They will have no effect on older versions when unsupported. Do not assume the attributes are available to read. Despite their names, a true value does not indicate that the -corresponding function will be used, only that that it may be. +corresponding function will be used, only that it may be. Please file issues any time you have to use these private knobs with a way to reproduce the issue you were seeing. Link to that issue from a comment in your From 6cd7572f859a32a1f4626644c3e8139055df59e3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 12 Mar 2023 12:48:25 -0500 Subject: [PATCH 25/50] Optimize fmean() weighted average (#102626) --- Lib/statistics.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 07d1fd5ba6e98e..7d5d750193a5ab 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -136,9 +136,9 @@ from decimal import Decimal from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right -from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum +from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod from functools import reduce -from operator import mul, itemgetter +from operator import itemgetter from collections import Counter, namedtuple, defaultdict _SQRT2 = sqrt(2.0) @@ -496,28 +496,26 @@ def fmean(data, weights=None): >>> fmean([3.5, 4.0, 5.25]) 4.25 """ - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + n = 0 + def count(iterable): + nonlocal n + for n, x in enumerate(iterable, start=1): + yield x + data = count(data) total = fsum(data) if not n: raise StatisticsError('fmean requires at least one data point') return total / n - try: - num_weights = len(weights) - except TypeError: + if not isinstance(weights, (list, tuple)): weights = list(weights) - num_weights = len(weights) - num = fsum(map(mul, data, weights)) - if n != num_weights: + try: + num = sumprod(data, weights) + except ValueError: raise StatisticsError('data and weights must be the same length') den = fsum(weights) if not den: From f6ca71a42268dcd890bd1930501b6c7e6d7ce66e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D1=82=D0=B0=D0=BB=D0=B8=D0=B9=20=D0=94=D0=BC?= =?UTF-8?q?=D0=B8=D1=82=D1=80=D0=B8=D0=B5=D0=B2?= Date: Sun, 12 Mar 2023 21:03:59 +0300 Subject: [PATCH 26/50] Fix duplicated words in the docs (GH-102629) With grep utility found some duplicated words Automerge-Triggered-By: GH:AlexWaygood --- Doc/library/base64.rst | 2 +- Doc/reference/compound_stmts.rst | 2 +- Misc/NEWS.d/3.11.0a6.rst | 2 +- Misc/NEWS.d/3.12.0a1.rst | 5 ++--- Misc/NEWS.d/3.12.0a2.rst | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 4ca3768f827c6b..d5b6af8c1928ef 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -58,7 +58,7 @@ The modern interface provides: This allows an application to e.g. generate URL or filesystem safe Base64 strings. The default is ``None``, for which the standard Base64 alphabet is used. - May assert or raise a a :exc:`ValueError` if the length of *altchars* is not 2. Raises a + May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. Raises a :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 52bb0b22b042f6..f0a8936c35bf4a 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -816,7 +816,7 @@ keyword against a subject. Syntax: If the OR pattern fails, the AS pattern fails. Otherwise, the AS pattern binds the subject to the name on the right of the as keyword and succeeds. -``capture_pattern`` cannot be a a ``_``. +``capture_pattern`` cannot be a ``_``. In simple terms ``P as NAME`` will match with ``P``, and on success it will set ``NAME = ``. diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 68b80e46690dc9..8621edcfb04bb3 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -1055,7 +1055,7 @@ Patch by Victor Stinner. .. section: Build Building Python now requires support for floating point Not-a-Number (NaN): -remove the ``Py_NO_NAN`` macro. Patch by by Victor Stinner. +remove the ``Py_NO_NAN`` macro. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 2a943fe481af99..075e8da825a331 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -4559,8 +4559,7 @@ The minimum Sphinx version required to build the documentation is now 3.2. .. section: Documentation Augmented documentation of asyncio.create_task(). Clarified the need to keep -strong references to tasks and added a code snippet detailing how to to -this. +strong references to tasks and added a code snippet detailing how to do this. .. @@ -5066,7 +5065,7 @@ are stubs and not functional. .. nonce: Bsswsc .. section: Build -Add a new ``--with-dsymutil`` configure option to to link debug information +Add a new ``--with-dsymutil`` configure option to link debug information in macOS. Patch by Pablo Galindo. .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 41ad8cd22b5d89..117be21a3221b6 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -546,7 +546,7 @@ Clean up refleak on failed module initialisation in :mod:`_zoneinfo` .. nonce: qc_KHr .. section: Library -Clean up refleaks on failed module initialisation in in :mod:`_pickle` +Clean up refleaks on failed module initialisation in :mod:`_pickle` .. From 5d677c556f03a34d1c2d86e4cc96025870c20c12 Mon Sep 17 00:00:00 2001 From: gaogaotiantian Date: Sun, 12 Mar 2023 16:09:55 -0700 Subject: [PATCH 27/50] GH-101673: Fix pdb bug where local variable changes are lost after longlist (#101674) --- Lib/pdb.py | 13 ++------- Lib/test/test_pdb.py | 29 +++++++++++++++++++ ...-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst | 1 + 3 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 78d0ce537f1fc2..f11fc55536810f 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -107,15 +107,6 @@ def find_function(funcname, filename): return funcname, filename, lineno return None -def getsourcelines(obj): - lines, lineno = inspect.findsource(obj) - if inspect.isframe(obj) and obj.f_globals is obj.f_locals: - # must be a module frame: do not try to cut a block out of it - return lines, 1 - elif inspect.ismodule(obj): - return lines, 1 - return inspect.getblock(lines[lineno:]), lineno+1 - def lasti2lineno(code, lasti): linestarts = list(dis.findlinestarts(code)) linestarts.reverse() @@ -1357,7 +1348,7 @@ def do_longlist(self, arg): filename = self.curframe.f_code.co_filename breaklist = self.get_file_breaks(filename) try: - lines, lineno = getsourcelines(self.curframe) + lines, lineno = inspect.getsourcelines(self.curframe) except OSError as err: self.error(err) return @@ -1373,7 +1364,7 @@ def do_source(self, arg): except: return try: - lines, lineno = getsourcelines(obj) + lines, lineno = inspect.getsourcelines(obj) except (OSError, TypeError) as err: self.error(err) return diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 48f419e62fbbed..d91bd0b2f03a0f 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1474,6 +1474,35 @@ def test_pdb_issue_gh_94215(): (Pdb) continue """ +def test_pdb_issue_gh_101673(): + """See GH-101673 + + Make sure ll won't revert local variable assignment + + >>> def test_function(): + ... a = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... '!a = 2', + ... 'll', + ... 'p a', + ... 'continue' + ... ]): + ... test_function() + --Return-- + > (3)test_function()->None + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) !a = 2 + (Pdb) ll + 1 def test_function(): + 2 a = 1 + 3 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p a + 2 + (Pdb) continue + """ + @support.requires_subprocess() class PdbTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst b/Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst new file mode 100644 index 00000000000000..4e673ba9811568 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst @@ -0,0 +1 @@ +Fix a :mod:`pdb` bug where ``ll`` clears the changes to local variables. From 78e4e6c3d71980d4e6687f07afa6ddfc83e29b04 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 13 Mar 2023 11:24:52 +0200 Subject: [PATCH 28/50] gh-101100: Fix Sphinx warnings in `turtle` module (#102340) Co-authored-by: C.A.M. Gerlach --- Doc/library/turtle.rst | 58 ++++++++++++++++++++++-------------------- Lib/turtle.py | 2 +- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index b7eb569c18a6c4..05392d04e52263 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -1214,7 +1214,7 @@ Appearance will be displayed stretched according to its stretchfactors: *stretch_wid* is stretchfactor perpendicular to its orientation, *stretch_len* is stretchfactor in direction of its orientation, *outline* determines the width - of the shapes's outline. + of the shape's outline. .. doctest:: :skipif: _tkinter is None @@ -1545,7 +1545,7 @@ below: 1. Create an empty Shape object of type "compound". 2. Add as many components to this object as desired, using the - :meth:`addcomponent` method. + :meth:`~Shape.addcomponent` method. For example: @@ -2125,7 +2125,7 @@ Public classes :param cv: a :class:`tkinter.Canvas` - Provides screen oriented methods like :func:`setbg` etc. that are described + Provides screen oriented methods like :func:`bgcolor` etc. that are described above. .. class:: Screen() @@ -2315,7 +2315,9 @@ of this module or which better fits to your needs, e.g. for use in a classroom, you can prepare a configuration file ``turtle.cfg`` which will be read at import time and modify the configuration according to its settings. -The built in configuration would correspond to the following turtle.cfg:: +The built in configuration would correspond to the following ``turtle.cfg``: + +.. code-block:: ini width = 0.5 height = 0.75 @@ -2340,15 +2342,15 @@ The built in configuration would correspond to the following turtle.cfg:: Short explanation of selected entries: -- The first four lines correspond to the arguments of the :meth:`Screen.setup` +- The first four lines correspond to the arguments of the :func:`Screen.setup ` method. - Line 5 and 6 correspond to the arguments of the method - :meth:`Screen.screensize`. + :func:`Screen.screensize `. - *shape* can be any of the built-in shapes, e.g: arrow, turtle, etc. For more info try ``help(shape)``. -- If you want to use no fillcolor (i.e. make the turtle transparent), you have +- If you want to use no fill color (i.e. make the turtle transparent), you have to write ``fillcolor = ""`` (but all nonempty strings must not have quotes in - the cfg-file). + the cfg file). - If you want to reflect the turtle its state, you have to use ``resizemode = auto``. - If you set e.g. ``language = italian`` the docstringdict @@ -2398,6 +2400,8 @@ The :mod:`turtledemo` package directory contains: The demo scripts are: +.. currentmodule:: turtle + .. tabularcolumns:: |l|L|L| +----------------+------------------------------+-----------------------+ @@ -2469,20 +2473,20 @@ Have fun! Changes since Python 2.6 ======================== -- The methods :meth:`Turtle.tracer`, :meth:`Turtle.window_width` and - :meth:`Turtle.window_height` have been eliminated. +- The methods :func:`Turtle.tracer `, :func:`Turtle.window_width ` and + :func:`Turtle.window_height ` have been eliminated. Methods with these names and functionality are now available only as methods of :class:`Screen`. The functions derived from these remain available. (In fact already in Python 2.6 these methods were merely duplications of the corresponding - :class:`TurtleScreen`/:class:`Screen`-methods.) + :class:`TurtleScreen`/:class:`Screen` methods.) -- The method :meth:`Turtle.fill` has been eliminated. - The behaviour of :meth:`begin_fill` and :meth:`end_fill` - have changed slightly: now every filling-process must be completed with an +- The method :func:`!Turtle.fill` has been eliminated. + The behaviour of :func:`begin_fill` and :func:`end_fill` + have changed slightly: now every filling process must be completed with an ``end_fill()`` call. -- A method :meth:`Turtle.filling` has been added. It returns a boolean +- A method :func:`Turtle.filling ` has been added. It returns a boolean value: ``True`` if a filling process is under way, ``False`` otherwise. This behaviour corresponds to a ``fill()`` call without arguments in Python 2.6. @@ -2490,23 +2494,23 @@ Changes since Python 2.6 Changes since Python 3.0 ======================== -- The methods :meth:`Turtle.shearfactor`, :meth:`Turtle.shapetransform` and - :meth:`Turtle.get_shapepoly` have been added. Thus the full range of +- The :class:`Turtle` methods :func:`shearfactor`, :func:`shapetransform` and + :func:`get_shapepoly` have been added. Thus the full range of regular linear transforms is now available for transforming turtle shapes. - :meth:`Turtle.tiltangle` has been enhanced in functionality: it now can - be used to get or set the tiltangle. :meth:`Turtle.settiltangle` has been + :func:`tiltangle` has been enhanced in functionality: it now can + be used to get or set the tilt angle. :func:`settiltangle` has been deprecated. -- The method :meth:`Screen.onkeypress` has been added as a complement to - :meth:`Screen.onkey` which in fact binds actions to the keyrelease event. - Accordingly the latter has got an alias: :meth:`Screen.onkeyrelease`. +- The :class:`Screen` method :func:`onkeypress` has been added as a complement to + :func:`onkey`. As the latter binds actions to the key release event, + an alias: :func:`onkeyrelease` was also added for it. -- The method :meth:`Screen.mainloop` has been added. So when working only - with Screen and Turtle objects one must not additionally import - :func:`mainloop` anymore. +- The method :func:`Screen.mainloop ` has been added, + so there is no longer a need to use the standalone :func:`mainloop` function + when working with :class:`Screen` and :class:`Turtle` objects. -- Two input methods has been added :meth:`Screen.textinput` and - :meth:`Screen.numinput`. These popup input dialogs and return +- Two input methods have been added: :func:`Screen.textinput ` and + :func:`Screen.numinput `. These pop up input dialogs and return strings and numbers respectively. - Two example scripts :file:`tdemo_nim.py` and :file:`tdemo_round_dance.py` diff --git a/Lib/turtle.py b/Lib/turtle.py index 6abf9f7f65af0d..1b369327bc8eff 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -954,7 +954,7 @@ def __repr__(self): class TurtleScreen(TurtleScreenBase): - """Provides screen oriented methods like setbg etc. + """Provides screen oriented methods like bgcolor etc. Only relies upon the methods of TurtleScreenBase and NOT upon components of the underlying graphics toolkit - From 233e32f93614255bf5fc7c93cd98af453e58cc98 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 Mar 2023 10:34:54 +0000 Subject: [PATCH 29/50] GH-102300: Reuse objects with refcount == 1 in float specialized binary ops. (GH-102301) --- ...-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst | 1 + Python/bytecodes.c | 15 +++----------- Python/ceval_macros.h | 20 +++++++++++++++++++ Python/generated_cases.c.h | 15 +++----------- 4 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst new file mode 100644 index 00000000000000..4227014582b7e6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst @@ -0,0 +1 @@ +Reuse operands with refcount of 1 in float specializations of BINARY_OP. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index dc2a20f22b6038..178519fcb63ced 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -195,10 +195,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; - prod = PyFloat_FromDouble(dprod); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - ERROR_IF(prod == NULL, error); + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); } inst(BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- sub)) { @@ -218,10 +215,7 @@ dummy_func( DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; - sub = PyFloat_FromDouble(dsub); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - ERROR_IF(sub == NULL, error); + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); } inst(BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { @@ -278,10 +272,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; - sum = PyFloat_FromDouble(dsum); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - ERROR_IF(sum == NULL, error); + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); } inst(BINARY_OP_ADD_INT, (unused/1, left, right -- sum)) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index ac1fec77daca8a..98b72ec1b36428 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -350,3 +350,23 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define KWNAMES_LEN() \ (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames))) + +#define DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dval, result) \ +do { \ + if (Py_REFCNT(left) == 1) { \ + ((PyFloatObject *)left)->ob_fval = (dval); \ + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);\ + result = (left); \ + } \ + else if (Py_REFCNT(right) == 1) {\ + ((PyFloatObject *)right)->ob_fval = (dval); \ + _Py_DECREF_NO_DEALLOC(left); \ + result = (right); \ + }\ + else { \ + result = PyFloat_FromDouble(dval); \ + if ((result) == NULL) goto error; \ + _Py_DECREF_NO_DEALLOC(left); \ + _Py_DECREF_NO_DEALLOC(right); \ + } \ +} while (0) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3839aeecb912a1..21073cbf978ce0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -263,10 +263,7 @@ STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; - prod = PyFloat_FromDouble(dprod); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - if (prod == NULL) goto pop_2_error; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -300,10 +297,7 @@ DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; - sub = PyFloat_FromDouble(dsub); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - if (sub == NULL) goto pop_2_error; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -372,10 +366,7 @@ STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; - sum = PyFloat_FromDouble(dsum); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - if (sum == NULL) goto pop_2_error; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; From 9a8b66b58c74236959a01d579e2c156d9c2e7cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=AD=E4=B9=9D=E9=BC=8E?= <109224573@qq.com> Date: Mon, 13 Mar 2023 21:15:52 +0800 Subject: [PATCH 30/50] Fix broken link to MSDN (#102355) --- Lib/socket.py | 2 +- Lib/test/support/socket_helper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/socket.py b/Lib/socket.py index 3a4f94de9cc03a..321fcda51505c1 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -910,7 +910,7 @@ def create_server(address, *, family=AF_INET, backlog=None, reuse_port=False, # address, effectively preventing this one from accepting # connections. Also, it may set the process in a state where # it'll no longer respond to any signals or graceful kills. - # See: msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx + # See: https://learn.microsoft.com/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse if os.name not in ('nt', 'cygwin') and \ hasattr(_socket, 'SO_REUSEADDR'): try: diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index d2960c9e333474..78409238db8954 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -63,7 +63,7 @@ def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): http://bugs.python.org/issue2550 for more info. The following site also has a very thorough description about the implications of both REUSEADDR and EXCLUSIVEADDRUSE on Windows: - http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx) + https://learn.microsoft.com/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse XXX: although this approach is a vast improvement on previous attempts to elicit unused ports, it rests heavily on the assumption that the ephemeral From ca01cae1e91446c70fa569d4502c7cbba381e7ff Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 13 Mar 2023 15:56:24 +0000 Subject: [PATCH 31/50] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives (#102631) --- Python/ceval.c | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index b422d0ed34ede3..92235519c993b6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -13,7 +13,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES -#include "pycore_pyerrors.h" // _PyErr_Fetch(), _PyErr_GetRaisedException() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -1783,18 +1783,15 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) if (exc == NULL) { /* Reraise */ _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); - value = exc_info->exc_value; - if (Py_IsNone(value) || value == NULL) { + exc = exc_info->exc_value; + if (Py_IsNone(exc) || exc == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "No active exception to reraise"); return 0; } - assert(PyExceptionInstance_Check(value)); - type = PyExceptionInstance_Class(value); - Py_XINCREF(type); - Py_XINCREF(value); - PyObject *tb = PyException_GetTraceback(value); /* new ref */ - _PyErr_Restore(tstate, type, value, tb); + Py_INCREF(exc); + assert(PyExceptionInstance_Check(exc)); + _PyErr_SetRaisedException(tstate, exc); return 1; } @@ -2035,28 +2032,27 @@ call_exc_trace(Py_tracefunc func, PyObject *self, PyThreadState *tstate, _PyInterpreterFrame *f) { - PyObject *type, *value, *traceback, *orig_traceback, *arg; - int err; - _PyErr_Fetch(tstate, &type, &value, &orig_traceback); - if (value == NULL) { - value = Py_NewRef(Py_None); + PyObject *exc = _PyErr_GetRaisedException(tstate); + assert(exc && PyExceptionInstance_Check(exc)); + PyObject *type = PyExceptionInstance_Class(exc); + PyObject *traceback = PyException_GetTraceback(exc); + if (traceback == NULL) { + traceback = Py_NewRef(Py_None); } - _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback); - traceback = (orig_traceback != NULL) ? orig_traceback : Py_None; - arg = PyTuple_Pack(3, type, value, traceback); + PyObject *arg = PyTuple_Pack(3, type, exc, traceback); + Py_XDECREF(traceback); + if (arg == NULL) { - _PyErr_Restore(tstate, type, value, orig_traceback); + _PyErr_SetRaisedException(tstate, exc); return; } - err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg); + int err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg); Py_DECREF(arg); if (err == 0) { - _PyErr_Restore(tstate, type, value, orig_traceback); + _PyErr_SetRaisedException(tstate, exc); } else { - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(orig_traceback); + Py_XDECREF(exc); } } From 634cb61909b1392f0eef3e749931777adb54f16e Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 13 Mar 2023 15:59:20 +0000 Subject: [PATCH 32/50] gh-87092: refactor assemble() to a number of separate functions, which do not need the compiler struct (#102562) --- Lib/test/support/bytecode_helper.py | 2 +- Lib/test/test_peepholer.py | 43 ++- Python/compile.c | 534 +++++++++++++++------------- 3 files changed, 318 insertions(+), 261 deletions(-) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 190fe8723b1fb5..1d9b889c920986 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -125,7 +125,7 @@ def complete_insts_info(self, insts): assert isinstance(item, tuple) inst = list(reversed(item)) opcode = dis.opmap[inst.pop()] - oparg = inst.pop() if opcode in self.HAS_ARG_OR_TARGET else 0 + oparg = inst.pop() loc = inst + [-1] * (4 - len(inst)) res.append((opcode, oparg, *loc)) return res diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index aea234e38705a8..9ff017da53c2b1 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -995,15 +995,19 @@ def test_conditional_jump_forward_non_const_condition(self): ('LOAD_CONST', 2, 13), lbl, ('LOAD_CONST', 3, 14), + ('RETURN_VALUE', 14), ] - expected = [ + expected_insts = [ ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), - ('LOAD_CONST', 2, 13), + ('LOAD_CONST', 1, 13), lbl, - ('LOAD_CONST', 3, 14) + ('RETURN_CONST', 2, 14), ] - self.cfg_optimization_test(insts, expected, consts=list(range(5))) + self.cfg_optimization_test(insts, + expected_insts, + consts=[0, 1, 2, 3, 4], + expected_consts=[0, 2, 3]) def test_conditional_jump_forward_const_condition(self): # The unreachable branch of the jump is removed, the jump @@ -1015,26 +1019,32 @@ def test_conditional_jump_forward_const_condition(self): ('LOAD_CONST', 2, 13), lbl, ('LOAD_CONST', 3, 14), + ('RETURN_VALUE', 14), ] - expected = [ - ('NOP', None, 11), - ('NOP', None, 12), - ('LOAD_CONST', 3, 14) + expected_insts = [ + ('NOP', 11), + ('NOP', 12), + ('RETURN_CONST', 1, 14), ] - self.cfg_optimization_test(insts, expected, consts=list(range(5))) + self.cfg_optimization_test(insts, + expected_insts, + consts=[0, 1, 2, 3, 4], + expected_consts=[0, 3]) def test_conditional_jump_backward_non_const_condition(self): insts = [ lbl1 := self.Label(), ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), - ('LOAD_CONST', 2, 13), + ('LOAD_NAME', 2, 13), + ('RETURN_VALUE', 13), ] expected = [ lbl := self.Label(), ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl, 12), - ('LOAD_CONST', 2, 13) + ('LOAD_NAME', 2, 13), + ('RETURN_VALUE', 13), ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) @@ -1042,16 +1052,17 @@ def test_conditional_jump_backward_const_condition(self): # The unreachable branch of the jump is removed insts = [ lbl1 := self.Label(), - ('LOAD_CONST', 1, 11), + ('LOAD_CONST', 3, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_CONST', 2, 13), + ('RETURN_VALUE', 13), ] - expected = [ + expected_insts = [ lbl := self.Label(), - ('NOP', None, 11), - ('JUMP', lbl, 12) + ('NOP', 11), + ('JUMP', lbl, 12), ] - self.cfg_optimization_test(insts, expected, consts=list(range(5))) + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) if __name__ == "__main__": diff --git a/Python/compile.c b/Python/compile.c index 45c97b4f8ef0e6..29e55b8b30c56b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -308,7 +308,7 @@ typedef struct basicblock_ { block, not to be confused with b_next, which is next by control flow. */ struct basicblock_ *b_list; /* The label of this block if it is a jump target, -1 otherwise */ - int b_label; + jump_target_label b_label; /* Exception stack at start of block, used by assembler to create the exception handling table */ ExceptStack *b_exceptstack; /* pointer to an array of instructions, initially NULL */ @@ -501,7 +501,7 @@ instr_sequence_next_inst(instr_sequence *seq) { static jump_target_label instr_sequence_new_label(instr_sequence *seq) { - jump_target_label lbl = {seq->s_next_free_label++}; + jump_target_label lbl = {++seq->s_next_free_label}; return lbl; } @@ -606,6 +606,14 @@ instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { instruction *instr = &seq->s_instrs[i]; RETURN_IF_ERROR(cfg_builder_addop(g, instr->i_opcode, instr->i_oparg, instr->i_loc)); } + int nblocks = 0; + for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { + nblocks++; + } + if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { + PyErr_NoMemory(); + return ERROR; + } return SUCCESS; } @@ -937,6 +945,7 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) static bool cfg_builder_check(cfg_builder *g) { + assert(g->g_entryblock->b_iused > 0); for (basicblock *block = g->g_block_list; block != NULL; block = block->b_list) { assert(!_PyMem_IsPtrFreed(block)); if (block->b_instr != NULL) { @@ -1086,7 +1095,7 @@ cfg_builder_new_block(cfg_builder *g) /* Extend the singly linked list of blocks with new block. */ b->b_list = g->g_block_list; g->g_block_list = b; - b->b_label = -1; + b->b_label = NO_LABEL; return b; } @@ -1267,11 +1276,21 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) static bool cfg_builder_current_block_is_terminated(cfg_builder *g) { - if (IS_LABEL(g->g_current_label)) { + struct cfg_instr *last = basicblock_last_instr(g->g_curblock); + if (last && IS_TERMINATOR_OPCODE(last->i_opcode)) { return true; } - struct cfg_instr *last = basicblock_last_instr(g->g_curblock); - return last && IS_TERMINATOR_OPCODE(last->i_opcode); + if (IS_LABEL(g->g_current_label)) { + if (last || IS_LABEL(g->g_curblock->b_label)) { + return true; + } + else { + /* current block is empty, label it */ + g->g_curblock->b_label = g->g_current_label; + g->g_current_label = NO_LABEL; + } + } + return false; } static int @@ -1282,7 +1301,7 @@ cfg_builder_maybe_start_new_block(cfg_builder *g) if (b == NULL) { return ERROR; } - b->b_label = g->g_current_label.id; + b->b_label = g->g_current_label; g->g_current_label = NO_LABEL; cfg_builder_use_next_block(g, b); } @@ -1432,50 +1451,49 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) } static Py_ssize_t -compiler_add_const(struct compiler *c, PyObject *o) +compiler_add_const(PyObject *const_cache, struct compiler_unit *u, PyObject *o) { - PyObject *key = merge_consts_recursive(c->c_const_cache, o); + assert(PyDict_CheckExact(const_cache)); + PyObject *key = merge_consts_recursive(const_cache, o); if (key == NULL) { return ERROR; } - Py_ssize_t arg = dict_add_o(c->u->u_consts, key); + Py_ssize_t arg = dict_add_o(u->u_consts, key); Py_DECREF(key); return arg; } static int -compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) +compiler_addop_load_const(PyObject *const_cache, struct compiler_unit *u, location loc, PyObject *o) { - Py_ssize_t arg = compiler_add_const(c, o); + Py_ssize_t arg = compiler_add_const(const_cache, u, o); if (arg < 0) { return ERROR; } - return codegen_addop_i(INSTR_SEQUENCE(c), LOAD_CONST, arg, loc); + return codegen_addop_i(&u->u_instr_sequence, LOAD_CONST, arg, loc); } static int -compiler_addop_o(struct compiler *c, location loc, +compiler_addop_o(struct compiler_unit *u, location loc, int opcode, PyObject *dict, PyObject *o) { Py_ssize_t arg = dict_add_o(dict, o); if (arg < 0) { return ERROR; } - return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); + return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); } static int -compiler_addop_name(struct compiler *c, location loc, +compiler_addop_name(struct compiler_unit *u, location loc, int opcode, PyObject *dict, PyObject *o) { - Py_ssize_t arg; - - PyObject *mangled = _Py_Mangle(c->u->u_private, o); + PyObject *mangled = _Py_Mangle(u->u_private, o); if (!mangled) { return ERROR; } - arg = dict_add_o(dict, mangled); + Py_ssize_t arg = dict_add_o(dict, mangled); Py_DECREF(mangled); if (arg < 0) { return ERROR; @@ -1488,7 +1506,7 @@ compiler_addop_name(struct compiler *c, location loc, arg <<= 1; arg |= 1; } - return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); + return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); } /* Add an opcode with an integer argument */ @@ -1509,7 +1527,7 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) static int codegen_addop_j(instr_sequence *seq, location loc, - int opcode, jump_target_label target) + int opcode, jump_target_label target) { assert(IS_LABEL(target)); assert(IS_JUMP_OPCODE(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); @@ -1527,7 +1545,7 @@ codegen_addop_j(instr_sequence *seq, location loc, } #define ADDOP_LOAD_CONST(C, LOC, O) \ - RETURN_IF_ERROR(compiler_addop_load_const((C), (LOC), (O))) + RETURN_IF_ERROR(compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), (O))) /* Same as ADDOP_LOAD_CONST, but steals a reference. */ #define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ @@ -1535,7 +1553,7 @@ codegen_addop_j(instr_sequence *seq, location loc, if (__new_const == NULL) { \ return ERROR; \ } \ - if (compiler_addop_load_const((C), (LOC), __new_const) < 0) { \ + if (compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), __new_const) < 0) { \ Py_DECREF(__new_const); \ return ERROR; \ } \ @@ -1544,7 +1562,7 @@ codegen_addop_j(instr_sequence *seq, location loc, #define ADDOP_N(C, LOC, OP, O, TYPE) { \ assert(!HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - if (compiler_addop_o((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O)) < 0) { \ + if (compiler_addop_o((C)->u, (LOC), (OP), (C)->u->u_ ## TYPE, (O)) < 0) { \ Py_DECREF((O)); \ return ERROR; \ } \ @@ -1552,7 +1570,7 @@ codegen_addop_j(instr_sequence *seq, location loc, } #define ADDOP_NAME(C, LOC, OP, O, TYPE) \ - RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) + RETURN_IF_ERROR(compiler_addop_name((C)->u, (LOC), (OP), (C)->u->u_ ## TYPE, (O))) #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -2556,7 +2574,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (c->c_optimize < 2) { docstring = _PyAST_GetDocString(body); } - if (compiler_add_const(c, docstring ? docstring : Py_None) < 0) { + if (compiler_add_const(c->c_const_cache, c->u, docstring ? docstring : Py_None) < 0) { compiler_exit_scope(c); return ERROR; } @@ -2924,7 +2942,7 @@ compiler_lambda(struct compiler *c, expr_ty e) /* Make None the first constant, so the lambda can't have a docstring. */ - RETURN_IF_ERROR(compiler_add_const(c, Py_None)); + RETURN_IF_ERROR(compiler_add_const(c->c_const_cache, c->u, Py_None)); c->u->u_argcount = asdl_seq_LEN(args->args); c->u->u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); @@ -4915,7 +4933,7 @@ compiler_call_simple_kw_helper(struct compiler *c, location loc, keyword_ty kw = asdl_seq_GET(keywords, i); PyTuple_SET_ITEM(names, i, Py_NewRef(kw->arg)); } - Py_ssize_t arg = compiler_add_const(c, names); + Py_ssize_t arg = compiler_add_const(c->c_const_cache, c->u, names); if (arg < 0) { return ERROR; } @@ -7372,7 +7390,7 @@ push_cold_blocks_to_end(cfg_builder *g, int code_flags) { if (explicit_jump == NULL) { return ERROR; } - basicblock_addop(explicit_jump, JUMP, b->b_next->b_label, NO_LOCATION); + basicblock_addop(explicit_jump, JUMP, b->b_next->b_label.id, NO_LOCATION); explicit_jump->b_cold = 1; explicit_jump->b_next = b->b_next; b->b_next = explicit_jump; @@ -7676,13 +7694,33 @@ assemble_emit_location(struct assembler* a, location loc, int isize) return write_location_info_entry(a, loc, isize); } -/* assemble_emit() +static int +assemble_location_info(struct assembler *a, basicblock *entryblock, int firstlineno) +{ + a->a_lineno = firstlineno; + location loc = NO_LOCATION; + int size = 0; + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int j = 0; j < b->b_iused; j++) { + if (!same_location(loc, b->b_instr[j].i_loc)) { + RETURN_IF_ERROR(assemble_emit_location(a, loc, size)); + loc = b->b_instr[j].i_loc; + size = 0; + } + size += instr_size(&b->b_instr[j]); + } + } + RETURN_IF_ERROR(assemble_emit_location(a, loc, size)); + return SUCCESS; +} + +/* assemble_emit_instr() Extend the bytecode with a new instruction. Update lnotab if necessary. */ static int -assemble_emit(struct assembler *a, struct cfg_instr *i) +assemble_emit_instr(struct assembler *a, struct cfg_instr *i) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); _Py_CODEUNIT *code; @@ -7700,6 +7738,35 @@ assemble_emit(struct assembler *a, struct cfg_instr *i) return SUCCESS; } +static int merge_const_one(PyObject *const_cache, PyObject **obj); + +static int +assemble_emit(struct assembler *a, basicblock *entryblock, int first_lineno, + PyObject *const_cache) +{ + RETURN_IF_ERROR(assemble_init(a, first_lineno)); + + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int j = 0; j < b->b_iused; j++) { + RETURN_IF_ERROR(assemble_emit_instr(a, &b->b_instr[j])); + } + } + + RETURN_IF_ERROR(assemble_location_info(a, entryblock, a->a_lineno)); + + RETURN_IF_ERROR(assemble_exception_table(a, entryblock)); + + RETURN_IF_ERROR(_PyBytes_Resize(&a->a_except_table, a->a_except_table_off)); + RETURN_IF_ERROR(merge_const_one(const_cache, &a->a_except_table)); + + RETURN_IF_ERROR(_PyBytes_Resize(&a->a_linetable, a->a_location_off)); + RETURN_IF_ERROR(merge_const_one(const_cache, &a->a_linetable)); + + RETURN_IF_ERROR(_PyBytes_Resize(&a->a_bytecode, a->a_offset * sizeof(_Py_CODEUNIT))); + RETURN_IF_ERROR(merge_const_one(const_cache, &a->a_bytecode)); + return SUCCESS; +} + static int normalize_jumps_in_block(cfg_builder *g, basicblock *b) { struct cfg_instr *last = basicblock_last_instr(b); @@ -7760,7 +7827,7 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } - basicblock_addop(backwards_jump, JUMP, target->b_label, NO_LOCATION); + basicblock_addop(backwards_jump, JUMP, target->b_label.id, NO_LOCATION); backwards_jump->b_instr[0].i_target = target; last->i_opcode = reversed_opcode; last->i_target = b->b_next; @@ -7963,9 +8030,9 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) static int add_checks_for_loads_of_uninitialized_variables(basicblock *entryblock, - struct compiler *c) + int nlocals, + int nparams) { - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); if (nlocals == 0) { return SUCCESS; } @@ -7986,7 +8053,6 @@ add_checks_for_loads_of_uninitialized_variables(basicblock *entryblock, // First origin of being uninitialized: // The non-parameter locals in the entry block. - int nparams = (int)PyList_GET_SIZE(c->u->u_ste->ste_varnames); uint64_t start_mask = 0; for (int i = nparams; i < nlocals; i++) { start_mask |= (uint64_t)1 << i; @@ -8092,7 +8158,7 @@ compute_code_flags(struct compiler *c) static int merge_const_one(PyObject *const_cache, PyObject **obj) { - PyDict_CheckExact(const_cache); + assert(PyDict_CheckExact(const_cache)); PyObject *key = _PyCode_ConstantKey(*obj); if (key == NULL) { return ERROR; @@ -8122,29 +8188,29 @@ extern void _Py_set_localsplus_info(int, PyObject *, unsigned char, PyObject *, PyObject *); static void -compute_localsplus_info(struct compiler *c, int nlocalsplus, +compute_localsplus_info(struct compiler_unit *u, int nlocalsplus, PyObject *names, PyObject *kinds) { PyObject *k, *v; Py_ssize_t pos = 0; - while (PyDict_Next(c->u->u_varnames, &pos, &k, &v)) { + while (PyDict_Next(u->u_varnames, &pos, &k, &v)) { int offset = (int)PyLong_AS_LONG(v); assert(offset >= 0); assert(offset < nlocalsplus); // For now we do not distinguish arg kinds. _PyLocals_Kind kind = CO_FAST_LOCAL; - if (PyDict_GetItem(c->u->u_cellvars, k) != NULL) { + if (PyDict_GetItem(u->u_cellvars, k) != NULL) { kind |= CO_FAST_CELL; } _Py_set_localsplus_info(offset, k, kind, names, kinds); } - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); // This counter mirrors the fix done in fix_cell_offsets(). int numdropped = 0; pos = 0; - while (PyDict_Next(c->u->u_cellvars, &pos, &k, &v)) { - if (PyDict_GetItem(c->u->u_varnames, k) != NULL) { + while (PyDict_Next(u->u_cellvars, &pos, &k, &v)) { + if (PyDict_GetItem(u->u_varnames, k) != NULL) { // Skip cells that are already covered by locals. numdropped += 1; continue; @@ -8157,7 +8223,7 @@ compute_localsplus_info(struct compiler *c, int nlocalsplus, } pos = 0; - while (PyDict_Next(c->u->u_freevars, &pos, &k, &v)) { + while (PyDict_Next(u->u_freevars, &pos, &k, &v)) { int offset = (int)PyLong_AS_LONG(v); assert(offset >= 0); offset += nlocals - numdropped; @@ -8167,19 +8233,20 @@ compute_localsplus_info(struct compiler *c, int nlocalsplus, } static PyCodeObject * -makecode(struct compiler *c, struct assembler *a, PyObject *constslist, - int maxdepth, int nlocalsplus, int code_flags) +makecode(struct compiler_unit *u, struct assembler *a, PyObject *const_cache, + PyObject *constslist, int maxdepth, int nlocalsplus, int code_flags, + PyObject *filename) { PyCodeObject *co = NULL; PyObject *names = NULL; PyObject *consts = NULL; PyObject *localsplusnames = NULL; PyObject *localspluskinds = NULL; - names = dict_keys_inorder(c->u->u_names, 0); + names = dict_keys_inorder(u->u_names, 0); if (!names) { goto error; } - if (merge_const_one(c->c_const_cache, &names) < 0) { + if (merge_const_one(const_cache, &names) < 0) { goto error; } @@ -8187,17 +8254,17 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, if (consts == NULL) { goto error; } - if (merge_const_one(c->c_const_cache, &consts) < 0) { + if (merge_const_one(const_cache, &consts) < 0) { goto error; } - assert(c->u->u_posonlyargcount < INT_MAX); - assert(c->u->u_argcount < INT_MAX); - assert(c->u->u_kwonlyargcount < INT_MAX); - int posonlyargcount = (int)c->u->u_posonlyargcount; - int posorkwargcount = (int)c->u->u_argcount; + assert(u->u_posonlyargcount < INT_MAX); + assert(u->u_argcount < INT_MAX); + assert(u->u_kwonlyargcount < INT_MAX); + int posonlyargcount = (int)u->u_posonlyargcount; + int posorkwargcount = (int)u->u_argcount; assert(INT_MAX - posonlyargcount - posorkwargcount > 0); - int kwonlyargcount = (int)c->u->u_kwonlyargcount; + int kwonlyargcount = (int)u->u_kwonlyargcount; localsplusnames = PyTuple_New(nlocalsplus); if (localsplusnames == NULL) { @@ -8207,16 +8274,16 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, if (localspluskinds == NULL) { goto error; } - compute_localsplus_info(c, nlocalsplus, localsplusnames, localspluskinds); + compute_localsplus_info(u, nlocalsplus, localsplusnames, localspluskinds); struct _PyCodeConstructor con = { - .filename = c->c_filename, - .name = c->u->u_name, - .qualname = c->u->u_qualname ? c->u->u_qualname : c->u->u_name, + .filename = filename, + .name = u->u_name, + .qualname = u->u_qualname ? u->u_qualname : u->u_name, .flags = code_flags, .code = a->a_bytecode, - .firstlineno = c->u->u_firstlineno, + .firstlineno = u->u_firstlineno, .linetable = a->a_linetable, .consts = consts, @@ -8238,7 +8305,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, goto error; } - if (merge_const_one(c->c_const_cache, &localsplusnames) < 0) { + if (merge_const_one(const_cache, &localsplusnames) < 0) { goto error; } con.localsplusnames = localsplusnames; @@ -8289,7 +8356,7 @@ dump_basicblock(const basicblock *b) { const char *b_return = basicblock_returns(b) ? "return " : ""; fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, offset: %d %s\n", - b->b_label, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, + b->b_label.id, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, b->b_startdepth, b->b_offset, b_return); if (b->b_instr) { int i; @@ -8316,11 +8383,11 @@ static int duplicate_exits_without_lineno(cfg_builder *g); static int * -build_cellfixedoffsets(struct compiler *c) +build_cellfixedoffsets(struct compiler_unit *u) { - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(u->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(u->u_freevars); int noffsets = ncellvars + nfreevars; int *fixed = PyMem_New(int, noffsets); @@ -8334,8 +8401,8 @@ build_cellfixedoffsets(struct compiler *c) PyObject *varname, *cellindex; Py_ssize_t pos = 0; - while (PyDict_Next(c->u->u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(c->u->u_varnames, varname); + while (PyDict_Next(u->u_cellvars, &pos, &varname, &cellindex)) { + PyObject *varindex = PyDict_GetItem(u->u_varnames, varname); if (varindex != NULL) { assert(PyLong_AS_LONG(cellindex) < INT_MAX); assert(PyLong_AS_LONG(varindex) < INT_MAX); @@ -8349,17 +8416,17 @@ build_cellfixedoffsets(struct compiler *c) } static int -insert_prefix_instructions(struct compiler *c, basicblock *entryblock, +insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, int *fixed, int nfreevars, int code_flags) { - assert(c->u->u_firstlineno > 0); + assert(u->u_firstlineno > 0); /* Add the generator prefix instructions. */ if (code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { struct cfg_instr make_gen = { .i_opcode = RETURN_GENERATOR, .i_oparg = 0, - .i_loc = LOCATION(c->u->u_firstlineno, c->u->u_firstlineno, -1, -1), + .i_loc = LOCATION(u->u_firstlineno, u->u_firstlineno, -1, -1), .i_target = NULL, }; RETURN_IF_ERROR(insert_instruction(entryblock, 0, &make_gen)); @@ -8373,12 +8440,12 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, } /* Set up cells for any variable that escapes, to be put in a closure. */ - const int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); + const int ncellvars = (int)PyDict_GET_SIZE(u->u_cellvars); if (ncellvars) { - // c->u->u_cellvars has the cells out of order so we sort them + // u->u_cellvars has the cells out of order so we sort them // before adding the MAKE_CELL instructions. Note that we // adjust for arg cells, which come first. - const int nvars = ncellvars + (int)PyDict_GET_SIZE(c->u->u_varnames); + const int nvars = ncellvars + (int)PyDict_GET_SIZE(u->u_varnames); int *sorted = PyMem_RawCalloc(nvars, sizeof(int)); if (sorted == NULL) { PyErr_NoMemory(); @@ -8447,11 +8514,11 @@ guarantee_lineno_for_exits(basicblock *entryblock, int firstlineno) { } static int -fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) +fix_cell_offsets(struct compiler_unit *u, basicblock *entryblock, int *fixedmap) { - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(u->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(u->u_freevars); int noffsets = ncellvars + nfreevars; // First deal with duplicates (arg cells). @@ -8491,8 +8558,6 @@ fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) return numdropped; } -static void -propagate_line_numbers(basicblock *entryblock); #ifndef NDEBUG @@ -8589,30 +8654,30 @@ remove_redundant_jumps(cfg_builder *g) { } static int -prepare_localsplus(struct compiler* c, cfg_builder *g, int code_flags) +prepare_localsplus(struct compiler_unit* u, cfg_builder *g, int code_flags) { - assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); - assert(PyDict_GET_SIZE(c->u->u_cellvars) < INT_MAX); - assert(PyDict_GET_SIZE(c->u->u_freevars) < INT_MAX); - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); + assert(PyDict_GET_SIZE(u->u_varnames) < INT_MAX); + assert(PyDict_GET_SIZE(u->u_cellvars) < INT_MAX); + assert(PyDict_GET_SIZE(u->u_freevars) < INT_MAX); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(u->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(u->u_freevars); assert(INT_MAX - nlocals - ncellvars > 0); assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); int nlocalsplus = nlocals + ncellvars + nfreevars; - int* cellfixedoffsets = build_cellfixedoffsets(c); + int* cellfixedoffsets = build_cellfixedoffsets(u); if (cellfixedoffsets == NULL) { return ERROR; } // This must be called before fix_cell_offsets(). - if (insert_prefix_instructions(c, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { + if (insert_prefix_instructions(u, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { PyMem_Free(cellfixedoffsets); return ERROR; } - int numdropped = fix_cell_offsets(c, g->g_entryblock, cellfixedoffsets); + int numdropped = fix_cell_offsets(u, g->g_entryblock, cellfixedoffsets); PyMem_Free(cellfixedoffsets); // At this point we're done with it. cellfixedoffsets = NULL; if (numdropped < 0) { @@ -8636,176 +8701,127 @@ add_return_at_end(struct compiler *c, int addNone) return SUCCESS; } -static PyCodeObject * -assemble(struct compiler *c, int addNone) -{ - PyCodeObject *co = NULL; - PyObject *consts = NULL; - cfg_builder g_; - cfg_builder *g = &g_; - struct assembler a; - memset(&a, 0, sizeof(struct assembler)); - - int code_flags = compute_code_flags(c); - if (code_flags < 0) { - return NULL; - } - - if (add_return_at_end(c, addNone) < 0) { - return NULL; - } - - /** Preprocessing **/ - if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), g) < 0) { - goto error; - } - assert(cfg_builder_check(g)); - - int nblocks = 0; - for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { - nblocks++; - } - if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { - PyErr_NoMemory(); - goto error; - } +static void propagate_line_numbers(basicblock *entryblock); +static int +resolve_line_numbers(struct compiler_unit *u, cfg_builder *g) +{ /* Set firstlineno if it wasn't explicitly set. */ - if (!c->u->u_firstlineno) { + if (!u->u_firstlineno) { if (g->g_entryblock->b_instr && g->g_entryblock->b_instr->i_loc.lineno) { - c->u->u_firstlineno = g->g_entryblock->b_instr->i_loc.lineno; + u->u_firstlineno = g->g_entryblock->b_instr->i_loc.lineno; } else { - c->u->u_firstlineno = 1; + u->u_firstlineno = 1; } } + RETURN_IF_ERROR(duplicate_exits_without_lineno(g)); + propagate_line_numbers(g->g_entryblock); + guarantee_lineno_for_exits(g->g_entryblock, u->u_firstlineno); + return SUCCESS; +} +static int +optimize_code_unit(cfg_builder *g, PyObject *consts, PyObject *const_cache, + int code_flags, int nlocals, int nparams) +{ + assert(cfg_builder_check(g)); + /** Preprocessing **/ /* Map labels to targets and mark exception handlers */ - if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { - goto error; - } - if (mark_except_handlers(g->g_entryblock) < 0) { - goto error; - } - if (label_exception_targets(g->g_entryblock)) { - goto error; - } + RETURN_IF_ERROR(translate_jump_labels_to_targets(g->g_entryblock)); + RETURN_IF_ERROR(mark_except_handlers(g->g_entryblock)); + RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); /** Optimization **/ - consts = consts_dict_keys_inorder(c->u->u_consts); + RETURN_IF_ERROR(optimize_cfg(g, consts, const_cache)); + RETURN_IF_ERROR(remove_unused_consts(g->g_entryblock, consts)); + RETURN_IF_ERROR( + add_checks_for_loads_of_uninitialized_variables( + g->g_entryblock, nlocals, nparams)); + + RETURN_IF_ERROR(push_cold_blocks_to_end(g, code_flags)); + return SUCCESS; +} + +static PyCodeObject * +assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, + int code_flags, PyObject *filename) +{ + PyCodeObject *co = NULL; + PyObject *consts = consts_dict_keys_inorder(u->u_consts); if (consts == NULL) { goto error; } - if (optimize_cfg(g, consts, c->c_const_cache)) { - goto error; - } - if (remove_unused_consts(g->g_entryblock, consts) < 0) { + cfg_builder g; + if (instr_sequence_to_cfg(&u->u_instr_sequence, &g) < 0) { goto error; } - if (add_checks_for_loads_of_uninitialized_variables(g->g_entryblock, c) < 0) { + int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); + if (optimize_code_unit(&g, consts, const_cache, code_flags, nlocals, nparams) < 0) { goto error; } - /** line numbers (TODO: move this before optimization stage) */ - if (duplicate_exits_without_lineno(g) < 0) { - goto error; - } - propagate_line_numbers(g->g_entryblock); - guarantee_lineno_for_exits(g->g_entryblock, c->u->u_firstlineno); + /** Assembly **/ - if (push_cold_blocks_to_end(g, code_flags) < 0) { + if (resolve_line_numbers(u, &g) < 0) { goto error; } - /** Assembly **/ - - int nlocalsplus = prepare_localsplus(c, g, code_flags); + int nlocalsplus = prepare_localsplus(u, &g, code_flags); if (nlocalsplus < 0) { goto error; } - int maxdepth = stackdepth(g->g_entryblock, code_flags); + int maxdepth = stackdepth(g.g_entryblock, code_flags); if (maxdepth < 0) { goto error; } /* TO DO -- For 3.12, make sure that `maxdepth <= MAX_ALLOWED_STACK_USE` */ - convert_exception_handlers_to_nops(g->g_entryblock); + convert_exception_handlers_to_nops(g.g_entryblock); /* Order of basic blocks must have been determined by now */ - if (normalize_jumps(g) < 0) { + if (normalize_jumps(&g) < 0) { goto error; } - assert(no_redundant_jumps(g)); - assert(opcode_metadata_is_sane(g)); + assert(no_redundant_jumps(&g)); + assert(opcode_metadata_is_sane(&g)); /* Can't modify the bytecode after computing jump offsets. */ - assemble_jump_offsets(g->g_entryblock); + assemble_jump_offsets(g.g_entryblock); - /* Create assembler */ - if (assemble_init(&a, c->u->u_firstlineno) < 0) { - goto error; - } - - /* Emit code. */ - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) { - if (assemble_emit(&a, &b->b_instr[j]) < 0) { - goto error; - } - } + struct assembler a; + int res = assemble_emit(&a, g.g_entryblock, u->u_firstlineno, const_cache); + if (res == SUCCESS) { + co = makecode(u, &a, const_cache, consts, maxdepth, nlocalsplus, + code_flags, filename); } + assemble_free(&a); - /* Emit location info */ - a.a_lineno = c->u->u_firstlineno; - location loc = NO_LOCATION; - int size = 0; - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) { - if (!same_location(loc, b->b_instr[j].i_loc)) { - if (assemble_emit_location(&a, loc, size)) { - goto error; - } - loc = b->b_instr[j].i_loc; - size = 0; - } - size += instr_size(&b->b_instr[j]); - } - } - if (assemble_emit_location(&a, loc, size)) { - goto error; - } + error: + Py_XDECREF(consts); + cfg_builder_fini(&g); + return co; +} - if (assemble_exception_table(&a, g->g_entryblock) < 0) { - goto error; - } - if (_PyBytes_Resize(&a.a_except_table, a.a_except_table_off) < 0) { - goto error; - } - if (merge_const_one(c->c_const_cache, &a.a_except_table) < 0) { - goto error; - } +static PyCodeObject * +assemble(struct compiler *c, int addNone) +{ + struct compiler_unit *u = c->u; + PyObject *const_cache = c->c_const_cache; + PyObject *filename = c->c_filename; - if (_PyBytes_Resize(&a.a_linetable, a.a_location_off) < 0) { - goto error; - } - if (merge_const_one(c->c_const_cache, &a.a_linetable) < 0) { - goto error; + int code_flags = compute_code_flags(c); + if (code_flags < 0) { + return NULL; } - if (_PyBytes_Resize(&a.a_bytecode, a.a_offset * sizeof(_Py_CODEUNIT)) < 0) { - goto error; - } - if (merge_const_one(c->c_const_cache, &a.a_bytecode) < 0) { - goto error; + if (add_return_at_end(c, addNone) < 0) { + return NULL; } - co = makecode(c, &a, consts, maxdepth, nlocalsplus, code_flags); - error: - Py_XDECREF(consts); - cfg_builder_fini(g); - assemble_free(&a); - return co; + return assemble_code_unit(u, const_cache, code_flags, filename); } static PyObject* @@ -9529,8 +9545,8 @@ translate_jump_labels_to_targets(basicblock *entryblock) { int max_label = -1; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - if (b->b_label > max_label) { - max_label = b->b_label; + if (b->b_label.id > max_label) { + max_label = b->b_label.id; } } size_t mapsize = sizeof(basicblock *) * (max_label + 1); @@ -9541,8 +9557,8 @@ translate_jump_labels_to_targets(basicblock *entryblock) } memset(label2block, 0, mapsize); for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - if (b->b_label >= 0) { - label2block[b->b_label] = b; + if (b->b_label.id >= 0) { + label2block[b->b_label.id] = b; } } for (basicblock *b = entryblock; b != NULL; b = b->b_next) { @@ -9554,7 +9570,7 @@ translate_jump_labels_to_targets(basicblock *entryblock) assert(lbl >= 0 && lbl <= max_label); instr->i_target = label2block[lbl]; assert(instr->i_target != NULL); - assert(instr->i_target->b_label == lbl); + assert(instr->i_target->b_label.id == lbl); } } } @@ -9828,9 +9844,15 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) if (PyErr_Occurred()) { goto error; } - int oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; + int oparg; + if (HAS_ARG(opcode)) { + oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); + if (PyErr_Occurred()) { + goto error; + } + } + else { + oparg = 0; } location loc; loc.lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 2)); @@ -9851,7 +9873,10 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) } RETURN_IF_ERROR(cfg_builder_addop(g, opcode, oparg, loc)); } - + struct cfg_instr *last = basicblock_last_instr(g->g_curblock); + if (last && !IS_TERMINATOR_OPCODE(last->i_opcode)) { + RETURN_IF_ERROR(cfg_builder_addop(g, RETURN_VALUE, 0, NO_LOCATION)); + } PyMem_Free(is_target); return SUCCESS; error: @@ -9859,6 +9884,38 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) return ERROR; } +static PyObject * +instr_sequence_to_instructions(instr_sequence *seq) { + PyObject *instructions = PyList_New(0); + if (instructions == NULL) { + return NULL; + } + for (int i = 0; i < seq->s_used; i++) { + instruction *instr = &seq->s_instrs[i]; + location loc = instr->i_loc; + int arg = HAS_TARGET(instr->i_opcode) ? + seq->s_labelmap[instr->i_oparg] : instr->i_oparg; + + PyObject *inst_tuple = Py_BuildValue( + "(iiiiii)", instr->i_opcode, arg, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + if (inst_tuple == NULL) { + goto error; + } + + int res = PyList_Append(instructions, inst_tuple); + Py_DECREF(inst_tuple); + if (res != 0) { + goto error; + } + } + return instructions; +error: + Py_XDECREF(instructions); + return NULL; +} + static PyObject * cfg_to_instructions(cfg_builder *g) { @@ -9868,7 +9925,7 @@ cfg_to_instructions(cfg_builder *g) } int lbl = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - b->b_label = lbl; + b->b_label = (jump_target_label){lbl}; lbl += b->b_iused; } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { @@ -9876,7 +9933,7 @@ cfg_to_instructions(cfg_builder *g) struct cfg_instr *instr = &b->b_instr[i]; location loc = instr->i_loc; int arg = HAS_TARGET(instr->i_opcode) ? - instr->i_target->b_label : instr->i_oparg; + instr->i_target->b_label.id : instr->i_oparg; PyObject *inst_tuple = Py_BuildValue( "(iiiiii)", instr->i_opcode, arg, @@ -9932,19 +9989,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, goto finally; } - cfg_builder g; - if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), &g) < 0) { - goto finally; - } - if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { - goto finally; - } - - res = cfg_to_instructions(&g); + res = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); finally: compiler_exit_scope(c); - cfg_builder_fini(&g); compiler_free(c); _PyArena_Free(arena); return res; @@ -9954,7 +10002,11 @@ PyObject * _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) { PyObject *res = NULL; - PyObject *const_cache = NULL; + PyObject *const_cache = PyDict_New(); + if (const_cache == NULL) { + return NULL; + } + cfg_builder g; memset(&g, 0, sizeof(cfg_builder)); if (cfg_builder_init(&g) < 0) { @@ -9963,19 +10015,13 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) if (instructions_to_cfg(instructions, &g) < 0) { goto error; } - const_cache = PyDict_New(); - if (const_cache == NULL) { - goto error; - } - if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { - goto error; - } - if (optimize_cfg(&g, consts, const_cache) < 0) { + int code_flags = 0, nlocals = 0, nparams = 0; + if (optimize_code_unit(&g, consts, const_cache, code_flags, nlocals, nparams) < 0) { goto error; } res = cfg_to_instructions(&g); error: - Py_XDECREF(const_cache); + Py_DECREF(const_cache); cfg_builder_fini(&g); return res; } From 2d370da5702f07d044f0e612df68172852b89f42 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 Mar 2023 18:35:37 +0000 Subject: [PATCH 33/50] GH-100987: Don't cache references to the names and consts array in `_PyEval_EvalFrameDefault`. (#102640) * Rename local variables, names and consts, from the interpeter loop. Will allow non-code objects in frames for better introspection of C builtins and extensions. * Remove unused dummy variables. --- Python/bytecodes.c | 44 +++++++++++++++++------------------- Python/ceval.c | 7 ------ Python/generated_cases.c.h | 46 +++++++++++++++++++------------------- 3 files changed, 44 insertions(+), 53 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 178519fcb63ced..a715bfd3289ec9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -70,8 +70,6 @@ dummy_func( unsigned int oparg, _Py_atomic_int * const eval_breaker, _PyCFrame cframe, - PyObject *names, - PyObject *consts, _Py_CODEUNIT *next_instr, PyObject **stack_pointer, PyObject *kwnames, @@ -115,7 +113,7 @@ dummy_func( } inst(LOAD_CONST, (-- value)) { - value = GETITEM(consts, oparg); + value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); } @@ -554,7 +552,7 @@ dummy_func( } inst(RETURN_CONST, (--)) { - PyObject *retval = GETITEM(consts, oparg); + PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -844,7 +842,7 @@ dummy_func( } inst(STORE_NAME, (v -- )) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { @@ -862,7 +860,7 @@ dummy_func( } inst(DELETE_NAME, (--)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { @@ -956,7 +954,7 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -967,7 +965,7 @@ dummy_func( #else (void)counter; // Unused. #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); @@ -975,21 +973,21 @@ dummy_func( } inst(DELETE_ATTR, (owner --)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); Py_DECREF(owner); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); ERROR_IF(err, error); } inst(DELETE_GLOBAL, (--)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); // Can't use ERROR_IF here. @@ -1003,7 +1001,7 @@ dummy_func( } inst(LOAD_NAME, ( -- v)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { _PyErr_Format(tstate, PyExc_SystemError, @@ -1074,7 +1072,7 @@ dummy_func( _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); @@ -1082,7 +1080,7 @@ dummy_func( STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { @@ -1436,7 +1434,7 @@ dummy_func( _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -1444,7 +1442,7 @@ dummy_func( STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg >> 1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ PyObject* meth = NULL; @@ -1525,7 +1523,7 @@ dummy_func( PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); uint16_t hint = index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -1616,7 +1614,7 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(names, oparg >> 1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -1661,7 +1659,7 @@ dummy_func( PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyObject *old_value; uint64_t new_version; @@ -1855,14 +1853,14 @@ dummy_func( } inst(IMPORT_NAME, (level, fromlist -- res)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(IMPORT_FROM, (from -- from, res)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); ERROR_IF(res == NULL, error); } @@ -2360,8 +2358,8 @@ dummy_func( inst(KW_NAMES, (--)) { assert(kwnames == NULL); - assert(oparg < PyTuple_GET_SIZE(consts)); - kwnames = GETITEM(consts, oparg); + assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); + kwnames = GETITEM(frame->f_code->co_consts, oparg); } // Cache layout: counter/1, func_version/2, min_args/1 diff --git a/Python/ceval.c b/Python/ceval.c index 92235519c993b6..7d60cf987e9c47 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -773,18 +773,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Local "register" variables. * These are cached values from the frame and code object. */ - PyObject *names; - PyObject *consts; _Py_CODEUNIT *next_instr; PyObject **stack_pointer; /* Sets the above local variables from the frame */ #define SET_LOCALS_FROM_FRAME() \ - { \ - PyCodeObject *co = frame->f_code; \ - names = co->co_names; \ - consts = co->co_consts; \ - } \ assert(_PyInterpreterFrame_LASTI(frame) >= -1); \ /* Jump back to the last instruction executed... */ \ next_instr = frame->prev_instr + 1; \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 21073cbf978ce0..631b7844f9ed88 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -50,7 +50,7 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - value = GETITEM(consts, oparg); + value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); STACK_GROW(1); stack_pointer[-1] = value; @@ -101,7 +101,7 @@ oparg = (next_instr++)->op.arg; { PyObject *value; - value = GETITEM(consts, oparg); + value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); _tmp_1 = value; } @@ -150,7 +150,7 @@ PyObject *_tmp_2; { PyObject *value; - value = GETITEM(consts, oparg); + value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); _tmp_2 = value; } @@ -744,7 +744,7 @@ } TARGET(RETURN_CONST) { - PyObject *retval = GETITEM(consts, oparg); + PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1083,7 +1083,7 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { @@ -1103,7 +1103,7 @@ } TARGET(DELETE_NAME) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { @@ -1217,7 +1217,7 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -1228,7 +1228,7 @@ #else (void)counter; // Unused. #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); @@ -1240,7 +1240,7 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); Py_DECREF(owner); if (err) goto pop_1_error; @@ -1250,7 +1250,7 @@ TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); if (err) goto pop_1_error; @@ -1259,7 +1259,7 @@ } TARGET(DELETE_GLOBAL) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); // Can't use ERROR_IF here. @@ -1275,7 +1275,7 @@ TARGET(LOAD_NAME) { PyObject *v; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { _PyErr_Format(tstate, PyExc_SystemError, @@ -1347,7 +1347,7 @@ _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); @@ -1355,7 +1355,7 @@ STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { @@ -1797,7 +1797,7 @@ _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -1805,7 +1805,7 @@ STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg >> 1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ PyObject* meth = NULL; @@ -1916,7 +1916,7 @@ PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); uint16_t hint = index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -2040,7 +2040,7 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(names, oparg >> 1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -2096,7 +2096,7 @@ PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyObject *old_value; uint64_t new_version; @@ -2349,7 +2349,7 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); Py_DECREF(fromlist); @@ -2362,7 +2362,7 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; STACK_GROW(1); @@ -2997,8 +2997,8 @@ TARGET(KW_NAMES) { assert(kwnames == NULL); - assert(oparg < PyTuple_GET_SIZE(consts)); - kwnames = GETITEM(consts, oparg); + assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); + kwnames = GETITEM(frame->f_code->co_consts, oparg); DISPATCH(); } From 85ba8a3e03707092800cbf2a29d95e0b495e3cb7 Mon Sep 17 00:00:00 2001 From: chgnrdv <52372310+chgnrdv@users.noreply.github.com> Date: Mon, 13 Mar 2023 22:25:17 +0300 Subject: [PATCH 34/50] gh-102650: Remove duplicate include directives from multiple source files (#102651) Remove duplicate include directives from multiple source files --- Mac/Tools/pythonw.c | 1 - Modules/_hashopenssl.c | 3 +-- Modules/arraymodule.c | 1 - Modules/signalmodule.c | 4 +--- Programs/_testembed.c | 1 - 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index 78813e818e7dac..9dfb77f6ff41c3 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -27,7 +27,6 @@ #include #include #include -#include #include diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 82398547f9b372..ee8c588020118c 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -32,12 +32,11 @@ /* EVP is the preferred interface to hashing in OpenSSL */ #include #include -#include +#include // FIPS_mode() /* We use the object interface to discover what hashes OpenSSL supports. */ #include #include -#include // FIPS_mode() #ifndef OPENSSL_THREADS # error "OPENSSL_THREADS is not defined, Python requires thread-safe OpenSSL" diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 114c69a033593c..798a7629257966 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -13,7 +13,6 @@ #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "structmember.h" // PyMemberDef #include // offsetof() -#include /*[clinic input] module array diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 0e472e1ee4f9dd..fdd1450050fa1b 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -13,7 +13,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_signal.h" +#include "pycore_signal.h" // Py_NSIG #ifndef MS_WINDOWS # include "posixmodule.h" @@ -28,8 +28,6 @@ # endif #endif -#include "pycore_signal.h" // Py_NSIG - #ifdef HAVE_SIGNAL_H # include #endif diff --git a/Programs/_testembed.c b/Programs/_testembed.c index a6ce3f7b200550..00717114b40286 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -9,7 +9,6 @@ #include "pycore_initconfig.h" // _PyConfig_InitCompatConfig() #include "pycore_runtime.h" // _PyRuntime #include "pycore_import.h" // _PyImport_FrozenBootstrap -#include #include #include #include // putenv() From 71e37d907905b0504c5bb7b25681adeea2157492 Mon Sep 17 00:00:00 2001 From: T Date: Tue, 14 Mar 2023 04:46:35 +0800 Subject: [PATCH 35/50] gh-98169 dataclasses.astuple support DefaultDict (#98170) Co-authored-by: Pieter Eendebak --- Lib/dataclasses.py | 25 ++++++++++++------- Lib/test/test_dataclasses.py | 21 +++++++++++++--- ...2-10-10-19-14-51.gh-issue-98169.DBWIxL.rst | 2 ++ 3 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index f4617b1dbdac66..7c3285cf440a39 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1321,15 +1321,14 @@ def _asdict_inner(obj, dict_factory): # generator (which is not true for namedtuples, handled # above). return type(obj)(_asdict_inner(v, dict_factory) for v in obj) - elif isinstance(obj, dict) and hasattr(type(obj), 'default_factory'): - # obj is a defaultdict, which has a different constructor from - # dict as it requires the default_factory as its first arg. - # https://bugs.python.org/issue35540 - result = type(obj)(getattr(obj, 'default_factory')) - for k, v in obj.items(): - result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) - return result elif isinstance(obj, dict): + if hasattr(type(obj), 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = type(obj)(getattr(obj, 'default_factory')) + for k, v in obj.items(): + result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) + return result return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) for k, v in obj.items()) @@ -1382,7 +1381,15 @@ def _astuple_inner(obj, tuple_factory): # above). return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) elif isinstance(obj, dict): - return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + obj_type = type(obj) + if hasattr(obj_type, 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = obj_type(getattr(obj, 'default_factory')) + for k, v in obj.items(): + result[_astuple_inner(k, tuple_factory)] = _astuple_inner(v, tuple_factory) + return result + return obj_type((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) for k, v in obj.items()) else: return copy.deepcopy(obj) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 76bed0c3314673..46d4e0fedad2f2 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1706,19 +1706,17 @@ class C: def test_helper_asdict_defaultdict(self): # Ensure asdict() does not throw exceptions when a # defaultdict is a member of a dataclass - @dataclass class C: mp: DefaultDict[str, List] - dd = defaultdict(list) dd["x"].append(12) c = C(mp=dd) d = asdict(c) - assert d == {"mp": {"x": [12]}} - assert d["mp"] is not c.mp # make sure defaultdict is copied + self.assertEqual(d, {"mp": {"x": [12]}}) + self.assertTrue(d["mp"] is not c.mp) # make sure defaultdict is copied def test_helper_astuple(self): # Basic tests for astuple(), it should return a new tuple. @@ -1847,6 +1845,21 @@ class C: t = astuple(c, tuple_factory=list) self.assertEqual(t, ['outer', T(1, ['inner', T(11, 12, 13)], 2)]) + def test_helper_astuple_defaultdict(self): + # Ensure astuple() does not throw exceptions when a + # defaultdict is a member of a dataclass + @dataclass + class C: + mp: DefaultDict[str, List] + + dd = defaultdict(list) + dd["x"].append(12) + c = C(mp=dd) + t = astuple(c) + + self.assertEqual(t, ({"x": [12]},)) + self.assertTrue(t[0] is not dd) # make sure defaultdict is copied + def test_dynamic_class_creation(self): cls_dict = {'__annotations__': {'x': int, 'y': int}, } diff --git a/Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst b/Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst new file mode 100644 index 00000000000000..24c3aeecc83f18 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst @@ -0,0 +1,2 @@ +Fix :func:`dataclasses.astuple` crash when :class:`collections.defaultdict` +is present in the attributes. From d97757f793ea53dda3cc6882b4a92d3e921b17c9 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 14 Mar 2023 00:42:05 +0300 Subject: [PATCH 36/50] gh-102069: Fix `__weakref__` descriptor generation for custom dataclasses (#102075) --- Lib/dataclasses.py | 3 +++ Lib/test/test_dataclasses.py | 15 +++++++++++---- ...2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 7c3285cf440a39..82b08fc017884f 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1189,6 +1189,9 @@ def _add_slots(cls, is_frozen, weakref_slot): # Remove __dict__ itself. cls_dict.pop('__dict__', None) + # Clear existing `__weakref__` descriptor, it belongs to a previous type: + cls_dict.pop('__weakref__', None) # gh-102069 + # And finally create the class. qualname = getattr(cls, '__qualname__', None) cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 46d4e0fedad2f2..46f33043c27071 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -3175,6 +3175,8 @@ class A: with self.assertRaisesRegex(TypeError, "cannot create weak reference"): weakref.ref(a) + with self.assertRaises(AttributeError): + a.__weakref__ def test_slots_weakref(self): @dataclass(slots=True, weakref_slot=True) @@ -3183,7 +3185,9 @@ class A: self.assertIn("__weakref__", A.__slots__) a = A(1) - weakref.ref(a) + a_ref = weakref.ref(a) + + self.assertIs(a.__weakref__, a_ref) def test_slots_weakref_base_str(self): class Base: @@ -3249,7 +3253,8 @@ class A(Base): self.assertIn("__weakref__", Base.__slots__) self.assertNotIn("__weakref__", A.__slots__) a = A(1) - weakref.ref(a) + a_ref = weakref.ref(a) + self.assertIs(a.__weakref__, a_ref) def test_weakref_slot_subclass_no_weakref_slot(self): @dataclass(slots=True, weakref_slot=True) @@ -3265,7 +3270,8 @@ class A(Base): self.assertIn("__weakref__", Base.__slots__) self.assertNotIn("__weakref__", A.__slots__) a = A(1) - weakref.ref(a) + a_ref = weakref.ref(a) + self.assertIs(a.__weakref__, a_ref) def test_weakref_slot_normal_base_weakref_slot(self): class Base: @@ -3280,7 +3286,8 @@ class A(Base): self.assertIn("__weakref__", Base.__slots__) self.assertNotIn("__weakref__", A.__slots__) a = A(1) - weakref.ref(a) + a_ref = weakref.ref(a) + self.assertIs(a.__weakref__, a_ref) class TestDescriptors(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst b/Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst new file mode 100644 index 00000000000000..04c87e515cca93 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst @@ -0,0 +1 @@ +Fix ``__weakref__`` descriptor generation for custom dataclasses. From 74885a08dbc7cc768d15711752957096d6a5a350 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Mar 2023 15:50:16 -0600 Subject: [PATCH 37/50] gh-101524: Fix the ChannelID tp_name (gh-102655) https://github.com/python/cpython/issues/101524 --- Modules/_xxinterpchannelsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index a0cd4a2363fb53..fead12c963da26 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -1806,7 +1806,7 @@ static PyType_Slot ChannelIDType_slots[] = { }; static PyType_Spec ChannelIDType_spec = { - .name = "_xxsubinterpreters.ChannelID", + .name = MODULE_NAME ".ChannelID", .basicsize = sizeof(channelid), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), From 959ea2f9e967042954878f7af5c85053238212a7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Mar 2023 16:01:44 -0600 Subject: [PATCH 38/50] gh-101659: Avoid Allocation for Shared Exceptions in the _xxsubinterpreters Module (gh-102659) https://github.com/python/cpython/issues/101659 --- Modules/_xxsubinterpretersmodule.c | 123 ++++++++++++----------------- 1 file changed, 49 insertions(+), 74 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 79dbe3474ba9e8..76fb87fa3a34e1 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -15,14 +15,14 @@ #define MODULE_NAME "_xxsubinterpreters" -static char * +static const char * _copy_raw_string(PyObject *strobj) { const char *str = PyUnicode_AsUTF8(strobj); if (str == NULL) { return NULL; } - char *copied = PyMem_Malloc(strlen(str)+1); + char *copied = PyMem_RawMalloc(strlen(str)+1); if (copied == NULL) { PyErr_NoMemory(); return NULL; @@ -128,7 +128,7 @@ clear_module_state(module_state *state) /* data-sharing-specific code ***********************************************/ struct _sharednsitem { - char *name; + const char *name; _PyCrossInterpreterData data; }; @@ -152,7 +152,7 @@ static void _sharednsitem_clear(struct _sharednsitem *item) { if (item->name != NULL) { - PyMem_Free(item->name); + PyMem_RawFree((void *)item->name); item->name = NULL; } (void)_release_xid_data(&item->data, 1); @@ -258,96 +258,74 @@ _sharedns_apply(_sharedns *shared, PyObject *ns) // of the exception in the calling interpreter. typedef struct _sharedexception { - char *name; - char *msg; + const char *name; + const char *msg; } _sharedexception; -static _sharedexception * -_sharedexception_new(void) -{ - _sharedexception *err = PyMem_NEW(_sharedexception, 1); - if (err == NULL) { - PyErr_NoMemory(); - return NULL; - } - err->name = NULL; - err->msg = NULL; - return err; -} +static const struct _sharedexception no_exception = { + .name = NULL, + .msg = NULL, +}; static void _sharedexception_clear(_sharedexception *exc) { if (exc->name != NULL) { - PyMem_Free(exc->name); + PyMem_RawFree((void *)exc->name); } if (exc->msg != NULL) { - PyMem_Free(exc->msg); + PyMem_RawFree((void *)exc->msg); } } -static void -_sharedexception_free(_sharedexception *exc) -{ - _sharedexception_clear(exc); - PyMem_Free(exc); -} - -static _sharedexception * -_sharedexception_bind(PyObject *exc) +static const char * +_sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) { assert(exc != NULL); - char *failure = NULL; - - _sharedexception *err = _sharedexception_new(); - if (err == NULL) { - goto finally; - } + const char *failure = NULL; - PyObject *name = PyUnicode_FromFormat("%S", Py_TYPE(exc)); - if (name == NULL) { + PyObject *nameobj = PyUnicode_FromFormat("%S", Py_TYPE(exc)); + if (nameobj == NULL) { failure = "unable to format exception type name"; - goto finally; + goto error; } - err->name = _copy_raw_string(name); - Py_DECREF(name); - if (err->name == NULL) { + sharedexc->name = _copy_raw_string(nameobj); + Py_DECREF(nameobj); + if (sharedexc->name == NULL) { if (PyErr_ExceptionMatches(PyExc_MemoryError)) { failure = "out of memory copying exception type name"; } else { failure = "unable to encode and copy exception type name"; } - goto finally; + goto error; } if (exc != NULL) { - PyObject *msg = PyUnicode_FromFormat("%S", exc); - if (msg == NULL) { + PyObject *msgobj = PyUnicode_FromFormat("%S", exc); + if (msgobj == NULL) { failure = "unable to format exception message"; - goto finally; + goto error; } - err->msg = _copy_raw_string(msg); - Py_DECREF(msg); - if (err->msg == NULL) { + sharedexc->msg = _copy_raw_string(msgobj); + Py_DECREF(msgobj); + if (sharedexc->msg == NULL) { if (PyErr_ExceptionMatches(PyExc_MemoryError)) { failure = "out of memory copying exception message"; } else { failure = "unable to encode and copy exception message"; } - goto finally; + goto error; } } -finally: - if (failure != NULL) { - PyErr_Clear(); - if (err->name != NULL) { - PyMem_Free(err->name); - err->name = NULL; - } - err->msg = failure; - } - return err; + return NULL; + +error: + assert(failure != NULL); + PyErr_Clear(); + _sharedexception_clear(sharedexc); + *sharedexc = no_exception; + return failure; } static void @@ -430,7 +408,7 @@ _ensure_not_running(PyInterpreterState *interp) static int _run_script(PyInterpreterState *interp, const char *codestr, - _sharedns *shared, _sharedexception **exc) + _sharedns *shared, _sharedexception *sharedexc) { PyObject *excval = NULL; PyObject *main_mod = _PyInterpreterState_GetMainModule(interp); @@ -462,22 +440,20 @@ _run_script(PyInterpreterState *interp, const char *codestr, Py_DECREF(result); // We throw away the result. } - *exc = NULL; + *sharedexc = no_exception; return 0; error: excval = PyErr_GetRaisedException(); - _sharedexception *sharedexc = _sharedexception_bind(excval); - Py_XDECREF(excval); - if (sharedexc == NULL) { - fprintf(stderr, "RunFailedError: script raised an uncaught exception"); + const char *failure = _sharedexception_bind(excval, sharedexc); + if (failure != NULL) { + fprintf(stderr, + "RunFailedError: script raised an uncaught exception (%s)", + failure); PyErr_Clear(); - sharedexc = NULL; - } - else { - assert(!PyErr_Occurred()); } - *exc = sharedexc; + Py_XDECREF(excval); + assert(!PyErr_Occurred()); return -1; } @@ -505,7 +481,7 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, } // Run the script. - _sharedexception *exc = NULL; + _sharedexception exc; int result = _run_script(interp, codestr, shared, &exc); // Switch back. @@ -514,10 +490,9 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, } // Propagate any exception out to the caller. - if (exc != NULL) { + if (exc.name != NULL) { assert(state != NULL); - _sharedexception_apply(exc, state->RunFailedError); - _sharedexception_free(exc); + _sharedexception_apply(&exc, state->RunFailedError); } else if (result != 0) { // We were unable to allocate a shared exception. From 392f2ad3cbf2e1f24656fe0410a9b65882257582 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 13 Mar 2023 15:08:45 -0700 Subject: [PATCH 39/50] gh-98831: Use DECREF_INPUTS() more (#102409) --- Python/bytecodes.c | 80 +++++++++++++++----------------------- Python/generated_cases.c.h | 15 ++++--- 2 files changed, 38 insertions(+), 57 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a715bfd3289ec9..74582ecbbda103 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -384,8 +384,7 @@ dummy_func( if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - Py_DECREF(dict); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(true, error); } Py_INCREF(res); // Do this before DECREF'ing dict, sub @@ -420,7 +419,7 @@ dummy_func( inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { int err = PySet_Add(set, v); - Py_DECREF(v); + DECREF_INPUTS(); ERROR_IF(err, error); PREDICT(JUMP_BACKWARD); } @@ -899,7 +898,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); + DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -910,7 +909,7 @@ dummy_func( STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); + DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { @@ -921,7 +920,7 @@ dummy_func( for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { @@ -932,14 +931,14 @@ dummy_func( for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + DECREF_INPUTS(); } inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); + DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -967,22 +966,21 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); + DECREF_INPUTS(); ERROR_IF(err, error); } inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - Py_DECREF(owner); + DECREF_INPUTS(); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); + DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1249,9 +1247,7 @@ dummy_func( inst(BUILD_STRING, (pieces[oparg] -- str)) { str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int i = 0; i < oparg; i++) { - Py_DECREF(pieces[i]); - } + DECREF_INPUTS(); ERROR_IF(str == NULL, error); } @@ -1314,10 +1310,7 @@ dummy_func( if (map == NULL) goto error; - for (int i = 0; i < oparg; i++) { - Py_DECREF(values[i*2]); - Py_DECREF(values[i*2+1]); - } + DECREF_INPUTS(); ERROR_IF(map == NULL, error); } @@ -1373,10 +1366,7 @@ dummy_func( map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - Py_DECREF(keys); - for (int i = 0; i < oparg; i++) { - Py_DECREF(values[i]); - } + DECREF_INPUTS(); ERROR_IF(map == NULL, error); } @@ -1464,7 +1454,7 @@ dummy_func( NULL | meth | arg1 | ... | argN */ - Py_DECREF(owner); + DECREF_INPUTS(); ERROR_IF(meth == NULL, error); res2 = NULL; res = meth; @@ -1473,7 +1463,7 @@ dummy_func( else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - Py_DECREF(owner); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } } @@ -1492,7 +1482,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - Py_DECREF(owner); + DECREF_INPUTS(); } inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { @@ -1509,7 +1499,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - Py_DECREF(owner); + DECREF_INPUTS(); } inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { @@ -1540,7 +1530,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - Py_DECREF(owner); + DECREF_INPUTS(); } inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { @@ -1554,7 +1544,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - Py_DECREF(owner); + DECREF_INPUTS(); } inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, cls -- res2 if (oparg & 1), res)) { @@ -1570,7 +1560,7 @@ dummy_func( res = descr; assert(res != NULL); Py_INCREF(res); - Py_DECREF(cls); + DECREF_INPUTS(); } inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused if (oparg & 1), unused)) { @@ -1707,8 +1697,7 @@ dummy_func( STAT_INC(COMPARE_OP, deferred); assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - Py_DECREF(left); - Py_DECREF(right); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -1734,8 +1723,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); - Py_DECREF(left); - Py_DECREF(right); + DECREF_INPUTS(); ERROR_IF(cond == NULL, error); assert(next_instr[1].op.code == POP_JUMP_IF_FALSE || next_instr[1].op.code == POP_JUMP_IF_TRUE); @@ -1885,7 +1873,7 @@ dummy_func( } else { int err = PyObject_IsTrue(cond); - Py_DECREF(cond); + DECREF_INPUTS(); if (err == 0) { JUMPBY(oparg); } @@ -1905,7 +1893,7 @@ dummy_func( } else { int err = PyObject_IsTrue(cond); - Py_DECREF(cond); + DECREF_INPUTS(); if (err > 0) { JUMPBY(oparg); } @@ -1917,7 +1905,7 @@ dummy_func( inst(POP_JUMP_IF_NOT_NONE, (value -- )) { if (!Py_IsNone(value)) { - Py_DECREF(value); + DECREF_INPUTS(); JUMPBY(oparg); } else { @@ -1931,7 +1919,7 @@ dummy_func( JUMPBY(oparg); } else { - Py_DECREF(value); + DECREF_INPUTS(); } } @@ -2065,7 +2053,7 @@ dummy_func( if (iter == NULL) { goto error; } - Py_DECREF(iterable); + DECREF_INPUTS(); } PREDICT(LOAD_CONST); } @@ -2940,9 +2928,7 @@ dummy_func( assert(PyTuple_CheckExact(callargs)); result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); + DECREF_INPUTS(); assert(PEEK(3 + (oparg & 1)) == NULL); ERROR_IF(result == NULL, error); @@ -3009,9 +2995,7 @@ dummy_func( inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) { slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); + DECREF_INPUTS(); ERROR_IF(slice == NULL, error); } @@ -3057,8 +3041,7 @@ dummy_func( } else { /* Actually call format(). */ result = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_XDECREF(fmt_spec); + DECREF_INPUTS(); ERROR_IF(result == NULL, error); } } @@ -3084,8 +3067,7 @@ dummy_func( assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 631b7844f9ed88..b1dbb58d956367 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1568,8 +1568,8 @@ PyObject **pieces = (stack_pointer - oparg); PyObject *str; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int i = 0; i < oparg; i++) { - Py_DECREF(pieces[i]); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); } if (str == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1665,9 +1665,8 @@ if (map == NULL) goto error; - for (int i = 0; i < oparg; i++) { - Py_DECREF(values[i*2]); - Py_DECREF(values[i*2+1]); + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); } if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } STACK_SHRINK(oparg*2); @@ -1732,10 +1731,10 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - Py_DECREF(keys); - for (int i = 0; i < oparg; i++) { - Py_DECREF(values[i]); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); } + Py_DECREF(keys); if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } STACK_SHRINK(oparg); stack_pointer[-1] = map; From 61479d46848bc7a7f9b571b0b09c4a4b4436d839 Mon Sep 17 00:00:00 2001 From: Blind4Basics <32236948+Blind4Basics@users.noreply.github.com> Date: Mon, 13 Mar 2023 23:35:37 +0100 Subject: [PATCH 40/50] gh-102627: Replace address pointing toward malicious web page (#102630) * Replace known bad address pointing toward a malicious web page. Co-authored-by: C.A.M. Gerlach Co-authored-by: Hugo van Kemenade --- Doc/library/concurrent.futures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index c543c849585b7f..09c9fc4e6e227a 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -202,7 +202,7 @@ ThreadPoolExecutor Example 'http://www.cnn.com/', 'http://europe.wsj.com/', 'http://www.bbc.co.uk/', - 'http://some-made-up-domain.com/'] + 'http://nonexistant-subdomain.python.org/'] # Retrieve a single page and report the URL and contents def load_url(url, timeout): From 457e4d1a516c2b83edeff2f255f4cd6e7b114feb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 13 Mar 2023 20:06:43 -0500 Subject: [PATCH 41/50] GH-102670: Use sumprod() to simplify, speed up, and improve accuracy of statistics functions (GH-102649) --- Lib/statistics.py | 26 ++++++++++--------- Lib/test/test_statistics.py | 12 ++++++++- ...-03-13-18-27-00.gh-issue-102670.GyoThv.rst | 2 ++ 3 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst diff --git a/Lib/statistics.py b/Lib/statistics.py index 7d5d750193a5ab..6bd214bbfe2ff5 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1036,7 +1036,7 @@ def covariance(x, y, /): raise StatisticsError('covariance requires at least two data points') xbar = fsum(x) / n ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + sxy = sumprod((xi - xbar for xi in x), (yi - ybar for yi in y)) return sxy / (n - 1) @@ -1074,11 +1074,14 @@ def correlation(x, y, /, *, method='linear'): start = (n - 1) / -2 # Center rankings around zero x = _rank(x, start=start) y = _rank(y, start=start) - xbar = fsum(x) / n - ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((d := xi - xbar) * d for xi in x) - syy = fsum((d := yi - ybar) * d for yi in y) + else: + xbar = fsum(x) / n + ybar = fsum(y) / n + x = [xi - xbar for xi in x] + y = [yi - ybar for yi in y] + sxy = sumprod(x, y) + sxx = sumprod(x, x) + syy = sumprod(y, y) try: return sxy / sqrt(sxx * syy) except ZeroDivisionError: @@ -1131,14 +1134,13 @@ def linear_regression(x, y, /, *, proportional=False): raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') - if proportional: - sxy = fsum(xi * yi for xi, yi in zip(x, y)) - sxx = fsum(xi * xi for xi in x) - else: + if not proportional: xbar = fsum(x) / n ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((d := xi - xbar) * d for xi in x) + x = [xi - xbar for xi in x] # List because used three times below + y = (yi - ybar for yi in y) # Generator because only used once below + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float + sxx = sumprod(x, x) try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 31a3cb6b53a6f2..f0fa6454b1f91a 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1,4 +1,4 @@ -"""Test suite for statistics module, including helper NumericTestCase and +x = """Test suite for statistics module, including helper NumericTestCase and approx_equal function. """ @@ -2610,6 +2610,16 @@ def test_proportional(self): self.assertAlmostEqual(slope, 20 + 1/150) self.assertEqual(intercept, 0.0) + def test_float_output(self): + x = [Fraction(2, 3), Fraction(3, 4)] + y = [Fraction(4, 5), Fraction(5, 6)] + slope, intercept = statistics.linear_regression(x, y) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) + slope, intercept = statistics.linear_regression(x, y, proportional=True) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) + class TestNormalDist: # General note on precision: The pdf(), cdf(), and overlap() methods diff --git a/Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst b/Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst new file mode 100644 index 00000000000000..3de09f86754f3e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst @@ -0,0 +1,2 @@ +Optimized fmean(), correlation(), covariance(), and linear_regression() +using the new math.sumprod() function. From cbd3fbfb6e5c1cc96bbeb99483a580f165b01671 Mon Sep 17 00:00:00 2001 From: Jacob Bower <1978924+jbower-fb@users.noreply.github.com> Date: Mon, 13 Mar 2023 20:35:54 -0500 Subject: [PATCH 42/50] gh-102013: Add PyUnstable_GC_VisitObjects (#102014) --- Doc/c-api/gcsupport.rst | 33 +++++++++ Include/objimpl.h | 19 +++++ ...-02-18-00-55-14.gh-issue-102013.83mrtI.rst | 1 + Modules/_testcapimodule.c | 69 +++++++++++++++++++ Modules/gcmodule.c | 24 +++++++ 5 files changed, 146 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 8c90d1e8991c10..cb5d64a50487fe 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -228,3 +228,36 @@ garbage collection runs. Returns the current state, 0 for disabled and 1 for enabled. .. versionadded:: 3.10 + + +Querying Garbage Collector State +-------------------------------- + +The C-API provides the following interface for querying information about +the garbage collector. + +.. c:function:: void PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) + + Run supplied *callback* on all live GC-capable objects. *arg* is passed through to + all invocations of *callback*. + + .. warning:: + If new objects are (de)allocated by the callback it is undefined if they + will be visited. + + Garbage collection is disabled during operation. Explicitly running a collection + in the callback may lead to undefined behaviour e.g. visiting the same objects + multiple times or not at all. + + .. versionadded:: 3.12 + +.. c:type:: int (*gcvisitobjects_t)(PyObject *object, void *arg) + + Type of the visitor function to be passed to :c:func:`PyUnstable_GC_VisitObjects`. + *arg* is the same as the *arg* passed to ``PyUnstable_GC_VisitObjects``. + Return ``0`` to continue iteration, return ``1`` to stop iteration. Other return + values are reserved for now so behavior on returning anything else is undefined. + + .. versionadded:: 3.12 + + diff --git a/Include/objimpl.h b/Include/objimpl.h index dde8df34835328..ef871c5ea93ebe 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -157,6 +157,25 @@ PyAPI_FUNC(int) PyGC_Enable(void); PyAPI_FUNC(int) PyGC_Disable(void); PyAPI_FUNC(int) PyGC_IsEnabled(void); + +#if !defined(Py_LIMITED_API) +/* Visit all live GC-capable objects, similar to gc.get_objects(None). The + * supplied callback is called on every such object with the void* arg set + * to the supplied arg. Returning 0 from the callback ends iteration, returning + * 1 allows iteration to continue. Returning any other value may result in + * undefined behaviour. + * + * If new objects are (de)allocated by the callback it is undefined if they + * will be visited. + + * Garbage collection is disabled during operation. Explicitly running a + * collection in the callback may lead to undefined behaviour e.g. visiting the + * same objects multiple times or not at all. + */ +typedef int (*gcvisitobjects_t)(PyObject*, void*); +PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); +#endif + /* Test if a type has a GC head */ #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) diff --git a/Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst b/Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst new file mode 100644 index 00000000000000..0350237ebc7390 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst @@ -0,0 +1 @@ +Add a new (unstable) C-API function for iterating over GC'able objects using a callback: ``PyUnstable_VisitObjects``. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ea67017a1ba3b1..f45d0312e94411 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3310,6 +3310,73 @@ function_set_kw_defaults(PyObject *self, PyObject *args) Py_RETURN_NONE; } +struct gc_visit_state_basic { + PyObject *target; + int found; +}; + +static int +gc_visit_callback_basic(PyObject *obj, void *arg) +{ + struct gc_visit_state_basic *state = (struct gc_visit_state_basic *)arg; + if (obj == state->target) { + state->found = 1; + return 0; + } + return 1; +} + +static PyObject * +test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + PyObject *obj; + struct gc_visit_state_basic state; + + obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + state.target = obj; + state.found = 0; + + PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); + Py_DECREF(obj); + if (!state.found) { + PyErr_SetString( + PyExc_AssertionError, + "test_gc_visit_objects_basic: Didn't find live list"); + return NULL; + } + Py_RETURN_NONE; +} + +static int +gc_visit_callback_exit_early(PyObject *obj, void *arg) + { + int *visited_i = (int *)arg; + (*visited_i)++; + if (*visited_i == 2) { + return 0; + } + return 1; +} + +static PyObject * +test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + int visited_i = 0; + PyUnstable_GC_VisitObjects(gc_visit_callback_exit_early, &visited_i); + if (visited_i != 2) { + PyErr_SetString( + PyExc_AssertionError, + "test_gc_visit_objects_exit_early: did not exit when expected"); + } + Py_RETURN_NONE; +} + + static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyMethodDef TestMethods[] = { @@ -3452,6 +3519,8 @@ static PyMethodDef TestMethods[] = { {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, + {"test_gc_visit_objects_basic", test_gc_visit_objects_basic, METH_NOARGS, NULL}, + {"test_gc_visit_objects_exit_early", test_gc_visit_objects_exit_early, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 5879c5e11fe14a..4eaa5490b6134c 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2401,3 +2401,27 @@ PyObject_GC_IsFinalized(PyObject *obj) } return 0; } + +void +PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +{ + size_t i; + GCState *gcstate = get_gc_state(); + int origenstate = gcstate->enabled; + gcstate->enabled = 0; + for (i = 0; i < NUM_GENERATIONS; i++) { + PyGC_Head *gc_list, *gc; + gc_list = GEN_HEAD(gcstate, i); + for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + Py_INCREF(op); + int res = callback(op, arg); + Py_DECREF(op); + if (!res) { + goto done; + } + } + } +done: + gcstate->enabled = origenstate; +} From 7bdb331b67c4aee513e01794ba7dd2c3be43d3ca Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Tue, 14 Mar 2023 11:07:59 +0900 Subject: [PATCH 43/50] doc: Remove a duplicate 'versionchanged' in library/asyncio-task (gh-102677) --- Doc/library/asyncio-task.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 9b984243282268..5f1449e1b599ef 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -814,9 +814,6 @@ Waiting Primitives Raises :exc:`TimeoutError` if the timeout occurs before all Futures are done. - .. versionchanged:: 3.10 - Removed the *loop* parameter. - Example:: for coro in as_completed(aws): From 3d872a74c8c16d4a077c2223f678b1f8f7e0e988 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 14 Mar 2023 14:22:21 +0530 Subject: [PATCH 44/50] GH-100227: cleanup initialization of global interned dict (#102682) --- Objects/unicodeobject.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 2d50f9c340f2f3..b9fb53147b9b51 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14533,6 +14533,15 @@ _PyUnicode_InitGlobalObjects(PyInterpreterState *interp) return _PyStatus_OK(); } + // Initialize the global interned dict + PyObject *interned = PyDict_New(); + if (interned == NULL) { + PyErr_Clear(); + return _PyStatus_ERR("failed to create interned dict"); + } + + set_interned_dict(interned); + /* Intern statically allocated string identifiers and deepfreeze strings. * This must be done before any module initialization so that statically * allocated string identifiers are used instead of heap allocated strings. @@ -14600,14 +14609,7 @@ PyUnicode_InternInPlace(PyObject **p) } PyObject *interned = get_interned_dict(); - if (interned == NULL) { - interned = PyDict_New(); - if (interned == NULL) { - PyErr_Clear(); /* Don't leave an exception */ - return; - } - set_interned_dict(interned); - } + assert(interned != NULL); PyObject *t = PyDict_SetDefault(interned, s, s); if (t == NULL) { From 88c262c086077377b40dfae5e46f597e28ffe3c9 Mon Sep 17 00:00:00 2001 From: T Date: Tue, 14 Mar 2023 17:23:52 +0800 Subject: [PATCH 45/50] gh-100315: clarification to `__slots__` docs. (#102621) refer to tp_itemsize in discussion on "variable-length" built-in types --- Doc/reference/datamodel.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index f447bbb1216d52..1865d09fcaa127 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1944,8 +1944,10 @@ Notes on using *__slots__* descriptor directly from the base class). This renders the meaning of the program undefined. In the future, a check may be added to prevent this. -* Nonempty *__slots__* does not work for classes derived from "variable-length" - built-in types such as :class:`int`, :class:`bytes` and :class:`tuple`. +* :exc:`TypeError` will be raised if nonempty *__slots__* are defined for a + class derived from a + :c:member:`"variable-length" built-in type ` such as + :class:`int`, :class:`bytes`, and :class:`tuple`. * Any non-string :term:`iterable` may be assigned to *__slots__*. From a703f743dbf2675948e59c44fa9d7112f7825100 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 14 Mar 2023 15:21:38 +0530 Subject: [PATCH 46/50] GH-94851: check unicode consistency of static strings in debug mode (#102684) --- .../internal/pycore_unicodeobject_generated.h | 664 ++++++++++++++++++ Tools/build/generate_global_objects.py | 1 + 2 files changed, 665 insertions(+) diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 52af37a8e60aa8..fea9b6dbb1a75f 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -13,1332 +13,1996 @@ static inline void _PyUnicode_InitStaticStrings(void) { PyObject *string; string = &_Py_ID(CANCELLED); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(FINISHED); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(False); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(JSONDecodeError); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(PENDING); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(Py_Repr); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(TextIOWrapper); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(True); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(WarningMessage); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_WindowsConsoleIO); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__IOBase_closed); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__abc_tpflags__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__abs__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__abstractmethods__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__add__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__aenter__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__aexit__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__aiter__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__all__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__and__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__anext__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__annotations__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__args__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__asyncio_running_event_loop__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__await__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__bases__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__bool__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__build_class__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__builtins__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__bytes__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__call__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__cantrace__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__class__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__class_getitem__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__classcell__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__complex__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__contains__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__copy__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ctypes_from_outparam__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__del__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__delattr__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__delete__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__delitem__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__dict__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__dictoffset__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__dir__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__divmod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__doc__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__enter__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__eq__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__exit__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__file__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__float__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__floordiv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__format__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__fspath__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ge__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__get__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getattr__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getattribute__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getinitargs__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getitem__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getnewargs__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getnewargs_ex__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getstate__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__gt__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__hash__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__iadd__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__iand__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ifloordiv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ilshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__imatmul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__imod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__import__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__imul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__index__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__init__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__init_subclass__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__instancecheck__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__int__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__invert__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ior__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ipow__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__irshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__isabstractmethod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__isub__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__iter__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__itruediv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ixor__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__le__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__len__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__length_hint__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__lltrace__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__loader__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__lshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__lt__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__main__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__matmul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__missing__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__mod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__module__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__mro_entries__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__mul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__name__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ne__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__neg__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__new__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__newobj__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__newobj_ex__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__next__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__notes__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__or__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__orig_class__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__origin__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__package__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__parameters__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__path__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__pos__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__pow__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__prepare__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__qualname__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__radd__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rand__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rdivmod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reduce__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reduce_ex__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__repr__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reversed__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rfloordiv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rlshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rmatmul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rmod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rmul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ror__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__round__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rpow__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rrshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rsub__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rtruediv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rxor__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__set__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__set_name__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__setattr__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__setitem__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__setstate__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__sizeof__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__slotnames__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__slots__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__spec__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__str__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__sub__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__subclasscheck__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__subclasshook__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__truediv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__trunc__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__typing_is_unpacked_typevartuple__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__typing_prepare_subst__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__typing_subst__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__typing_unpacked_tuple_args__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__warningregistry__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__weaklistoffset__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__weakref__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__xor__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_abc_impl); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_abstract_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_active); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_annotation); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_anonymous_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_argtypes_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_as_parameter_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_asyncio_future_blocking); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_blksize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_bootstrap); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_check_retval_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_dealloc_warn); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_feature_version); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_fields_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_finalizing); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_find_and_load); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_fix_up_module); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_flags_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_get_sourcefile); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_handle_fromlist); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_initializing); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_io); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_is_text_encoding); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_length_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_limbo); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_lock_unlock_module); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_loop); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_needs_com_addref_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_pack_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_restype_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_showwarnmsg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_shutdown); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_slotnames); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_strptime_datetime); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_swappedbytes_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_type_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_uninitialized_submodules); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_warn_unawaited_coroutine); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_xoptions); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(a); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(abs_tol); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(access); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(add); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(add_done_callback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(after_in_child); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(after_in_parent); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(aggregate_class); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(append); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(argdefs); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(arguments); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(argv); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(as_integer_ratio); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ast); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(attribute); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(authorizer_callback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(autocommit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(b); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(backtick); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(base); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(before); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(big); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(binary_form); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(block); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffer); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffer_callback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffer_size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffering); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffers); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(bufsize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(builtins); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(byteorder); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(bytes); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(bytes_per_sep); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(c); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(c_call); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(c_exception); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(c_return); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cached_statements); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cadata); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cafile); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(call); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(call_exception_handler); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(call_soon); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cancel); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(capath); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(category); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cb_type); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(certfile); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(check_same_thread); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(clear); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(close); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(closed); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(closefd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(closure); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_argcount); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_cellvars); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_code); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_consts); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_exceptiontable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_filename); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_firstlineno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_flags); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_freevars); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_kwonlyargcount); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_linetable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_name); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_names); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_nlocals); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_posonlyargcount); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_qualname); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_stacksize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_varnames); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(code); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(command); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(comment_factory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(consts); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(context); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cookie); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(copy); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(copyreg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(coro); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(count); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cwd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(d); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(data); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(database); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(decode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(decoder); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(default); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(defaultaction); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(delete); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(depth); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(detect_types); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(deterministic); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(device); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dict); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dictcomp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(difference_update); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(digest); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(digest_size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(digestmod); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dir_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(discard); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dispatch_table); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(displayhook); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dklen); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(doc); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dont_inherit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dst); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dst_dir_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(duration); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(e); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(effective_ids); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(element_factory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(encode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(encoding); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(end); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(end_lineno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(end_offset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(endpos); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(env); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(errors); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(event); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(eventmask); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(exc_type); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(exc_value); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(excepthook); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(exception); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(exp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(extend); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(facility); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(factory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(false); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(family); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fanout); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fd2); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fdel); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fget); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(file); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(file_actions); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(filename); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fileno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(filepath); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fillvalue); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(filters); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(final); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(find_class); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fix_imports); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(flags); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(flush); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(follow_symlinks); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(format); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(frequency); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(from_param); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fromlist); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fromtimestamp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fromutc); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(func); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(future); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(generation); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(genexpr); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get_debug); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get_event_loop); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get_loop); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get_source); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(getattr); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(getstate); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(gid); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(globals); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(groupindex); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(groups); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(handle); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(hash_name); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(header); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(headers); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(hi); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(hook); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(id); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ident); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ignore); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(imag); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(importlib); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(in_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(incoming); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(indexgroup); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(inf); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(inheritable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(initial); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(initial_bytes); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(initial_value); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(initval); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(inner_size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(input); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(insert_comments); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(insert_pis); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(instructions); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(intern); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(intersection); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(isatty); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(isinstance); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(isoformat); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(isolation_level); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(istext); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(item); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(items); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(iter); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(iterable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(iterations); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(join); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(jump); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(keepends); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(key); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(keyfile); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(keys); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(kind); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(kw); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(kw1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(kw2); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(lambda); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last_node); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last_traceback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last_type); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last_value); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(latin1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(leaf_size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(len); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(length); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(level); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(limit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(line); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(line_buffering); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(lineno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(listcomp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(little); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(lo); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(locale); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(locals); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(logoption); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(loop); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mapping); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(match); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(max_length); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxdigits); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxevents); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxmem); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxsplit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxvalue); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(memLevel); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(memlimit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(message); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(metaclass); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(method); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mod); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(module); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(module_globals); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(modules); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mro); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(msg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mycmp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n_arg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n_fields); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n_sequence_fields); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n_unnamed_fields); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(name); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(name_from); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(namespace_separator); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(namespaces); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(narg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ndigits); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(new_limit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(newline); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(newlines); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(next); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(node_depth); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(node_offset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ns); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(nstype); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(nt); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(null); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(number); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(obj); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(object); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(offset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(offset_dst); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(offset_src); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(on_type_read); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(onceregistry); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(only_keys); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(oparg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(opcode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(open); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(opener); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(operation); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(optimize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(options); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(order); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(out_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(outgoing); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(overlapped); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(owner); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(p); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pages); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(parent); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(password); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(path); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pattern); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(peek); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(persistent_id); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(persistent_load); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(person); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pi_factory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pid); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(policy); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pos); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pos1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pos2); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(posix); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(print_file_and_line); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(priority); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(progress); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(progress_handler); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(proto); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(protocol); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ps1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ps2); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(query); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(quotetabs); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(r); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(raw); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(read); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(read1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readall); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readinto); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readinto1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readline); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readonly); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(real); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reducer_override); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(registry); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(rel_tol); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reload); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(repl); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(replace); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reserved); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(resetids); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(return); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reverse); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reversed); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(s); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(salt); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sched_priority); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(scheduler); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(seek); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(seekable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(selectors); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(self); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(send); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sep); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sequence); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(server_hostname); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(server_side); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(session); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setcomp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setpgroup); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setsid); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setsigdef); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setsigmask); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setstate); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(shape); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(show_cmd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(signed); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sizehint); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(skip_file_prefixes); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sleep); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sock); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sort); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sound); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(source); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(source_traceback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(src); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(src_dir_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(stacklevel); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(start); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(statement); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(status); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(stderr); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(stdin); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(stdout); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(step); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(store_name); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(strategy); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(strftime); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(strict); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(strict_mode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(string); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sub_key); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(symmetric_difference_update); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tabsize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tag); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(target); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(target_is_directory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(task); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tb_frame); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tb_lasti); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tb_lineno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tb_next); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tell); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(template); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(term); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(text); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(threading); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(throw); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(timeout); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(times); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(timetuple); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(top); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(trace_callback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(traceback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(trailers); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(translate); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(true); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(truncate); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(twice); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(txt); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(type); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tz); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tzname); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(uid); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(unlink); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(unraisablehook); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(uri); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(usedforsecurity); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(value); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(values); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(version); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(volume); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(warnings); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(warnoptions); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(wbits); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(week); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(weekday); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(which); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(who); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(withdata); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(writable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(write); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(write_through); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(x); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(year); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(zdict); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); } /* End auto-generated code */ diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index 9ceae89878cda8..1f53f02d41ef39 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -360,6 +360,7 @@ def generate_static_strings_initializer(identifiers, strings): # This use of _Py_ID() is ignored by iter_global_strings() # since iter_files() ignores .h files. printer.write(f'string = &_Py_ID({i});') + printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') printer.write(f'PyUnicode_InternInPlace(&string);') # XXX What about "strings"? printer.write(END) From 1ff81c0cb67215694f084e51c4d35ae53b9f5cf9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Mar 2023 10:05:54 -0600 Subject: [PATCH 47/50] gh-81057: Add a CI Check for New Unsupported C Global Variables (gh-102506) This will keep us from adding new unsupported (i.e. non-const) C global variables, which would break interpreter isolation. FYI, historically it is very uncommon for new global variables to get added. Furthermore, it is rare for new code to break the c-analyzer. So the check should almost always pass unnoticed. Note that I've removed test_check_c_globals. A test wasn't a great fit conceptually and was super slow on debug builds. A CI check is a better fit. This also resolves gh-100237. https://github.com/python/cpython/issues/81057 --- .github/workflows/build.yml | 3 + Lib/test/test_check_c_globals.py | 34 ---------- Makefile.pre.in | 6 ++ .../c_parser/preprocessor/common.py | 6 +- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 23 ++++--- Tools/c-analyzer/cpython/__main__.py | 62 ++++++++++++++++--- Tools/c-analyzer/cpython/_parser.py | 9 ++- Tools/c-analyzer/cpython/ignored.tsv | 1 + 8 files changed, 90 insertions(+), 54 deletions(-) delete mode 100644 Lib/test/test_check_c_globals.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2241b0b8aa409e..4e5328282f1224 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -111,6 +111,9 @@ jobs: run: make smelly - name: Check limited ABI symbols run: make check-limited-abi + - name: Check for unsupported C global variables + if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME + run: make check-c-globals build_win32: name: 'Windows (x86)' diff --git a/Lib/test/test_check_c_globals.py b/Lib/test/test_check_c_globals.py deleted file mode 100644 index 670be52422f799..00000000000000 --- a/Lib/test/test_check_c_globals.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -import test.test_tools -from test.support.warnings_helper import save_restore_warnings_filters - - -# TODO: gh-92584: c-analyzer uses distutils which was removed in Python 3.12 -raise unittest.SkipTest("distutils has been removed in Python 3.12") - - -test.test_tools.skip_if_missing('c-analyzer') -with test.test_tools.imports_under_tool('c-analyzer'): - # gh-95349: Save/restore warnings filters to leave them unchanged. - # Importing the c-analyzer imports docutils which imports pkg_resources - # which adds a warnings filter. - with save_restore_warnings_filters(): - from cpython.__main__ import main - - -class ActualChecks(unittest.TestCase): - - # XXX Also run the check in "make check". - #@unittest.expectedFailure - # Failing on one of the buildbots (see https://bugs.python.org/issue36876). - @unittest.skip('activate this once all the globals have been resolved') - def test_check_c_globals(self): - try: - main('check', {}) - except NotImplementedError: - raise unittest.SkipTest('not supported on this host') - - -if __name__ == '__main__': - # Test needs to be a package, so we can do relative imports. - unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 1a1853bf3d7871..59762165c10036 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2560,6 +2560,12 @@ distclean: clobber docclean smelly: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/smelly.py +# Check if any unsupported C global variables have been added. +check-c-globals: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/c-analyzer/check-c-globals.py \ + --format summary \ + --traceback + # Find files with funny names funny: find $(SUBDIRS) $(SUBDIRSTOO) \ diff --git a/Tools/c-analyzer/c_parser/preprocessor/common.py b/Tools/c-analyzer/c_parser/preprocessor/common.py index 4291a066337545..dbe1edeef38527 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/common.py +++ b/Tools/c-analyzer/c_parser/preprocessor/common.py @@ -115,15 +115,15 @@ def converted_error(tool, argv, filename): def convert_error(tool, argv, filename, stderr, rc): error = (stderr.splitlines()[0], rc) if (_expected := is_os_mismatch(filename, stderr)): - logger.debug(stderr.strip()) + logger.info(stderr.strip()) raise OSMismatchError(filename, _expected, argv, error, tool) elif (_missing := is_missing_dep(stderr)): - logger.debug(stderr.strip()) + logger.info(stderr.strip()) raise MissingDependenciesError(filename, (_missing,), argv, error, tool) elif '#error' in stderr: # XXX Ignore incompatible files. error = (stderr.splitlines()[1], rc) - logger.debug(stderr.strip()) + logger.info(stderr.strip()) raise ErrorDirectiveError(filename, argv, error, tool) else: # Try one more time, with stderr written to the terminal. diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 7ef1a8afc3b135..24c1b0e9b9d48c 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -6,6 +6,11 @@ TOOL = 'gcc' +META_FILES = { + '', + '', +} + # https://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html # flags: # 1 start of a new file @@ -75,11 +80,15 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): # The first line is special. # The next two lines are consistent. - for expected in [ - f'# 1 "{reqfile}"', - '# 1 ""', - '# 1 ""', - ]: + firstlines = [ + f'# 0 "{reqfile}"', + '# 0 ""', + '# 0 ""', + ] + if text.startswith('# 1 '): + # Some preprocessors emit a lineno of 1 for line-less entries. + firstlines = [l.replace('# 0 ', '# 1 ') for l in firstlines] + for expected in firstlines: line = next(lines) if line != expected: raise NotImplementedError((line, expected)) @@ -121,7 +130,7 @@ def _iter_top_include_lines(lines, topfile, cwd, # _parse_marker_line() that the preprocessor reported lno as 1. lno = 1 for line in lines: - if line == '# 1 "" 2': + if line == '# 0 "" 2' or line == '# 1 "" 2': # We're done with this top-level include. return @@ -174,8 +183,8 @@ def _parse_marker_line(line, reqfile=None): return None, None, None lno, origfile, flags = m.groups() lno = int(lno) + assert origfile not in META_FILES, (line,) assert lno > 0, (line, lno) - assert origfile not in ('', ''), (line,) flags = set(int(f) for f in flags.split()) if flags else () if 1 in flags: diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 2b9e4233b95ac4..fe7a16726f45a9 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -1,5 +1,6 @@ import logging import sys +import textwrap from c_common.fsutil import expand_filenames, iter_files_by_suffix from c_common.scriptutil import ( @@ -26,6 +27,39 @@ logger = logging.getLogger(__name__) +CHECK_EXPLANATION = textwrap.dedent(''' + ------------------------- + + Non-constant global variables are generally not supported + in the CPython repo. We use a tool to analyze the C code + and report if any unsupported globals are found. The tool + may be run manually with: + + ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE] + + Occasionally the tool is unable to parse updated code. + If this happens then add the file to the "EXCLUDED" list + in Tools/c-analyzer/cpython/_parser.py and create a new + issue for fixing the tool (and CC ericsnowcurrently + on the issue). + + If the tool reports an unsupported global variable and + it is actually const (and thus supported) then first try + fixing the declaration appropriately in the code. If that + doesn't work then add the variable to the "should be const" + section of Tools/c-analyzer/cpython/ignored.tsv. + + If the tool otherwise reports an unsupported global variable + then first try to make it non-global, possibly adding to + PyInterpreterState (for core code) or module state (for + extension modules). In an emergency, you can add the + variable to Tools/c-analyzer/cpython/globals-to-fix.tsv + to get CI passing, but doing so should be avoided. If + this course it taken, be sure to create an issue for + eliminating the global (and CC ericsnowcurrently). +''') + + def _resolve_filenames(filenames): if filenames: resolved = (_files.resolve_filename(f) for f in filenames) @@ -123,14 +157,26 @@ def _cli_check(parser, **kwargs): def cmd_check(filenames=None, **kwargs): filenames = _resolve_filenames(filenames) kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print) - c_analyzer.cmd_check( - filenames, - relroot=REPO_ROOT, - _analyze=_analyzer.analyze, - _CHECKS=CHECKS, - file_maxsizes=_parser.MAX_SIZES, - **kwargs - ) + try: + c_analyzer.cmd_check( + filenames, + relroot=REPO_ROOT, + _analyze=_analyzer.analyze, + _CHECKS=CHECKS, + file_maxsizes=_parser.MAX_SIZES, + **kwargs + ) + except SystemExit as exc: + num_failed = exc.args[0] if getattr(exc, 'args', None) else None + if isinstance(num_failed, int): + if num_failed > 0: + sys.stderr.flush() + print(CHECK_EXPLANATION, flush=True) + raise # re-raise + except Exception: + sys.stderr.flush() + print(CHECK_EXPLANATION, flush=True) + raise # re-raise def cmd_analyze(filenames=None, **kwargs): diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index e7764165d36c4c..6da4fbbf4224f1 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -106,15 +106,20 @@ def clean_lines(text): * ./Include/internal Modules/_decimal/**/*.c Modules/_decimal/libmpdec +Modules/_elementtree.c Modules/expat Modules/_hacl/*.c Modules/_hacl/include Modules/_hacl/*.h Modules/_hacl/include -Modules/_tkinter.c /usr/include/tcl8.6 Modules/md5module.c Modules/_hacl/include Modules/sha1module.c Modules/_hacl/include Modules/sha2module.c Modules/_hacl/include -Modules/tkappinit.c /usr/include/tcl Objects/stringlib/*.h Objects +# possible system-installed headers, just in case +Modules/_tkinter.c /usr/include/tcl8.6 +Modules/_uuidmodule.c /usr/include/uuid +Modules/nismodule.c /usr/include/tirpc +Modules/tkappinit.c /usr/include/tcl + # @end=tsv@ ''')[1:] diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 700ddf2851839e..048112dd992555 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -228,6 +228,7 @@ Modules/_xxinterpchannelsmodule.c - _channelid_end_recv - Modules/_xxinterpchannelsmodule.c - _channelid_end_send - Modules/_zoneinfo.c - DAYS_BEFORE_MONTH - Modules/_zoneinfo.c - DAYS_IN_MONTH - +Modules/_xxsubinterpretersmodule.c - no_exception - Modules/arraymodule.c - descriptors - Modules/arraymodule.c - emptybuf - Modules/cjkcodecs/_codecs_cn.c - _mapping_list - From 80abd62647b2a36947a11a6a8e395061be6f0c61 Mon Sep 17 00:00:00 2001 From: Paul Watson Date: Tue, 14 Mar 2023 13:40:12 -0500 Subject: [PATCH 48/50] gh-102354: change python3 to python in docs examples (#102696) --- Doc/library/__main__.rst | 2 +- Doc/library/importlib.metadata.rst | 2 +- Doc/library/pdb.rst | 4 ++-- Doc/tutorial/venv.rst | 2 +- Doc/using/venv-create.inc | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index 6a2a7a7317f711..761c88710f9891 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -259,7 +259,7 @@ one mentioned below are preferred. See :mod:`venv` for an example of a package with a minimal ``__main__.py`` in the standard library. It doesn't contain a ``if __name__ == '__main__'`` - block. You can invoke it with ``python3 -m venv [directory]``. + block. You can invoke it with ``python -m venv [directory]``. See :mod:`runpy` for more details on the :option:`-m` flag to the interpreter executable. diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 988d1a317f5960..6e084101995e25 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -73,7 +73,7 @@ something into it: .. code-block:: shell-session - $ python3 -m venv example + $ python -m venv example $ source example/bin/activate (example) $ python -m pip install wheel diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 4ae12a5d03a78d..21c6ca8622dceb 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -58,7 +58,7 @@ of the debugger is:: :file:`pdb.py` can also be invoked as a script to debug other scripts. For example:: - python3 -m pdb myscript.py + python -m pdb myscript.py When invoked as a script, pdb will automatically enter post-mortem debugging if the program being debugged exits abnormally. After post-mortem debugging (or @@ -72,7 +72,7 @@ useful than quitting the debugger upon program's exit. .. versionadded:: 3.7 :file:`pdb.py` now accepts a ``-m`` option that execute modules similar to the way - ``python3 -m`` does. As with a script, the debugger will pause execution just + ``python -m`` does. As with a script, the debugger will pause execution just before the first line of the module. diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index 05f0e6bbcc1b04..d1bba098d7d23b 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -44,7 +44,7 @@ whichever version you want. To create a virtual environment, decide upon a directory where you want to place it, and run the :mod:`venv` module as a script with the directory path:: - python3 -m venv tutorial-env + python -m venv tutorial-env This will create the ``tutorial-env`` directory if it doesn't exist, and also create directories inside it containing a copy of the Python diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index d535b254f05698..43ee6b7807d57e 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -1,7 +1,7 @@ Creation of :ref:`virtual environments ` is done by executing the command ``venv``:: - python3 -m venv /path/to/new/virtual/environment + python -m venv /path/to/new/virtual/environment Running this command creates the target directory (creating any parent directories that don't exist already) and places a ``pyvenv.cfg`` file in it From cdb21ba74d933e262bc1696b6ce78b50cb5a4443 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Mar 2023 14:01:35 -0600 Subject: [PATCH 49/50] gh-102660: Handle m_copy Specially for the sys and builtins Modules (gh-102661) It doesn't make sense to use multi-phase init for these modules. Using a per-interpreter "m_copy" (instead of PyModuleDef.m_base.m_copy) makes this work okay. (This came up while working on gh-101660.) Note that we might instead end up disallowing re-load for sys/builtins since they are so special. https://github.com/python/cpython/issues/102660 --- Include/internal/pycore_interp.h | 1 + Python/bltinmodule.c | 3 +++ Python/import.c | 35 ++++++++++++++++++++++++++++---- Python/pystate.c | 1 + Python/sysmodule.c | 8 ++++++++ 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 9efed0a1cf90c2..84303318d21811 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -111,6 +111,7 @@ struct _is { PyObject *dict; /* Stores per-interpreter state */ + PyObject *sysdict_copy; PyObject *builtins_copy; // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 12ca0ba6c4873c..a8a34620b9bcdf 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3098,6 +3098,9 @@ _PyBuiltin_Init(PyInterpreterState *interp) } Py_DECREF(debug); + /* m_copy of Py_None means it is copied some other way. */ + builtinsmodule.m_base.m_copy = Py_NewRef(Py_None); + return mod; #undef ADD_TO_ALL #undef SETBUILTIN diff --git a/Python/import.c b/Python/import.c index 1bf4199e125aa5..612fee243bd9e7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -978,6 +978,16 @@ _PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) return 0; } +static inline int +match_mod_name(PyObject *actual, const char *expected) +{ + if (PyUnicode_CompareWithASCIIString(actual, expected) == 0) { + return 1; + } + assert(!PyErr_Occurred()); + return 0; +} + static int fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) { @@ -1001,7 +1011,8 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) // when the extension module doesn't support sub-interpreters. // XXX Why special-case the main interpreter? if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { - if (def->m_size == -1) { + /* m_copy of Py_None means it is copied some other way. */ + if (def->m_size == -1 && def->m_base.m_copy != Py_None) { if (def->m_base.m_copy) { /* Somebody already imported the module, likely under a different name. @@ -1055,18 +1066,34 @@ import_find_extension(PyThreadState *tstate, PyObject *name, PyObject *modules = MODULES(tstate->interp); if (def->m_size == -1) { + PyObject *m_copy = def->m_base.m_copy; /* Module does not support repeated initialization */ - if (def->m_base.m_copy == NULL) + if (m_copy == NULL) { return NULL; + } + else if (m_copy == Py_None) { + if (match_mod_name(name, "sys")) { + m_copy = tstate->interp->sysdict_copy; + } + else if (match_mod_name(name, "builtins")) { + m_copy = tstate->interp->builtins_copy; + } + else { + _PyErr_SetString(tstate, PyExc_ImportError, "missing m_copy"); + return NULL; + } + } + /* m_copy of Py_None means it is copied some other way. */ mod = import_add_module(tstate, name); - if (mod == NULL) + if (mod == NULL) { return NULL; + } mdict = PyModule_GetDict(mod); if (mdict == NULL) { Py_DECREF(mod); return NULL; } - if (PyDict_Update(mdict, def->m_base.m_copy)) { + if (PyDict_Update(mdict, m_copy)) { Py_DECREF(mod); return NULL; } diff --git a/Python/pystate.c b/Python/pystate.c index 28606e4f32f71c..3a2966c54a4c3b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -805,6 +805,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) assert(interp->imports.importlib == NULL); assert(interp->imports.import_func == NULL); + Py_CLEAR(interp->sysdict_copy); Py_CLEAR(interp->builtins_copy); Py_CLEAR(interp->dict); #ifdef HAVE_FORK diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 764fb70bae6c38..fc0550266bf1af 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3425,12 +3425,20 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return _PyStatus_ERR("failed to create a module object"); } + /* m_copy of Py_None means it is copied some other way. */ + sysmodule.m_base.m_copy = Py_NewRef(Py_None); + PyObject *sysdict = PyModule_GetDict(sysmod); if (sysdict == NULL) { goto error; } interp->sysdict = Py_NewRef(sysdict); + interp->sysdict_copy = PyDict_Copy(sysdict); + if (interp->sysdict_copy == NULL) { + goto error; + } + if (PyDict_SetItemString(sysdict, "modules", modules) < 0) { goto error; } From d77c48740f5cd17597693bd0d27e32db725fa3a0 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Wed, 15 Mar 2023 05:20:14 +0900 Subject: [PATCH 50/50] gh-102674: Remove _specialization_stats from Lib/opcode.py (#102685) It's not use except in a test, so move it there instead. --- Lib/opcode.py | 8 -------- Lib/test/test__opcode.py | 3 +-- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 23529d87a09ef9..d4a2d554cf67de 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -377,14 +377,6 @@ def pseudo_op(name, op, real_ops): _specialized_instructions = [ opcode for family in _specializations.values() for opcode in family ] -_specialization_stats = [ - "success", - "failure", - "hit", - "deferred", - "miss", - "deopt", -] _cache_format = { "LOAD_GLOBAL": { diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index db831069c7aeb8..fb4ab15f7041ed 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -69,8 +69,7 @@ def test_stack_effect_jump(self): class SpecializationStatsTests(unittest.TestCase): def test_specialization_stats(self): - stat_names = opcode._specialization_stats - + stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] specialized_opcodes = [ op.lower() for op in opcode._specializations