From e69215f76680b36e009ad4ab94c0653fc7445d21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 20:57:41 +0200 Subject: [PATCH 1/2] gh-106320: Remove private _PyDict function Move private _PyDict functions to the internal C API (pycore_dict.h): * _PyDictView_Intersect() * _PyDictView_New() * _PyDict_ContainsId() * _PyDict_DelItemId() * _PyDict_DelItem_KnownHash() * _PyDict_GetItemIdWithError() * _PyDict_GetItem_KnownHash() * _PyDict_HasSplitTable() * _PyDict_NewPresized() * _PyDict_Next() * _PyDict_Pop() * _PyDict_SetItemId() * _PyDict_SetItem_KnownHash() * _PyDict_SizeOf() No longer export most of these functions. Move also the _PyDictViewObject to the internal C API. Move dict_getitem_knownhash() function from _testcapi to the _testinternalcapi extension. Update test_capi.test_dict for this move. --- Include/cpython/dictobject.h | 30 ----------------------- Include/internal/pycore_dict.h | 45 ++++++++++++++++++++++++++++++++-- Lib/test/test_dict.py | 4 +-- Modules/_asynciomodule.c | 1 + Modules/_collectionsmodule.c | 1 + Modules/_ctypes/stgdict.c | 1 + Modules/_sre/sre.c | 1 + Modules/_testcapimodule.c | 21 ---------------- Modules/_testinternalcapi.c | 22 +++++++++++++++++ Modules/_threadmodule.c | 4 +-- Python/import.c | 1 + 11 files changed, 74 insertions(+), 57 deletions(-) diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index f44880991f4136..b05ca3ef453816 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -32,19 +32,8 @@ typedef struct { PyDictValues *ma_values; } PyDictObject; -PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); -PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, - PyObject *item, Py_hash_t hash); -PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); - -PyAPI_FUNC(int) _PyDict_Next( - PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { @@ -56,26 +45,7 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key); -PyAPI_FUNC(int) _PyDict_ContainsId(PyObject *, _Py_Identifier *); - -PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); -PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); -PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *, PyObject *, PyObject *); -#define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) - -PyAPI_FUNC(int) _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); - -PyAPI_FUNC(int) _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); - -/* _PyDictView */ - -typedef struct { - PyObject_HEAD - PyDictObject *dv_dict; -} _PyDictViewObject; -PyAPI_FUNC(PyObject *) _PyDictView_New(PyObject *, PyTypeObject *); -PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); /* Dictionary watchers */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index df7bc7e58f6c97..b20da98bba9bfd 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -14,15 +14,44 @@ extern "C" { // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); -extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); - extern int _PyDict_DelItemIf(PyObject *mp, PyObject *key, int (*predicate)(PyObject *value)); +// "KnownHash" variants +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, + Py_hash_t hash); +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, + PyObject *item, Py_hash_t hash); +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, + Py_hash_t hash); +extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); + +// "Id" variants +extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, + _Py_Identifier *key); +extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); +extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); +extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); + +extern int _PyDict_Next( + PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); + extern int _PyDict_HasOnlyStringKeys(PyObject *mp); extern void _PyDict_MaybeUntrack(PyObject *mp); +extern PyObject* _PyDict_NewPresized(Py_ssize_t minused); + +// Export for '_ctypes' shared extension +PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); + +extern PyObject* _PyDict_Pop(PyObject *, PyObject *, PyObject *); + +#define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) + /* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, the first occurrence of a key wins, if override is 1, the last occurrence of a key wins, if override is 2, a KeyError with conflicting key as @@ -32,6 +61,18 @@ extern int _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); extern void _PyDict_DebugMallocStats(FILE *out); + +/* _PyDictView */ + +typedef struct { + PyObject_HEAD + PyDictObject *dv_dict; +} _PyDictViewObject; + +extern PyObject* _PyDictView_New(PyObject *, PyTypeObject *); +extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other); + + /* runtime lifecycle */ extern void _PyDict_Fini(PyInterpreterState *interp); diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index fbc6ce8282de3c..eab64b4f9106c1 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1582,8 +1582,8 @@ class CAPITest(unittest.TestCase): # Test _PyDict_GetItem_KnownHash() @support.cpython_only def test_getitem_knownhash(self): - _testcapi = import_helper.import_module('_testcapi') - dict_getitem_knownhash = _testcapi.dict_getitem_knownhash + _testinternalcapi = import_helper.import_module('_testinternalcapi') + dict_getitem_knownhash = _testinternalcapi.dict_getitem_knownhash d = {'x': 1, 'y': 2, 'z': 3} self.assertEqual(dict_getitem_knownhash(d, 'x', hash('x')), 1) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 6266dc8e3555f8..c66a8623413f4b 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3,6 +3,7 @@ #endif #include "Python.h" +#include "pycore_dict.h" // _PyDict_GetItem_KnownHash() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index f2915f83b9d968..c8cd53de5e2262 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dict.h" // _PyDict_GetItem_KnownHash() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_typeobject.h" // _PyType_GetModuleState() diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 3348ebd6593d2f..54291a71667e38 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -9,6 +9,7 @@ #endif #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dict.h" // _PyDict_SizeOf() #include #ifdef MS_WIN32 # include diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index c4e43a0db0f5d3..3872c3663c7294 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -39,6 +39,7 @@ static const char copyright[] = " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; #include "Python.h" +#include "pycore_dict.h" // _PyDict_Next() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 6a1eba3a0de217..20b96320f4c339 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -271,26 +271,6 @@ test_dict_iteration(PyObject* self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -static PyObject* -dict_getitem_knownhash(PyObject *self, PyObject *args) -{ - PyObject *mp, *key, *result; - Py_ssize_t hash; - - if (!PyArg_ParseTuple(args, "OOn:dict_getitem_knownhash", - &mp, &key, &hash)) { - return NULL; - } - - result = _PyDict_GetItem_KnownHash(mp, key, (Py_hash_t)hash); - if (result == NULL && !PyErr_Occurred()) { - _PyErr_SetKeyError(key); - return NULL; - } - - return Py_XNewRef(result); -} - /* Issue #4701: Check that PyObject_Hash implicitly calls * PyType_Ready if it hasn't already been called */ @@ -3353,7 +3333,6 @@ static PyMethodDef TestMethods[] = { {"test_sizeof_c_types", test_sizeof_c_types, METH_NOARGS}, {"test_list_api", test_list_api, METH_NOARGS}, {"test_dict_iteration", test_dict_iteration, METH_NOARGS}, - {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, {"test_lazy_hash_inheritance", test_lazy_hash_inheritance,METH_NOARGS}, {"test_xincref_doesnt_leak",test_xincref_doesnt_leak, METH_NOARGS}, {"test_incref_doesnt_leak", test_incref_doesnt_leak, METH_NOARGS}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 9b45d59987dca6..3e3dfeca037c7f 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1572,6 +1572,27 @@ new_hamt(PyObject *self, PyObject *args) } +static PyObject* +dict_getitem_knownhash(PyObject *self, PyObject *args) +{ + PyObject *mp, *key, *result; + Py_ssize_t hash; + + if (!PyArg_ParseTuple(args, "OOn:dict_getitem_knownhash", + &mp, &key, &hash)) { + return NULL; + } + + result = _PyDict_GetItem_KnownHash(mp, key, (Py_hash_t)hash); + if (result == NULL && !PyErr_Occurred()) { + _PyErr_SetKeyError(key); + return NULL; + } + + return Py_XNewRef(result); +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1637,6 +1658,7 @@ static PyMethodDef module_functions[] = { {"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS}, {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, + {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 984747c44d7a57..229abfba82d1ad 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -4,14 +4,14 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_MakePendingCalls() +#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_interp.h" // _PyInterpreterState.threads.count #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_weakref.h" // _PyWeakref_GET_REF() -#include // offsetof() - +#include // offsetof() #ifdef HAVE_SIGNAL_H # include // SIGINT #endif diff --git a/Python/import.c b/Python/import.c index 5681deb3c4fc73..48090d05240188 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,6 +1,7 @@ /* Module definition and import implementation */ #include "Python.h" +#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() From 814f867e4b54f699a4fef58d56383d4ac41bfcc1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 21:23:27 +0200 Subject: [PATCH 2/2] Export _PyDict_Pop() for _socket extension --- Include/internal/pycore_dict.h | 3 ++- Modules/socketmodule.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index b20da98bba9bfd..e79fb207cf75b1 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -48,7 +48,8 @@ extern PyObject* _PyDict_NewPresized(Py_ssize_t minused); // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); -extern PyObject* _PyDict_Pop(PyObject *, PyObject *, PyObject *); +// Export for '_socket' shared extension (Windows remove_unusable_flags()) +PyAPI_FUNC(PyObject*) _PyDict_Pop(PyObject *, PyObject *, PyObject *); #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 392a56d5ab7d3e..d1ac1ffec844a0 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -106,6 +106,7 @@ Local naming conventions: #endif #include "Python.h" +#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState