Skip to content

Commit

Permalink
Add Python 3.11 support (#3231)
Browse files Browse the repository at this point in the history
Even though Python 3.11 is at the development stage now, there are several breaking changes that we want to address. Namely,

* Replace "Py_TYPE(obj) = type" with:
  "Py_SET_TYPE(obj, type)"
* Replace "Py_REFCNT(Py_None) += n" with:
  "Py_SET_REFCNT(Py_None, Py_REFCNT(Py_None) + n)"
* Add pythoncapi_compat.h to get Py_SET_TYPE() and Py_SET_REFCNT() on
  Python 3.9 and older. File copied from:
  https://github.com/pythoncapi/pythoncapi_compat

On Python 3.10, Py_REFCNT() can no longer be used to set a reference
count:

* https://docs.python.org/dev/c-api/structures.html#c.Py_REFCNT
* https://docs.python.org/dev/whatsnew/3.10.html#id2

On Python 3.11, Py_TYPE() can no longer be used to set an object
type:

* https://docs.python.org/dev/c-api/structures.html#c.Py_TYPE
* https://docs.python.org/dev/whatsnew/3.11.html#id2
  • Loading branch information
vstinner authored Feb 7, 2022
1 parent 50a9a64 commit 02f1311
Show file tree
Hide file tree
Showing 5 changed files with 405 additions and 8 deletions.
9 changes: 5 additions & 4 deletions src/core/buffer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <cstring> // std::strerror, std::memcpy
#include <mutex> // std::mutex, std::lock_guard
#include "buffer.h"
#include "lib/pythoncapi_compat.h" // Py_SET_REFCNT()
#include "mmm.h" // MemoryMapWorker, MemoryMapManager
#include "python/pybuffer.h" // py::buffer
#include "utils/alloc.h" // dt::malloc, dt::realloc
Expand Down Expand Up @@ -235,7 +236,7 @@ class BufferImpl
PyObject** elements = static_cast<PyObject**>(data_);
for (size_t i = 0; i < n; ++i) {
XAssert(elements[i] != nullptr);
XAssert(elements[i]->ob_refcnt > 0);
XAssert(Py_REFCNT(elements[i]) > 0);
}
}
}
Expand Down Expand Up @@ -929,7 +930,7 @@ class Mmap_BufferImpl : public BufferImpl, MemoryMapWorker {
for (size_t i = 0; i < n; ++i) {
data[i] = Py_None;
}
Py_None->ob_refcnt += n;
Py_SET_REFCNT(Py_None, Py_REFCNT(Py_None) + n);
}
impl_->contains_pyobjects_ = true;
return *this;
Expand All @@ -954,7 +955,7 @@ class Mmap_BufferImpl : public BufferImpl, MemoryMapWorker {
if (n_new > n_old) {
PyObject** data = static_cast<PyObject**>(xptr());
for (size_t i = n_old; i < n_new; ++i) data[i] = Py_None;
Py_None->ob_refcnt += n_new - n_old;
Py_SET_REFCNT(Py_None, Py_REFCNT(Py_None) + n_new - n_old);
}
} else {
impl_->resize(newsize);
Expand Down Expand Up @@ -1018,7 +1019,7 @@ class Mmap_BufferImpl : public BufferImpl, MemoryMapWorker {
size_t i = 0;
for (; i < n_copy; ++i) Py_INCREF(newdata[i]);
for (; i < n_new; ++i) newdata[i] = Py_None;
Py_None->ob_refcnt += n_new - n_copy;
Py_SET_REFCNT(Py_None, Py_REFCNT(Py_None) + n_new - n_copy);
}
impl_->release(); // noexcept
}
Expand Down
5 changes: 3 additions & 2 deletions src/core/column/const_na.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//------------------------------------------------------------------------------
#include "lib/pythoncapi_compat.h" // Py_SET_REFCNT()
#include "column/const.h"
#include "column/sentinel_fw.h"
#include "column/sentinel_str.h"
Expand Down Expand Up @@ -84,7 +85,7 @@ static Column _fw_col(size_t nrows, SType stype) {
});

if (std::is_same<T, PyObject*>::value) {
Py_None->ob_refcnt += nrows;
Py_SET_REFCNT(Py_None, Py_REFCNT(Py_None) + nrows);
buf.set_pyobjects(/* clear_data= */ false);
}
return Column(new ColClass(nrows, stype, std::move(buf)));
Expand All @@ -101,7 +102,7 @@ static Column _special_col(size_t nrows) {
});

if (std::is_same<T, PyObject*>::value) {
Py_None->ob_refcnt += nrows;
Py_SET_REFCNT(Py_None, Py_REFCNT(Py_None) + nrows);
buf.set_pyobjects(/* clear_data= */ false);
}
return Column(new ColClass(nrows, std::move(buf)));
Expand Down
Loading

0 comments on commit 02f1311

Please sign in to comment.