Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make wrapped C++ functions pickleable #30099

Merged
merged 39 commits into from
Feb 20, 2024
Merged
Show file tree
Hide file tree
Changes from 38 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
e4b1218
Add `test_*_repr()` to test behavior with different Python versions.
Feb 8, 2024
9d613e9
Adjust expected repr for PyPy
Feb 8, 2024
ebc1d23
Adjust another expected repr for PyPy
Feb 8, 2024
fdc9126
Try again: undo mistaken adjustment for PyPy
Feb 8, 2024
9f1d6a0
Give up on test_pytypes test_capsule_with_name_repr (not sufficiently…
Feb 8, 2024
04e16a2
`_wrapped_simple_callable` proof of concept
Feb 9, 2024
c53a281
Add `module_::def_as_native()`
Feb 9, 2024
424ace3
Resolve PyPy `TypeError: cannot create weak reference to builtin_func…
Feb 9, 2024
f9fd9d0
Replace `PyCapsule` with `function_record_PyObject`.
Feb 10, 2024
0723dda
function_record_PyTypeObject: Replace C++20 designated initializers w…
Feb 12, 2024
694ebbd
Introduce `PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID` and use along with…
Feb 12, 2024
16c19d4
Move `std::once_flag` out of `inline` function (in hopes that that fi…
Feb 12, 2024
ca31ae2
`tp_vectorcall` was introduced only with Python 3.8
Feb 12, 2024
b5d3bf5
clang-tidy auto-fixes
Feb 12, 2024
9e263c5
Disable `-Wmissing-field-initializers`. Guard `PyType_Ready(&function…
Feb 12, 2024
6b45525
Give up on the `std::call_once` idea, for Python 3.6 compatibility (i…
Feb 12, 2024
2c876d6
Add `__reduce_ex__` to `function_record_PyTypeObject`. Add `_pybind11…
Feb 12, 2024
3a5730a
Move `function_record_PyTypeObject_PyType_Ready()` call in `get_inter…
Feb 12, 2024
fe1b774
gcc 4.8.5 and 7.5.0 reject `PYBIND11_WARNING_DISABLE_GCC("-Wmissing-f…
Feb 12, 2024
1679fe0
`function_record_PyTypeObject_PyType_Ready()`, `get_pybind11_detail_f…
Feb 13, 2024
7359b5a
gcc 4.8.5 and 7.5.0 reject `PYBIND11_WARNING_DISABLE_GCC("-Wcast-func…
Feb 13, 2024
dc50802
Python 3.6, 3.7: Skip `get_pybind11_detail_function_record_pickle_hel…
Feb 13, 2024
fe87588
New version of `_function_record_pickle_helper`, using `collections.n…
Feb 13, 2024
554d529
Explicit `str(tup_obj[1])` to fix 🐍 3 • centos:7 • x64 segfault
Feb 13, 2024
84f9c35
Factor out detail/function_record_pyobject.h
Feb 13, 2024
808cbbf
Use PYBIND11_NAMESPACE_BEGIN/END for function_record_PyTypeObject_met…
Feb 13, 2024
61973df
Factor out function_record_PyTypeObject_methods::tp_name_impl, mainly…
Feb 13, 2024
9e22ddd
Simplify implementation of UNEXPECTED CALL functions.
Feb 13, 2024
d701abb
Factor out `detail::get_scope_module()`
Feb 13, 2024
d148a74
IncludeCleaner fixes (Google toolchain).
Feb 13, 2024
0d210a5
Comment out unreachable code (to resolve MSVC Werrors).
Feb 14, 2024
0e7bb10
Use built-in `eval()` instead of `function_record_pickle_helper()`
Feb 14, 2024
1607d54
Remove `function_record_pickle_helper()`
Feb 14, 2024
8b77252
Mark `internals::function_record_capsule_name` as OBSOLETE.
Feb 14, 2024
0731b71
Add comment pointing to google/pywrapcc#30099
Feb 14, 2024
02b31b1
Archive experimental code from video meet with @rainwoodman 2024-02-15
Feb 16, 2024
bcdfc69
Add a pickle roundtrip test starting with `m.simple_callable.__self__…
Feb 16, 2024
5793372
PyPy does not have `m.simple_callable.__self__`
Feb 16, 2024
1c24995
Change "UNUSUAL" comment as suggested by @rainwoodman (only very slig…
Feb 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ set(PYBIND11_HEADERS
include/pybind11/detail/cross_extension_shared_state.h
include/pybind11/detail/descr.h
include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
include/pybind11/detail/function_record_pyobject.h
include/pybind11/detail/init.h
include/pybind11/detail/internals.h
include/pybind11/detail/native_enum_data.h
Expand Down
1 change: 1 addition & 0 deletions include/pybind11/attr.h
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ struct argument_record {

/// Internal data structure which holds metadata about a bound function (signature, overloads,
/// etc.)
#define PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID "v1" // PLEASE UPDATE if the struct is changed.
struct function_record {
function_record()
: is_constructor(false), is_new_style_constructor(false), is_stateless(false),
Expand Down
205 changes: 205 additions & 0 deletions include/pybind11/detail/function_record_pyobject.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
// Copyright (c) 2024 The Pybind Development Team.
// All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

// For background see the description of PR google/pywrapcc#30099.

#pragma once

#include "../attr.h"
#include "../pytypes.h"
#include "common.h"

#include <cstring>

PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
PYBIND11_NAMESPACE_BEGIN(detail)

struct function_record_PyObject {
PyObject_HEAD
function_record *cpp_func_rec;
};

PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods)

PyObject *tp_new_impl(PyTypeObject *type, PyObject *args, PyObject *kwds);
PyObject *tp_alloc_impl(PyTypeObject *type, Py_ssize_t nitems);
int tp_init_impl(PyObject *self, PyObject *args, PyObject *kwds);
void tp_dealloc_impl(PyObject *self);
void tp_free_impl(void *self);

static PyObject *reduce_ex_impl(PyObject *self, PyObject *, PyObject *);

PYBIND11_WARNING_PUSH
#if defined(__GNUC__) && __GNUC__ >= 8
PYBIND11_WARNING_DISABLE_GCC("-Wcast-function-type")
#endif
static PyMethodDef tp_methods_impl[]
= {{"__reduce_ex__", (PyCFunction) reduce_ex_impl, METH_VARARGS | METH_KEYWORDS, nullptr},
{nullptr, nullptr, 0, nullptr}};
PYBIND11_WARNING_POP

// Note that this name is versioned.
constexpr char tp_name_impl[]
= "pybind11_detail_function_record_" PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID
"_" PYBIND11_PLATFORM_ABI_ID_V4;

PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods)

// Designated initializers are a C++20 feature:
// https://en.cppreference.com/w/cpp/language/aggregate_initialization#Designated_initializers
// MSVC rejects them unless /std:c++20 is used (error code C7555).
PYBIND11_WARNING_PUSH
PYBIND11_WARNING_DISABLE_CLANG("-Wmissing-field-initializers")
#if defined(__GNUC__) && __GNUC__ >= 8
PYBIND11_WARNING_DISABLE_GCC("-Wmissing-field-initializers")
#endif
static PyTypeObject function_record_PyTypeObject = {
PyVarObject_HEAD_INIT(nullptr, 0)
/* const char *tp_name */ function_record_PyTypeObject_methods::tp_name_impl,
/* Py_ssize_t tp_basicsize */ sizeof(function_record_PyObject),
/* Py_ssize_t tp_itemsize */ 0,
/* destructor tp_dealloc */ function_record_PyTypeObject_methods::tp_dealloc_impl,
/* Py_ssize_t tp_vectorcall_offset */ 0,
/* getattrfunc tp_getattr */ nullptr,
/* setattrfunc tp_setattr */ nullptr,
/* PyAsyncMethods *tp_as_async */ nullptr,
/* reprfunc tp_repr */ nullptr,
/* PyNumberMethods *tp_as_number */ nullptr,
/* PySequenceMethods *tp_as_sequence */ nullptr,
/* PyMappingMethods *tp_as_mapping */ nullptr,
/* hashfunc tp_hash */ nullptr,
/* ternaryfunc tp_call */ nullptr,
/* reprfunc tp_str */ nullptr,
/* getattrofunc tp_getattro */ nullptr,
/* setattrofunc tp_setattro */ nullptr,
/* PyBufferProcs *tp_as_buffer */ nullptr,
/* unsigned long tp_flags */ Py_TPFLAGS_DEFAULT,
/* const char *tp_doc */ nullptr,
/* traverseproc tp_traverse */ nullptr,
/* inquiry tp_clear */ nullptr,
/* richcmpfunc tp_richcompare */ nullptr,
/* Py_ssize_t tp_weaklistoffset */ 0,
/* getiterfunc tp_iter */ nullptr,
/* iternextfunc tp_iternext */ nullptr,
/* struct PyMethodDef *tp_methods */ function_record_PyTypeObject_methods::tp_methods_impl,
/* struct PyMemberDef *tp_members */ nullptr,
/* struct PyGetSetDef *tp_getset */ nullptr,
/* struct _typeobject *tp_base */ nullptr,
/* PyObject *tp_dict */ nullptr,
/* descrgetfunc tp_descr_get */ nullptr,
/* descrsetfunc tp_descr_set */ nullptr,
/* Py_ssize_t tp_dictoffset */ 0,
/* initproc tp_init */ function_record_PyTypeObject_methods::tp_init_impl,
/* allocfunc tp_alloc */ function_record_PyTypeObject_methods::tp_alloc_impl,
/* newfunc tp_new */ function_record_PyTypeObject_methods::tp_new_impl,
/* freefunc tp_free */ function_record_PyTypeObject_methods::tp_free_impl,
/* inquiry tp_is_gc */ nullptr,
/* PyObject *tp_bases */ nullptr,
/* PyObject *tp_mro */ nullptr,
/* PyObject *tp_cache */ nullptr,
/* PyObject *tp_subclasses */ nullptr,
/* PyObject *tp_weaklist */ nullptr,
/* destructor tp_del */ nullptr,
/* unsigned int tp_version_tag */ 0,
/* destructor tp_finalize */ nullptr,
#if PY_VERSION_HEX >= 0x03080000
/* vectorcallfunc tp_vectorcall */ nullptr,
#endif
};
PYBIND11_WARNING_POP

static bool function_record_PyTypeObject_PyType_Ready_first_call = true;

inline void function_record_PyTypeObject_PyType_Ready() {
if (function_record_PyTypeObject_PyType_Ready_first_call) {
if (PyType_Ready(&function_record_PyTypeObject) < 0) {
throw error_already_set();
}
function_record_PyTypeObject_PyType_Ready_first_call = false;
}
}

inline bool is_function_record_PyObject(PyObject *obj) {
if (PyType_Check(obj) != 0) {
return false;
}
PyTypeObject *obj_type = Py_TYPE(obj);
// Fast path (pointer comparison).
if (obj_type == &function_record_PyTypeObject) {
return true;
}
// This works across extension modules. Note that tp_name is versioned.
if (strcmp(obj_type->tp_name, function_record_PyTypeObject.tp_name) == 0) {
return true;
}
return false;
}

inline function_record *function_record_ptr_from_PyObject(PyObject *obj) {
if (is_function_record_PyObject(obj)) {
return ((detail::function_record_PyObject *) obj)->cpp_func_rec;
}
return nullptr;
}

inline object function_record_PyObject_New() {
auto *py_func_rec = PyObject_New(function_record_PyObject, &function_record_PyTypeObject);
if (py_func_rec == nullptr) {
throw error_already_set();
}
py_func_rec->cpp_func_rec = nullptr; // For clarity/purity. Redundant in practice.
return reinterpret_steal<object>((PyObject *) py_func_rec);
}

PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods)

// Guard against accidents & oversights, in particular when porting to future Python versions.
inline PyObject *tp_new_impl(PyTypeObject *, PyObject *, PyObject *) {
pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_new_impl");
// return nullptr; // Unreachable.
}

inline PyObject *tp_alloc_impl(PyTypeObject *, Py_ssize_t) {
pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_alloc_impl");
// return nullptr; // Unreachable.
}

inline int tp_init_impl(PyObject *, PyObject *, PyObject *) {
pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_init_impl");
// return -1; // Unreachable.
}

// The implementation needs the definition of `class cpp_function`.
void tp_dealloc_impl(PyObject *self);

inline void tp_free_impl(void *) {
pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_free_impl");
}

inline PyObject *reduce_ex_impl(PyObject *self, PyObject *, PyObject *) {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I worry this implementation violated the protocol: the returned function call should have created an object of the same type as the one that was reduce_ex-d. aka:

reduce = obj.__reduce_ex__()
type(reduce[0](*reduce[1])) == type(obj)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Quoting from the Python documentation:

  • A callable object that will be called to create the initial version of the object.
  • A tuple of arguments for the callable object. An empty tuple must be given if the callable does not accept any argument.

There is no mention that the callable has to be of the same type is the object being pickled. Such a restriction would be completely artificial and severely limiting:

There are types that for good reasons are not referenced from any importable module. There is no way that pickle.load() could find them.

This is the case for Boost.Python and pybind11 functions.

Going deeper into the weeds:

Python has __builtins__ as home for some similar types. I was thinking of adding the pybind11 function type to __builtins__, but I was told that future Python versions will make this impossible.

Even deeper:

We could create something like __pybind11__builtins__ (I'd actually call that __pybind11_internals__), which would have the benefit of solving problems like here, BUT, how does that spring into existence? — Python creates __builtins__ on startup. We don't have that luxury.

Also note that there may be multiple pybind11 versions / ABIs in use simultaneously in any given Python interpreter.

Conclusion: I don't think there is any practical way to implement a callable of the "correct" pybind11 function record type that pickle.load() could find.

Copy link

@rainwoodman rainwoodman Feb 15, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My main concern is the type of the return value of the callable, which is supposed to be the "initial version of the object". From what I read the current approach is:

  1. function_recorder's reduce_ex callable returns a module object, even though there is a PyObject for function_record itself.
  2. Another Python side object's "callable" looks up the attr from the module object to return the callable. (I could not quite find the relevant implementation however).

I argue that step 1's callable should have returned the PyObject for function_record instead, because of the "returns initial value" requirement. As function_record has a name, and a scope, I think we should have enough information for that. The records can be either attached to a pybind11_builtin module, or the actual scope where the CCallable is attached.

Then step 2 can be adjusted accordingly.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"Initial" here refers to what happens when the object is unpickled.

When the object is unpickled: the callable creates an object, then calls __getstate__ (if there is one) to mutate the "initial" object.


The step 2. is something we have no control over.

Because it is coded into the pickle implementation how it handles built-in method objects repr(m.simple_callable):

<built-in method simple_callable of pybind11_detail_function_record_v1__gcc_libstdcpp_cxxabi1018 object at 0x...>

Unless we want to completely change how pybind11 functions are implemented (nanobind actually did that!), we are forced to design the __reduce_ex__ of the function_record to produce:

A callable that returns an (intermediate) object for with getattr(obj, 'simple_callable)` returns our function object.

The intermediate object could be anything that has that behavior.

Additional important requirement: it needs to assume that the module with the function was not imported already, otherwise the pickle load step (we have to assume it's in a new process) will fail.

In other words, in or before the getattr(obj, 'simple_callable') step the module needs to be imported, and we cannot rely on any side-effects of importing the module before we trigger the import:

Yes, the function object has a name and a scope, but the type object does not have a scope.

This will be a problem even for nanobind, which doesn't use PyCFunction but has it's own nanobind.nb_func type. To stay with that concrete example: unless nanobind exists as an importable module, the pickle load stage cannot work. This is what I meant by the "spring into existence" problem mentioned before.

I was thinking of giving giving the function_record type an __init__ that could act as the callable in the pickle mechanism, but that's the exact same problem: how does it spring into existence in the pickle load stage?

Note that this PR solves that problem in a very minimalistic and clean fashion:

(<built-in function eval>, ("__import__('importlib').import_module('pybind11_tests.pickling')",))

It could be even better (avoid the eval) if we had something like __import_module__, which could then act as the callable directly:

(<built-in function __import_module__>, ("pybind11_tests.pickling",))

Unfortunately that does not exist. I actually tried under this PR to make such a callable myself, and to inject it into the rec->scope when a cpp_function is built (so that it springs into existence just in time), but that fell flat because stubgen picks it up (>500 TGP failures because we have ~130 auto-generated but checked-in stubgen files). See message of the commit that replaced the "pickle helper" callable with eval:

0e7bb10

I had _function_record_pickle_helper_abi_stamp as the name at that time. I also tried __function_record_pickle_helper_abi_stamp and __function_record_pickle_helper_abi_stamp__ but stubgen stubbornly picked it up no matter what. The eval approach was born out of an intense search for a solution that does not make it necessary to change stubgen, or worse, 130+ checked-in auto-generated files all over the codebase.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks a lot @rainwoodman for taking the time to go through this yesterday 1:1, and for correcting my understanding of your concern.

I captured the experimental code we worked on in 02b31b1 (exactly as it was when we left off).

Our conclusion is captured in the long comment added with bcdfc69. Could you please take another look?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for adding the comment and the test. This looks good!

// Deliberately ignoring the arguments for simplicity (expected is `protocol: int`).
const function_record *rec = function_record_ptr_from_PyObject(self);
if (rec == nullptr) {
pybind11_fail(
"FATAL: function_record_PyTypeObject reduce_ex_impl(): cannot obtain cpp_func_rec.");
}
if (rec->name != nullptr && rec->name[0] != '\0' && rec->scope
&& PyModule_Check(rec->scope.ptr()) != 0) {
object scope_module = get_scope_module(rec->scope);
if (scope_module) {
return make_tuple(reinterpret_borrow<object>(PyEval_GetBuiltins())["eval"],
make_tuple(str("__import__('importlib').import_module('")
+ scope_module + str("')")))
.release()
.ptr();
}
}
set_error(PyExc_RuntimeError, repr(self) + str(" is not pickleable."));
return nullptr;
}

PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods)

PYBIND11_NAMESPACE_END(detail)
PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)
1 change: 1 addition & 0 deletions include/pybind11/detail/internals.h
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ struct internals {
# if PYBIND11_INTERNALS_VERSION > 4
// Note that we have to use a std::string to allocate memory to ensure a unique address
// We want unique addresses since we use pointer equality to compare function records
// OBSOLETE: google/pywrapcc#30099
std::string function_record_capsule_name = internals_function_record_capsule_name;
# endif

Expand Down
11 changes: 2 additions & 9 deletions include/pybind11/functional.h
Original file line number Diff line number Diff line change
Expand Up @@ -96,15 +96,8 @@ struct type_caster<std::function<Return(Args...)>> {
auto *cfunc_self = PyCFunction_GET_SELF(cfunc.ptr());
if (cfunc_self == nullptr) {
PyErr_Clear();
} else if (isinstance<capsule>(cfunc_self)) {
auto c = reinterpret_borrow<capsule>(cfunc_self);

function_record *rec = nullptr;
// Check that we can safely reinterpret the capsule into a function_record
if (detail::is_function_record_capsule(c)) {
rec = c.get_pointer<function_record>();
}

} else {
function_record *rec = function_record_ptr_from_PyObject(cfunc_self);
while (rec != nullptr) {
if (rec->is_stateless
&& same_type(typeid(function_type),
Expand Down
74 changes: 32 additions & 42 deletions include/pybind11/pybind11.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#pragma once

#include "detail/class.h"
#include "detail/function_record_pyobject.h"
#include "detail/init.h"
#include "detail/native_enum_data.h"
#include "detail/smart_holder_sfinae_hooks_only.h"
Expand All @@ -20,6 +21,7 @@
#include "options.h"
#include "typing.h"

#include <cassert>
#include <cstdlib>
#include <cstring>
#include <memory>
Expand Down Expand Up @@ -528,20 +530,11 @@ class cpp_function : public function {
if (rec->sibling) {
if (PyCFunction_Check(rec->sibling.ptr())) {
auto *self = PyCFunction_GET_SELF(rec->sibling.ptr());
if (!isinstance<capsule>(self)) {
chain = detail::function_record_ptr_from_PyObject(self);
if (chain && !chain->scope.is(rec->scope)) {
/* Never append a method to an overload chain of a parent class;
instead, hide the parent's overloads in this case */
chain = nullptr;
} else {
auto rec_capsule = reinterpret_borrow<capsule>(self);
if (detail::is_function_record_capsule(rec_capsule)) {
chain = rec_capsule.get_pointer<detail::function_record>();
/* Never append a method to an overload chain of a parent class;
instead, hide the parent's overloads in this case */
if (!chain->scope.is(rec->scope)) {
chain = nullptr;
}
} else {
chain = nullptr;
}
}
}
// Don't trigger for things like the default __init__, which are wrapper_descriptors
Expand All @@ -561,21 +554,14 @@ class cpp_function : public function {
= reinterpret_cast<PyCFunction>(reinterpret_cast<void (*)()>(dispatcher));
rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS;

capsule rec_capsule(unique_rec.release(),
detail::get_function_record_capsule_name(),
[](void *ptr) { destruct((detail::function_record *) ptr); });
detail::function_record_PyTypeObject_PyType_Ready(); // Call-once initialization.
object py_func_rec = detail::function_record_PyObject_New();
((detail::function_record_PyObject *) py_func_rec.ptr())->cpp_func_rec
= unique_rec.release();
guarded_strdup.release();

object scope_module;
if (rec->scope) {
if (hasattr(rec->scope, "__module__")) {
scope_module = rec->scope.attr("__module__");
} else if (hasattr(rec->scope, "__name__")) {
scope_module = rec->scope.attr("__name__");
}
}

m_ptr = PyCFunction_NewEx(rec->def, rec_capsule.ptr(), scope_module.ptr());
object scope_module = detail::get_scope_module(rec->scope);
m_ptr = PyCFunction_NewEx(rec->def, py_func_rec.ptr(), scope_module.ptr());
if (!m_ptr) {
pybind11_fail("cpp_function::cpp_function(): Could not allocate function object");
}
Expand Down Expand Up @@ -604,9 +590,9 @@ class cpp_function : public function {
// chain.
chain_start = rec;
rec->next = chain;
auto rec_capsule
= reinterpret_borrow<capsule>(((PyCFunctionObject *) m_ptr)->m_self);
rec_capsule.set_pointer(unique_rec.release());
auto *py_func_rec
= (detail::function_record_PyObject *) PyCFunction_GET_SELF(m_ptr);
py_func_rec->cpp_func_rec = unique_rec.release();
guarded_strdup.release();
} else {
// Or end of chain (normal behavior)
Expand Down Expand Up @@ -680,6 +666,8 @@ class cpp_function : public function {
}
}

friend void detail::function_record_PyTypeObject_methods::tp_dealloc_impl(PyObject *);

/// When a cpp_function is GCed, release any memory allocated by pybind11
static void destruct(detail::function_record *rec, bool free_strings = true) {
// If on Python 3.9, check the interpreter "MICRO" (patch) version.
Expand Down Expand Up @@ -729,13 +717,11 @@ class cpp_function : public function {
/// Main dispatch logic for calls to functions bound using pybind11
static PyObject *dispatcher(PyObject *self, PyObject *args_in, PyObject *kwargs_in) {
using namespace detail;
assert(isinstance<capsule>(self));
const function_record *overloads = function_record_ptr_from_PyObject(self);
assert(overloads != nullptr);

/* Iterator over the list of potentially admissible overloads */
const function_record *overloads = reinterpret_cast<function_record *>(
PyCapsule_GetPointer(self, get_function_record_capsule_name())),
*current_overload = overloads;
assert(overloads != nullptr);
const function_record *current_overload = overloads;

/* Need to know how many arguments + keyword arguments there are to pick the right
overload */
Expand Down Expand Up @@ -1209,6 +1195,17 @@ class cpp_function : public function {

PYBIND11_NAMESPACE_BEGIN(detail)

PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods)

// This implementation needs the definition of `class cpp_function`.
inline void tp_dealloc_impl(PyObject *self) {
auto *py_func_rec = (function_record_PyObject *) self;
cpp_function::destruct(py_func_rec->cpp_func_rec);
py_func_rec->cpp_func_rec = nullptr;
}

PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods)

/// Instance creation function for all pybind11 types. It only allocates space for the
/// C++ object, but doesn't call the constructor -- an `__init__` function must do that.
extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *) {
Expand Down Expand Up @@ -2288,14 +2285,7 @@ class class_ : public detail::generic_type {
if (!func_self) {
throw error_already_set();
}
if (!isinstance<capsule>(func_self)) {
return nullptr;
}
auto cap = reinterpret_borrow<capsule>(func_self);
if (!detail::is_function_record_capsule(cap)) {
return nullptr;
}
return cap.get_pointer<detail::function_record>();
return detail::function_record_ptr_from_PyObject(func_self.ptr());
}
};

Expand Down
Loading
Loading