How can I set a PyObject* pointer to None?

Question

I want to use a pointer to change the value of a python object, here is what I tried:

This works:

def main():
     cdef int i
     cdef int* iptr
     i = 5
     iptr = &i
     iptr[0] = 1
     print(i)  # this works, prints 1

and this:

cdef class A:
     cdef int i
     def __init__(self, i): self.i = i

def main():
     cdef int* ptr
     cdef A a

     a = A(5)
     ptr = &a.i
     ptr[0] = 1
     print(a.i)  # this works too

But when I try to use similar code with objects, I can't get it to work:

from cpython cimport PyObject

cdef class A:
     cdef object o
     def __init__(self, o): self.o = o

def main():
     cdef PyObject* ptr
     cdef A a

     a = A(list())
     # I also tried with `<void*>a.o`, `<PyObject*>a` and `<void*>a`
     ptr = <PyObject*>a.o

     # this segfaults
     # ptr[0] = dict()
     # print(a.o)

     # this doesnt compile: Cannot convert Python object to 'PyObject'
     # ptr[0] = None
     # print(a.o)

     # this doesnt compile: incompatible types when assigning to type ‘PyObject’ from type ‘struct PyObject *’
     # ptr[0] = <PyObject>None
     # print(a.o)

     # this doesnt compile: Cannot assign type 'PyObject *' to 'PyObject'
     # ptr[0] = <PyObject*>None
     # print(a.o)

Answer 1

You can do this if you are willing to declare the object reference you want to change as a PyObject *, rather than as an object. Also, in order for this to work, you must manage the reference counting on the object yourself. That is the bargain you are making -- once you give up the Python way of getting and setting variables, you are in the land of C style memory management where a mistake can cause you to leak memory, access invalid data, have your objects deleted before you are done with them, and have mysterious bugs show up long after your memory management sin has been committed. Therefore, I'd really recommend against going this route, and instead use the standard pythonic way of setting variables to reference objects.

However, if you really must, this code works as you want your example to work.

from cpython cimport PyObject
from cpython.ref cimport Py_INCREF, Py_XDECREF


cdef class A:
    cdef PyObject *o

    def __init__(self, o):
        cdef PyObject *tmp

        tmp = self.o
        Py_INCREF(o)
        self.o = <PyObject *>o
        Py_XDECREF(tmp)


def main():
    cdef PyObject **ptr
    cdef PyObject *tmp
    cdef A a

    a = A(list())
    ptr = &a.o

    new_obj = dict()

    tmp = ptr[0]
    Py_INCREF(new_obj)
    ptr[0] = <PyObject *>new_obj
    Py_XDECREF(tmp)

    print <object>a.o

Notice that whenever we change the PyObject reference, we first increment the reference on the new object, change the pointer, and then decerement the reference to the old object. I'd recommend you always follow that pattern -- decrementing before incrementing can cause bugs where you might free an object you are still using if you are resetting a variable to reference the same object.

Also, be sure to have the GIL held whenever manipulating Python objects, including updating their reference counts.

There seems to be no way to convince Cython to give you the pointer to an object reference, unless you declare the object reference in the C style. (i.e. PyObject * rather than object -- though those two things are really the same when your program is running. The difference is only at compile time, where Cython will do automatic reference management for object variables but not PyObject * variables.)

Answer 2

Okay, so I decided to dig a little bit on the cythonized code and this is what I got:

cdef class A:
    cdef object o

This will create a new PyObject __pyx_obj_1p_A and a type __pyx_tp_new_1p_A (and a static pointer __pyx_ptype_1p_A):

struct __pyx_obj_1p_A {
    PyObject_HEAD
    PyObject *o;
};

static PyObject *__pyx_tp_new_1p_A(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) {
    struct __pyx_obj_1p_A *p;
    PyObject *o;

    if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) {
        o = (*t->tp_alloc)(t, 0);
    }else{
        o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0);
    } 

    if (unlikely(!o)) {
        return 0;
    }

    p = (struct __pyx_obj_1p_A *)o;
    p->o = Py_None;
    Py_INCREF(Py_None);
    return o;
}

static PyTypeObject __pyx_type_1p_A = {
    # members of the type struct
    __pyx_tp_new_1p_A, # tp_new
    # the rest of the members
}

A block of code like:

from cpython cimport PyObject
from cython import address

cdef A a
cdef void* vptr
cdef PyObject* ptr
cdef PyObject** aptr
a = A()
ptr = <PyObject*>a.o
vptr = <void*>a.o
aptr = address(ptr)
aptr = &ptr

Roughly corresponds to:

__pyx_v_a = (struct __pyx_obj_1p_A *) PyObject_Call((PyObject*)&__pyx_type_1p_A, args, NULL);
__Pyx_GOTREF(__pyx_v_a);
__pyx_v_ptr = ((PyObject *)__pyx_v_a->o);
__pyx_v_vptr = ((void *)__pyx_v_a->o);
__pyx_v_aptr = (&__pyx_v_ptr);
__pyx_v_aptr = (&__pyx_v_ptr);

So, my problem is that cython forces the use of a temporary variable, and I get the address of the variable on the stack, not the struct attribute which is just wrong. The program might segfault depending on the optimization, for me with `-O2' the program segfaulted.