Why does the memoryview assignment from array have python interactions?

Question

When compiled with cython -a my-file.pyx, this simple line of cdef is annotated as yellow in the html file.

# my-file.pyx
from cpython.array cimport array

def f(double[:] xyz):
    cdef double[:] inv2 = array('d', [xyz[0]*3, xyz[1], xyz[2]*3])

Is this correct? I was expecting this line to have no python interactions.

I actually don't know how to tell if the code still has python interactions except for the coloring of the lines in the html files. How can I tell if there is still any improvement to be done when a line is yellow?

The corresponding c code is

  __pyx_t_1 = 0;
  __pyx_t_2 = -1;
  if (__pyx_t_1 < 0) {
    __pyx_t_1 += __pyx_v_xyz.shape[0];
    if (unlikely(__pyx_t_1 < 0)) __pyx_t_2 = 0;
  } else if (unlikely(__pyx_t_1 >= __pyx_v_xyz.shape[0])) __pyx_t_2 = 0;
  if (unlikely(__pyx_t_2 != -1)) {
    __Pyx_RaiseBufferIndexError(__pyx_t_2);
    __PYX_ERR(0, 5, __pyx_L1_error)
  }
  __pyx_t_3 = PyFloat_FromDouble(((*((double *) ( /* dim=0 */ (__pyx_v_xyz.data + __pyx_t_1 * __pyx_v_xyz.strides[0]) ))) * 3.0)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_3);
  __pyx_t_1 = 1;
  __pyx_t_2 = -1;
  if (__pyx_t_1 < 0) {
    __pyx_t_1 += __pyx_v_xyz.shape[0];
    if (unlikely(__pyx_t_1 < 0)) __pyx_t_2 = 0;
  } else if (unlikely(__pyx_t_1 >= __pyx_v_xyz.shape[0])) __pyx_t_2 = 0;
  if (unlikely(__pyx_t_2 != -1)) {
    __Pyx_RaiseBufferIndexError(__pyx_t_2);
    __PYX_ERR(0, 5, __pyx_L1_error)
  }
  __pyx_t_4 = PyFloat_FromDouble((*((double *) ( /* dim=0 */ (__pyx_v_xyz.data + __pyx_t_1 * __pyx_v_xyz.strides[0]) )))); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_4);
  __pyx_t_1 = 2;
  __pyx_t_2 = -1;
  if (__pyx_t_1 < 0) {
    __pyx_t_1 += __pyx_v_xyz.shape[0];
    if (unlikely(__pyx_t_1 < 0)) __pyx_t_2 = 0;
  } else if (unlikely(__pyx_t_1 >= __pyx_v_xyz.shape[0])) __pyx_t_2 = 0;
  if (unlikely(__pyx_t_2 != -1)) {
    __Pyx_RaiseBufferIndexError(__pyx_t_2);
    __PYX_ERR(0, 5, __pyx_L1_error)
  }
  __pyx_t_5 = PyFloat_FromDouble(((*((double *) ( /* dim=0 */ (__pyx_v_xyz.data + __pyx_t_1 * __pyx_v_xyz.strides[0]) ))) * 3.0)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __pyx_t_6 = PyList_New(3); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __Pyx_GIVEREF(__pyx_t_3);
  PyList_SET_ITEM(__pyx_t_6, 0, __pyx_t_3);
  __Pyx_GIVEREF(__pyx_t_4);
  PyList_SET_ITEM(__pyx_t_6, 1, __pyx_t_4);
  __Pyx_GIVEREF(__pyx_t_5);
  PyList_SET_ITEM(__pyx_t_6, 2, __pyx_t_5);
  __pyx_t_3 = 0;
  __pyx_t_4 = 0;
  __pyx_t_5 = 0;
  __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_5);
  __Pyx_INCREF(__pyx_n_s_d);
  __Pyx_GIVEREF(__pyx_n_s_d);
  PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_n_s_d);
  __Pyx_GIVEREF(__pyx_t_6);
  PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_6);
  __pyx_t_6 = 0;
  __pyx_t_6 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_7cpython_5array_array), __pyx_t_5, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 5, __pyx_L1_error)
  __Pyx_GOTREF(__pyx_t_6);
  __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
  __pyx_t_7 = __Pyx_PyObject_to_MemoryviewSlice_ds_double(__pyx_t_6, PyBUF_WRITABLE); if (unlikely(!__pyx_t_7.memview)) __PYX_ERR(0, 5, __pyx_L1_error)
  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
  __pyx_v_inv2 = __pyx_t_7;
  __pyx_t_7.memview = NULL;
  __pyx_t_7.data = NULL;`

You are creating an `array` object and then taking the memoryview of that. Why are you doing that instead of initializing the array directly? https://stackoverflow.com/questions/25974975/cython-c-array-initialization — MegaIng, Aug 15 '23 at 15:30
You are right, I should. I am cythonizing some old numpy code, and just replaced numpy arrays by array.array for no good reason ... (The cython documents often uses them) — nos, Aug 15 '23 at 15:49

score 0 · Answer 1 · answered Aug 15 '23 at 16:42

A memoryview is simply an efficient way of accessing the data of a Python object that supports the buffer protocol (and the bit that's actually optimized is the indexing).

cpython.array is just a Python object that supports the buffer protocol.

So the line

cdef double[:] inv2 = array('d', [xyz[0]*3, xyz[1], xyz[2]*3])

needs to:

Call a Python function with the arguments 'd' (a Python string), and a three-long list of values. Cython has a little bit of special visibility into the array.array, but not too much.
Check that the returned Python object supports the buffer protocol with a type of double.
Do a bit of reference counting so the lifetime of the Python object will be at least as long as that of the memoryview.

I don't think there's a quick way of creating an empty array.array without needing a bunch of Python objects, but I think ideally you'd create an empty array and then fill in the elements.

# Numpy just as an illustrative example
cdef double[:] inv2 = np.empty((3,), dtype=np.double)
inv2[0] = xyz[0]*3
inv2[1] = xyz[1]
inv2[2] = xyz[2]*3

That way all the maths is kept in C (but call to np.empty is still a Python call, and for a 3-long array may end up with more overhead than what you have now).

score 0 · Answer 2 · answered Aug 16 '23 at 14:07

As pointed out by @megalng, one better way is to use

# my-file.pyx

def f(double[:] xyz):
    cdef double[3] inv2 = [xyz[0]*3, xyz[1], xyz[2]*3]

It translates to

  __pyx_t_1 = 0;
  __pyx_t_2 = -1;
  if (__pyx_t_1 < 0) {
    __pyx_t_2 = 0;
  } else if (unlikely(__pyx_t_1 >= __pyx_v_xyz.shape[0])) __pyx_t_2 = 0;
  if (unlikely(__pyx_t_2 != -1)) {
    __Pyx_RaiseBufferIndexError(__pyx_t_2);
    __PYX_ERR(0, 10, __pyx_L1_error)
  }
  __pyx_t_3 = 1;
  __pyx_t_2 = -1;
  if (__pyx_t_3 < 0) {
    __pyx_t_2 = 0;
  } else if (unlikely(__pyx_t_3 >= __pyx_v_xyz.shape[0])) __pyx_t_2 = 0;
  if (unlikely(__pyx_t_2 != -1)) {
    __Pyx_RaiseBufferIndexError(__pyx_t_2);
    __PYX_ERR(0, 10, __pyx_L1_error)
  }
  __pyx_t_4 = 2;
  __pyx_t_2 = -1;
  if (__pyx_t_4 < 0) {
    __pyx_t_2 = 0;
  } else if (unlikely(__pyx_t_4 >= __pyx_v_xyz.shape[0])) __pyx_t_2 = 0;
  if (unlikely(__pyx_t_2 != -1)) {
    __Pyx_RaiseBufferIndexError(__pyx_t_2);
    __PYX_ERR(0, 10, __pyx_L1_error)
  }
  __pyx_t_5[0] = ((*((double *) ( /* dim=0 */ (__pyx_v_xyz.data + __pyx_t_1 * __pyx_v_xyz.strides[0]) ))) * 3.0);
  __pyx_t_5[1] = (*((double *) ( /* dim=0 */ (__pyx_v_xyz.data + __pyx_t_3 * __pyx_v_xyz.strides[0]) )));
  __pyx_t_5[2] = ((*((double *) ( /* dim=0 */ (__pyx_v_xyz.data + __pyx_t_4 * __pyx_v_xyz.strides[0]) ))) * 3.0);
  memcpy(&(__pyx_v_inv2[0]), __pyx_t_5, sizeof(__pyx_v_inv2[0]) * (3));

which is less lines and it shows up less yellow in the html file.

If I use C array

def f3(double[:] xyz):
    cdef double inv[3]
    inv[0] = xyz[0]*3
    inv[1] = xyz[1]
    inv[2] = xyz[2]*3

then all those lines are not yellow. But unfortunately I cannot assign them directly in one line

Strangely, in the cython document, it calls cdef int[3][3][3] carr a C array (note the location of brackets)

# Memoryview on a C array
cdef int[3][3][3] carr
cdef int [:, :, :] carr_view = carr

Why does the memoryview assignment from array have python interactions?

2 Answers2