For the needs of my daily work, I have to wrap with SWIG some C++ classes located in a DSO. Everything was fine until I decided for performance reasons to use Python buffer protocol, that leads me to use the -builtin flag of the SWIG tool. I've managed to get thing working but when I try to serialize/deserialize with pickle my wrapped classes, I get the following error:
Traceback (most recent call last):
File "test.py", line 23, in <module>
s = pickle.dumps(a) # crash here
_pickle.PicklingError: Can't pickle <class 'mymodule.A'>: attribute lookup A on imp failed
Here is a MWE exposing the issue.
First, the SWIG interface file mymodule.i wraps an inlined C++ class A.
// File mymodule.i
%module mymodule
%feature("python:tp_str") A "A_str" ;
%feature("python:tp_repr") A "A_str" ;
%{
#include <iostream>
#include <sstream>
// forward declarations
class A ;
static std::ostream & operator << (std::ostream &, const A &) ;
static int A_getI(const A &) ;
static void A_setI(A &, int) ;
class A
{
int i_ ;
public:
A(int i) : i_(i) {}
friend std::ostream & operator << (std::ostream &, const A &) ;
friend int A_getI(const A &) ;
friend void A_setI(A &, int) ;
} ;
std::ostream & operator << (std::ostream & os, const A & a)
{
return os << "A(" << a.i_ << ")" ;
}
int A_getI(const A & a)
{
return a.i_ ;
}
void A_setI(A & a, int i)
{
a.i_ = i ;
}
static PyObject * A_str(SwigPyObject * obj)
{
A * p_a = reinterpret_cast<A *>(obj->ptr) ;
std::ostringstream oss ;
oss << *p_a ;
return PyUnicode_FromString(oss.str().c_str()) ;
}
%}
class A
{
public:
A(int i) ;
} ;
%extend A
{
PyObject * __getstate__()
{
PyObject * tuple = PyTuple_New(1) ;
PyTuple_SetItem(tuple, 0, PyLong_FromLong(A_getI(*self))) ;
return tuple ;
}
void __setstate__(PyObject * state)
{
if (PyTuple_Check(state)) {
PyObject * val = PyTuple_GetItem(state, 0) ;
if (PyLong_Check(val)) {
int i = (int) PyLong_AsLong(val) ;
A_setI(*self, i) ;
}
}
}
}
Then, the test.py I use:
#!/usr/bin/env python3
# File: test.py
from mymodule import *
# a first object a
a = A(10)
print("a=",a)
# we manually get the internal state of a
sa = a.__getstate__()
print("sa=",sa)
# a second object b with a different internal state
b = A(-1)
print("b=",b)
# we manually set a's internal state to b
b.__setstate__(sa)
print("b=",b)
# now we try to dump a's state using pickle...
import pickle
s = pickle.dumps(a) # crash here
print(s)
I compile the module:
swig3.0 -python -py3 -c++ -builtin -o mymodulePYTHON_wrap.cxx mymodule.i
g++ -D_mymodule_EXPORTS -DSWIG_REV="" -fPIC -I/usr/include/python3.4m -std=c++11 -o mymodulePYTHON_wrap.cxx.o -c mymodulePYTHON_wrap.cxx
g++ -fPIC -shared -Wl,-soname,_mymodule.so -o _mymodule.so mymodulePYTHON_wrap.cxx.o
I run the test:
python3 test.py
And I get the following output:
a= A(10)
sa= (10,)
b= A(-1)
b= A(10)
Traceback (most recent call last):
File "test.py", line 23, in <module>
s = pickle.dumps(a) # crash here
_pickle.PicklingError: Can't pickle <class 'mymodule.A'>: attribute lookup A on imp failed
What is the reason of the error ?
How can I solve my problem ?
I've also tried to:
- read the SWIG and the Python documentation
- search the Web
- read most of the articles related to this problem on SO but without better knowledge (the closer to my problem seems to be Serialize SWIG extension with dill except that Mark McKems' answer leaves me hopeless, What's the exact usage of __reduce__ in Pickler, Pickle Cython Class with C pointers)
- use a
__reduce__method returning what's expected (it worked) but the same error occurs - use dill instead of pickle, the same
- change the protocol to the highest, the same
The versions of the my tools are quite old because I have to use a Debian jessie:
- Python 3.4.2
- G++ 4.9.2
- SWIG 3.0.2
I'm stuck. Any help would be appreciated.
EDIT
The mymodule.py file generated by SWIG (WITH -builtin flag):
# This file was automatically generated by SWIG (http://www.swig.org).
# Version 3.0.2
#
# Do not make changes to this file unless you know what you are doing--modify
# the SWIG interface file instead.
from sys import version_info
if version_info >= (2,6,0):
def swig_import_helper():
from os.path import dirname
import imp
fp = None
try:
fp, pathname, description = imp.find_module('_mymodule', [dirname(__file__)])
except ImportError:
import _mymodule
return _mymodule
if fp is not None:
try:
_mod = imp.load_module('_mymodule', fp, pathname, description)
finally:
fp.close()
return _mod
_mymodule = swig_import_helper()
del swig_import_helper
else:
import _mymodule
del version_info
from _mymodule import *
try:
_swig_property = property
except NameError:
pass # Python < 2.2 doesn't have 'property'.
def _swig_setattr_nondynamic(self,class_type,name,value,static=1):
if (name == "thisown"): return self.this.own(value)
if (name == "this"):
if type(value).__name__ == 'SwigPyObject':
self.__dict__[name] = value
return
method = class_type.__swig_setmethods__.get(name,None)
if method: return method(self,value)
if (not static):
self.__dict__[name] = value
else:
raise AttributeError("You cannot add attributes to %s" % self)
def _swig_setattr(self,class_type,name,value):
return _swig_setattr_nondynamic(self,class_type,name,value,0)
def _swig_getattr(self,class_type,name):
if (name == "thisown"): return self.this.own()
method = class_type.__swig_getmethods__.get(name,None)
if method: return method(self)
raise AttributeError(name)
def _swig_repr(self):
try: strthis = "proxy of " + self.this.__repr__()
except: strthis = ""
return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)
try:
_object = object
_newclass = 1
except AttributeError:
class _object : pass
_newclass = 0
# This file is compatible with both classic and new-style classes.
The mymodule.py file WITHOUT -builtin SWIG flag:
# This file was automatically generated by SWIG (http://www.swig.org).
# Version 3.0.2
#
# Do not make changes to this file unless you know what you are doing--modify
# the SWIG interface file instead.
from sys import version_info
if version_info >= (2,6,0):
def swig_import_helper():
from os.path import dirname
import imp
fp = None
try:
fp, pathname, description = imp.find_module('_mymodule', [dirname(__file__)])
except ImportError:
import _mymodule
return _mymodule
if fp is not None:
try:
_mod = imp.load_module('_mymodule', fp, pathname, description)
finally:
fp.close()
return _mod
_mymodule = swig_import_helper()
del swig_import_helper
else:
import _mymodule
del version_info
try:
_swig_property = property
except NameError:
pass # Python < 2.2 doesn't have 'property'.
def _swig_setattr_nondynamic(self,class_type,name,value,static=1):
if (name == "thisown"): return self.this.own(value)
if (name == "this"):
if type(value).__name__ == 'SwigPyObject':
self.__dict__[name] = value
return
method = class_type.__swig_setmethods__.get(name,None)
if method: return method(self,value)
if (not static):
self.__dict__[name] = value
else:
raise AttributeError("You cannot add attributes to %s" % self)
def _swig_setattr(self,class_type,name,value):
return _swig_setattr_nondynamic(self,class_type,name,value,0)
def _swig_getattr(self,class_type,name):
if (name == "thisown"): return self.this.own()
method = class_type.__swig_getmethods__.get(name,None)
if method: return method(self)
raise AttributeError(name)
def _swig_repr(self):
try: strthis = "proxy of " + self.this.__repr__()
except: strthis = ""
return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)
try:
_object = object
_newclass = 1
except AttributeError:
class _object : pass
_newclass = 0
class A(_object):
__swig_setmethods__ = {}
__setattr__ = lambda self, name, value: _swig_setattr(self, A, name, value)
__swig_getmethods__ = {}
__getattr__ = lambda self, name: _swig_getattr(self, A, name)
__repr__ = _swig_repr
def __init__(self, *args):
this = _mymodule.new_A(*args)
try: self.this.append(this)
except: self.this = this
def __getstate__(self) -> "PyObject *" : return _mymodule.A___getstate__(self)
def __setstate__(self, *args) -> "void" : return _mymodule.A___setstate__(self, *args)
__swig_destroy__ = _mymodule.delete_A
__del__ = lambda self : None;
A_swigregister = _mymodule.A_swigregister
A_swigregister(A)
# This file is compatible with both classic and new-style classes.
