I recently hit on the idea of using some form of metaprogramming to replace the (admittedly limit) boilerplate code needed to persist and then load objects using pickle. So you'd get something like
class MyClass(object):
# Something to add save-ability (maybe a decorator or metaclass)...
# More code...
my_instance = MyClass('/path/to/persistent_file')
my_instance.do_some_stuff()
my_instance.save()
my_instance._do_some_more_stuff()
my_instance.save()
I attemped to accomplish this by using a metaclass. Unfortunately it did not turn out as well as I had hoped. The __init__ methods became complicated because this happens:
- The user calls MyClass(path, other_args) to either open an existing persistent instance or, if none exists at path, create a new one
- The system finds a file at path and calls pickle.load() to read it
- pickle.load() makes another call to MyClass.__init__ to build the object based on what it read from disk
MyClass.__init__'s ability to tell #1 from #3 is basically a hack in my implementation. Is there a clean way to do this (whether using metaclasses, decorators, or whatever)?
EDIT: @roippi asked, "It sounds like you're trying to reimplement shelve, no?" Yes, this exercise started as an effort to reimplement shelve, but allowing arbitrary keys, which is basically what my SyncPickleDict is. Then I found myself wanting to generalize it. I've fixed the example to use non-string keys to make this a little clearer.
#!/usr/bin/python2.7
import pickle
import os
import datetime
from collections import OrderedDict
class SyncPickleParent(object):
def __init__ (self, filepath, *args, **kwargs):
print "here and self is", self
self.__filepath = filepath
super(SyncPickleParent, self).__init__(*args, **kwargs)
def sync_pickle(self):
with open(self.__filepath, "w") as ofile:
pickle.dump(self, ofile)
class SyncPickle(type):
def __new__(meta, name, bases, dct):
return super(SyncPickle, meta).__new__(meta, name, (SyncPickleParent,)+bases, dct)
def __call__(cls, filepath, force_init=False, force_overwrite=False, *args, **kwds):
if force_init:
return type.__call__(cls, filepath, *args, **kwds)
if not force_overwrite:
try:
with open(filepath) as ifile:
red = pickle.load(ifile)
red._SyncPickleParent__filepath = filepath
return red
except IOError as ioe:
pass
result = type.__call__(cls, filepath, *args, **kwds)
result.sync_pickle() # Check that file is writable
return result
class SyncPickleDict(OrderedDict):
__metaclass__ = SyncPickle
def __init__(self, *args, **kwargs):
print "in init; args={}; kwargs={}".format(args, kwargs)
super(SyncPickleDict, self).__init__(*args, **kwargs)
def __reduce__(self):
# Yuck. The tuple output by __reduce__ has to have two bogus
# arguments
return (self.__class__, ("dummy", True, False, tuple(self.items())))
if "__main__" == __name__:
spd = SyncPickleDict(os.path.expanduser("~/tmp/bar.dat"))
spd[(1,2,3)] = 'foobar'
spd['access_count'] = spd.get('access_count', 0) + 1
spd[datetime.datetime.now()] = "A new key every time"
spd.sync_pickle()
print "\n".join(map(str, spd.iteritems()))