How to tell for which object attribute pickle fails?

I had the same problem as you, but my classes were a bit more complicated (i.e. a large tree of similar objects) so the printing didn’t help much so I hacked together a helper function. It is not complete and is only intended for use with pickling protocol 2:
It was enough so I could locate my problems. If you want to extend it to cover everything, the protocol is described at http://www.python.org/dev/peps/pep-0307/ i’ve made this post editable so everybody can update the code.

import pickle
def get_pickling_errors(obj,seen=None):
    if seen == None:
        seen = []
    try:
        state = obj.__getstate__()
    except AttributeError:
        return
    if state == None:
        return
    if isinstance(state,tuple):
        if not isinstance(state[0],dict):
            state=state[1]
        else:
            state=state[0].update(state[1])
    result = {}    
    for i in state:
        try:
            pickle.dumps(state[i],protocol=2)
        except pickle.PicklingError:
            if not state[i] in seen:
                seen.append(state[i])
                result[i]=get_pickling_errors(state[i],seen)
    return result

An example of the usage where K is the object that doesn’t pickle

>>> get_pickling_errors(K)
{'_gen': {}, '_base': {'_gens': None}}

This means that the attibute K._gen is not picklable and the same goes for K._base._gens.

I have found that if you subclass Pickler and wrap the Pickler.save() method in a try, except block

import pickle
class MyPickler (pickle.Pickler):
    def save(self, obj):
        try:
            pickle.Pickler.save(self, obj)
        except Exception, e:
            import pdb;pdb.set_trace()

Then call it like so

import StringIO
output = StringIO.StringIO()
MyPickler(output).dump(thingee)

If you use dill, your example doesn’t fail to pickle…

>>> import dill
>>> import new
>>> 
>>> class Test(object):
...     pass
... 
>>> def test_func(self):
...     pass
... 
>>> test = Test()
>>> dill.dumps(test)
'x80x02cdill.dilln_create_typenqx00(cdill.dilln_load_typenqx01Ux08TypeTypeqx02x85qx03Rqx04Ux04Testqx05hx01UnObjectTypeqx06x85qx07Rqx08x85qt}qn(Ur__slotnames__qx0b]qx0cUn__module__qrUx08__main__qx0eUx07__doc__qx0fNutqx10Rqx11)x81qx12}qx13b.'
>>> test.test_meth = new.instancemethod(test_func, test)
>>> dill.dumps(test)
'x80x02cdill.dilln_create_typenqx00(cdill.dilln_load_typenqx01Ux08TypeTypeqx02x85qx03Rqx04Ux04Testqx05hx01UnObjectTypeqx06x85qx07Rqx08x85qt}qn(Ur__slotnames__qx0b]qx0cUn__module__qrUx08__main__qx0eUx07__doc__qx0fNutqx10Rqx11)x81qx12}qx13Uttest_methqx14hx01UnMethodTypeqx15x85qx16Rqx17cdill.dilln_create_functionnqx18(cdill.dilln_unmarshalnqx19Ubcx01x00x00x00x01x00x00x00x01x00x00x00Cx00x00x00sx04x00x00x00dx00x00S(x01x00x00x00N(x00x00x00x00(x01x00x00x00tx04x00x00x00self(x00x00x00x00(x00x00x00x00sx07x00x00x00<stdin>ttx00x00x00test_funcx01x00x00x00sx02x00x00x00x00x01qx1ax85qx1bRqx1cc__builtin__n__main__nUttest_funcqx1dNN}qx1etqx1fRq hx12Nx87q!Rq"sb.'

So we have to find something that dill can’t pickle…

>>> class Unpicklable(object):
...   def breakme(self):
...     self.x = iter(set())
... 
>>> u = Unpicklable()
>>> dill.dumps(u)
'x80x02cdill.dilln_create_typenqx00(cdill.dilln_load_typenqx01Ux08TypeTypeqx02x85qx03Rqx04Ux0bUnpicklableqx05hx01UnObjectTypeqx06x85qx07Rqx08x85qt}qn(Ur__slotnames__qx0b]qx0cUn__module__qrUx08__main__qx0eUx07breakmeqx0fcdill.dilln_create_functionnqx10(cdill.dilln_unmarshalnqx11Uxafcx01x00x00x00x02x00x00x00x02x00x00x00Cx00x00x00s"x00x00x00dx01x00dx00x00lx00x00}x01x00tx01x00tx02x00x83x00x00x83x01x00|x00x00_x03x00dx00x00S(x02x00x00x00Nixffxffxffxff(x04x00x00x00ttx00x00x00itertoolstx04x00x00x00itertx03x00x00x00settx01x00x00x00x(x02x00x00x00tx04x00x00x00selfRx00x00x00x00(x00x00x00x00(x00x00x00x00sx07x00x00x00<stdin>tx07x00x00x00breakmex02x00x00x00sx04x00x00x00x00x01x0cx01qx12x85qx13Rqx14c__builtin__n__main__nhx0fNN}qx15tqx16Rqx17Ux07__doc__qx18Nutqx19Rqx1a)x81qx1b}qx1cb.'
>>> u.breakme()
>>> dill.dumps(u)
Traceback (most recent call last):
…(snip)… 
pickle.PicklingError: Can't pickle <type 'setiterator'>: it's not found as __builtin__.setiterator
>>>

If the error message wasn’t good, I could use dill.detect to see which what unpicklable objects the top-level object contains.

>>> dill.detect.badobjects(u, depth=1)
{'__hash__': <method-wrapper '__hash__' of Unpicklable object at 0x10a37b350>, '__setattr__': <method-wrapper '__setattr__' of Unpicklable object at 0x10a37b350>, '__reduce_ex__': <built-in method __reduce_ex__ of Unpicklable object at 0x10a37b350>, '__reduce__': <built-in method __reduce__ of Unpicklable object at 0x10a37b350>, '__str__': <method-wrapper '__str__' of Unpicklable object at 0x10a37b350>, '__format__': <built-in method __format__ of Unpicklable object at 0x10a37b350>, '__getattribute__': <method-wrapper '__getattribute__' of Unpicklable object at 0x10a37b350>, '__delattr__': <method-wrapper '__delattr__' of Unpicklable object at 0x10a37b350>, 'breakme': <bound method Unpicklable.breakme of <__main__.Unpicklable object at 0x10a37b350>>, '__repr__': <method-wrapper '__repr__' of Unpicklable object at 0x10a37b350>, '__dict__': {'x': <setiterator object at 0x10a370820>}, 'x': <setiterator object at 0x10a370820>, '__sizeof__': <built-in method __sizeof__ of Unpicklable object at 0x10a37b350>, '__init__': <method-wrapper '__init__' of Unpicklable object at 0x10a37b350>}
>>> dill.detect.badtypes(u, depth=1)
{'__hash__': <type 'method-wrapper'>, '__setattr__': <type 'method-wrapper'>, '__reduce_ex__': <type 'builtin_function_or_method'>, '__reduce__': <type 'builtin_function_or_method'>, '__str__': <type 'method-wrapper'>, '__format__': <type 'builtin_function_or_method'>, '__getattribute__': <type 'method-wrapper'>, '__delattr__': <type 'method-wrapper'>, 'breakme': <type 'instancemethod'>, '__repr__': <type 'method-wrapper'>, '__dict__': <type 'dict'>, 'x': <type 'setiterator'>, '__sizeof__': <type 'builtin_function_or_method'>, '__init__': <type 'method-wrapper'>}
>>> set(dill.detect.badtypes(u, depth=1).values())
set([<type 'dict'>, <type 'method-wrapper'>, <type 'instancemethod'>, <type 'setiterator'>, <type 'builtin_function_or_method'>])

dill doesn’t rely on the __getstate__ method being present, although maybe it should utilize it if it exists. You can also use objgraph to get a picture of all the object dependencies that are used to build the thing that doesn’t pickle. It can help you sort out what the root of the problem is, based on the above information.

See dill.detect use in tracking down unpicklable items in this issue: https://github.com/uqfoundation/dill/issues/58

I was puzzled as well by the general error..
Digging a bit more, the doc explains how class instances are pickled https://docs.python.org/3/library/pickle.html#pickling-class-instances

Based on doc, I implemented the following util to understand which attributes are causing problems while pickling a custom object obj

def pickle_get_attributes(obj):
    return obj.__class__, obj.__dict__


def pickle_restore_attributes(cls, attributes):
    obj = cls.__new__(cls)
    obj.__dict__.update(attributes)
    return obj


def check_if_pickle(obj):
    obj_class, obj_attrs_dict = pickle_get_attributes(obj)

    obj_attrs_dict["__class__"] = obj_class

    for nm, attr in obj_attrs_dict.items():
        print(f"{nm} {type(attr)}")
        try:
            pickle_str = pickle.dumps(attr)
        except TypeError as e:
            print(f"CANNOT PICKLE {nm} of type {type(attr)}")

check_if_pickle(obj)