ImportError: No module named 'pandas.indexes'
Question:
Importing pandas didn’t throw the error, but rather trying to read a picked pandas dataframe as such:
import numpy as np
import pandas as pd
import matplotlib
import seaborn as sns
sns.set(style="white")
control_data = pd.read_pickle('null_report.pickle')
test_data = pd.read_pickle('test_report.pickle')
The traceback is 165 lines with three concurrent exceptions (whatever that means). Is read_pickle
not compatible with pandas version 17.1 I’m running? How do I unpickle my dataframe for use?
Below is a copy of the traceback:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
45 with open(path, 'rb') as fh:
---> 46 return pkl.load(fh)
47 except (Exception) as e:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
51 with open(path, 'rb') as fh:
---> 52 return pc.load(fh, encoding=encoding, compat=False)
53
C:UserstestAnaconda3libsite-packagespandascompatpickle_compat.py in load(fh, encoding, compat, is_verbose)
115
--> 116 return up.load()
117 except:
C:UserstestAnaconda3libpickle.py in load(self)
1038 assert isinstance(key, bytes_types)
-> 1039 dispatch[key[0]](self)
1040 except _Stop as stopinst:
C:UserstestAnaconda3libpickle.py in load_stack_global(self)
1342 raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343 self.append(self.find_class(module, name))
1344 dispatch[STACK_GLOBAL[0]] = load_stack_global
C:UserstestAnaconda3libpickle.py in find_class(self, module, name)
1383 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384 __import__(module, level=0)
1385 if self.proto >= 4:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in read_pickle(path)
59 try:
---> 60 return try_read(path)
61 except:
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
56 with open(path, 'rb') as fh:
---> 57 return pc.load(fh, encoding=encoding, compat=True)
58
C:UserstestAnaconda3libsite-packagespandascompatpickle_compat.py in load(fh, encoding, compat, is_verbose)
115
--> 116 return up.load()
117 except:
C:UserstestAnaconda3libpickle.py in load(self)
1038 assert isinstance(key, bytes_types)
-> 1039 dispatch[key[0]](self)
1040 except _Stop as stopinst:
C:UserstestAnaconda3libpickle.py in load_stack_global(self)
1342 raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343 self.append(self.find_class(module, name))
1344 dispatch[STACK_GLOBAL[0]] = load_stack_global
C:UserstestAnaconda3libpickle.py in find_class(self, module, name)
1383 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384 __import__(module, level=0)
1385 if self.proto >= 4:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
45 with open(path, 'rb') as fh:
---> 46 return pkl.load(fh)
47 except (Exception) as e:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
51 with open(path, 'rb') as fh:
---> 52 return pc.load(fh, encoding=encoding, compat=False)
53
C:UserstestAnaconda3libsite-packagespandascompatpickle_compat.py in load(fh, encoding, compat, is_verbose)
115
--> 116 return up.load()
117 except:
C:UserstestAnaconda3libpickle.py in load(self)
1038 assert isinstance(key, bytes_types)
-> 1039 dispatch[key[0]](self)
1040 except _Stop as stopinst:
C:UserstestAnaconda3libpickle.py in load_stack_global(self)
1342 raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343 self.append(self.find_class(module, name))
1344 dispatch[STACK_GLOBAL[0]] = load_stack_global
C:UserstestAnaconda3libpickle.py in find_class(self, module, name)
1383 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384 __import__(module, level=0)
1385 if self.proto >= 4:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
<ipython-input-17-3b05fe7d20a4> in <module>()
3 # test_data = np.genfromtxt(fh, usecols=2)
4
----> 5 control_data = pd.read_pickle('null_report.pickle')
6 test_data = pd.read_pickle('test_report.pickle')
7
C:UserstestAnaconda3libsite-packagespandasiopickle.py in read_pickle(path)
61 except:
62 if PY3:
---> 63 return try_read(path, encoding='latin1')
64 raise
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
55 except:
56 with open(path, 'rb') as fh:
---> 57 return pc.load(fh, encoding=encoding, compat=True)
58
59 try:
C:UserstestAnaconda3libsite-packagespandascompatpickle_compat.py in load(fh, encoding, compat, is_verbose)
114 up.is_verbose = is_verbose
115
--> 116 return up.load()
117 except:
118 raise
C:UserstestAnaconda3libpickle.py in load(self)
1037 raise EOFError
1038 assert isinstance(key, bytes_types)
-> 1039 dispatch[key[0]](self)
1040 except _Stop as stopinst:
1041 return stopinst.value
C:UserstestAnaconda3libpickle.py in load_stack_global(self)
1341 if type(name) is not str or type(module) is not str:
1342 raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343 self.append(self.find_class(module, name))
1344 dispatch[STACK_GLOBAL[0]] = load_stack_global
1345
C:UserstestAnaconda3libpickle.py in find_class(self, module, name)
1382 elif module in _compat_pickle.IMPORT_MAPPING:
1383 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384 __import__(module, level=0)
1385 if self.proto >= 4:
1386 return _getattribute(sys.modules[module], name)[0]
ImportError: No module named 'pandas.indexes'
I also tried loading the pickle file from pickle directly:
via_pickle = pickle.load( open( 'null_report.pickle', "rb" ) )
and got the same error:
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
<ipython-input-23-ba2e3adae1c4> in <module>()
1
----> 2 via_pickle = pickle.load( open( 'null_report.pickle', "rb" ) )
3
4 # control_data = pd.read_pickle('null_report.pickle')
5 # test_data = pd.read_pickle('test_report.pickle')
ImportError: No module named 'pandas.indexes'
Answers:
I had this problem from trying to open a pickled dataframe made with pandas 0.18.1 using pandas 0.17.1.
If you are using pip, upgrade pandas with:
pip install --upgrade pandas
If you are using a library like anaconda, use:
conda upgrade pandas
If you need to have both versions of pandas on your machine, consider using virtualenv
Here is the solution without updating pandas or whatever your using.
If you’re using python2
import cPickle
with open('filename.pkl', 'rb') as fo:
dict = cPickle.load(fo, encoding='latin1’)
If you’re using python3
import pickle
with open('filename.pkl', 'rb') as fo:
dict = pickle.load(fo, encoding='latin1’)
Saving and loading in different versions of pandas
using pickle
often does not work. Instead, use pandas.HDFStore
.
When I needed to update pandas but also needed some data saved with pickle in previous versions, I went back and re-saved that data in HDF format instead, when nothing else would work. No problems anymore.
Works for any sort of pandas data structure it seems, even multi-indexed dataframes! In short, if pickling fails after a version upgrade, try HDFStore; it’s more reliable (and more efficient!).
I had this error when I created a pkl file with python 2.7 and was trying to read it with python 3.6
I did:
pd.read_pickle('foo.pkl')
and it worked
In the pandas 0.23.4, there is a better way to fix the problem. Use the pandas.read_pickle
to read the fileobject, like:
pd.read_pickle(open('test_report.pickle', 'rb'))
If you want to read pickled text instead of a file, do
import io
pd.read_pickle(io.BytesIO(pickled_text))
If you face the error – ValueError: Unrecognized compression type: infer
,
explicitly mention the compression type. It could be one of None(no compression), gzip, bz2, xz or zip(depending upon file extension).
pd.read_pickle(io.BytesIO(pickled_text), compression=None)
A flexible way to deal with internal API changes that break unpickling is to implement a custom Unpickler instance.
For example, the pandas.indexes
module has been moved to pandas.core.indexes
. We can write an Unpickler, that adapts the module path accordingly. To do that, we can overwrite the method find_class
:
import sys
class Unpickler(pickle.Unpickler):
def find_class(self, module, name):
'''This method gets called for every module pickle tries to load.'''
# python 2 --> 3 compatibility: __builtin__ has been renamed to builtins
if module == '__builtin__':
module = 'builtins'
# pandas compatibility: in newer versions, pandas.indexes has been moved to pandas.core.indexes
if 'pandas.indexes' in module:
module = module.replace('pandas.indexes', 'pandas.core.indexes')
__import__(module)
return getattr(sys.modules[module], name)
with open('/path/to/pickle.pkl', 'rb') as file:
pdf = Unpickler(file).load()
Importing pandas didn’t throw the error, but rather trying to read a picked pandas dataframe as such:
import numpy as np
import pandas as pd
import matplotlib
import seaborn as sns
sns.set(style="white")
control_data = pd.read_pickle('null_report.pickle')
test_data = pd.read_pickle('test_report.pickle')
The traceback is 165 lines with three concurrent exceptions (whatever that means). Is read_pickle
not compatible with pandas version 17.1 I’m running? How do I unpickle my dataframe for use?
Below is a copy of the traceback:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
45 with open(path, 'rb') as fh:
---> 46 return pkl.load(fh)
47 except (Exception) as e:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
51 with open(path, 'rb') as fh:
---> 52 return pc.load(fh, encoding=encoding, compat=False)
53
C:UserstestAnaconda3libsite-packagespandascompatpickle_compat.py in load(fh, encoding, compat, is_verbose)
115
--> 116 return up.load()
117 except:
C:UserstestAnaconda3libpickle.py in load(self)
1038 assert isinstance(key, bytes_types)
-> 1039 dispatch[key[0]](self)
1040 except _Stop as stopinst:
C:UserstestAnaconda3libpickle.py in load_stack_global(self)
1342 raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343 self.append(self.find_class(module, name))
1344 dispatch[STACK_GLOBAL[0]] = load_stack_global
C:UserstestAnaconda3libpickle.py in find_class(self, module, name)
1383 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384 __import__(module, level=0)
1385 if self.proto >= 4:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in read_pickle(path)
59 try:
---> 60 return try_read(path)
61 except:
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
56 with open(path, 'rb') as fh:
---> 57 return pc.load(fh, encoding=encoding, compat=True)
58
C:UserstestAnaconda3libsite-packagespandascompatpickle_compat.py in load(fh, encoding, compat, is_verbose)
115
--> 116 return up.load()
117 except:
C:UserstestAnaconda3libpickle.py in load(self)
1038 assert isinstance(key, bytes_types)
-> 1039 dispatch[key[0]](self)
1040 except _Stop as stopinst:
C:UserstestAnaconda3libpickle.py in load_stack_global(self)
1342 raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343 self.append(self.find_class(module, name))
1344 dispatch[STACK_GLOBAL[0]] = load_stack_global
C:UserstestAnaconda3libpickle.py in find_class(self, module, name)
1383 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384 __import__(module, level=0)
1385 if self.proto >= 4:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
45 with open(path, 'rb') as fh:
---> 46 return pkl.load(fh)
47 except (Exception) as e:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
51 with open(path, 'rb') as fh:
---> 52 return pc.load(fh, encoding=encoding, compat=False)
53
C:UserstestAnaconda3libsite-packagespandascompatpickle_compat.py in load(fh, encoding, compat, is_verbose)
115
--> 116 return up.load()
117 except:
C:UserstestAnaconda3libpickle.py in load(self)
1038 assert isinstance(key, bytes_types)
-> 1039 dispatch[key[0]](self)
1040 except _Stop as stopinst:
C:UserstestAnaconda3libpickle.py in load_stack_global(self)
1342 raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343 self.append(self.find_class(module, name))
1344 dispatch[STACK_GLOBAL[0]] = load_stack_global
C:UserstestAnaconda3libpickle.py in find_class(self, module, name)
1383 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384 __import__(module, level=0)
1385 if self.proto >= 4:
ImportError: No module named 'pandas.indexes'
During handling of the above exception, another exception occurred:
ImportError Traceback (most recent call last)
<ipython-input-17-3b05fe7d20a4> in <module>()
3 # test_data = np.genfromtxt(fh, usecols=2)
4
----> 5 control_data = pd.read_pickle('null_report.pickle')
6 test_data = pd.read_pickle('test_report.pickle')
7
C:UserstestAnaconda3libsite-packagespandasiopickle.py in read_pickle(path)
61 except:
62 if PY3:
---> 63 return try_read(path, encoding='latin1')
64 raise
C:UserstestAnaconda3libsite-packagespandasiopickle.py in try_read(path, encoding)
55 except:
56 with open(path, 'rb') as fh:
---> 57 return pc.load(fh, encoding=encoding, compat=True)
58
59 try:
C:UserstestAnaconda3libsite-packagespandascompatpickle_compat.py in load(fh, encoding, compat, is_verbose)
114 up.is_verbose = is_verbose
115
--> 116 return up.load()
117 except:
118 raise
C:UserstestAnaconda3libpickle.py in load(self)
1037 raise EOFError
1038 assert isinstance(key, bytes_types)
-> 1039 dispatch[key[0]](self)
1040 except _Stop as stopinst:
1041 return stopinst.value
C:UserstestAnaconda3libpickle.py in load_stack_global(self)
1341 if type(name) is not str or type(module) is not str:
1342 raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343 self.append(self.find_class(module, name))
1344 dispatch[STACK_GLOBAL[0]] = load_stack_global
1345
C:UserstestAnaconda3libpickle.py in find_class(self, module, name)
1382 elif module in _compat_pickle.IMPORT_MAPPING:
1383 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384 __import__(module, level=0)
1385 if self.proto >= 4:
1386 return _getattribute(sys.modules[module], name)[0]
ImportError: No module named 'pandas.indexes'
I also tried loading the pickle file from pickle directly:
via_pickle = pickle.load( open( 'null_report.pickle', "rb" ) )
and got the same error:
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
<ipython-input-23-ba2e3adae1c4> in <module>()
1
----> 2 via_pickle = pickle.load( open( 'null_report.pickle', "rb" ) )
3
4 # control_data = pd.read_pickle('null_report.pickle')
5 # test_data = pd.read_pickle('test_report.pickle')
ImportError: No module named 'pandas.indexes'
I had this problem from trying to open a pickled dataframe made with pandas 0.18.1 using pandas 0.17.1.
If you are using pip, upgrade pandas with:
pip install --upgrade pandas
If you are using a library like anaconda, use:
conda upgrade pandas
If you need to have both versions of pandas on your machine, consider using virtualenv
Here is the solution without updating pandas or whatever your using.
If you’re using python2
import cPickle
with open('filename.pkl', 'rb') as fo:
dict = cPickle.load(fo, encoding='latin1’)
If you’re using python3
import pickle
with open('filename.pkl', 'rb') as fo:
dict = pickle.load(fo, encoding='latin1’)
Saving and loading in different versions of pandas
using pickle
often does not work. Instead, use pandas.HDFStore
.
When I needed to update pandas but also needed some data saved with pickle in previous versions, I went back and re-saved that data in HDF format instead, when nothing else would work. No problems anymore.
Works for any sort of pandas data structure it seems, even multi-indexed dataframes! In short, if pickling fails after a version upgrade, try HDFStore; it’s more reliable (and more efficient!).
I had this error when I created a pkl file with python 2.7 and was trying to read it with python 3.6
I did:
pd.read_pickle('foo.pkl')
and it worked
In the pandas 0.23.4, there is a better way to fix the problem. Use the pandas.read_pickle
to read the fileobject, like:
pd.read_pickle(open('test_report.pickle', 'rb'))
If you want to read pickled text instead of a file, do
import io
pd.read_pickle(io.BytesIO(pickled_text))
If you face the error – ValueError: Unrecognized compression type: infer
,
explicitly mention the compression type. It could be one of None(no compression), gzip, bz2, xz or zip(depending upon file extension).
pd.read_pickle(io.BytesIO(pickled_text), compression=None)
A flexible way to deal with internal API changes that break unpickling is to implement a custom Unpickler instance.
For example, the pandas.indexes
module has been moved to pandas.core.indexes
. We can write an Unpickler, that adapts the module path accordingly. To do that, we can overwrite the method find_class
:
import sys
class Unpickler(pickle.Unpickler):
def find_class(self, module, name):
'''This method gets called for every module pickle tries to load.'''
# python 2 --> 3 compatibility: __builtin__ has been renamed to builtins
if module == '__builtin__':
module = 'builtins'
# pandas compatibility: in newer versions, pandas.indexes has been moved to pandas.core.indexes
if 'pandas.indexes' in module:
module = module.replace('pandas.indexes', 'pandas.core.indexes')
__import__(module)
return getattr(sys.modules[module], name)
with open('/path/to/pickle.pkl', 'rb') as file:
pdf = Unpickler(file).load()