Unable to load files using pickle and multiple modules
Question:
I’m trying to create a user system, which uses a setting and Gui module, and when the GUI module requests for the file to load up using pickle, I keep getting an attribute error. this is from the settings module:
import pickle
import hashlib
class User(object):
def __init__(self, fname, lname, dob, gender):
self.firstname = fname
self.lastname = lname
self._dob = dob
self.gender = gender
self.type = 'General'
self._username = ''
self._hashkey = ''
def Report(self):
print("Full Name: {0} {1}nDate of Birth: {2}nGender: {3}nAccess Level: {4}".format(self.firstname,self.lastname, self._dob, self.gender, self.type))
print(self._username)
def Genusername(self):
self._username = str(str(self._dob)[:2] + self.firstname[:2] + self.lastname[:2])
saveUsers(users)
def Genhashkey(self, password):
encoded = password.encode('utf-8','strict')
return hashlib.sha256(encoded).hexdigest()
def Verifypassword(self, password):
if self._hashkey == self.Genhashkey(password):
return True
else:
return False
class SAdmin(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Stock Admin'
class Manager(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Manager'
def saveUsers(users):
with open('user_data.pkl', 'wb') as file:
pickle.dump(users, file, -1) # PICKLE HIGHEST LEVEL PROTOCOL
def loadUsers(users):
try:
with open('user_data.pkl', 'rb') as file:
temp = pickle.load(file)
for item in temp:
users.append(item)
except IOError:
saveUsers([])
def userReport(users):
for user in users:
print(user.firstname, user.lastname)
def addUser(users):
fname = input('What is your First Name?n > ')
lname = input('What is your Last Name?n > ')
dob = int(input('Please enter your date of birth in the following format, example 12211996n> '))
gender = input("What is your gender? 'M' or 'F'n >")
level = input("Enter the access level given to this user 'G', 'A', 'M'n > ")
password = input("Enter a password:n > ")
if level == 'G':
usertype = User
if level == 'A':
usertype = SAdmin
if level == 'M':
usertype = Manager
users.append(usertype(fname, lname, dob, gender))
user = users[len(users)-1]
user.Genusername()
user._hashkey = user.Genhashkey(password)
saveUsers(users)
def deleteUser(users):
userReport(users)
delete = input('Please type in the First Name of the user do you wish to delete:n > ')
for user in users:
if user.firstname == delete:
users.remove(user)
saveUsers(users)
def changePass(users):
userReport(users)
change = input('Please type in the First Name of the user you wish to change the password for :n > ')
for user in users:
if user.firstname == change:
oldpass = input('Please type in your old password:n > ')
newpass = input('Please type in your new password:n > ')
if user.Verifypassword(oldpass):
user._hashkey = user.Genhashkey(newpass)
saveUsers(users)
else:
print('Your old password does not match!')
def verifyUser(username, password):
for user in users:
if user._username == username and user.Verifypassword(password):
return True
else:
return False
if __name__ == '__main__':
users = []
loadUsers(users)
and this is the GUI module:
from PyQt4 import QtGui, QtCore
import Settings
class loginWindow(QtGui.QDialog):
def __init__(self):
super().__init__()
self.initUI()
def initUI(self):
self.lbl1 = QtGui.QLabel('Username')
self.lbl2 = QtGui.QLabel('Password')
self.username = QtGui.QLineEdit()
self.password = QtGui.QLineEdit()
self.okButton = QtGui.QPushButton("OK")
self.okButton.clicked.connect(self.tryLogin)
self.cancelButton = QtGui.QPushButton("Cancel")
grid = QtGui.QGridLayout()
grid.setSpacing(10)
grid.addWidget(self.lbl1, 1, 0)
grid.addWidget(self.username, 1, 1)
grid.addWidget(self.lbl2, 2, 0)
grid.addWidget(self.password, 2, 1)
grid.addWidget(self.okButton, 3, 1)
grid.addWidget(self.cancelButton, 3, 0)
self.setLayout(grid)
self.setGeometry(300, 300, 2950, 150)
self.setWindowTitle('Login')
self.show()
def tryLogin(self):
print(self.username.text(), self.password.text())
if Settings.verifyUser(self.username.text(),self.password.text()):
print('it Woks')
else:
QtGui.QMessageBox.warning(
self, 'Error', 'Incorrect Username or Password')
class Window(QtGui.QMainWindow):
def __init__(self):
super().__init__()
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
users = []
Settings.loadUsers(users)
if loginWindow().exec_() == QtGui.QDialog.Accepted:
window = Window()
window.show()
sys.exit(app.exec_())
each user is a class and are put into a list and then the list is saved using pickle when I load up just the settings file and verify the login everything works fine but when I open up the GUI module and try to verify it doesn’t let me, the error I’m getting:
Traceback (most recent call last):
File "C:Users`ProgramLoginGUI.py", line 53, in <module>
Settings.loadUsers(users)
File "C:UsersProgramSettings.py", line 51, in loadUsers
temp = pickle.load(file)
AttributeError: Can't get attribute 'Manager' on <module '__main__' (built-in)>
Answers:
The issue is that you’re pickling objects defined in Settings by actually running the ‘Settings’ module, then you’re trying to unpickle the objects from the GUI module.
Remember that pickle doesn’t actually store information about how a class/object is constructed, and needs access to the class when unpickling. See wiki on using Pickle for more details.
In the pkl data, you see that the object being referenced is __main__.Manager, as the ‘Settings’ module was main when you created the pickle file (i.e. you ran the ‘Settings’ module as the main script to invoke the addUser function).
Then, you try unpickling in ‘Gui’ – so that module has the name __main__, and you’re importing Setting within that module. So of course the Manager class will actually be Settings.Manager. But the pkl file doesn’t know this, and looks for the Manager class within __main__, and throws an AttributeError because it doesn’t exist (Settings.Manager does, but __main__.Manager doesn’t).
Here’s a minimal code set to demonstrate.
The class_def.py module:
import pickle
class Foo(object):
def __init__(self, name):
self.name = name
def main():
foo = Foo('a')
with open('test_data.pkl', 'wb') as f:
pickle.dump([foo], f, -1)
if __name__=='__main__':
main()
You run the above to generate the pickle data.
The main_module.py module:
import pickle
import class_def
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
You run the above to attempt to open the pickle file, and this throws roughly the same error that you were seeing. (Slightly different, but I’m guessing that’s because I’m on Python 2.7)
The solution is either:
- You make the class available within the namespace of the top-level module (i.e. GUI or main_module) through an explicit import, or
- You create the pickle file from the same top-level module as the one that you will open it in (i.e. call
Settings.addUser from GUI, or class_def.main from main_module). This means that the pkl file will save the objects as Settings.Manager or class_def.Foo, which can then be found in the GUI`main_module` namespace.
Option 1 example:
import pickle
import class_def
from class_def import Foo # Import Foo into main_module's namespace explicitly
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
Option 2 example:
import pickle
import class_def
if __name__=='__main__':
class_def.main() # Objects are being pickled with main_module as the top-level
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
Please first read the answer mentioned by zehnpaard to know the reason for the attribute error. Other than the solution he already provided, in python3 you can use the pickle.Unpickler class and override the find_class method as mentioned below:
import pickle
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
if name == 'Manager':
from settings import Manager
return Manager
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
If you have a class defined outside the module, whose object is in pickle data,
you have to import the class
from outside_module import DefinedClass1, DefinedClass2, DefinedClass3
with open('pickle_file.pkl', 'rb') as f:
pickle_data = pickle.load(f)
If you’re still getting this error even after importing the appropriate classes in the loading module (zehnpaard’s solution #1), then the find_class function of pickle.Unpickler can be overwritten and explicitly directed to look in the current module’s namespace.
import pickle
from settings import Manager
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
try:
return super().find_class(__name__, name)
except AttributeError:
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
## No exception trying to get 'Manager'
Note: This method loses the relative-import path information stored in module. So, be careful of namespace collisions in your pickled classes.
if you use dill dump/load model will work
import dill
from sklearn.preprocessing import FunctionTransformer
sp_clf = FunctionTransformer(lambda X:X.astype('float').fillna(0).applymap(abs))
with open('temp.joblib','wb') as io:
dill.dump(sp_clf,io)
with open('temp.joblib','rb') as io:
dd=dill.load(io)
I’m trying to create a user system, which uses a setting and Gui module, and when the GUI module requests for the file to load up using pickle, I keep getting an attribute error. this is from the settings module:
import pickle
import hashlib
class User(object):
def __init__(self, fname, lname, dob, gender):
self.firstname = fname
self.lastname = lname
self._dob = dob
self.gender = gender
self.type = 'General'
self._username = ''
self._hashkey = ''
def Report(self):
print("Full Name: {0} {1}nDate of Birth: {2}nGender: {3}nAccess Level: {4}".format(self.firstname,self.lastname, self._dob, self.gender, self.type))
print(self._username)
def Genusername(self):
self._username = str(str(self._dob)[:2] + self.firstname[:2] + self.lastname[:2])
saveUsers(users)
def Genhashkey(self, password):
encoded = password.encode('utf-8','strict')
return hashlib.sha256(encoded).hexdigest()
def Verifypassword(self, password):
if self._hashkey == self.Genhashkey(password):
return True
else:
return False
class SAdmin(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Stock Admin'
class Manager(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Manager'
def saveUsers(users):
with open('user_data.pkl', 'wb') as file:
pickle.dump(users, file, -1) # PICKLE HIGHEST LEVEL PROTOCOL
def loadUsers(users):
try:
with open('user_data.pkl', 'rb') as file:
temp = pickle.load(file)
for item in temp:
users.append(item)
except IOError:
saveUsers([])
def userReport(users):
for user in users:
print(user.firstname, user.lastname)
def addUser(users):
fname = input('What is your First Name?n > ')
lname = input('What is your Last Name?n > ')
dob = int(input('Please enter your date of birth in the following format, example 12211996n> '))
gender = input("What is your gender? 'M' or 'F'n >")
level = input("Enter the access level given to this user 'G', 'A', 'M'n > ")
password = input("Enter a password:n > ")
if level == 'G':
usertype = User
if level == 'A':
usertype = SAdmin
if level == 'M':
usertype = Manager
users.append(usertype(fname, lname, dob, gender))
user = users[len(users)-1]
user.Genusername()
user._hashkey = user.Genhashkey(password)
saveUsers(users)
def deleteUser(users):
userReport(users)
delete = input('Please type in the First Name of the user do you wish to delete:n > ')
for user in users:
if user.firstname == delete:
users.remove(user)
saveUsers(users)
def changePass(users):
userReport(users)
change = input('Please type in the First Name of the user you wish to change the password for :n > ')
for user in users:
if user.firstname == change:
oldpass = input('Please type in your old password:n > ')
newpass = input('Please type in your new password:n > ')
if user.Verifypassword(oldpass):
user._hashkey = user.Genhashkey(newpass)
saveUsers(users)
else:
print('Your old password does not match!')
def verifyUser(username, password):
for user in users:
if user._username == username and user.Verifypassword(password):
return True
else:
return False
if __name__ == '__main__':
users = []
loadUsers(users)
and this is the GUI module:
from PyQt4 import QtGui, QtCore
import Settings
class loginWindow(QtGui.QDialog):
def __init__(self):
super().__init__()
self.initUI()
def initUI(self):
self.lbl1 = QtGui.QLabel('Username')
self.lbl2 = QtGui.QLabel('Password')
self.username = QtGui.QLineEdit()
self.password = QtGui.QLineEdit()
self.okButton = QtGui.QPushButton("OK")
self.okButton.clicked.connect(self.tryLogin)
self.cancelButton = QtGui.QPushButton("Cancel")
grid = QtGui.QGridLayout()
grid.setSpacing(10)
grid.addWidget(self.lbl1, 1, 0)
grid.addWidget(self.username, 1, 1)
grid.addWidget(self.lbl2, 2, 0)
grid.addWidget(self.password, 2, 1)
grid.addWidget(self.okButton, 3, 1)
grid.addWidget(self.cancelButton, 3, 0)
self.setLayout(grid)
self.setGeometry(300, 300, 2950, 150)
self.setWindowTitle('Login')
self.show()
def tryLogin(self):
print(self.username.text(), self.password.text())
if Settings.verifyUser(self.username.text(),self.password.text()):
print('it Woks')
else:
QtGui.QMessageBox.warning(
self, 'Error', 'Incorrect Username or Password')
class Window(QtGui.QMainWindow):
def __init__(self):
super().__init__()
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
users = []
Settings.loadUsers(users)
if loginWindow().exec_() == QtGui.QDialog.Accepted:
window = Window()
window.show()
sys.exit(app.exec_())
each user is a class and are put into a list and then the list is saved using pickle when I load up just the settings file and verify the login everything works fine but when I open up the GUI module and try to verify it doesn’t let me, the error I’m getting:
Traceback (most recent call last):
File "C:Users`ProgramLoginGUI.py", line 53, in <module>
Settings.loadUsers(users)
File "C:UsersProgramSettings.py", line 51, in loadUsers
temp = pickle.load(file)
AttributeError: Can't get attribute 'Manager' on <module '__main__' (built-in)>
The issue is that you’re pickling objects defined in Settings by actually running the ‘Settings’ module, then you’re trying to unpickle the objects from the GUI module.
Remember that pickle doesn’t actually store information about how a class/object is constructed, and needs access to the class when unpickling. See wiki on using Pickle for more details.
In the pkl data, you see that the object being referenced is __main__.Manager, as the ‘Settings’ module was main when you created the pickle file (i.e. you ran the ‘Settings’ module as the main script to invoke the addUser function).
Then, you try unpickling in ‘Gui’ – so that module has the name __main__, and you’re importing Setting within that module. So of course the Manager class will actually be Settings.Manager. But the pkl file doesn’t know this, and looks for the Manager class within __main__, and throws an AttributeError because it doesn’t exist (Settings.Manager does, but __main__.Manager doesn’t).
Here’s a minimal code set to demonstrate.
The class_def.py module:
import pickle
class Foo(object):
def __init__(self, name):
self.name = name
def main():
foo = Foo('a')
with open('test_data.pkl', 'wb') as f:
pickle.dump([foo], f, -1)
if __name__=='__main__':
main()
You run the above to generate the pickle data.
The main_module.py module:
import pickle
import class_def
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
You run the above to attempt to open the pickle file, and this throws roughly the same error that you were seeing. (Slightly different, but I’m guessing that’s because I’m on Python 2.7)
The solution is either:
- You make the class available within the namespace of the top-level module (i.e. GUI or main_module) through an explicit import, or
- You create the pickle file from the same top-level module as the one that you will open it in (i.e. call
Settings.addUserfrom GUI, orclass_def.mainfrom main_module). This means that the pkl file will save the objects asSettings.Managerorclass_def.Foo, which can then be found in theGUI`main_module` namespace.
Option 1 example:
import pickle
import class_def
from class_def import Foo # Import Foo into main_module's namespace explicitly
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
Option 2 example:
import pickle
import class_def
if __name__=='__main__':
class_def.main() # Objects are being pickled with main_module as the top-level
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
Please first read the answer mentioned by zehnpaard to know the reason for the attribute error. Other than the solution he already provided, in python3 you can use the pickle.Unpickler class and override the find_class method as mentioned below:
import pickle
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
if name == 'Manager':
from settings import Manager
return Manager
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
If you have a class defined outside the module, whose object is in pickle data,
you have to import the class
from outside_module import DefinedClass1, DefinedClass2, DefinedClass3
with open('pickle_file.pkl', 'rb') as f:
pickle_data = pickle.load(f)
If you’re still getting this error even after importing the appropriate classes in the loading module (zehnpaard’s solution #1), then the find_class function of pickle.Unpickler can be overwritten and explicitly directed to look in the current module’s namespace.
import pickle
from settings import Manager
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
try:
return super().find_class(__name__, name)
except AttributeError:
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
## No exception trying to get 'Manager'
Note: This method loses the relative-import path information stored in module. So, be careful of namespace collisions in your pickled classes.
if you use dill dump/load model will work
import dill
from sklearn.preprocessing import FunctionTransformer
sp_clf = FunctionTransformer(lambda X:X.astype('float').fillna(0).applymap(abs))
with open('temp.joblib','wb') as io:
dill.dump(sp_clf,io)
with open('temp.joblib','rb') as io:
dd=dill.load(io)