Accessing an attribute of a multiprocessing Proxy of a class
Question:
I have a class that I want to share in a read-only fashion with children processes in a pool, so I prepared a proxy of a class but it didn’t work. The following is a simplified example of my problem.
from multiprocessing.managers import BaseManager
class TestClass:
def __init__(self, a):
self.a = a
def b(self):
print self.a
class MyManager(BaseManager): pass
MyManager.register('test', TestClass)
if __name__ == '__main__':
manager = MyManager()
manager.start()
t = TestClass(1)
print t.a
mt = manager.test(2)
mt.b()
mt.a
When I run this code I get:
1
2
Traceback (most recent call last):
File "multiprocess_example_stackexchange.py", line 20, in <module>
mt.a
AttributeError: 'AutoProxy[test]' object has no attribute 'a'
It seems that I cannot access the attribute of a shared object directly via a proxy. Is the only way using a method that gets the attribute, or am I doing something wrong?
Answers:
The Proxy objects used by multiprocessing.BaseManager and its sub-classes normally only expose methods from the objects they’re referring to, not attributes. Now, there is multiprocessing.Manager().Namespace, which provides a Proxy sub-class that does provide access to attributes, rather than methods. We can create our own Proxy type which inherits from that, which enables access to all our attributes, as well as access to our b function:
from multiprocessing.managers import BaseManager, NamespaceProxy
class TestClass(object):
def __init__(self, a):
self.a = a
def b(self):
print self.a
class MyManager(BaseManager): pass
class TestProxy(NamespaceProxy):
# We need to expose the same __dunder__ methods as NamespaceProxy,
# in addition to the b method.
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__', 'b')
def b(self):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('b')
MyManager.register('test', TestClass, TestProxy)
if __name__ == '__main__':
manager = MyManager()
manager.start()
t = TestClass(1)
print t.a
mt = manager.test(2)
print mt.a
mt.a = 5
mt.b()
Output:
1
2
5
Edit:
If you want to be able to dynamically add methods from your original class to a Proxy class, you can do something like this:
from multiprocessing.managers import BaseManager, NamespaceProxy
import inspect
class TestClass(object):
def __init__(self, a):
self.a = a
def b(self):
print self.a
class AnotherClass(object):
def __init__(self, a):
self.a = a
def c(self):
print self.a
class MyManager(BaseManager): pass
class ProxyBase(NamespaceProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
class TestProxy(ProxyBase): pass
class AnotherProxy(ProxyBase): pass
def register_proxy(name, cls, proxy):
for attr in dir(cls):
if inspect.ismethod(getattr(cls, attr)) and not attr.startswith("__"):
proxy._exposed_ += (attr,)
setattr(proxy, attr,
lambda s: object.__getattribute__(s, '_callmethod')(attr))
MyManager.register(name, cls, proxy)
register_proxy('test', TestClass, TestProxy)
register_proxy('another', AnotherClass, AnotherProxy)
if __name__ == '__main__':
manager = MyManager()
manager.start()
mt = manager.test(2)
ma = manager.another(3)
mt.b()
ma.c()
mt.a = 5
ma.a = 6
mt.b()
ma.c()
Here’s a less verbose alternative that I found to work well in practice. Not sure if there are any disadvantages.
class TestClass:
def __init__(self, a):
self.a = a
def b(self):
print self.a
def wrap_test_class(*args, **kwargs):
obj = TestClass(*args, **kwargs)
obj.get_a = lambda: obj.a
return obj
class MyManager(BaseManager): pass
MyManager.register('test', wrap_test_class)
This allows you to access a by calling proxy_object.get_a()
This is an example of passing parameters (example: __getitem__) or not (example: __len__):
class TestProxy(NamespaceProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__','__len__','__getitem__')
def __len__(self):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('__len__')
def __getitem__(self,index):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('__getitem__',(index,))
After spending few hours to reading the source codes, here is the simplest ways to implement the proxy class to expose all attributes and methods:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
self._callmethod(name, args)
return wrapper
return result
BaseManager.register('Test', Test, TestProxy)
manager = BaseManager()
test = manager.Test()
Also, here is an auto proxy method:
def Proxy(target):
dic = {'types': types}
exec('''def __getattr__(self, key):
result = self._callmethod('__getattribute__', (key,))
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
self._callmethod(key, args)
return wrapper
return result''', dic)
proxyName = target.__name__ + "Proxy"
ProxyType = type(proxyName, (NamespaceProxy,), dic)
ProxyType._exposed_ = tuple(dir(target))
return ProxyType
TestProxy = Proxy(Test)
BaseManager.register('Test', Test, TestProxy)
manager = BaseManager()
test = manager.Test()
Since I do not have enough reputation to comment, I am posting an answer. The otherwise excellent answer by @shtse8 has a bug. I wanted to point this out since this page is one of the top hits when you search for queries regarding NamespaceProxy and the said answer has been used by others as well.
The problem lies in the below code:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
self._callmethod(name, args) # Result not returned
return wrapper
return result
If you use this class (or the equivalent ‘auto proxy’ method mentioned in the answer) to create proxy objects, then all functions which are programmed to return a value will always return a NoneType instead (if you access the functions from the proxy). Moreover, they will not pass on any keyword arguments either. This is because in the wrapper the result of the method call use kwargs and neither is the call itself being returned. Therefore, we need to pass kwargs and put a return in the line self._callmethod(name, args).
The TestProxy class should then become:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(name, args, kwargs) # Note the return here
return wrapper
return result
The ‘auto proxy’ function would then become:
def Proxy(target):
dic = {'types': types}
exec('''def __getattr__(self, key):
result = self._callmethod('__getattribute__', (key,))
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(key, args, kwargs)
return wrapper
return result''', dic)
proxyName = target.__name__ + "Proxy"
ProxyType = type(proxyName, (NamespaceProxy,), dic)
ProxyType._exposed_ = tuple(dir(target))
return ProxyType
Update: Edited code and explanation to pass kwargs as well. Check Viktor’s answer below
Charcit’s solution was working for me except I made a small completion/bugfixing. There kwargs cannot be passed to the called methods. So the fixed version:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(name, args, kwargs) # args and kwargs!
return wrapper
return result
Didn’t test the ‘autoproxy’ method but this fix should apply there too:
def Proxy(target):
dic = {'types': types}
exec('''def __getattr__(self, key):
result = self._callmethod('__getattribute__', (key,))
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(key, args, kwargs)
return wrapper
return result''', dic)
proxyName = target.__name__ + "Proxy"
ProxyType = type(proxyName, (NamespaceProxy,), dic)
ProxyType._exposed_ = tuple(dir(target))
return ProxyType
I was originally using the answer by @shtse8 (as modified by @Charchit-Agarwal and @Viktor), which mostly worked for me, but it was trying to pickle the methods returned by the __getattr__ function, which didn’t work in some use cases.
So here is another solution which is closer to the implementation of NamespaceProxy, but adds in the public methods the same way AutoProxy does.
def MakeProxyWithAttrs(target):
""" Create a derived NamespaceProxy class for `target`. """
# This bit follows what multiprocessing.managers.MakeProxy normally does.
dic = {}
public_methods = [m for m in dir(target) if m[0] != '_']
for meth in public_methods:
exec('''def %s(self, *args, **kwds):
return self._callmethod(%r, args, kwds)
'''%(meth,meth), dic)
# NamespaceProxy starts with __getattribute__ defined, so subclass from that
# rather than BaseProxy, as MakeProxy normally does.
proxy_name = target.__name__ + "_Proxy"
ProxyType = type(proxy_name, (NamespaceProxy,), dic)
# Expose all the public methods and also __getattribute__ and __setattr__.
ProxyType._exposed_ = tuple(public_methods + ['__getattribute__', '__setattr__'])
return ProxyType
class Manager(multiprocessing.managers.BaseManager): pass
test_proxy = MakeProxyWithAttrs(test_func)
Manager.register('test', test_func, test_proxy)
manager = Manager()
manager.start()
test = manager.test()
I have a class that I want to share in a read-only fashion with children processes in a pool, so I prepared a proxy of a class but it didn’t work. The following is a simplified example of my problem.
from multiprocessing.managers import BaseManager
class TestClass:
def __init__(self, a):
self.a = a
def b(self):
print self.a
class MyManager(BaseManager): pass
MyManager.register('test', TestClass)
if __name__ == '__main__':
manager = MyManager()
manager.start()
t = TestClass(1)
print t.a
mt = manager.test(2)
mt.b()
mt.a
When I run this code I get:
1
2
Traceback (most recent call last):
File "multiprocess_example_stackexchange.py", line 20, in <module>
mt.a
AttributeError: 'AutoProxy[test]' object has no attribute 'a'
It seems that I cannot access the attribute of a shared object directly via a proxy. Is the only way using a method that gets the attribute, or am I doing something wrong?
The Proxy objects used by multiprocessing.BaseManager and its sub-classes normally only expose methods from the objects they’re referring to, not attributes. Now, there is multiprocessing.Manager().Namespace, which provides a Proxy sub-class that does provide access to attributes, rather than methods. We can create our own Proxy type which inherits from that, which enables access to all our attributes, as well as access to our b function:
from multiprocessing.managers import BaseManager, NamespaceProxy
class TestClass(object):
def __init__(self, a):
self.a = a
def b(self):
print self.a
class MyManager(BaseManager): pass
class TestProxy(NamespaceProxy):
# We need to expose the same __dunder__ methods as NamespaceProxy,
# in addition to the b method.
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__', 'b')
def b(self):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('b')
MyManager.register('test', TestClass, TestProxy)
if __name__ == '__main__':
manager = MyManager()
manager.start()
t = TestClass(1)
print t.a
mt = manager.test(2)
print mt.a
mt.a = 5
mt.b()
Output:
1
2
5
Edit:
If you want to be able to dynamically add methods from your original class to a Proxy class, you can do something like this:
from multiprocessing.managers import BaseManager, NamespaceProxy
import inspect
class TestClass(object):
def __init__(self, a):
self.a = a
def b(self):
print self.a
class AnotherClass(object):
def __init__(self, a):
self.a = a
def c(self):
print self.a
class MyManager(BaseManager): pass
class ProxyBase(NamespaceProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
class TestProxy(ProxyBase): pass
class AnotherProxy(ProxyBase): pass
def register_proxy(name, cls, proxy):
for attr in dir(cls):
if inspect.ismethod(getattr(cls, attr)) and not attr.startswith("__"):
proxy._exposed_ += (attr,)
setattr(proxy, attr,
lambda s: object.__getattribute__(s, '_callmethod')(attr))
MyManager.register(name, cls, proxy)
register_proxy('test', TestClass, TestProxy)
register_proxy('another', AnotherClass, AnotherProxy)
if __name__ == '__main__':
manager = MyManager()
manager.start()
mt = manager.test(2)
ma = manager.another(3)
mt.b()
ma.c()
mt.a = 5
ma.a = 6
mt.b()
ma.c()
Here’s a less verbose alternative that I found to work well in practice. Not sure if there are any disadvantages.
class TestClass:
def __init__(self, a):
self.a = a
def b(self):
print self.a
def wrap_test_class(*args, **kwargs):
obj = TestClass(*args, **kwargs)
obj.get_a = lambda: obj.a
return obj
class MyManager(BaseManager): pass
MyManager.register('test', wrap_test_class)
This allows you to access a by calling proxy_object.get_a()
This is an example of passing parameters (example: __getitem__) or not (example: __len__):
class TestProxy(NamespaceProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__','__len__','__getitem__')
def __len__(self):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('__len__')
def __getitem__(self,index):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('__getitem__',(index,))
After spending few hours to reading the source codes, here is the simplest ways to implement the proxy class to expose all attributes and methods:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
self._callmethod(name, args)
return wrapper
return result
BaseManager.register('Test', Test, TestProxy)
manager = BaseManager()
test = manager.Test()
Also, here is an auto proxy method:
def Proxy(target):
dic = {'types': types}
exec('''def __getattr__(self, key):
result = self._callmethod('__getattribute__', (key,))
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
self._callmethod(key, args)
return wrapper
return result''', dic)
proxyName = target.__name__ + "Proxy"
ProxyType = type(proxyName, (NamespaceProxy,), dic)
ProxyType._exposed_ = tuple(dir(target))
return ProxyType
TestProxy = Proxy(Test)
BaseManager.register('Test', Test, TestProxy)
manager = BaseManager()
test = manager.Test()
Since I do not have enough reputation to comment, I am posting an answer. The otherwise excellent answer by @shtse8 has a bug. I wanted to point this out since this page is one of the top hits when you search for queries regarding NamespaceProxy and the said answer has been used by others as well.
The problem lies in the below code:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
self._callmethod(name, args) # Result not returned
return wrapper
return result
If you use this class (or the equivalent ‘auto proxy’ method mentioned in the answer) to create proxy objects, then all functions which are programmed to return a value will always return a NoneType instead (if you access the functions from the proxy). Moreover, they will not pass on any keyword arguments either. This is because in the wrapper the result of the method call use kwargs and neither is the call itself being returned. Therefore, we need to pass kwargs and put a return in the line self._callmethod(name, args).
The TestProxy class should then become:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(name, args, kwargs) # Note the return here
return wrapper
return result
The ‘auto proxy’ function would then become:
def Proxy(target):
dic = {'types': types}
exec('''def __getattr__(self, key):
result = self._callmethod('__getattribute__', (key,))
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(key, args, kwargs)
return wrapper
return result''', dic)
proxyName = target.__name__ + "Proxy"
ProxyType = type(proxyName, (NamespaceProxy,), dic)
ProxyType._exposed_ = tuple(dir(target))
return ProxyType
Update: Edited code and explanation to pass kwargs as well. Check Viktor’s answer below
Charcit’s solution was working for me except I made a small completion/bugfixing. There kwargs cannot be passed to the called methods. So the fixed version:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(name, args, kwargs) # args and kwargs!
return wrapper
return result
Didn’t test the ‘autoproxy’ method but this fix should apply there too:
def Proxy(target):
dic = {'types': types}
exec('''def __getattr__(self, key):
result = self._callmethod('__getattribute__', (key,))
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(key, args, kwargs)
return wrapper
return result''', dic)
proxyName = target.__name__ + "Proxy"
ProxyType = type(proxyName, (NamespaceProxy,), dic)
ProxyType._exposed_ = tuple(dir(target))
return ProxyType
I was originally using the answer by @shtse8 (as modified by @Charchit-Agarwal and @Viktor), which mostly worked for me, but it was trying to pickle the methods returned by the __getattr__ function, which didn’t work in some use cases.
So here is another solution which is closer to the implementation of NamespaceProxy, but adds in the public methods the same way AutoProxy does.
def MakeProxyWithAttrs(target):
""" Create a derived NamespaceProxy class for `target`. """
# This bit follows what multiprocessing.managers.MakeProxy normally does.
dic = {}
public_methods = [m for m in dir(target) if m[0] != '_']
for meth in public_methods:
exec('''def %s(self, *args, **kwds):
return self._callmethod(%r, args, kwds)
'''%(meth,meth), dic)
# NamespaceProxy starts with __getattribute__ defined, so subclass from that
# rather than BaseProxy, as MakeProxy normally does.
proxy_name = target.__name__ + "_Proxy"
ProxyType = type(proxy_name, (NamespaceProxy,), dic)
# Expose all the public methods and also __getattribute__ and __setattr__.
ProxyType._exposed_ = tuple(public_methods + ['__getattribute__', '__setattr__'])
return ProxyType
class Manager(multiprocessing.managers.BaseManager): pass
test_proxy = MakeProxyWithAttrs(test_func)
Manager.register('test', test_func, test_proxy)
manager = Manager()
manager.start()
test = manager.test()