Python: Getting a traceback from a multiprocessing.Process
Question:
I am trying to get hold of a traceback object from a multiprocessing.Process.
Unfortunately passing the exception info through a pipe does not work because traceback objects can not be pickled:
def foo(pipe_to_parent):
try:
raise Exception('xxx')
except:
pipe_to_parent.send(sys.exc_info())
to_child, to_self = multiprocessing.Pipe()
process = multiprocessing.Process(target = foo, args = (to_self,))
process.start()
exc_info = to_child.recv()
process.join()
print traceback.format_exception(*exc_info)
to_child.close()
to_self.close()
Traceback:
Traceback (most recent call last):
File "/usr/lib/python2.6/multiprocessing/process.py", line 231, in _bootstrap
self.run()
File "/usr/lib/python2.6/multiprocessing/process.py", line 88, in run
self._target(*self._args, **self._kwargs)
File "foo", line 7, in foo
to_parent.send(sys.exc_info())
PicklingError: Can't pickle <type 'traceback'>: attribute lookup __builtin__.traceback failed
Is there another way to access the exception info? I’d like to avoid passing the formatted string.
Answers:
It seems to be difficult to made picklable the traceback object.
But you can only send the 2 first items of sys.exc_info()
, and a preformated traceback information with the traceback.extract_tb method :
import multiprocessing
import sys
import traceback
def foo(pipe_to_parent):
try:
raise Exception('xxx')
except:
except_type, except_class, tb = sys.exc_info()
pipe_to_parent.send((except_type, except_class, traceback.extract_tb(tb)))
to_child, to_self = multiprocessing.Pipe()
process = multiprocessing.Process(target = foo, args = (to_self,))
process.start()
exc_info = to_child.recv()
process.join()
print exc_info
to_child.close()
to_self.close()
which give you :
(<type 'exceptions.Exception'>, Exception('xxx',), [('test_tb.py', 7, 'foo', "raise Exception('xxx')")])
And then, you’ll be able to grab more informations about the exception cause (filename, line number where exception raised, method name and the statement that raise the exception)
Since multiprocessing
does print the string contents of exceptions raised in child processes, you can wrap all your child process code in a try-except that catches any exceptions, formats the relavent stack traces, and raises a new Exception
that holds all the relevant information in its string:
An example of a function I use with multiprocessing.map
:
def run_functor(functor):
"""
Given a no-argument functor, run it and return its result. We can
use this with multiprocessing.map and map it over a list of job
functors to do them.
Handles getting more than multiprocessing's pitiful exception output
"""
try:
# This is where you do your actual work
return functor()
except:
# Put all exception text into an exception and raise that
raise Exception("".join(traceback.format_exception(*sys.exc_info())))
What you get is a stack trace with another formatted stack trace as the error message, which helps with debugging.
Using tblib
you can pass wrapped exceptions and reraise them later:
import tblib.pickling_support
tblib.pickling_support.install()
from multiprocessing import Pool
import sys
class ExceptionWrapper(object):
def __init__(self, ee):
self.ee = ee
__, __, self.tb = sys.exc_info()
def re_raise(self):
raise self.ee.with_traceback(self.tb)
# for Python 2 replace the previous line by:
# raise self.ee, None, self.tb
# example of how to use ExceptionWrapper
def inverse(i):
""" will fail for i == 0 """
try:
return 1.0 / i
except Exception as e:
return ExceptionWrapper(e)
def main():
p = Pool(1)
results = p.map(inverse, [0, 1, 2, 3])
for result in results:
if isinstance(result, ExceptionWrapper):
result.re_raise()
if __name__ == "__main__":
main()
So, if you catch an exception in your remote process, wrap it with ExceptionWrapper
and then pass it back. Calling re_raise()
in the main process will do the work.
This is a variation of this excellent answer. Both are relying on tblib for storing the traceback.
However, instead of having to return the exception object (as asked for by the OP), the worker
function can be left as-is and is just wrapped in try
/except
to store exceptions for re-raise.
import tblib.pickling_support
tblib.pickling_support.install()
import sys
class DelayedException(Exception):
def __init__(self, ee):
self.ee = ee
__, __, self.tb = sys.exc_info()
super(DelayedException, self).__init__(str(ee))
def re_raise(self):
raise self.ee, None, self.tb
Example
def worker():
try:
raise ValueError('Something went wrong.')
except Exception as e:
raise DelayedException(e)
if __name__ == '__main__':
import multiprocessing
pool = multiprocessing.Pool()
try:
pool.imap(worker, [1, 2, 3])
except DelayedException as e:
e.re_raise()
Python 3
In Python 3, now the get
method of multiprocessing.pool.Async
returns full traceback, see http://bugs.python.org/issue13831.
Python 2
Use the traceback.format_exc
(which means formatted expetion) to get the traceback string.
It would be much more covenient with making a decorator as below.
def full_traceback(func):
import traceback, functools
@functools.wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
msg = "{}nnOriginal {}".format(e, traceback.format_exc())
raise type(e)(msg)
return wrapper
Example:
def func0():
raise NameError("func0 exception")
def func1():
return func0()
# Key is here!
@full_traceback
def main(i):
return func1()
if __name__ == '__main__':
from multiprocessing import Pool
pool = Pool(4)
try:
results = pool.map_async(main, range(5)).get(1e5)
finally:
pool.close()
pool.join()
The traceback with the decorator:
Traceback (most recent call last):
File "bt.py", line 34, in <module>
results = pool.map_async(main, range(5)).get(1e5)
File "/opt/anaconda/lib/python2.7/multiprocessing/pool.py", line 567, in get
raise self._value
NameError: Exception in func0
Original Traceback (most recent call last):
File "bt.py", line 13, in wrapper
return func(*args, **kwargs)
File "bt.py", line 27, in main
return func1()
File "bt.py", line 23, in func1
return func0()
File "bt.py", line 20, in func0
raise NameError("Exception in func0")
NameError: Exception in func0
The traceback without the decorator:
Traceback (most recent call last):
File "bt.py", line 34, in <module>
results = pool.map_async(main, range(5)).get(1e5)
File "/opt/anaconda/lib/python2.7/multiprocessing/pool.py", line 567, in get
raise self._value
NameError: Exception in func0
The same solutions as @Syrtis Major and @interfect but, tested with Python 3.6:
import sys
import traceback
import functools
def catch_remote_exceptions(wrapped_function):
""" https://stackoverflow.com/questions/6126007/python-getting-a-traceback """
@functools.wraps(wrapped_function)
def new_function(*args, **kwargs):
try:
return wrapped_function(*args, **kwargs)
except:
raise Exception( "".join(traceback.format_exception(*sys.exc_info())) )
return new_function
Usage:
class ProcessLocker(object):
@catch_remote_exceptions
def __init__(self):
super().__init__()
@catch_remote_exceptions
def create_process_locks(self, total_processes):
self.process_locks = []
# ...
I am trying to get hold of a traceback object from a multiprocessing.Process.
Unfortunately passing the exception info through a pipe does not work because traceback objects can not be pickled:
def foo(pipe_to_parent):
try:
raise Exception('xxx')
except:
pipe_to_parent.send(sys.exc_info())
to_child, to_self = multiprocessing.Pipe()
process = multiprocessing.Process(target = foo, args = (to_self,))
process.start()
exc_info = to_child.recv()
process.join()
print traceback.format_exception(*exc_info)
to_child.close()
to_self.close()
Traceback:
Traceback (most recent call last):
File "/usr/lib/python2.6/multiprocessing/process.py", line 231, in _bootstrap
self.run()
File "/usr/lib/python2.6/multiprocessing/process.py", line 88, in run
self._target(*self._args, **self._kwargs)
File "foo", line 7, in foo
to_parent.send(sys.exc_info())
PicklingError: Can't pickle <type 'traceback'>: attribute lookup __builtin__.traceback failed
Is there another way to access the exception info? I’d like to avoid passing the formatted string.
It seems to be difficult to made picklable the traceback object.
But you can only send the 2 first items of sys.exc_info()
, and a preformated traceback information with the traceback.extract_tb method :
import multiprocessing
import sys
import traceback
def foo(pipe_to_parent):
try:
raise Exception('xxx')
except:
except_type, except_class, tb = sys.exc_info()
pipe_to_parent.send((except_type, except_class, traceback.extract_tb(tb)))
to_child, to_self = multiprocessing.Pipe()
process = multiprocessing.Process(target = foo, args = (to_self,))
process.start()
exc_info = to_child.recv()
process.join()
print exc_info
to_child.close()
to_self.close()
which give you :
(<type 'exceptions.Exception'>, Exception('xxx',), [('test_tb.py', 7, 'foo', "raise Exception('xxx')")])
And then, you’ll be able to grab more informations about the exception cause (filename, line number where exception raised, method name and the statement that raise the exception)
Since multiprocessing
does print the string contents of exceptions raised in child processes, you can wrap all your child process code in a try-except that catches any exceptions, formats the relavent stack traces, and raises a new Exception
that holds all the relevant information in its string:
An example of a function I use with multiprocessing.map
:
def run_functor(functor):
"""
Given a no-argument functor, run it and return its result. We can
use this with multiprocessing.map and map it over a list of job
functors to do them.
Handles getting more than multiprocessing's pitiful exception output
"""
try:
# This is where you do your actual work
return functor()
except:
# Put all exception text into an exception and raise that
raise Exception("".join(traceback.format_exception(*sys.exc_info())))
What you get is a stack trace with another formatted stack trace as the error message, which helps with debugging.
Using tblib
you can pass wrapped exceptions and reraise them later:
import tblib.pickling_support
tblib.pickling_support.install()
from multiprocessing import Pool
import sys
class ExceptionWrapper(object):
def __init__(self, ee):
self.ee = ee
__, __, self.tb = sys.exc_info()
def re_raise(self):
raise self.ee.with_traceback(self.tb)
# for Python 2 replace the previous line by:
# raise self.ee, None, self.tb
# example of how to use ExceptionWrapper
def inverse(i):
""" will fail for i == 0 """
try:
return 1.0 / i
except Exception as e:
return ExceptionWrapper(e)
def main():
p = Pool(1)
results = p.map(inverse, [0, 1, 2, 3])
for result in results:
if isinstance(result, ExceptionWrapper):
result.re_raise()
if __name__ == "__main__":
main()
So, if you catch an exception in your remote process, wrap it with ExceptionWrapper
and then pass it back. Calling re_raise()
in the main process will do the work.
This is a variation of this excellent answer. Both are relying on tblib for storing the traceback.
However, instead of having to return the exception object (as asked for by the OP), the worker
function can be left as-is and is just wrapped in try
/except
to store exceptions for re-raise.
import tblib.pickling_support
tblib.pickling_support.install()
import sys
class DelayedException(Exception):
def __init__(self, ee):
self.ee = ee
__, __, self.tb = sys.exc_info()
super(DelayedException, self).__init__(str(ee))
def re_raise(self):
raise self.ee, None, self.tb
Example
def worker():
try:
raise ValueError('Something went wrong.')
except Exception as e:
raise DelayedException(e)
if __name__ == '__main__':
import multiprocessing
pool = multiprocessing.Pool()
try:
pool.imap(worker, [1, 2, 3])
except DelayedException as e:
e.re_raise()
Python 3
In Python 3, now the get
method of multiprocessing.pool.Async
returns full traceback, see http://bugs.python.org/issue13831.
Python 2
Use the traceback.format_exc
(which means formatted expetion) to get the traceback string.
It would be much more covenient with making a decorator as below.
def full_traceback(func):
import traceback, functools
@functools.wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
msg = "{}nnOriginal {}".format(e, traceback.format_exc())
raise type(e)(msg)
return wrapper
Example:
def func0():
raise NameError("func0 exception")
def func1():
return func0()
# Key is here!
@full_traceback
def main(i):
return func1()
if __name__ == '__main__':
from multiprocessing import Pool
pool = Pool(4)
try:
results = pool.map_async(main, range(5)).get(1e5)
finally:
pool.close()
pool.join()
The traceback with the decorator:
Traceback (most recent call last):
File "bt.py", line 34, in <module>
results = pool.map_async(main, range(5)).get(1e5)
File "/opt/anaconda/lib/python2.7/multiprocessing/pool.py", line 567, in get
raise self._value
NameError: Exception in func0
Original Traceback (most recent call last):
File "bt.py", line 13, in wrapper
return func(*args, **kwargs)
File "bt.py", line 27, in main
return func1()
File "bt.py", line 23, in func1
return func0()
File "bt.py", line 20, in func0
raise NameError("Exception in func0")
NameError: Exception in func0
The traceback without the decorator:
Traceback (most recent call last):
File "bt.py", line 34, in <module>
results = pool.map_async(main, range(5)).get(1e5)
File "/opt/anaconda/lib/python2.7/multiprocessing/pool.py", line 567, in get
raise self._value
NameError: Exception in func0
The same solutions as @Syrtis Major and @interfect but, tested with Python 3.6:
import sys
import traceback
import functools
def catch_remote_exceptions(wrapped_function):
""" https://stackoverflow.com/questions/6126007/python-getting-a-traceback """
@functools.wraps(wrapped_function)
def new_function(*args, **kwargs):
try:
return wrapped_function(*args, **kwargs)
except:
raise Exception( "".join(traceback.format_exception(*sys.exc_info())) )
return new_function
Usage:
class ProcessLocker(object):
@catch_remote_exceptions
def __init__(self):
super().__init__()
@catch_remote_exceptions
def create_process_locks(self, total_processes):
self.process_locks = []
# ...