Timeout on subprocess readline in Python
Question:
I have a small issue that I’m not quite sure how to solve. Here is a minimal example:
What I have
scan_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while(some_criterium):
line = scan_process.stdout.readline()
some_criterium = do_something(line)
What I would like
scan_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while(some_criterium):
line = scan_process.stdout.readline()
if nothing_happens_after_10s:
break
else:
some_criterium = do_something(line)
I read a line from a subprocess and do something with it. How can I exit if no line arrived after a fixed time interval?
Answers:
I used something a bit more general in Python (if I remember correctly, also pieced together from Stack Overflow questions, but I cannot recall which ones).
import thread
from threading import Timer
def run_with_timeout(timeout, default, f, *args, **kwargs):
if not timeout:
return f(*args, **kwargs)
try:
timeout_timer = Timer(timeout, thread.interrupt_main)
timeout_timer.start()
result = f(*args, **kwargs)
return result
except KeyboardInterrupt:
return default
finally:
timeout_timer.cancel()
Be warned, though. This uses an interrupt to stop whatever function you give it. This might not be a good idea for all functions and it also prevents you from closing the program with Ctrl + C during the timeout (i.e. Ctrl + C will be handled as a timeout).
You could use this and call it like:
scan_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while(some_criterium):
line = run_with_timeout(timeout, None, scan_process.stdout.readline)
if line is None:
break
else:
some_criterium = do_something(line)
It might be a bit overkill, though. I suspect there is a simpler option for your case that I don’t know.
Thanks for all the answers!
I found a way to solve my problem by simply using select.poll to peek into standard output.
import select
...
scan_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
poll_obj = select.poll()
poll_obj.register(scan_process.stdout, select.POLLIN)
while(some_criterium and not time_limit):
poll_result = poll_obj.poll(0)
if poll_result:
line = scan_process.stdout.readline()
some_criterium = do_something(line)
update(time_limit)
Try using signal.alarm:
#timeout.py
import signal, sys
def timeout(sig, frm):
print "This is taking too long..."
sys.exit(1)
signal.signal(signal.SIGALRM, timeout)
signal.alarm(10)
byte = 0
while 'IT' not in open('/dev/urandom').read(2):
byte += 2
print "I got IT in %s byte(s)!" % byte
A couple of runs to show it works:
$ python timeout.py
This is taking too long...
$ python timeout.py
I got IT in 4672 byte(s)!
For a more detailed example, see pGuides.
While Tom’s solution works, using select()
in the C idiom is more compact, this is the equivalent of your answer:
from select import select
scan_process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1) # Line buffered
while some_criterium and not time_limit:
poll_result = select([scan_process.stdout], [], [], time_limit)[0]
The rest is the same.
See pydoc select.select
.
[Note: this is Unix-specific, as are some of the other answers.]
[Note 2: edited to add line buffering as per OP request]
[Note 3: the line buffering may not be reliable in all circumstances, leading to readline() blocking]
Here’s a portable solution that enforces the timeout for reading a single line using asyncio
:
#!/usr/bin/env python3
import asyncio
import sys
from asyncio.subprocess import PIPE, STDOUT
async def run_command(*args, timeout=None):
# Start child process
# NOTE: universal_newlines parameter is not supported
process = await asyncio.create_subprocess_exec(*args,
stdout=PIPE, stderr=STDOUT)
# Read line (sequence of bytes ending with b'n') asynchronously
while True:
try:
line = await asyncio.wait_for(process.stdout.readline(), timeout)
except asyncio.TimeoutError:
pass
else:
if not line: # EOF
break
elif do_something(line):
continue # While some criterium is satisfied
process.kill() # Timeout or some criterion is not satisfied
break
return await process.wait() # Wait for the child process to exit
if sys.platform == "win32":
loop = asyncio.ProactorEventLoop() # For subprocess' pipes on Windows
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop()
returncode = loop.run_until_complete(run_command("cmd", "arg 1", "arg 2",
timeout=10))
loop.close()
A portable solution is to use a thread to kill the child process if reading a line takes too long:
#!/usr/bin/env python3
from subprocess import Popen, PIPE, STDOUT
timeout = 10
with Popen(command, stdout=PIPE, stderr=STDOUT,
universal_newlines=True) as process: # text mode
# kill process in timeout seconds unless the timer is restarted
watchdog = WatchdogTimer(timeout, callback=process.kill, daemon=True)
watchdog.start()
for line in process.stdout:
# don't invoke the watcthdog callback if do_something() takes too long
with watchdog.blocked:
if not do_something(line): # some criterium is not satisfied
process.kill()
break
watchdog.restart() # restart timer just before reading the next line
watchdog.cancel()
where WatchdogTimer
class is like threading.Timer
that can be restarted and/or blocked:
from threading import Event, Lock, Thread
from subprocess import Popen, PIPE, STDOUT
from time import monotonic # use time.time or monotonic.monotonic on Python 2
class WatchdogTimer(Thread):
"""Run *callback* in *timeout* seconds unless the timer is restarted."""
def __init__(self, timeout, callback, *args, timer=monotonic, **kwargs):
super().__init__(**kwargs)
self.timeout = timeout
self.callback = callback
self.args = args
self.timer = timer
self.cancelled = Event()
self.blocked = Lock()
def run(self):
self.restart() # don't start timer until `.start()` is called
# wait until timeout happens or the timer is canceled
while not self.cancelled.wait(self.deadline - self.timer()):
# don't test the timeout while something else holds the lock
# allow the timer to be restarted while blocked
with self.blocked:
if self.deadline <= self.timer() and not self.cancelled.is_set():
return self.callback(*self.args) # on timeout
def restart(self):
"""Restart the watchdog timer."""
self.deadline = self.timer() + self.timeout
def cancel(self):
self.cancelled.set()
I have a small issue that I’m not quite sure how to solve. Here is a minimal example:
What I have
scan_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while(some_criterium):
line = scan_process.stdout.readline()
some_criterium = do_something(line)
What I would like
scan_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while(some_criterium):
line = scan_process.stdout.readline()
if nothing_happens_after_10s:
break
else:
some_criterium = do_something(line)
I read a line from a subprocess and do something with it. How can I exit if no line arrived after a fixed time interval?
I used something a bit more general in Python (if I remember correctly, also pieced together from Stack Overflow questions, but I cannot recall which ones).
import thread
from threading import Timer
def run_with_timeout(timeout, default, f, *args, **kwargs):
if not timeout:
return f(*args, **kwargs)
try:
timeout_timer = Timer(timeout, thread.interrupt_main)
timeout_timer.start()
result = f(*args, **kwargs)
return result
except KeyboardInterrupt:
return default
finally:
timeout_timer.cancel()
Be warned, though. This uses an interrupt to stop whatever function you give it. This might not be a good idea for all functions and it also prevents you from closing the program with Ctrl + C during the timeout (i.e. Ctrl + C will be handled as a timeout).
You could use this and call it like:
scan_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while(some_criterium):
line = run_with_timeout(timeout, None, scan_process.stdout.readline)
if line is None:
break
else:
some_criterium = do_something(line)
It might be a bit overkill, though. I suspect there is a simpler option for your case that I don’t know.
Thanks for all the answers!
I found a way to solve my problem by simply using select.poll to peek into standard output.
import select
...
scan_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
poll_obj = select.poll()
poll_obj.register(scan_process.stdout, select.POLLIN)
while(some_criterium and not time_limit):
poll_result = poll_obj.poll(0)
if poll_result:
line = scan_process.stdout.readline()
some_criterium = do_something(line)
update(time_limit)
Try using signal.alarm:
#timeout.py
import signal, sys
def timeout(sig, frm):
print "This is taking too long..."
sys.exit(1)
signal.signal(signal.SIGALRM, timeout)
signal.alarm(10)
byte = 0
while 'IT' not in open('/dev/urandom').read(2):
byte += 2
print "I got IT in %s byte(s)!" % byte
A couple of runs to show it works:
$ python timeout.py
This is taking too long...
$ python timeout.py
I got IT in 4672 byte(s)!
For a more detailed example, see pGuides.
While Tom’s solution works, using select()
in the C idiom is more compact, this is the equivalent of your answer:
from select import select
scan_process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1) # Line buffered
while some_criterium and not time_limit:
poll_result = select([scan_process.stdout], [], [], time_limit)[0]
The rest is the same.
See pydoc select.select
.
[Note: this is Unix-specific, as are some of the other answers.]
[Note 2: edited to add line buffering as per OP request]
[Note 3: the line buffering may not be reliable in all circumstances, leading to readline() blocking]
Here’s a portable solution that enforces the timeout for reading a single line using asyncio
:
#!/usr/bin/env python3
import asyncio
import sys
from asyncio.subprocess import PIPE, STDOUT
async def run_command(*args, timeout=None):
# Start child process
# NOTE: universal_newlines parameter is not supported
process = await asyncio.create_subprocess_exec(*args,
stdout=PIPE, stderr=STDOUT)
# Read line (sequence of bytes ending with b'n') asynchronously
while True:
try:
line = await asyncio.wait_for(process.stdout.readline(), timeout)
except asyncio.TimeoutError:
pass
else:
if not line: # EOF
break
elif do_something(line):
continue # While some criterium is satisfied
process.kill() # Timeout or some criterion is not satisfied
break
return await process.wait() # Wait for the child process to exit
if sys.platform == "win32":
loop = asyncio.ProactorEventLoop() # For subprocess' pipes on Windows
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop()
returncode = loop.run_until_complete(run_command("cmd", "arg 1", "arg 2",
timeout=10))
loop.close()
A portable solution is to use a thread to kill the child process if reading a line takes too long:
#!/usr/bin/env python3
from subprocess import Popen, PIPE, STDOUT
timeout = 10
with Popen(command, stdout=PIPE, stderr=STDOUT,
universal_newlines=True) as process: # text mode
# kill process in timeout seconds unless the timer is restarted
watchdog = WatchdogTimer(timeout, callback=process.kill, daemon=True)
watchdog.start()
for line in process.stdout:
# don't invoke the watcthdog callback if do_something() takes too long
with watchdog.blocked:
if not do_something(line): # some criterium is not satisfied
process.kill()
break
watchdog.restart() # restart timer just before reading the next line
watchdog.cancel()
where WatchdogTimer
class is like threading.Timer
that can be restarted and/or blocked:
from threading import Event, Lock, Thread
from subprocess import Popen, PIPE, STDOUT
from time import monotonic # use time.time or monotonic.monotonic on Python 2
class WatchdogTimer(Thread):
"""Run *callback* in *timeout* seconds unless the timer is restarted."""
def __init__(self, timeout, callback, *args, timer=monotonic, **kwargs):
super().__init__(**kwargs)
self.timeout = timeout
self.callback = callback
self.args = args
self.timer = timer
self.cancelled = Event()
self.blocked = Lock()
def run(self):
self.restart() # don't start timer until `.start()` is called
# wait until timeout happens or the timer is canceled
while not self.cancelled.wait(self.deadline - self.timer()):
# don't test the timeout while something else holds the lock
# allow the timer to be restarted while blocked
with self.blocked:
if self.deadline <= self.timer() and not self.cancelled.is_set():
return self.callback(*self.args) # on timeout
def restart(self):
"""Restart the watchdog timer."""
self.deadline = self.timer() + self.timeout
def cancel(self):
self.cancelled.set()