Python: How to read stdout of subprocess in a nonblocking way
Question:
I am trying to make a simple python script that starts a subprocess and monitors its standard output. Here is a snippet from the code:
process = subprocess.Popen([path_to_exe, os.path.join(temp_dir,temp_file)], stdout=subprocess.PIPE)
while True:
output=process.stdout.readline()
print "test"
The problem is that the script hangs on output=process.stdout.readline()
and that the line print "test"
only executes after the subprocess is terminated.
Is there a way to read standard output and print it without having to wait for the subprocess to terminate?
The subprocess which I am starting is a Windows binary for which I do not have the source code.
I have found several similar questions, but the answers are only applicable on Linux or in case I have the source of the suprocess I am starting.
Answers:
You could try this:
import subprocess
import os
""" Continuously print command output """
""" Will only work if there are newline characters in the output. """
def run_cmd(command):
popen = subprocess.Popen(command, stdout=subprocess.PIPE)
return iter(popen.stdout.readline, b"")
for line in run_cmd([path_to_exe, os.path.join(temp_dir,temp_file)]):
print(line), # the comma keeps python from adding an empty line
Check select module
import subprocess
import select
import time
x=subprocess.Popen(['/bin/bash','-c',"while true; do sleep 5; echo yes; done"],stdout=subprocess.PIPE)
y=select.poll()
y.register(x.stdout,select.POLLIN)
while True:
if y.poll(1):
print x.stdout.readline()
else:
print "nothing here"
time.sleep(1)
EDIT:
Threaded Solution for non posix systems:
import subprocess
from threading import Thread
import time
linebuffer=[]
x=subprocess.Popen(['/bin/bash','-c',"while true; do sleep 5; echo yes; done"],stdout=subprocess.PIPE)
def reader(f,buffer):
while True:
line=f.readline()
if line:
buffer.append(line)
else:
break
t=Thread(target=reader,args=(x.stdout,linebuffer))
t.daemon=True
t.start()
while True:
if linebuffer:
print linebuffer.pop(0)
else:
print "nothing here"
time.sleep(1)
As of python 3.5, you can use os.set_blocking()
as follows.
import os
import subprocess
process = subprocess.Popen([path_to_exe, os.path.join(temp_dir,temp_file)], stdout=subprocess.PIPE)
os.set_blocking(process.stdout.fileno(), False) #<----- HERE
while True:
output=process.stdout.readline()
print "test"
if process.poll() is not None:
break
rc = process.poll()
Note that os.set_blocking()
is only available on UNIX platforms.
I am trying to make a simple python script that starts a subprocess and monitors its standard output. Here is a snippet from the code:
process = subprocess.Popen([path_to_exe, os.path.join(temp_dir,temp_file)], stdout=subprocess.PIPE)
while True:
output=process.stdout.readline()
print "test"
The problem is that the script hangs on output=process.stdout.readline()
and that the line print "test"
only executes after the subprocess is terminated.
Is there a way to read standard output and print it without having to wait for the subprocess to terminate?
The subprocess which I am starting is a Windows binary for which I do not have the source code.
I have found several similar questions, but the answers are only applicable on Linux or in case I have the source of the suprocess I am starting.
You could try this:
import subprocess
import os
""" Continuously print command output """
""" Will only work if there are newline characters in the output. """
def run_cmd(command):
popen = subprocess.Popen(command, stdout=subprocess.PIPE)
return iter(popen.stdout.readline, b"")
for line in run_cmd([path_to_exe, os.path.join(temp_dir,temp_file)]):
print(line), # the comma keeps python from adding an empty line
Check select module
import subprocess
import select
import time
x=subprocess.Popen(['/bin/bash','-c',"while true; do sleep 5; echo yes; done"],stdout=subprocess.PIPE)
y=select.poll()
y.register(x.stdout,select.POLLIN)
while True:
if y.poll(1):
print x.stdout.readline()
else:
print "nothing here"
time.sleep(1)
EDIT:
Threaded Solution for non posix systems:
import subprocess
from threading import Thread
import time
linebuffer=[]
x=subprocess.Popen(['/bin/bash','-c',"while true; do sleep 5; echo yes; done"],stdout=subprocess.PIPE)
def reader(f,buffer):
while True:
line=f.readline()
if line:
buffer.append(line)
else:
break
t=Thread(target=reader,args=(x.stdout,linebuffer))
t.daemon=True
t.start()
while True:
if linebuffer:
print linebuffer.pop(0)
else:
print "nothing here"
time.sleep(1)
As of python 3.5, you can use os.set_blocking()
as follows.
import os
import subprocess
process = subprocess.Popen([path_to_exe, os.path.join(temp_dir,temp_file)], stdout=subprocess.PIPE)
os.set_blocking(process.stdout.fileno(), False) #<----- HERE
while True:
output=process.stdout.readline()
print "test"
if process.poll() is not None:
break
rc = process.poll()
Note that os.set_blocking()
is only available on UNIX platforms.