Interactive input/output using Python
Question:
I have a program that interacts with the user (acts like a shell), and I want to run it using the Python subprocess module interactively.
That means, I want the possibility to write to standard input and immediately get the output from standard output. I tried many solutions offered here, but none of them seems to work for my needs.
The code I’ve written is based on Running an interactive command from within Python.
import Queue
import threading
import subprocess
def enqueue_output(out, queue):
for line in iter(out.readline, b''):
queue.put(line)
out.close()
def getOutput(outQueue):
outStr = ''
try:
while True: # Adds output from the queue until it is empty
outStr += outQueue.get_nowait()
except Queue.Empty:
return outStr
p = subprocess.Popen("./a.out", stdin=subprocess.PIPE, stout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize = 1)
#p = subprocess.Popen("./a.out", stdin=subprocess.PIPE, stout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, universal_newlines=True)
outQueue = Queue()
errQueue = Queue()
outThread = Thread(target=enqueue_output, args=(p.stdout, outQueue))
errThread = Thread(target=enqueue_output, args=(p.stderr, errQueue))
outThread.daemon = True
errThread.daemon = True
outThread.start()
errThread.start()
p.stdin.write("1n")
p.stdin.flush()
errors = getOutput(errQueue)
output = getOutput(outQueue)
p.stdin.write("5n")
p.stdin.flush()
erros = getOutput(errQueue)
output = getOutput(outQueue)
The problem is that the queue remains empty, as if there is no output.
Only if I write to standard input all the input that the program needs to execute and terminate, then I get the output (which is not what I want). For example, if I do something like:
p.stdin.write("1n5n")
errors = getOutput(errQueue)
output = getOutput(outQueue)
Is there a way to do what I want to do?
The script will run on a Linux machine. I changed my script and deleted the universal_newlines=True + set the bufsize to 1 and flushed standard input immediately after write. Still I don’t get any output.
Second try:
I tried this solution and it works for me:
from subprocess import Popen, PIPE
fw = open("tmpout", "wb")
fr = open("tmpout", "r")
p = Popen("./a.out", stdin = PIPE, stdout = fw, stderr = fw, bufsize = 1)
p.stdin.write("1n")
out = fr.read()
p.stdin.write("5n")
out = fr.read()
fw.close()
fr.close()
Answers:
Two solutions for this issue on Linux:
First one is to use a file to write the output to, and read from it simultaneously:
from subprocess import Popen, PIPE
fw = open("tmpout", "wb")
fr = open("tmpout", "r")
p = Popen("./a.out", stdin = PIPE, stdout = fw, stderr = fw, bufsize = 1)
p.stdin.write("1n")
out = fr.read()
p.stdin.write("5n")
out = fr.read()
fw.close()
fr.close()
Second, as J.F. Sebastian offered, is to make p.stdout and p.stderr pipes non-blocking using fnctl module:
import os
import fcntl
from subprocess import Popen, PIPE
def setNonBlocking(fd):
"""
Set the file description of the given file descriptor to non-blocking.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFL)
flags = flags | os.O_NONBLOCK
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
p = Popen("./a.out", stdin = PIPE, stdout = PIPE, stderr = PIPE, bufsize = 1)
setNonBlocking(p.stdout)
setNonBlocking(p.stderr)
p.stdin.write("1n")
while True:
try:
out1 = p.stdout.read()
except IOError:
continue
else:
break
out1 = p.stdout.read()
p.stdin.write("5n")
while True:
try:
out2 = p.stdout.read()
except IOError:
continue
else:
break
None of the current answers worked for me. At the end, I’ve got this working:
import subprocess
def start(executable_file):
return subprocess.Popen(
executable_file,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def read(process):
return process.stdout.readline().decode("utf-8").strip()
def write(process, message):
process.stdin.write(f"{message.strip()}n".encode("utf-8"))
process.stdin.flush()
def terminate(process):
process.stdin.close()
process.terminate()
process.wait(timeout=0.2)
process = start("./dummy.py")
write(process, "hello dummy")
print(read(process))
terminate(process)
Tested with this dummy.py script:
#!/usr/bin/env python3.6
import random
import time
while True:
message = input()
time.sleep(random.uniform(0.1, 1.0)) # simulates process time
print(message[::-1])
The caveats are (all managed in the functions):
- Input/output always lines with newline.
- Flush child’s stdin after every write.
- Use
readline() from child’s stdout.
It’s a pretty simple solution in my opinion (not mine, I found it here: https://eli.thegreenplace.net/2017/interacting-with-a-long-running-child-process-in-python/). I was using Python 3.6.
Here is an interactive shell. You have to run read() on a separate thread, otherwise it will block the write()
import sys
import os
import subprocess
from subprocess import Popen, PIPE
import threading
class LocalShell(object):
def __init__(self):
pass
def run(self):
env = os.environ.copy()
p = Popen('/bin/bash', stdin=PIPE, stdout=PIPE, stderr=subprocess.STDOUT, shell=True, env=env)
sys.stdout.write("Started Local Terminal...rnrn")
def writeall(p):
while True:
# print("read data: ")
data = p.stdout.read(1).decode("utf-8")
if not data:
break
sys.stdout.write(data)
sys.stdout.flush()
writer = threading.Thread(target=writeall, args=(p,))
writer.start()
try:
while True:
d = sys.stdin.read(1)
if not d:
break
self._write(p, d.encode())
except EOFError:
pass
def _write(self, process, message):
process.stdin.write(message)
process.stdin.flush()
shell = LocalShell()
shell.run()
As I know, the simplest way is to make dedicated threads on both sides: two for stdout/stderr on parent process and one for stdin on child process. Part of such example please find here.
I have a program that interacts with the user (acts like a shell), and I want to run it using the Python subprocess module interactively.
That means, I want the possibility to write to standard input and immediately get the output from standard output. I tried many solutions offered here, but none of them seems to work for my needs.
The code I’ve written is based on Running an interactive command from within Python.
import Queue
import threading
import subprocess
def enqueue_output(out, queue):
for line in iter(out.readline, b''):
queue.put(line)
out.close()
def getOutput(outQueue):
outStr = ''
try:
while True: # Adds output from the queue until it is empty
outStr += outQueue.get_nowait()
except Queue.Empty:
return outStr
p = subprocess.Popen("./a.out", stdin=subprocess.PIPE, stout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize = 1)
#p = subprocess.Popen("./a.out", stdin=subprocess.PIPE, stout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, universal_newlines=True)
outQueue = Queue()
errQueue = Queue()
outThread = Thread(target=enqueue_output, args=(p.stdout, outQueue))
errThread = Thread(target=enqueue_output, args=(p.stderr, errQueue))
outThread.daemon = True
errThread.daemon = True
outThread.start()
errThread.start()
p.stdin.write("1n")
p.stdin.flush()
errors = getOutput(errQueue)
output = getOutput(outQueue)
p.stdin.write("5n")
p.stdin.flush()
erros = getOutput(errQueue)
output = getOutput(outQueue)
The problem is that the queue remains empty, as if there is no output.
Only if I write to standard input all the input that the program needs to execute and terminate, then I get the output (which is not what I want). For example, if I do something like:
p.stdin.write("1n5n")
errors = getOutput(errQueue)
output = getOutput(outQueue)
Is there a way to do what I want to do?
The script will run on a Linux machine. I changed my script and deleted the universal_newlines=True + set the bufsize to 1 and flushed standard input immediately after write. Still I don’t get any output.
Second try:
I tried this solution and it works for me:
from subprocess import Popen, PIPE
fw = open("tmpout", "wb")
fr = open("tmpout", "r")
p = Popen("./a.out", stdin = PIPE, stdout = fw, stderr = fw, bufsize = 1)
p.stdin.write("1n")
out = fr.read()
p.stdin.write("5n")
out = fr.read()
fw.close()
fr.close()
Two solutions for this issue on Linux:
First one is to use a file to write the output to, and read from it simultaneously:
from subprocess import Popen, PIPE
fw = open("tmpout", "wb")
fr = open("tmpout", "r")
p = Popen("./a.out", stdin = PIPE, stdout = fw, stderr = fw, bufsize = 1)
p.stdin.write("1n")
out = fr.read()
p.stdin.write("5n")
out = fr.read()
fw.close()
fr.close()
Second, as J.F. Sebastian offered, is to make p.stdout and p.stderr pipes non-blocking using fnctl module:
import os
import fcntl
from subprocess import Popen, PIPE
def setNonBlocking(fd):
"""
Set the file description of the given file descriptor to non-blocking.
"""
flags = fcntl.fcntl(fd, fcntl.F_GETFL)
flags = flags | os.O_NONBLOCK
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
p = Popen("./a.out", stdin = PIPE, stdout = PIPE, stderr = PIPE, bufsize = 1)
setNonBlocking(p.stdout)
setNonBlocking(p.stderr)
p.stdin.write("1n")
while True:
try:
out1 = p.stdout.read()
except IOError:
continue
else:
break
out1 = p.stdout.read()
p.stdin.write("5n")
while True:
try:
out2 = p.stdout.read()
except IOError:
continue
else:
break
None of the current answers worked for me. At the end, I’ve got this working:
import subprocess
def start(executable_file):
return subprocess.Popen(
executable_file,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def read(process):
return process.stdout.readline().decode("utf-8").strip()
def write(process, message):
process.stdin.write(f"{message.strip()}n".encode("utf-8"))
process.stdin.flush()
def terminate(process):
process.stdin.close()
process.terminate()
process.wait(timeout=0.2)
process = start("./dummy.py")
write(process, "hello dummy")
print(read(process))
terminate(process)
Tested with this dummy.py script:
#!/usr/bin/env python3.6
import random
import time
while True:
message = input()
time.sleep(random.uniform(0.1, 1.0)) # simulates process time
print(message[::-1])
The caveats are (all managed in the functions):
- Input/output always lines with newline.
- Flush child’s stdin after every write.
- Use
readline()from child’s stdout.
It’s a pretty simple solution in my opinion (not mine, I found it here: https://eli.thegreenplace.net/2017/interacting-with-a-long-running-child-process-in-python/). I was using Python 3.6.
Here is an interactive shell. You have to run read() on a separate thread, otherwise it will block the write()
import sys
import os
import subprocess
from subprocess import Popen, PIPE
import threading
class LocalShell(object):
def __init__(self):
pass
def run(self):
env = os.environ.copy()
p = Popen('/bin/bash', stdin=PIPE, stdout=PIPE, stderr=subprocess.STDOUT, shell=True, env=env)
sys.stdout.write("Started Local Terminal...rnrn")
def writeall(p):
while True:
# print("read data: ")
data = p.stdout.read(1).decode("utf-8")
if not data:
break
sys.stdout.write(data)
sys.stdout.flush()
writer = threading.Thread(target=writeall, args=(p,))
writer.start()
try:
while True:
d = sys.stdin.read(1)
if not d:
break
self._write(p, d.encode())
except EOFError:
pass
def _write(self, process, message):
process.stdin.write(message)
process.stdin.flush()
shell = LocalShell()
shell.run()
As I know, the simplest way is to make dedicated threads on both sides: two for stdout/stderr on parent process and one for stdin on child process. Part of such example please find here.