How to correctly redirect stdout, logging and tqdm into a PyQt widget
Question:
TL;DR
For answers, see:
- my 2019 initially own accepted answer using a text edit and stdout/stderr streams redirections, see https://stackoverflow.com/a/55082521/7237062
- my second answer, now marked as the accepted one: a derived and improved approach with a real QProgressBar ! https://stackoverflow.com/a/74091829/7237062
QUESTION
First of all, I know that a lot of questions are similar to this one.
But after spending so much time on it, I now look for help from the community.
I developed and use a bunch of python modules that rely on tqdm
.
I want them to be usable inside Jupyter, in console or with a GUI.
Everything works fine in Jupyter or console : there are no collisions between logging/prints and tqdm progress bars. Here is a sample code that shows the console/Jupyter behavior:
# coding=utf-8
from tqdm.auto import tqdm
import time
import logging
import sys
import datetime
__is_setup_done = False
def setup_logging(log_prefix):
global __is_setup_done
if __is_setup_done:
pass
else:
__log_file_name = "{}-{}_log_file.txt".format(log_prefix,
datetime.datetime.utcnow().isoformat().replace(":", "-"))
__log_format = '%(asctime)s - %(name)-30s - %(levelname)s - %(message)s'
__console_date_format = '%Y-%m-%d %H:%M:%S'
__file_date_format = '%Y-%m-%d %H-%M-%S'
root = logging.getLogger()
root.setLevel(logging.DEBUG)
console_formatter = logging.Formatter(__log_format, __console_date_format)
file_formatter = logging.Formatter(__log_format, __file_date_format)
file_handler = logging.FileHandler(__log_file_name, mode='a', delay=True)
# file_handler = TqdmLoggingHandler2(__log_file_name, mode='a', delay=True)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(file_formatter)
root.addHandler(file_handler)
tqdm_handler = TqdmLoggingHandler()
tqdm_handler.setLevel(logging.DEBUG)
tqdm_handler.setFormatter(console_formatter)
root.addHandler(tqdm_handler)
__is_setup_done = True
class TqdmLoggingHandler(logging.StreamHandler):
def __init__(self, level=logging.NOTSET):
logging.StreamHandler.__init__(self)
def emit(self, record):
msg = self.format(record)
tqdm.write(msg)
# from https://stackoverflow.com/questions/38543506/change-logging-print-function-to-tqdm-write-so-logging-doesnt-interfere-wit/38739634#38739634
self.flush()
def example_long_procedure():
setup_logging('long_procedure')
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
for i in tqdm(range(10), unit_scale=True, dynamic_ncols=True, file=sys.stdout):
time.sleep(.1)
__logger.info('foo {}'.format(i))
example_long_procedure()
The obtained output:
2019-03-07 22:22:27 - long_procedure - INFO - foo 0
2019-03-07 22:22:27 - long_procedure - INFO - foo 1
2019-03-07 22:22:27 - long_procedure - INFO - foo 2
2019-03-07 22:22:27 - long_procedure - INFO - foo 3
2019-03-07 22:22:27 - long_procedure - INFO - foo 4
2019-03-07 22:22:28 - long_procedure - INFO - foo 5
2019-03-07 22:22:28 - long_procedure - INFO - foo 6
2019-03-07 22:22:28 - long_procedure - INFO - foo 7
2019-03-07 22:22:28 - long_procedure - INFO - foo 8
2019-03-07 22:22:28 - long_procedure - INFO - foo 9
100%|¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦| 10.0/10.0 [00:01<00:00, 9.69it/s]
Now, I’m making a GUI with PyQt that uses code similar to above. Since processing may be long, I used threading in order to avoid freezing HMI during processing. I also used stdout
redirection using Queue() towards a Qt QWidget so the user can see what is happenning.
My current use case is 1 single thread that has logs and tqdm progress bars to redirect to 1 dedicated widget. (I’m not looking for multiple threads to feed the widget with multiple logs and multiple tqdm progress bar).
I managed to redirect stdout thanks to the informations from Redirecting stdout and stderr to a PyQt5 QTextEdit from a secondary thread.
However, only logger lines are redirected. TQDM progress bar is still directed to the console output.
Here is my current code:
# coding=utf-8
import time
import logging
import sys
import datetime
__is_setup_done = False
from queue import Queue
from PyQt5.QtCore import pyqtSlot, pyqtSignal, QObject, QThread, QMetaObject, Q_ARG, Qt
from PyQt5.QtGui import QTextCursor, QFont
from PyQt5.QtWidgets import QTextEdit, QPlainTextEdit, QWidget, QToolButton, QVBoxLayout, QApplication
from tqdm.auto import tqdm
class MainApp(QWidget):
def __init__(self):
super().__init__()
setup_logging(self.__class__.__name__)
self.__logger = logging.getLogger(self.__class__.__name__)
self.__logger.setLevel(logging.DEBUG)
# create console text queue
self.queue_console_text = Queue()
# redirect stdout to the queue
output_stream = WriteStream(self.queue_console_text)
sys.stdout = output_stream
layout = QVBoxLayout()
self.setMinimumWidth(500)
# GO button
self.btn_perform_actions = QToolButton(self)
self.btn_perform_actions.setText('Launch long processing')
self.btn_perform_actions.clicked.connect(self._btn_go_clicked)
self.console_text_edit = ConsoleTextEdit(self)
self.thread_initialize = QThread()
self.init_procedure_object = InitializationProcedures(self)
# create console text read thread + receiver object
self.thread_queue_listener = QThread()
self.console_text_receiver = ThreadConsoleTextQueueReceiver(self.queue_console_text)
# connect receiver object to widget for text update
self.console_text_receiver.queue_element_received_signal.connect(self.console_text_edit.append_text)
# attach console text receiver to console text thread
self.console_text_receiver.moveToThread(self.thread_queue_listener)
# attach to start / stop methods
self.thread_queue_listener.started.connect(self.console_text_receiver.run)
self.thread_queue_listener.finished.connect(self.console_text_receiver.finished)
self.thread_queue_listener.start()
layout.addWidget(self.btn_perform_actions)
layout.addWidget(self.console_text_edit)
self.setLayout(layout)
self.show()
@pyqtSlot()
def _btn_go_clicked(self):
# prepare thread for long operation
self.init_procedure_object.moveToThread(self.thread_initialize)
self.thread_initialize.started.connect(self.init_procedure_object.run)
self.thread_initialize.finished.connect(self.init_procedure_object.finished)
# start thread
self.btn_perform_actions.setEnabled(False)
self.thread_initialize.start()
class WriteStream(object):
def __init__(self, q: Queue):
self.queue = q
def write(self, text):
"""
Redirection of stream to the given queue
"""
self.queue.put(text)
def flush(self):
"""
Stream flush implementation
"""
pass
class ThreadConsoleTextQueueReceiver(QObject):
queue_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
self.queue_element_received_signal.emit('---> Console text queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_element_received_signal.emit(text)
@pyqtSlot()
def finished(self):
self.queue_element_received_signal.emit('---> Console text queue reception Stopped <---n')
class ConsoleTextEdit(QTextEdit):#QTextEdit):
def __init__(self, parent):
super(ConsoleTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setLineWidth(50)
self.setMinimumWidth(1200)
self.setFont(QFont('Consolas', 11))
self.flag = False
@pyqtSlot(str)
def append_text(self, text: str):
self.moveCursor(QTextCursor.End)
self.insertPlainText(text)
def long_procedure():
setup_logging('long_procedure')
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
for i in tqdm(range(10), unit_scale=True, dynamic_ncols=True):
time.sleep(.1)
__logger.info('foo {}'.format(i))
class InitializationProcedures(QObject):
def __init__(self, main_app: MainApp):
super(InitializationProcedures, self).__init__()
self._main_app = main_app
@pyqtSlot()
def run(self):
long_procedure()
@pyqtSlot()
def finished(self):
print("Thread finished !") # might call main window to do some stuff with buttons
self._main_app.btn_perform_actions.setEnabled(True)
def setup_logging(log_prefix):
global __is_setup_done
if __is_setup_done:
pass
else:
__log_file_name = "{}-{}_log_file.txt".format(log_prefix,
datetime.datetime.utcnow().isoformat().replace(":", "-"))
__log_format = '%(asctime)s - %(name)-30s - %(levelname)s - %(message)s'
__console_date_format = '%Y-%m-%d %H:%M:%S'
__file_date_format = '%Y-%m-%d %H-%M-%S'
root = logging.getLogger()
root.setLevel(logging.DEBUG)
console_formatter = logging.Formatter(__log_format, __console_date_format)
file_formatter = logging.Formatter(__log_format, __file_date_format)
file_handler = logging.FileHandler(__log_file_name, mode='a', delay=True)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(file_formatter)
root.addHandler(file_handler)
tqdm_handler = TqdmLoggingHandler()
tqdm_handler.setLevel(logging.DEBUG)
tqdm_handler.setFormatter(console_formatter)
root.addHandler(tqdm_handler)
__is_setup_done = True
class TqdmLoggingHandler(logging.StreamHandler):
def __init__(self, level=logging.NOTSET):
logging.StreamHandler.__init__(self)
def emit(self, record):
msg = self.format(record)
tqdm.write(msg)
# from https://stackoverflow.com/questions/38543506/change-logging-print-function-to-tqdm-write-so-logging-doesnt-interfere-wit/38739634#38739634
self.flush()
if __name__ == '__main__':
app = QApplication(sys.argv)
app.setStyle('Fusion')
tqdm.ncols = 50
ex = MainApp()
sys.exit(app.exec_())
I would like to obtain the exact behavior I would have had strictly invoking the code in console.
i.e. expected output in PyQt widget:
---> Console text queue reception Started <---
2019-03-07 19:42:19 - long_procedure - INFO - foo 0
2019-03-07 19:42:19 - long_procedure - INFO - foo 1
2019-03-07 19:42:19 - long_procedure - INFO - foo 2
2019-03-07 19:42:19 - long_procedure - INFO - foo 3
2019-03-07 19:42:19 - long_procedure - INFO - foo 4
2019-03-07 19:42:19 - long_procedure - INFO - foo 5
2019-03-07 19:42:20 - long_procedure - INFO - foo 6
2019-03-07 19:42:20 - long_procedure - INFO - foo 7
2019-03-07 19:42:20 - long_procedure - INFO - foo 8
2019-03-07 19:42:20 - long_procedure - INFO - foo 9
100%|################################| 10.0/10.0 [00:01<00:00, 9.16it/s]
Things I tried / explored with no success.
Option 1
This solution Display terminal output with tqdm in QPlainTextEdit does not give the expected results. It works well to redirect outputs containing only tqdm stuff.
The following code does not give the intended behavior, wether it is with QTextEdit or QPlainTextEdit. Only logger lines are redirected.
# code from this answer
# https://stackoverflow.com/questions/53381975/display-terminal-output-with-tqdm-in-qplaintextedit
@pyqtSlot(str)
def append_text(self, message: str):
if not hasattr(self, "flag"):
self.flag = False
message = message.replace('r', '').rstrip()
if message:
method = "replace_last_line" if self.flag else "append_text"
QMetaObject.invokeMethod(self,
method,
Qt.QueuedConnection,
Q_ARG(str, message))
self.flag = True
else:
self.flag = False
@pyqtSlot(str)
def replace_last_line(self, text):
cursor = self.textCursor()
cursor.movePosition(QTextCursor.End)
cursor.select(QTextCursor.BlockUnderCursor)
cursor.removeSelectedText()
cursor.insertBlock()
self.setTextCursor(cursor)
self.insertPlainText(text)
However, the above code + adding file=sys.stdout
to the tqdm call changes the behavior: tqdm output is redirected to the Qt widget. But in the end, only one line is displayed, and it is either a logger line or a tqdm line (it looks like it depends on which Qt widget I derived).
In the end, changing all tqdm invocations used modules should not be the preferred option.
So the other approach I found is to redirect stderr in the same stream/queue stdout is redirected to. Since tqdm writes to stderr by default, this way all tqdm outputs are redirected to widget.
But I still can’t figure out obtaining the exact output I’m looking for.
This question does not provide a clue on why behavior seems to differ between QTextEdit vs QPlainTextEdit
Option 2
This question Duplicate stdout, stderr in QTextEdit widget looks very similar to Display terminal output with tqdm in QPlainTextEdit and does not answer to my exact problem described above.
Option 3
Trying this solution using contextlib gave me an error due to no flush() method being defined. After fixing, I end up with only tqdm lines and no logger lines.
Option 4
I also tried to intercept the r character and implement a specific behavior, with not success.
Versions:
tqdm 4.28.1
pyqt 5.9.2
PyQt5 5.12
PyQt5_sip 4.19.14
Python 3.7.2
Answers:
EDIT 2019-mar-12: It seems to me that the answer is : it could probably be done, but requires a lots of effort in order to remember where which line comes from for the QTextEdit to behave as instended. Plus, since tdm writes to stderr by default, you would end up with catching all exceptions traces too.
That is why I’ll mark my own answer as solved: I find it more elegant to achieve the same purpose : show in pyqt what is happenning.
Here is my best shot to obtain something close to the intended behavior.
It does not exactly respond to the question, because I changed the GUI design.
So I won’t vote it as solved. Moreover, this is done all in one single python file. I plan to further challenge this solution to see if it works with real python modules doing tqdm imports.
I patched the basic tqdm class in a very ugly way. The main trick is to :
- dynamically change the tqdm module structure by storing the original tqdm class into a new name:
tqdm.orignal_class = tqdm.tqdm
- then inherit tqdm.original class
class TQDMPatch(tqdm.orignal_class):
- implement the constructor in order to force the file stream + any parameters to whatever you want:
super(TQDMPatch, self).__init__(... change some params ...)
. I gave my TQDM class a custom WriteStream()
that writes into a Queue()
- change the GUI strategy to intercept and redirect your custom tqdm stream to a separate Qt widget. My widget is making the assumption that all received prints contain
r
(which TQDM seems to be doing).
It is working both as in single python file and with multiple separated modules. In the latter case, imports order at startup is critical.
Screenshots:
Before launching processing
During processing
At end of processing
Here is the code
ALL-IN-ONE FILE
# coding=utf-8
import datetime
import logging
import sys
import time
from queue import Queue
from PyQt5.QtCore import pyqtSlot, pyqtSignal, QObject, QThread, Qt
from PyQt5.QtGui import QTextCursor, QFont
from PyQt5.QtWidgets import QTextEdit, QWidget, QToolButton, QVBoxLayout, QApplication, QLineEdit
# DEFINITION NEEDED FIRST ...
class WriteStream(object):
def __init__(self, q: Queue):
self.queue = q
def write(self, text):
self.queue.put(text)
def flush(self):
pass
# prepare queue and streams
queue_tqdm = Queue()
write_stream_tqdm = WriteStream(queue_tqdm)
################## START TQDM patch procedure ##################
import tqdm
# save original class into module
tqdm.orignal_class = tqdm.tqdm
class TQDMPatch(tqdm.orignal_class):
"""
Derive from original class
"""
def __init__(self, iterable=None, desc=None, total=None, leave=True,
file=None, ncols=None, mininterval=0.1, maxinterval=10.0,
miniters=None, ascii=None, disable=False, unit='it',
unit_scale=False, dynamic_ncols=False, smoothing=0.3,
bar_format=None, initial=0, position=None, postfix=None,
unit_divisor=1000, gui=False, **kwargs):
super(TQDMPatch, self).__init__(iterable, desc, total, leave,
write_stream_tqdm, # change any chosen file stream with our's
80, # change nb of columns (gui choice),
mininterval, maxinterval,
miniters, ascii, disable, unit,
unit_scale, False, smoothing,
bar_format, initial, position, postfix,
unit_divisor, gui, **kwargs)
print('TQDM Patch called') # check it works
@classmethod
def write(cls, s, file=None, end="n", nolock=False):
super(TQDMPatch, cls).write(s=s, file=file, end=end, nolock=nolock)
# all other tqdm.orignal_class @classmethod methods may need to be redefined !
# I mainly used tqdm.auto in my modules, so use that for patch
# unsure if this will work with all possible tqdm import methods
# might not work for tqdm_gui !
import tqdm.auto as AUTO
# change original class with the patched one, the original still exists
AUTO.tqdm = TQDMPatch
################## END of TQDM patch ##################
# normal MCVE code
__is_setup_done = False
class MainApp(QWidget):
def __init__(self):
super().__init__()
setup_logging(self.__class__.__name__)
self.__logger = logging.getLogger(self.__class__.__name__)
self.__logger.setLevel(logging.DEBUG)
# create stdout text queue
self.queue_std_out = Queue()
sys.stdout = WriteStream(self.queue_std_out)
layout = QVBoxLayout()
self.setMinimumWidth(500)
self.btn_perform_actions = QToolButton(self)
self.btn_perform_actions.setText('Launch long processing')
self.btn_perform_actions.clicked.connect(self._btn_go_clicked)
self.text_edit_std_out = StdOutTextEdit(self)
self.text_edit_tqdm = StdTQDMTextEdit(self)
self.thread_initialize = QThread()
self.init_procedure_object = InitializationProcedures(self)
# std out stream management
# create console text read thread + receiver object
self.thread_std_out_queue_listener = QThread()
self.std_out_text_receiver = ThreadStdOutStreamTextQueueReceiver(self.queue_std_out)
# connect receiver object to widget for text update
self.std_out_text_receiver.queue_std_out_element_received_signal.connect(self.text_edit_std_out.append_text)
# attach console text receiver to console text thread
self.std_out_text_receiver.moveToThread(self.thread_std_out_queue_listener)
# attach to start / stop methods
self.thread_std_out_queue_listener.started.connect(self.std_out_text_receiver.run)
self.thread_std_out_queue_listener.start()
# NEW: TQDM stream management
self.thread_tqdm_queue_listener = QThread()
self.tqdm_text_receiver = ThreadTQDMStreamTextQueueReceiver(queue_tqdm)
# connect receiver object to widget for text update
self.tqdm_text_receiver.queue_tqdm_element_received_signal.connect(self.text_edit_tqdm.set_tqdm_text)
# attach console text receiver to console text thread
self.tqdm_text_receiver.moveToThread(self.thread_tqdm_queue_listener)
# attach to start / stop methods
self.thread_tqdm_queue_listener.started.connect(self.tqdm_text_receiver.run)
self.thread_tqdm_queue_listener.start()
layout.addWidget(self.btn_perform_actions)
layout.addWidget(self.text_edit_std_out)
layout.addWidget(self.text_edit_tqdm)
self.setLayout(layout)
self.show()
@pyqtSlot()
def _btn_go_clicked(self):
# prepare thread for long operation
self.init_procedure_object.moveToThread(self.thread_initialize)
self.thread_initialize.started.connect(self.init_procedure_object.run)
self.thread_initialize.finished.connect(self.init_procedure_object.finished)
# start thread
self.btn_perform_actions.setEnabled(False)
self.thread_initialize.start()
class ThreadStdOutStreamTextQueueReceiver(QObject):
queue_std_out_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
self.queue_std_out_element_received_signal.emit('---> STD OUT Queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_std_out_element_received_signal.emit(text)
# NEW: dedicated receiving object for TQDM
class ThreadTQDMStreamTextQueueReceiver(QObject):
queue_tqdm_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
self.queue_tqdm_element_received_signal.emit('r---> TQDM Queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_tqdm_element_received_signal.emit(text)
class StdOutTextEdit(QTextEdit): # QTextEdit):
def __init__(self, parent):
super(StdOutTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setLineWidth(50)
self.setMinimumWidth(500)
self.setFont(QFont('Consolas', 11))
@pyqtSlot(str)
def append_text(self, text: str):
self.moveCursor(QTextCursor.End)
self.insertPlainText(text)
class StdTQDMTextEdit(QLineEdit):
def __init__(self, parent):
super(StdTQDMTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setEnabled(True)
self.setMinimumWidth(500)
self.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.setClearButtonEnabled(True)
self.setFont(QFont('Consolas', 11))
@pyqtSlot(str)
def set_tqdm_text(self, text: str):
new_text = text
if new_text.find('r') >= 0:
new_text = new_text.replace('r', '').rstrip()
if new_text:
self.setText(new_text)
else:
# we suppose that all TQDM prints have r
# so drop the rest
pass
def long_procedure():
# emulate import of modules
from tqdm.auto import tqdm
setup_logging('long_procedure')
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
tqdm_obect = tqdm(range(10), unit_scale=True, dynamic_ncols=True)
tqdm_obect.set_description("My progress bar description")
for i in tqdm_obect:
time.sleep(.1)
__logger.info('foo {}'.format(i))
class InitializationProcedures(QObject):
def __init__(self, main_app: MainApp):
super(InitializationProcedures, self).__init__()
self._main_app = main_app
@pyqtSlot()
def run(self):
long_procedure()
@pyqtSlot()
def finished(self):
print("Thread finished !") # might call main window to do some stuff with buttons
self._main_app.btn_perform_actions.setEnabled(True)
def setup_logging(log_prefix):
global __is_setup_done
if __is_setup_done:
pass
else:
__log_file_name = "{}-{}_log_file.txt".format(log_prefix,
datetime.datetime.utcnow().isoformat().replace(":", "-"))
__log_format = '%(asctime)s - %(name)-30s - %(levelname)s - %(message)s'
__console_date_format = '%Y-%m-%d %H:%M:%S'
__file_date_format = '%Y-%m-%d %H-%M-%S'
root = logging.getLogger()
root.setLevel(logging.DEBUG)
console_formatter = logging.Formatter(__log_format, __console_date_format)
file_formatter = logging.Formatter(__log_format, __file_date_format)
file_handler = logging.FileHandler(__log_file_name, mode='a', delay=True)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(file_formatter)
root.addHandler(file_handler)
tqdm_handler = TqdmLoggingHandler()
tqdm_handler.setLevel(logging.DEBUG)
tqdm_handler.setFormatter(console_formatter)
root.addHandler(tqdm_handler)
__is_setup_done = True
class TqdmLoggingHandler(logging.StreamHandler):
def __init__(self):
logging.StreamHandler.__init__(self)
def emit(self, record):
msg = self.format(record)
tqdm.tqdm.write(msg)
# from https://stackoverflow.com/questions/38543506/change-logging-print-function-to-tqdm-write-so-logging-doesnt-interfere-wit/38739634#38739634
self.flush()
if __name__ == '__main__':
app = QApplication(sys.argv)
app.setStyle('Fusion')
ex = MainApp()
sys.exit(app.exec_())
WITH PROPER SEPARATED MODULES
Same solution but with actual separated files.
MyPyQtGUI.py
, the program entry point
output_redirection_tools.py
the very first import that should be done during execution flow. Hosts all the magic.
config.py
, a config module hosting config elements
my_logging.py
, custom logging configuration
third_party_module_not_to_change.py
, sample version of some code I use but don’t want to change.
MyPyQtGUI.py
It is important to note that the very first import of the project should be import output_redirection_tools
since it does all the tqdm hack job.
# looks like an unused import, but it actually does the TQDM class trick to intercept prints
import output_redirection_tools # KEEP ME !!!
import logging
import sys
from PyQt5.QtCore import pyqtSlot, QObject, QThread, Qt
from PyQt5.QtGui import QTextCursor, QFont
from PyQt5.QtWidgets import QTextEdit, QWidget, QToolButton, QVBoxLayout, QApplication, QLineEdit
from config import config_dict, STDOUT_WRITE_STREAM_CONFIG, TQDM_WRITE_STREAM_CONFIG, STREAM_CONFIG_KEY_QUEUE,
STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER
from my_logging import setup_logging
import third_party_module_not_to_change
class MainApp(QWidget):
def __init__(self):
super().__init__()
setup_logging(self.__class__.__name__)
self.__logger = logging.getLogger(self.__class__.__name__)
self.__logger.setLevel(logging.DEBUG)
self.queue_std_out = config_dict[STDOUT_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_QUEUE]
self.queue_tqdm = config_dict[TQDM_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_QUEUE]
layout = QVBoxLayout()
self.setMinimumWidth(500)
self.btn_perform_actions = QToolButton(self)
self.btn_perform_actions.setText('Launch long processing')
self.btn_perform_actions.clicked.connect(self._btn_go_clicked)
self.text_edit_std_out = StdOutTextEdit(self)
self.text_edit_tqdm = StdTQDMTextEdit(self)
self.thread_initialize = QThread()
self.init_procedure_object = LongProcedureWrapper(self)
# std out stream management
# create console text read thread + receiver object
self.thread_std_out_queue_listener = QThread()
self.std_out_text_receiver = config_dict[STDOUT_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER]
# connect receiver object to widget for text update
self.std_out_text_receiver.queue_std_out_element_received_signal.connect(self.text_edit_std_out.append_text)
# attach console text receiver to console text thread
self.std_out_text_receiver.moveToThread(self.thread_std_out_queue_listener)
# attach to start / stop methods
self.thread_std_out_queue_listener.started.connect(self.std_out_text_receiver.run)
self.thread_std_out_queue_listener.start()
# NEW: TQDM stream management
self.thread_tqdm_queue_listener = QThread()
self.tqdm_text_receiver = config_dict[TQDM_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER]
# connect receiver object to widget for text update
self.tqdm_text_receiver.queue_tqdm_element_received_signal.connect(self.text_edit_tqdm.set_tqdm_text)
# attach console text receiver to console text thread
self.tqdm_text_receiver.moveToThread(self.thread_tqdm_queue_listener)
# attach to start / stop methods
self.thread_tqdm_queue_listener.started.connect(self.tqdm_text_receiver.run)
self.thread_tqdm_queue_listener.start()
layout.addWidget(self.btn_perform_actions)
layout.addWidget(self.text_edit_std_out)
layout.addWidget(self.text_edit_tqdm)
self.setLayout(layout)
self.show()
@pyqtSlot()
def _btn_go_clicked(self):
# prepare thread for long operation
self.init_procedure_object.moveToThread(self.thread_initialize)
self.thread_initialize.started.connect(self.init_procedure_object.run)
# start thread
self.btn_perform_actions.setEnabled(False)
self.thread_initialize.start()
class StdOutTextEdit(QTextEdit):
def __init__(self, parent):
super(StdOutTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setLineWidth(50)
self.setMinimumWidth(500)
self.setFont(QFont('Consolas', 11))
@pyqtSlot(str)
def append_text(self, text: str):
self.moveCursor(QTextCursor.End)
self.insertPlainText(text)
class StdTQDMTextEdit(QLineEdit):
def __init__(self, parent):
super(StdTQDMTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setEnabled(True)
self.setMinimumWidth(500)
self.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.setClearButtonEnabled(True)
self.setFont(QFont('Consolas', 11))
@pyqtSlot(str)
def set_tqdm_text(self, text: str):
new_text = text
if new_text.find('r') >= 0:
new_text = new_text.replace('r', '').rstrip()
if new_text:
self.setText(new_text)
else:
# we suppose that all TQDM prints have r, so drop the rest
pass
class LongProcedureWrapper(QObject):
def __init__(self, main_app: MainApp):
super(LongProcedureWrapper, self).__init__()
self._main_app = main_app
@pyqtSlot()
def run(self):
third_party_module_not_to_change.long_procedure()
if __name__ == '__main__':
app = QApplication(sys.argv)
app.setStyle('Fusion')
ex = MainApp()
sys.exit(app.exec_())
my_logging.py
import logging
import datetime
import tqdm
from config import config_dict, IS_SETUP_DONE
def setup_logging(log_prefix, force_debug_level=logging.DEBUG):
root = logging.getLogger()
root.setLevel(force_debug_level)
if config_dict[IS_SETUP_DONE]:
pass
else:
__log_file_name = "{}-{}_log_file.txt".format(log_prefix,
datetime.datetime.utcnow().isoformat().replace(":", "-"))
__log_format = '%(asctime)s - %(name)-30s - %(levelname)s - %(message)s'
__console_date_format = '%Y-%m-%d %H:%M:%S'
__file_date_format = '%Y-%m-%d %H-%M-%S'
console_formatter = logging.Formatter(__log_format, __console_date_format)
file_formatter = logging.Formatter(__log_format, __file_date_format)
file_handler = logging.FileHandler(__log_file_name, mode='a', delay=True)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(file_formatter)
root.addHandler(file_handler)
tqdm_handler = TqdmLoggingHandler()
tqdm_handler.setLevel(logging.DEBUG)
tqdm_handler.setFormatter(console_formatter)
root.addHandler(tqdm_handler)
config_dict[IS_SETUP_DONE] = True
class TqdmLoggingHandler(logging.StreamHandler):
def __init__(self):
logging.StreamHandler.__init__(self)
def emit(self, record):
msg = self.format(record)
tqdm.tqdm.write(msg)
self.flush()
output_redirection_tools.py
import sys
from queue import Queue
from PyQt5.QtCore import pyqtSlot, pyqtSignal, QObject
from config import config_dict, IS_STREAMS_REDIRECTION_SETUP_DONE, TQDM_WRITE_STREAM_CONFIG, STDOUT_WRITE_STREAM_CONFIG,
STREAM_CONFIG_KEY_QUEUE, STREAM_CONFIG_KEY_STREAM, STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER
class QueueWriteStream(object):
def __init__(self, q: Queue):
self.queue = q
def write(self, text):
self.queue.put(text)
def flush(self):
pass
def perform_tqdm_default_out_stream_hack(tqdm_file_stream, tqdm_nb_columns=None):
import tqdm
# save original class into module
tqdm.orignal_class = tqdm.tqdm
class TQDMPatch(tqdm.orignal_class):
"""
Derive from original class
"""
def __init__(self, iterable=None, desc=None, total=None, leave=True,
file=None, ncols=None, mininterval=0.1, maxinterval=10.0,
miniters=None, ascii=None, disable=False, unit='it',
unit_scale=False, dynamic_ncols=False, smoothing=0.3,
bar_format=None, initial=0, position=None, postfix=None,
unit_divisor=1000, gui=False, **kwargs):
super(TQDMPatch, self).__init__(iterable, desc, total, leave,
tqdm_file_stream, # change any chosen file stream with our's
tqdm_nb_columns, # change nb of columns (gui choice),
mininterval, maxinterval,
miniters, ascii, disable, unit,
unit_scale,
False, # change param
smoothing,
bar_format, initial, position, postfix,
unit_divisor, gui, **kwargs)
print('TQDM Patch called') # check it works
@classmethod
def write(cls, s, file=None, end="n", nolock=False):
super(TQDMPatch, cls).write(s=s, file=file, end=end, nolock=nolock)
#tqdm.orignal_class.write(s=s, file=file, end=end, nolock=nolock)
# all other tqdm.orignal_class @classmethod methods may need to be redefined !
# # I mainly used tqdm.auto in my modules, so use that for patch
# # unsure if this will work with all possible tqdm import methods
# # might not work for tqdm_gui !
import tqdm.auto as AUTO
#
# # change original class with the patched one, the original still exists
AUTO.tqdm = TQDMPatch
#tqdm.tqdm = TQDMPatch
def setup_streams_redirection(tqdm_nb_columns=None):
if config_dict[IS_STREAMS_REDIRECTION_SETUP_DONE]:
pass
else:
configure_tqdm_redirection(tqdm_nb_columns)
configure_std_out_redirection()
config_dict[IS_STREAMS_REDIRECTION_SETUP_DONE] = True
def configure_std_out_redirection():
queue_std_out = Queue()
config_dict[STDOUT_WRITE_STREAM_CONFIG] = {
STREAM_CONFIG_KEY_QUEUE: queue_std_out,
STREAM_CONFIG_KEY_STREAM: QueueWriteStream(queue_std_out),
STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER: StdOutTextQueueReceiver(q=queue_std_out)
}
perform_std_out_hack()
def perform_std_out_hack():
sys.stdout = config_dict[STDOUT_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_STREAM]
def configure_tqdm_redirection(tqdm_nb_columns=None):
queue_tqdm = Queue()
config_dict[TQDM_WRITE_STREAM_CONFIG] = {
STREAM_CONFIG_KEY_QUEUE: queue_tqdm,
STREAM_CONFIG_KEY_STREAM: QueueWriteStream(queue_tqdm),
STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER: TQDMTextQueueReceiver(q=queue_tqdm)
}
perform_tqdm_default_out_stream_hack(
tqdm_file_stream=config_dict[TQDM_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_STREAM],
tqdm_nb_columns=tqdm_nb_columns)
class StdOutTextQueueReceiver(QObject):
# we are forced to define 1 signal per class
# see https://stackoverflow.com/questions/50294652/how-to-create-pyqtsignals-dynamically
queue_std_out_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
self.queue_std_out_element_received_signal.emit('---> STD OUT Queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_std_out_element_received_signal.emit(text)
class TQDMTextQueueReceiver(QObject):
# we are forced to define 1 signal per class
# see https://stackoverflow.com/questions/50294652/how-to-create-pyqtsignals-dynamically
queue_tqdm_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
# we assume that all TQDM outputs start with r, so use that to show stream reception is started
self.queue_tqdm_element_received_signal.emit('r---> TQDM Queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_tqdm_element_received_signal.emit(text)
setup_streams_redirection()
config.py
IS_SETUP_DONE = 'is_setup_done'
TQDM_WRITE_STREAM_CONFIG = 'TQDM_WRITE_STREAM_CONFIG'
STDOUT_WRITE_STREAM_CONFIG = 'STDOUT_WRITE_STREAM_CONFIG'
IS_STREAMS_REDIRECTION_SETUP_DONE = 'IS_STREAMS_REDIRECTION_SETUP_DONE'
STREAM_CONFIG_KEY_QUEUE = 'queue'
STREAM_CONFIG_KEY_STREAM = 'write_stream'
STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER = 'qt_queue_receiver'
default_config_dict = {
IS_SETUP_DONE: False,
IS_STREAMS_REDIRECTION_SETUP_DONE: False,
TQDM_WRITE_STREAM_CONFIG: None,
STDOUT_WRITE_STREAM_CONFIG: None,
}
config_dict = default_config_dict
third_part_module_not_to_change.py
represents the kind of code I use and don’t want to / cannot change.
from tqdm.auto import tqdm
import logging
from my_logging import setup_logging
import time
def long_procedure():
setup_logging('long_procedure')
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
tqdm_obect = tqdm(range(10), unit_scale=True, dynamic_ncols=True)
tqdm_obect.set_description("My progress bar description")
for i in tqdm_obect:
time.sleep(.1)
__logger.info('foo {}'.format(i))
Using QProgressBar
Long after my inital anwser, I had to think about this again.
Don’t ask why, but this time I managed to get it with a QProgressBar 🙂
The trick (at least with TQDM 4.63.1 and higher), is that there is a property format_dict
with almost everything necessary for a progress bar. Maybe we already did have that before, but I missed it the first time …
Tested with:
tqdm=4.63.1
Qt=5.15.2; PyQt=5.15.6
coloredlogs=15.0.1
1. GIF showing the solution
2. How does it work?
As in my previous answer, we need:
- a queue
- a patched TQDM class
- a worker object to read queue and send signal to QProgressBar
New thing here are:
- a QProgressBar subclass
- we take advantage of the new TQDM context
with logging_redirect_tqdm():
which handles routing of logging traces
- use of a custom logging traces module, with compatibility with the coloredlogs module => provides a fancy QPlainTextEdit with logger coloredlogs 🙂
- no more tricks with stdout/stderr streams
Concerning the TQDM class patch, we redefine __init__
, but now we also define refresh
and close
(instead of using the file stream trick from my previous answer)0
__init__
stores a new tqdm instance attribute, the queue and sends a "{do_reset:true}" (to reset the QProgressBar and make it visible)
refresh
adds to queue format_dict
(it contains n
and total`)
close
adds to queue a string "close" (to hide the progress bar)
3. Full example (1 file)
import contextlib
import logging
import sys
from abc import ABC, abstractmethod
from queue import Queue
from PyQt5 import QtTest
from PyQt5.QtCore import PYQT_VERSION_STR, pyqtSignal, pyqtSlot, QObject, Qt, QT_VERSION_STR, QThread
from PyQt5.QtWidgets import QApplication, QPlainTextEdit, QProgressBar, QToolButton, QVBoxLayout, QWidget
__CONFIGURED = False
def setup_streams_redirection(tqdm_nb_columns=None):
if not __CONFIGURED:
tqdm_update_queue = Queue()
perform_tqdm_default_out_stream_hack(tqdm_update_queue=tqdm_update_queue, tqdm_nb_columns=tqdm_nb_columns)
return TQDMDataQueueReceiver(tqdm_update_queue)
def perform_tqdm_default_out_stream_hack(tqdm_update_queue: Queue, tqdm_nb_columns=None):
import tqdm
# save original class into module
tqdm.original_class = tqdm.std.tqdm
parent = tqdm.std.tqdm
class TQDMPatch(parent):
"""
Derive from original class
"""
def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None,
ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None,
ascii=None, disable=False, unit='it', unit_scale=False,
dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0,
position=None, postfix=None, unit_divisor=1000, write_bytes=None,
lock_args=None, nrows=None, colour=None, delay=0, gui=False,
**kwargs):
print('TQDM Patch called') # check it works
self.tqdm_update_queue = tqdm_update_queue
self.tqdm_update_queue.put({"do_reset": True})
super(TQDMPatch, self).__init__(iterable, desc, total, leave,
file, # no change here
ncols,
mininterval, maxinterval,
miniters, ascii, disable, unit,
unit_scale,
False, # change param ?
smoothing,
bar_format, initial, position, postfix,
unit_divisor, gui, **kwargs)
# def update(self, n=1):
# super(TQDMPatch, self).update(n=n)
# custom stuff ?
def refresh(self, nolock=False, lock_args=None):
super(TQDMPatch, self).refresh(nolock=nolock, lock_args=lock_args)
self.tqdm_update_queue.put(self.format_dict)
def close(self):
self.tqdm_update_queue.put({"close": True})
super(TQDMPatch, self).close()
# change original class with the patched one, the original still exists
tqdm.std.tqdm = TQDMPatch
tqdm.tqdm = TQDMPatch # may not be necessary
# for tqdm.auto users, maybe some additional stuff is needed
class TQDMDataQueueReceiver(QObject):
s_tqdm_object_received_signal = pyqtSignal(object)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
while True:
o = self.queue.get()
# noinspection PyUnresolvedReferences
self.s_tqdm_object_received_signal.emit(o)
class QTQDMProgressBar(QProgressBar):
def __init__(self, parent, tqdm_signal: pyqtSignal):
super(QTQDMProgressBar, self).__init__(parent)
self.setAlignment(Qt.AlignCenter)
self.setVisible(False)
# noinspection PyUnresolvedReferences
tqdm_signal.connect(self.do_it)
def do_it(self, e):
if not isinstance(e, dict):
return
do_reset = e.get("do_reset", False) # different from close, because we want visible=true
initial = e.get("initial", 0)
total = e.get("total", None)
n = e.get("n", None)
desc = e.get("prefix", None)
text = e.get("text", None)
do_close = e.get("close", False) # different from do_reset, we want visible=false
if do_reset:
self.reset()
if do_close:
self.reset()
self.setVisible(not do_close)
if initial:
self.setMinimum(initial)
else:
self.setMinimum(0)
if total:
self.setMaximum(total)
else:
self.setMaximum(0)
if n:
self.setValue(n)
if desc:
self.setFormat(f"{desc} %v/%m | %p %")
elif text:
self.setFormat(text)
else:
self.setFormat("%v/%m | %p")
def long_procedure():
# emulate late import of modules
from tqdm.auto import tqdm # don't import before patch !
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
tqdm_object = tqdm(range(10), unit_scale=True, dynamic_ncols=True)
tqdm_object.set_description("My progress bar description")
from tqdm.contrib.logging import logging_redirect_tqdm # don't import before patch !
with logging_redirect_tqdm():
for i in tqdm_object:
QtTest.QTest.qWait(200)
__logger.info(f'foo {i}')
class QtLoggingHelper(ABC):
@abstractmethod
def transform(self, msg: str):
raise NotImplementedError()
class QtLoggingBasic(QtLoggingHelper):
def transform(self, msg: str):
return msg
class QtLoggingColoredLogs(QtLoggingHelper):
def __init__(self):
# offensive programming: crash if necessary if import is not present
pass
def transform(self, msg: str):
import coloredlogs.converter
msg_html = coloredlogs.converter.convert(msg)
return msg_html
class QTextEditLogger(logging.Handler, QObject):
appendText = pyqtSignal(str)
def __init__(self,
logger_: logging.Logger,
formatter: logging.Formatter,
text_widget: QPlainTextEdit,
# table_widget: QTableWidget,
parent: QWidget):
super(QTextEditLogger, self).__init__()
super(QObject, self).__init__(parent=parent)
self.text_widget = text_widget
self.text_widget.setReadOnly(True)
# self.table_widget = table_widget
try:
self.helper = QtLoggingColoredLogs()
self.appendText.connect(self.text_widget.appendHtml)
logger_.info("Using QtLoggingColoredLogs")
except ImportError:
self.helper = QtLoggingBasic()
self.appendText.connect(self.text_widget.appendPlainText)
logger_.warning("Using QtLoggingBasic")
# logTextBox = QTextEditLogger(self)
# You can format what is printed to text box
self.setFormatter(formatter)
logger_.addHandler(self)
# You can control the logging level
self.setLevel(logging.DEBUG)
def emit(self, record: logging.LogRecord):
msg = self.format(record)
display_msg = self.helper.transform(msg=msg)
self.appendText.emit(display_msg)
# self.add_row(record)
class MainApp(QWidget):
def __init__(self):
super().__init__()
self.__logger = logging.getLogger(self.__class__.__name__)
self.__logger.setLevel(logging.DEBUG)
layout = QVBoxLayout()
self.setMinimumWidth(500)
self.btn_perform_actions = QToolButton(self)
self.btn_perform_actions.setText('Launch long processing')
self.btn_perform_actions.clicked.connect(self._btn_go_clicked)
self.thread_initialize = QThread()
self.init_procedure_object = LongProcedureWorker(self)
self.thread_tqdm_update_queue_listener = QThread()
# must be done before any TQDM import
self.tqdm_update_receiver = setup_streams_redirection()
self.tqdm_update_receiver.moveToThread(self.thread_tqdm_update_queue_listener)
self.thread_tqdm_update_queue_listener.started.connect(self.tqdm_update_receiver.run)
self.pb_tqdm = QTQDMProgressBar(self, tqdm_signal=self.tqdm_update_receiver.s_tqdm_object_received_signal)
layout.addWidget(self.pb_tqdm)
self.thread_tqdm_update_queue_listener.start()
self.plain_text_edit_logger = QPlainTextEdit(self)
LOG_FMT = "{asctime} | {levelname:10s} | {message}"
try:
import coloredlogs
FORMATTER = coloredlogs.ColoredFormatter(fmt=LOG_FMT, style="{")
except ImportError:
FORMATTER = logging.Formatter(fmt=LOG_FMT, style="{")
self.logging_ = QTextEditLogger(logger_=logging.getLogger(), # root logger, to intercept every log of app
formatter=FORMATTER,
text_widget=self.plain_text_edit_logger,
parent=self)
layout.addWidget(self.plain_text_edit_logger)
layout.addWidget(self.btn_perform_actions)
self.setLayout(layout)
import tqdm
self.__logger.info(f"tqdm {tqdm.__version__}")
self.__logger.info(f"Qt={QT_VERSION_STR}; PyQt={PYQT_VERSION_STR}")
with contextlib.suppress(ImportError):
import coloredlogs
self.__logger.info(f"coloredlogs {coloredlogs.__version__}")
# prepare thread for long operation
self.init_procedure_object.moveToThread(self.thread_initialize)
self.thread_initialize.started.connect(self.init_procedure_object.run)
self.init_procedure_object.finished.connect(self._init_procedure_finished)
self.init_procedure_object.finished.connect(self.thread_initialize.quit)
self.show()
@pyqtSlot()
def _btn_go_clicked(self):
# start thread
self.btn_perform_actions.setEnabled(False)
self.__logger.info("Launch Thread")
self.thread_initialize.start()
def _init_procedure_finished(self):
self.btn_perform_actions.setEnabled(True)
class LongProcedureWorker(QObject):
finished = pyqtSignal()
def __init__(self, main_app: MainApp):
super(LongProcedureWorker, self).__init__()
self._main_app = main_app
@pyqtSlot()
def run(self):
long_procedure()
self.finished.emit()
if __name__ == '__main__':
app = QApplication(sys.argv)
app.setStyle('Fusion')
ex = MainApp()
sys.exit(app.exec_())
TL;DR
For answers, see:
- my 2019 initially own accepted answer using a text edit and stdout/stderr streams redirections, see https://stackoverflow.com/a/55082521/7237062
- my second answer, now marked as the accepted one: a derived and improved approach with a real QProgressBar ! https://stackoverflow.com/a/74091829/7237062
QUESTION
First of all, I know that a lot of questions are similar to this one.
But after spending so much time on it, I now look for help from the community.
I developed and use a bunch of python modules that rely on tqdm
.
I want them to be usable inside Jupyter, in console or with a GUI.
Everything works fine in Jupyter or console : there are no collisions between logging/prints and tqdm progress bars. Here is a sample code that shows the console/Jupyter behavior:
# coding=utf-8
from tqdm.auto import tqdm
import time
import logging
import sys
import datetime
__is_setup_done = False
def setup_logging(log_prefix):
global __is_setup_done
if __is_setup_done:
pass
else:
__log_file_name = "{}-{}_log_file.txt".format(log_prefix,
datetime.datetime.utcnow().isoformat().replace(":", "-"))
__log_format = '%(asctime)s - %(name)-30s - %(levelname)s - %(message)s'
__console_date_format = '%Y-%m-%d %H:%M:%S'
__file_date_format = '%Y-%m-%d %H-%M-%S'
root = logging.getLogger()
root.setLevel(logging.DEBUG)
console_formatter = logging.Formatter(__log_format, __console_date_format)
file_formatter = logging.Formatter(__log_format, __file_date_format)
file_handler = logging.FileHandler(__log_file_name, mode='a', delay=True)
# file_handler = TqdmLoggingHandler2(__log_file_name, mode='a', delay=True)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(file_formatter)
root.addHandler(file_handler)
tqdm_handler = TqdmLoggingHandler()
tqdm_handler.setLevel(logging.DEBUG)
tqdm_handler.setFormatter(console_formatter)
root.addHandler(tqdm_handler)
__is_setup_done = True
class TqdmLoggingHandler(logging.StreamHandler):
def __init__(self, level=logging.NOTSET):
logging.StreamHandler.__init__(self)
def emit(self, record):
msg = self.format(record)
tqdm.write(msg)
# from https://stackoverflow.com/questions/38543506/change-logging-print-function-to-tqdm-write-so-logging-doesnt-interfere-wit/38739634#38739634
self.flush()
def example_long_procedure():
setup_logging('long_procedure')
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
for i in tqdm(range(10), unit_scale=True, dynamic_ncols=True, file=sys.stdout):
time.sleep(.1)
__logger.info('foo {}'.format(i))
example_long_procedure()
The obtained output:
2019-03-07 22:22:27 - long_procedure - INFO - foo 0
2019-03-07 22:22:27 - long_procedure - INFO - foo 1
2019-03-07 22:22:27 - long_procedure - INFO - foo 2
2019-03-07 22:22:27 - long_procedure - INFO - foo 3
2019-03-07 22:22:27 - long_procedure - INFO - foo 4
2019-03-07 22:22:28 - long_procedure - INFO - foo 5
2019-03-07 22:22:28 - long_procedure - INFO - foo 6
2019-03-07 22:22:28 - long_procedure - INFO - foo 7
2019-03-07 22:22:28 - long_procedure - INFO - foo 8
2019-03-07 22:22:28 - long_procedure - INFO - foo 9
100%|¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦¦| 10.0/10.0 [00:01<00:00, 9.69it/s]
Now, I’m making a GUI with PyQt that uses code similar to above. Since processing may be long, I used threading in order to avoid freezing HMI during processing. I also used stdout
redirection using Queue() towards a Qt QWidget so the user can see what is happenning.
My current use case is 1 single thread that has logs and tqdm progress bars to redirect to 1 dedicated widget. (I’m not looking for multiple threads to feed the widget with multiple logs and multiple tqdm progress bar).
I managed to redirect stdout thanks to the informations from Redirecting stdout and stderr to a PyQt5 QTextEdit from a secondary thread.
However, only logger lines are redirected. TQDM progress bar is still directed to the console output.
Here is my current code:
# coding=utf-8
import time
import logging
import sys
import datetime
__is_setup_done = False
from queue import Queue
from PyQt5.QtCore import pyqtSlot, pyqtSignal, QObject, QThread, QMetaObject, Q_ARG, Qt
from PyQt5.QtGui import QTextCursor, QFont
from PyQt5.QtWidgets import QTextEdit, QPlainTextEdit, QWidget, QToolButton, QVBoxLayout, QApplication
from tqdm.auto import tqdm
class MainApp(QWidget):
def __init__(self):
super().__init__()
setup_logging(self.__class__.__name__)
self.__logger = logging.getLogger(self.__class__.__name__)
self.__logger.setLevel(logging.DEBUG)
# create console text queue
self.queue_console_text = Queue()
# redirect stdout to the queue
output_stream = WriteStream(self.queue_console_text)
sys.stdout = output_stream
layout = QVBoxLayout()
self.setMinimumWidth(500)
# GO button
self.btn_perform_actions = QToolButton(self)
self.btn_perform_actions.setText('Launch long processing')
self.btn_perform_actions.clicked.connect(self._btn_go_clicked)
self.console_text_edit = ConsoleTextEdit(self)
self.thread_initialize = QThread()
self.init_procedure_object = InitializationProcedures(self)
# create console text read thread + receiver object
self.thread_queue_listener = QThread()
self.console_text_receiver = ThreadConsoleTextQueueReceiver(self.queue_console_text)
# connect receiver object to widget for text update
self.console_text_receiver.queue_element_received_signal.connect(self.console_text_edit.append_text)
# attach console text receiver to console text thread
self.console_text_receiver.moveToThread(self.thread_queue_listener)
# attach to start / stop methods
self.thread_queue_listener.started.connect(self.console_text_receiver.run)
self.thread_queue_listener.finished.connect(self.console_text_receiver.finished)
self.thread_queue_listener.start()
layout.addWidget(self.btn_perform_actions)
layout.addWidget(self.console_text_edit)
self.setLayout(layout)
self.show()
@pyqtSlot()
def _btn_go_clicked(self):
# prepare thread for long operation
self.init_procedure_object.moveToThread(self.thread_initialize)
self.thread_initialize.started.connect(self.init_procedure_object.run)
self.thread_initialize.finished.connect(self.init_procedure_object.finished)
# start thread
self.btn_perform_actions.setEnabled(False)
self.thread_initialize.start()
class WriteStream(object):
def __init__(self, q: Queue):
self.queue = q
def write(self, text):
"""
Redirection of stream to the given queue
"""
self.queue.put(text)
def flush(self):
"""
Stream flush implementation
"""
pass
class ThreadConsoleTextQueueReceiver(QObject):
queue_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
self.queue_element_received_signal.emit('---> Console text queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_element_received_signal.emit(text)
@pyqtSlot()
def finished(self):
self.queue_element_received_signal.emit('---> Console text queue reception Stopped <---n')
class ConsoleTextEdit(QTextEdit):#QTextEdit):
def __init__(self, parent):
super(ConsoleTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setLineWidth(50)
self.setMinimumWidth(1200)
self.setFont(QFont('Consolas', 11))
self.flag = False
@pyqtSlot(str)
def append_text(self, text: str):
self.moveCursor(QTextCursor.End)
self.insertPlainText(text)
def long_procedure():
setup_logging('long_procedure')
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
for i in tqdm(range(10), unit_scale=True, dynamic_ncols=True):
time.sleep(.1)
__logger.info('foo {}'.format(i))
class InitializationProcedures(QObject):
def __init__(self, main_app: MainApp):
super(InitializationProcedures, self).__init__()
self._main_app = main_app
@pyqtSlot()
def run(self):
long_procedure()
@pyqtSlot()
def finished(self):
print("Thread finished !") # might call main window to do some stuff with buttons
self._main_app.btn_perform_actions.setEnabled(True)
def setup_logging(log_prefix):
global __is_setup_done
if __is_setup_done:
pass
else:
__log_file_name = "{}-{}_log_file.txt".format(log_prefix,
datetime.datetime.utcnow().isoformat().replace(":", "-"))
__log_format = '%(asctime)s - %(name)-30s - %(levelname)s - %(message)s'
__console_date_format = '%Y-%m-%d %H:%M:%S'
__file_date_format = '%Y-%m-%d %H-%M-%S'
root = logging.getLogger()
root.setLevel(logging.DEBUG)
console_formatter = logging.Formatter(__log_format, __console_date_format)
file_formatter = logging.Formatter(__log_format, __file_date_format)
file_handler = logging.FileHandler(__log_file_name, mode='a', delay=True)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(file_formatter)
root.addHandler(file_handler)
tqdm_handler = TqdmLoggingHandler()
tqdm_handler.setLevel(logging.DEBUG)
tqdm_handler.setFormatter(console_formatter)
root.addHandler(tqdm_handler)
__is_setup_done = True
class TqdmLoggingHandler(logging.StreamHandler):
def __init__(self, level=logging.NOTSET):
logging.StreamHandler.__init__(self)
def emit(self, record):
msg = self.format(record)
tqdm.write(msg)
# from https://stackoverflow.com/questions/38543506/change-logging-print-function-to-tqdm-write-so-logging-doesnt-interfere-wit/38739634#38739634
self.flush()
if __name__ == '__main__':
app = QApplication(sys.argv)
app.setStyle('Fusion')
tqdm.ncols = 50
ex = MainApp()
sys.exit(app.exec_())
I would like to obtain the exact behavior I would have had strictly invoking the code in console.
i.e. expected output in PyQt widget:
---> Console text queue reception Started <---
2019-03-07 19:42:19 - long_procedure - INFO - foo 0
2019-03-07 19:42:19 - long_procedure - INFO - foo 1
2019-03-07 19:42:19 - long_procedure - INFO - foo 2
2019-03-07 19:42:19 - long_procedure - INFO - foo 3
2019-03-07 19:42:19 - long_procedure - INFO - foo 4
2019-03-07 19:42:19 - long_procedure - INFO - foo 5
2019-03-07 19:42:20 - long_procedure - INFO - foo 6
2019-03-07 19:42:20 - long_procedure - INFO - foo 7
2019-03-07 19:42:20 - long_procedure - INFO - foo 8
2019-03-07 19:42:20 - long_procedure - INFO - foo 9
100%|################################| 10.0/10.0 [00:01<00:00, 9.16it/s]
Things I tried / explored with no success.
Option 1
This solution Display terminal output with tqdm in QPlainTextEdit does not give the expected results. It works well to redirect outputs containing only tqdm stuff.
The following code does not give the intended behavior, wether it is with QTextEdit or QPlainTextEdit. Only logger lines are redirected.
# code from this answer
# https://stackoverflow.com/questions/53381975/display-terminal-output-with-tqdm-in-qplaintextedit
@pyqtSlot(str)
def append_text(self, message: str):
if not hasattr(self, "flag"):
self.flag = False
message = message.replace('r', '').rstrip()
if message:
method = "replace_last_line" if self.flag else "append_text"
QMetaObject.invokeMethod(self,
method,
Qt.QueuedConnection,
Q_ARG(str, message))
self.flag = True
else:
self.flag = False
@pyqtSlot(str)
def replace_last_line(self, text):
cursor = self.textCursor()
cursor.movePosition(QTextCursor.End)
cursor.select(QTextCursor.BlockUnderCursor)
cursor.removeSelectedText()
cursor.insertBlock()
self.setTextCursor(cursor)
self.insertPlainText(text)
However, the above code + adding file=sys.stdout
to the tqdm call changes the behavior: tqdm output is redirected to the Qt widget. But in the end, only one line is displayed, and it is either a logger line or a tqdm line (it looks like it depends on which Qt widget I derived).
In the end, changing all tqdm invocations used modules should not be the preferred option.
So the other approach I found is to redirect stderr in the same stream/queue stdout is redirected to. Since tqdm writes to stderr by default, this way all tqdm outputs are redirected to widget.
But I still can’t figure out obtaining the exact output I’m looking for.
This question does not provide a clue on why behavior seems to differ between QTextEdit vs QPlainTextEdit
Option 2
This question Duplicate stdout, stderr in QTextEdit widget looks very similar to Display terminal output with tqdm in QPlainTextEdit and does not answer to my exact problem described above.
Option 3
Trying this solution using contextlib gave me an error due to no flush() method being defined. After fixing, I end up with only tqdm lines and no logger lines.
Option 4
I also tried to intercept the r character and implement a specific behavior, with not success.
Versions:
tqdm 4.28.1
pyqt 5.9.2
PyQt5 5.12
PyQt5_sip 4.19.14
Python 3.7.2
EDIT 2019-mar-12: It seems to me that the answer is : it could probably be done, but requires a lots of effort in order to remember where which line comes from for the QTextEdit to behave as instended. Plus, since tdm writes to stderr by default, you would end up with catching all exceptions traces too.
That is why I’ll mark my own answer as solved: I find it more elegant to achieve the same purpose : show in pyqt what is happenning.
Here is my best shot to obtain something close to the intended behavior.
It does not exactly respond to the question, because I changed the GUI design.
So I won’t vote it as solved. Moreover, this is done all in one single python file. I plan to further challenge this solution to see if it works with real python modules doing tqdm imports.
I patched the basic tqdm class in a very ugly way. The main trick is to :
- dynamically change the tqdm module structure by storing the original tqdm class into a new name:
tqdm.orignal_class = tqdm.tqdm
- then inherit tqdm.original class
class TQDMPatch(tqdm.orignal_class):
- implement the constructor in order to force the file stream + any parameters to whatever you want:
super(TQDMPatch, self).__init__(... change some params ...)
. I gave my TQDM class a customWriteStream()
that writes into aQueue()
- change the GUI strategy to intercept and redirect your custom tqdm stream to a separate Qt widget. My widget is making the assumption that all received prints contain
r
(which TQDM seems to be doing).
It is working both as in single python file and with multiple separated modules. In the latter case, imports order at startup is critical.
Screenshots:
Before launching processing
During processing
At end of processing
Here is the code
ALL-IN-ONE FILE
# coding=utf-8
import datetime
import logging
import sys
import time
from queue import Queue
from PyQt5.QtCore import pyqtSlot, pyqtSignal, QObject, QThread, Qt
from PyQt5.QtGui import QTextCursor, QFont
from PyQt5.QtWidgets import QTextEdit, QWidget, QToolButton, QVBoxLayout, QApplication, QLineEdit
# DEFINITION NEEDED FIRST ...
class WriteStream(object):
def __init__(self, q: Queue):
self.queue = q
def write(self, text):
self.queue.put(text)
def flush(self):
pass
# prepare queue and streams
queue_tqdm = Queue()
write_stream_tqdm = WriteStream(queue_tqdm)
################## START TQDM patch procedure ##################
import tqdm
# save original class into module
tqdm.orignal_class = tqdm.tqdm
class TQDMPatch(tqdm.orignal_class):
"""
Derive from original class
"""
def __init__(self, iterable=None, desc=None, total=None, leave=True,
file=None, ncols=None, mininterval=0.1, maxinterval=10.0,
miniters=None, ascii=None, disable=False, unit='it',
unit_scale=False, dynamic_ncols=False, smoothing=0.3,
bar_format=None, initial=0, position=None, postfix=None,
unit_divisor=1000, gui=False, **kwargs):
super(TQDMPatch, self).__init__(iterable, desc, total, leave,
write_stream_tqdm, # change any chosen file stream with our's
80, # change nb of columns (gui choice),
mininterval, maxinterval,
miniters, ascii, disable, unit,
unit_scale, False, smoothing,
bar_format, initial, position, postfix,
unit_divisor, gui, **kwargs)
print('TQDM Patch called') # check it works
@classmethod
def write(cls, s, file=None, end="n", nolock=False):
super(TQDMPatch, cls).write(s=s, file=file, end=end, nolock=nolock)
# all other tqdm.orignal_class @classmethod methods may need to be redefined !
# I mainly used tqdm.auto in my modules, so use that for patch
# unsure if this will work with all possible tqdm import methods
# might not work for tqdm_gui !
import tqdm.auto as AUTO
# change original class with the patched one, the original still exists
AUTO.tqdm = TQDMPatch
################## END of TQDM patch ##################
# normal MCVE code
__is_setup_done = False
class MainApp(QWidget):
def __init__(self):
super().__init__()
setup_logging(self.__class__.__name__)
self.__logger = logging.getLogger(self.__class__.__name__)
self.__logger.setLevel(logging.DEBUG)
# create stdout text queue
self.queue_std_out = Queue()
sys.stdout = WriteStream(self.queue_std_out)
layout = QVBoxLayout()
self.setMinimumWidth(500)
self.btn_perform_actions = QToolButton(self)
self.btn_perform_actions.setText('Launch long processing')
self.btn_perform_actions.clicked.connect(self._btn_go_clicked)
self.text_edit_std_out = StdOutTextEdit(self)
self.text_edit_tqdm = StdTQDMTextEdit(self)
self.thread_initialize = QThread()
self.init_procedure_object = InitializationProcedures(self)
# std out stream management
# create console text read thread + receiver object
self.thread_std_out_queue_listener = QThread()
self.std_out_text_receiver = ThreadStdOutStreamTextQueueReceiver(self.queue_std_out)
# connect receiver object to widget for text update
self.std_out_text_receiver.queue_std_out_element_received_signal.connect(self.text_edit_std_out.append_text)
# attach console text receiver to console text thread
self.std_out_text_receiver.moveToThread(self.thread_std_out_queue_listener)
# attach to start / stop methods
self.thread_std_out_queue_listener.started.connect(self.std_out_text_receiver.run)
self.thread_std_out_queue_listener.start()
# NEW: TQDM stream management
self.thread_tqdm_queue_listener = QThread()
self.tqdm_text_receiver = ThreadTQDMStreamTextQueueReceiver(queue_tqdm)
# connect receiver object to widget for text update
self.tqdm_text_receiver.queue_tqdm_element_received_signal.connect(self.text_edit_tqdm.set_tqdm_text)
# attach console text receiver to console text thread
self.tqdm_text_receiver.moveToThread(self.thread_tqdm_queue_listener)
# attach to start / stop methods
self.thread_tqdm_queue_listener.started.connect(self.tqdm_text_receiver.run)
self.thread_tqdm_queue_listener.start()
layout.addWidget(self.btn_perform_actions)
layout.addWidget(self.text_edit_std_out)
layout.addWidget(self.text_edit_tqdm)
self.setLayout(layout)
self.show()
@pyqtSlot()
def _btn_go_clicked(self):
# prepare thread for long operation
self.init_procedure_object.moveToThread(self.thread_initialize)
self.thread_initialize.started.connect(self.init_procedure_object.run)
self.thread_initialize.finished.connect(self.init_procedure_object.finished)
# start thread
self.btn_perform_actions.setEnabled(False)
self.thread_initialize.start()
class ThreadStdOutStreamTextQueueReceiver(QObject):
queue_std_out_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
self.queue_std_out_element_received_signal.emit('---> STD OUT Queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_std_out_element_received_signal.emit(text)
# NEW: dedicated receiving object for TQDM
class ThreadTQDMStreamTextQueueReceiver(QObject):
queue_tqdm_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
self.queue_tqdm_element_received_signal.emit('r---> TQDM Queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_tqdm_element_received_signal.emit(text)
class StdOutTextEdit(QTextEdit): # QTextEdit):
def __init__(self, parent):
super(StdOutTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setLineWidth(50)
self.setMinimumWidth(500)
self.setFont(QFont('Consolas', 11))
@pyqtSlot(str)
def append_text(self, text: str):
self.moveCursor(QTextCursor.End)
self.insertPlainText(text)
class StdTQDMTextEdit(QLineEdit):
def __init__(self, parent):
super(StdTQDMTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setEnabled(True)
self.setMinimumWidth(500)
self.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.setClearButtonEnabled(True)
self.setFont(QFont('Consolas', 11))
@pyqtSlot(str)
def set_tqdm_text(self, text: str):
new_text = text
if new_text.find('r') >= 0:
new_text = new_text.replace('r', '').rstrip()
if new_text:
self.setText(new_text)
else:
# we suppose that all TQDM prints have r
# so drop the rest
pass
def long_procedure():
# emulate import of modules
from tqdm.auto import tqdm
setup_logging('long_procedure')
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
tqdm_obect = tqdm(range(10), unit_scale=True, dynamic_ncols=True)
tqdm_obect.set_description("My progress bar description")
for i in tqdm_obect:
time.sleep(.1)
__logger.info('foo {}'.format(i))
class InitializationProcedures(QObject):
def __init__(self, main_app: MainApp):
super(InitializationProcedures, self).__init__()
self._main_app = main_app
@pyqtSlot()
def run(self):
long_procedure()
@pyqtSlot()
def finished(self):
print("Thread finished !") # might call main window to do some stuff with buttons
self._main_app.btn_perform_actions.setEnabled(True)
def setup_logging(log_prefix):
global __is_setup_done
if __is_setup_done:
pass
else:
__log_file_name = "{}-{}_log_file.txt".format(log_prefix,
datetime.datetime.utcnow().isoformat().replace(":", "-"))
__log_format = '%(asctime)s - %(name)-30s - %(levelname)s - %(message)s'
__console_date_format = '%Y-%m-%d %H:%M:%S'
__file_date_format = '%Y-%m-%d %H-%M-%S'
root = logging.getLogger()
root.setLevel(logging.DEBUG)
console_formatter = logging.Formatter(__log_format, __console_date_format)
file_formatter = logging.Formatter(__log_format, __file_date_format)
file_handler = logging.FileHandler(__log_file_name, mode='a', delay=True)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(file_formatter)
root.addHandler(file_handler)
tqdm_handler = TqdmLoggingHandler()
tqdm_handler.setLevel(logging.DEBUG)
tqdm_handler.setFormatter(console_formatter)
root.addHandler(tqdm_handler)
__is_setup_done = True
class TqdmLoggingHandler(logging.StreamHandler):
def __init__(self):
logging.StreamHandler.__init__(self)
def emit(self, record):
msg = self.format(record)
tqdm.tqdm.write(msg)
# from https://stackoverflow.com/questions/38543506/change-logging-print-function-to-tqdm-write-so-logging-doesnt-interfere-wit/38739634#38739634
self.flush()
if __name__ == '__main__':
app = QApplication(sys.argv)
app.setStyle('Fusion')
ex = MainApp()
sys.exit(app.exec_())
WITH PROPER SEPARATED MODULES
Same solution but with actual separated files.
MyPyQtGUI.py
, the program entry pointoutput_redirection_tools.py
the very first import that should be done during execution flow. Hosts all the magic.config.py
, a config module hosting config elementsmy_logging.py
, custom logging configurationthird_party_module_not_to_change.py
, sample version of some code I use but don’t want to change.
MyPyQtGUI.py
It is important to note that the very first import of the project should be import output_redirection_tools
since it does all the tqdm hack job.
# looks like an unused import, but it actually does the TQDM class trick to intercept prints
import output_redirection_tools # KEEP ME !!!
import logging
import sys
from PyQt5.QtCore import pyqtSlot, QObject, QThread, Qt
from PyQt5.QtGui import QTextCursor, QFont
from PyQt5.QtWidgets import QTextEdit, QWidget, QToolButton, QVBoxLayout, QApplication, QLineEdit
from config import config_dict, STDOUT_WRITE_STREAM_CONFIG, TQDM_WRITE_STREAM_CONFIG, STREAM_CONFIG_KEY_QUEUE,
STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER
from my_logging import setup_logging
import third_party_module_not_to_change
class MainApp(QWidget):
def __init__(self):
super().__init__()
setup_logging(self.__class__.__name__)
self.__logger = logging.getLogger(self.__class__.__name__)
self.__logger.setLevel(logging.DEBUG)
self.queue_std_out = config_dict[STDOUT_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_QUEUE]
self.queue_tqdm = config_dict[TQDM_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_QUEUE]
layout = QVBoxLayout()
self.setMinimumWidth(500)
self.btn_perform_actions = QToolButton(self)
self.btn_perform_actions.setText('Launch long processing')
self.btn_perform_actions.clicked.connect(self._btn_go_clicked)
self.text_edit_std_out = StdOutTextEdit(self)
self.text_edit_tqdm = StdTQDMTextEdit(self)
self.thread_initialize = QThread()
self.init_procedure_object = LongProcedureWrapper(self)
# std out stream management
# create console text read thread + receiver object
self.thread_std_out_queue_listener = QThread()
self.std_out_text_receiver = config_dict[STDOUT_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER]
# connect receiver object to widget for text update
self.std_out_text_receiver.queue_std_out_element_received_signal.connect(self.text_edit_std_out.append_text)
# attach console text receiver to console text thread
self.std_out_text_receiver.moveToThread(self.thread_std_out_queue_listener)
# attach to start / stop methods
self.thread_std_out_queue_listener.started.connect(self.std_out_text_receiver.run)
self.thread_std_out_queue_listener.start()
# NEW: TQDM stream management
self.thread_tqdm_queue_listener = QThread()
self.tqdm_text_receiver = config_dict[TQDM_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER]
# connect receiver object to widget for text update
self.tqdm_text_receiver.queue_tqdm_element_received_signal.connect(self.text_edit_tqdm.set_tqdm_text)
# attach console text receiver to console text thread
self.tqdm_text_receiver.moveToThread(self.thread_tqdm_queue_listener)
# attach to start / stop methods
self.thread_tqdm_queue_listener.started.connect(self.tqdm_text_receiver.run)
self.thread_tqdm_queue_listener.start()
layout.addWidget(self.btn_perform_actions)
layout.addWidget(self.text_edit_std_out)
layout.addWidget(self.text_edit_tqdm)
self.setLayout(layout)
self.show()
@pyqtSlot()
def _btn_go_clicked(self):
# prepare thread for long operation
self.init_procedure_object.moveToThread(self.thread_initialize)
self.thread_initialize.started.connect(self.init_procedure_object.run)
# start thread
self.btn_perform_actions.setEnabled(False)
self.thread_initialize.start()
class StdOutTextEdit(QTextEdit):
def __init__(self, parent):
super(StdOutTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setLineWidth(50)
self.setMinimumWidth(500)
self.setFont(QFont('Consolas', 11))
@pyqtSlot(str)
def append_text(self, text: str):
self.moveCursor(QTextCursor.End)
self.insertPlainText(text)
class StdTQDMTextEdit(QLineEdit):
def __init__(self, parent):
super(StdTQDMTextEdit, self).__init__()
self.setParent(parent)
self.setReadOnly(True)
self.setEnabled(True)
self.setMinimumWidth(500)
self.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.setClearButtonEnabled(True)
self.setFont(QFont('Consolas', 11))
@pyqtSlot(str)
def set_tqdm_text(self, text: str):
new_text = text
if new_text.find('r') >= 0:
new_text = new_text.replace('r', '').rstrip()
if new_text:
self.setText(new_text)
else:
# we suppose that all TQDM prints have r, so drop the rest
pass
class LongProcedureWrapper(QObject):
def __init__(self, main_app: MainApp):
super(LongProcedureWrapper, self).__init__()
self._main_app = main_app
@pyqtSlot()
def run(self):
third_party_module_not_to_change.long_procedure()
if __name__ == '__main__':
app = QApplication(sys.argv)
app.setStyle('Fusion')
ex = MainApp()
sys.exit(app.exec_())
my_logging.py
import logging
import datetime
import tqdm
from config import config_dict, IS_SETUP_DONE
def setup_logging(log_prefix, force_debug_level=logging.DEBUG):
root = logging.getLogger()
root.setLevel(force_debug_level)
if config_dict[IS_SETUP_DONE]:
pass
else:
__log_file_name = "{}-{}_log_file.txt".format(log_prefix,
datetime.datetime.utcnow().isoformat().replace(":", "-"))
__log_format = '%(asctime)s - %(name)-30s - %(levelname)s - %(message)s'
__console_date_format = '%Y-%m-%d %H:%M:%S'
__file_date_format = '%Y-%m-%d %H-%M-%S'
console_formatter = logging.Formatter(__log_format, __console_date_format)
file_formatter = logging.Formatter(__log_format, __file_date_format)
file_handler = logging.FileHandler(__log_file_name, mode='a', delay=True)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(file_formatter)
root.addHandler(file_handler)
tqdm_handler = TqdmLoggingHandler()
tqdm_handler.setLevel(logging.DEBUG)
tqdm_handler.setFormatter(console_formatter)
root.addHandler(tqdm_handler)
config_dict[IS_SETUP_DONE] = True
class TqdmLoggingHandler(logging.StreamHandler):
def __init__(self):
logging.StreamHandler.__init__(self)
def emit(self, record):
msg = self.format(record)
tqdm.tqdm.write(msg)
self.flush()
output_redirection_tools.py
import sys
from queue import Queue
from PyQt5.QtCore import pyqtSlot, pyqtSignal, QObject
from config import config_dict, IS_STREAMS_REDIRECTION_SETUP_DONE, TQDM_WRITE_STREAM_CONFIG, STDOUT_WRITE_STREAM_CONFIG,
STREAM_CONFIG_KEY_QUEUE, STREAM_CONFIG_KEY_STREAM, STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER
class QueueWriteStream(object):
def __init__(self, q: Queue):
self.queue = q
def write(self, text):
self.queue.put(text)
def flush(self):
pass
def perform_tqdm_default_out_stream_hack(tqdm_file_stream, tqdm_nb_columns=None):
import tqdm
# save original class into module
tqdm.orignal_class = tqdm.tqdm
class TQDMPatch(tqdm.orignal_class):
"""
Derive from original class
"""
def __init__(self, iterable=None, desc=None, total=None, leave=True,
file=None, ncols=None, mininterval=0.1, maxinterval=10.0,
miniters=None, ascii=None, disable=False, unit='it',
unit_scale=False, dynamic_ncols=False, smoothing=0.3,
bar_format=None, initial=0, position=None, postfix=None,
unit_divisor=1000, gui=False, **kwargs):
super(TQDMPatch, self).__init__(iterable, desc, total, leave,
tqdm_file_stream, # change any chosen file stream with our's
tqdm_nb_columns, # change nb of columns (gui choice),
mininterval, maxinterval,
miniters, ascii, disable, unit,
unit_scale,
False, # change param
smoothing,
bar_format, initial, position, postfix,
unit_divisor, gui, **kwargs)
print('TQDM Patch called') # check it works
@classmethod
def write(cls, s, file=None, end="n", nolock=False):
super(TQDMPatch, cls).write(s=s, file=file, end=end, nolock=nolock)
#tqdm.orignal_class.write(s=s, file=file, end=end, nolock=nolock)
# all other tqdm.orignal_class @classmethod methods may need to be redefined !
# # I mainly used tqdm.auto in my modules, so use that for patch
# # unsure if this will work with all possible tqdm import methods
# # might not work for tqdm_gui !
import tqdm.auto as AUTO
#
# # change original class with the patched one, the original still exists
AUTO.tqdm = TQDMPatch
#tqdm.tqdm = TQDMPatch
def setup_streams_redirection(tqdm_nb_columns=None):
if config_dict[IS_STREAMS_REDIRECTION_SETUP_DONE]:
pass
else:
configure_tqdm_redirection(tqdm_nb_columns)
configure_std_out_redirection()
config_dict[IS_STREAMS_REDIRECTION_SETUP_DONE] = True
def configure_std_out_redirection():
queue_std_out = Queue()
config_dict[STDOUT_WRITE_STREAM_CONFIG] = {
STREAM_CONFIG_KEY_QUEUE: queue_std_out,
STREAM_CONFIG_KEY_STREAM: QueueWriteStream(queue_std_out),
STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER: StdOutTextQueueReceiver(q=queue_std_out)
}
perform_std_out_hack()
def perform_std_out_hack():
sys.stdout = config_dict[STDOUT_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_STREAM]
def configure_tqdm_redirection(tqdm_nb_columns=None):
queue_tqdm = Queue()
config_dict[TQDM_WRITE_STREAM_CONFIG] = {
STREAM_CONFIG_KEY_QUEUE: queue_tqdm,
STREAM_CONFIG_KEY_STREAM: QueueWriteStream(queue_tqdm),
STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER: TQDMTextQueueReceiver(q=queue_tqdm)
}
perform_tqdm_default_out_stream_hack(
tqdm_file_stream=config_dict[TQDM_WRITE_STREAM_CONFIG][STREAM_CONFIG_KEY_STREAM],
tqdm_nb_columns=tqdm_nb_columns)
class StdOutTextQueueReceiver(QObject):
# we are forced to define 1 signal per class
# see https://stackoverflow.com/questions/50294652/how-to-create-pyqtsignals-dynamically
queue_std_out_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
self.queue_std_out_element_received_signal.emit('---> STD OUT Queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_std_out_element_received_signal.emit(text)
class TQDMTextQueueReceiver(QObject):
# we are forced to define 1 signal per class
# see https://stackoverflow.com/questions/50294652/how-to-create-pyqtsignals-dynamically
queue_tqdm_element_received_signal = pyqtSignal(str)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
# we assume that all TQDM outputs start with r, so use that to show stream reception is started
self.queue_tqdm_element_received_signal.emit('r---> TQDM Queue reception Started <---n')
while True:
text = self.queue.get()
self.queue_tqdm_element_received_signal.emit(text)
setup_streams_redirection()
config.py
IS_SETUP_DONE = 'is_setup_done'
TQDM_WRITE_STREAM_CONFIG = 'TQDM_WRITE_STREAM_CONFIG'
STDOUT_WRITE_STREAM_CONFIG = 'STDOUT_WRITE_STREAM_CONFIG'
IS_STREAMS_REDIRECTION_SETUP_DONE = 'IS_STREAMS_REDIRECTION_SETUP_DONE'
STREAM_CONFIG_KEY_QUEUE = 'queue'
STREAM_CONFIG_KEY_STREAM = 'write_stream'
STREAM_CONFIG_KEY_QT_QUEUE_RECEIVER = 'qt_queue_receiver'
default_config_dict = {
IS_SETUP_DONE: False,
IS_STREAMS_REDIRECTION_SETUP_DONE: False,
TQDM_WRITE_STREAM_CONFIG: None,
STDOUT_WRITE_STREAM_CONFIG: None,
}
config_dict = default_config_dict
third_part_module_not_to_change.py
represents the kind of code I use and don’t want to / cannot change.
from tqdm.auto import tqdm
import logging
from my_logging import setup_logging
import time
def long_procedure():
setup_logging('long_procedure')
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
tqdm_obect = tqdm(range(10), unit_scale=True, dynamic_ncols=True)
tqdm_obect.set_description("My progress bar description")
for i in tqdm_obect:
time.sleep(.1)
__logger.info('foo {}'.format(i))
Using QProgressBar
Long after my inital anwser, I had to think about this again.
Don’t ask why, but this time I managed to get it with a QProgressBar 🙂
The trick (at least with TQDM 4.63.1 and higher), is that there is a property format_dict
with almost everything necessary for a progress bar. Maybe we already did have that before, but I missed it the first time …
Tested with:
tqdm=4.63.1
Qt=5.15.2; PyQt=5.15.6
coloredlogs=15.0.1
1. GIF showing the solution
2. How does it work?
As in my previous answer, we need:
- a queue
- a patched TQDM class
- a worker object to read queue and send signal to QProgressBar
New thing here are:
- a QProgressBar subclass
- we take advantage of the new TQDM context
with logging_redirect_tqdm():
which handles routing of logging traces - use of a custom logging traces module, with compatibility with the coloredlogs module => provides a fancy QPlainTextEdit with logger coloredlogs 🙂
- no more tricks with stdout/stderr streams
Concerning the TQDM class patch, we redefine __init__
, but now we also define refresh
and close
(instead of using the file stream trick from my previous answer)0
__init__
stores a new tqdm instance attribute, the queue and sends a "{do_reset:true}" (to reset the QProgressBar and make it visible)refresh
adds to queueformat_dict
(it containsn
and total`)close
adds to queue a string "close" (to hide the progress bar)
3. Full example (1 file)
import contextlib
import logging
import sys
from abc import ABC, abstractmethod
from queue import Queue
from PyQt5 import QtTest
from PyQt5.QtCore import PYQT_VERSION_STR, pyqtSignal, pyqtSlot, QObject, Qt, QT_VERSION_STR, QThread
from PyQt5.QtWidgets import QApplication, QPlainTextEdit, QProgressBar, QToolButton, QVBoxLayout, QWidget
__CONFIGURED = False
def setup_streams_redirection(tqdm_nb_columns=None):
if not __CONFIGURED:
tqdm_update_queue = Queue()
perform_tqdm_default_out_stream_hack(tqdm_update_queue=tqdm_update_queue, tqdm_nb_columns=tqdm_nb_columns)
return TQDMDataQueueReceiver(tqdm_update_queue)
def perform_tqdm_default_out_stream_hack(tqdm_update_queue: Queue, tqdm_nb_columns=None):
import tqdm
# save original class into module
tqdm.original_class = tqdm.std.tqdm
parent = tqdm.std.tqdm
class TQDMPatch(parent):
"""
Derive from original class
"""
def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None,
ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None,
ascii=None, disable=False, unit='it', unit_scale=False,
dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0,
position=None, postfix=None, unit_divisor=1000, write_bytes=None,
lock_args=None, nrows=None, colour=None, delay=0, gui=False,
**kwargs):
print('TQDM Patch called') # check it works
self.tqdm_update_queue = tqdm_update_queue
self.tqdm_update_queue.put({"do_reset": True})
super(TQDMPatch, self).__init__(iterable, desc, total, leave,
file, # no change here
ncols,
mininterval, maxinterval,
miniters, ascii, disable, unit,
unit_scale,
False, # change param ?
smoothing,
bar_format, initial, position, postfix,
unit_divisor, gui, **kwargs)
# def update(self, n=1):
# super(TQDMPatch, self).update(n=n)
# custom stuff ?
def refresh(self, nolock=False, lock_args=None):
super(TQDMPatch, self).refresh(nolock=nolock, lock_args=lock_args)
self.tqdm_update_queue.put(self.format_dict)
def close(self):
self.tqdm_update_queue.put({"close": True})
super(TQDMPatch, self).close()
# change original class with the patched one, the original still exists
tqdm.std.tqdm = TQDMPatch
tqdm.tqdm = TQDMPatch # may not be necessary
# for tqdm.auto users, maybe some additional stuff is needed
class TQDMDataQueueReceiver(QObject):
s_tqdm_object_received_signal = pyqtSignal(object)
def __init__(self, q: Queue, *args, **kwargs):
QObject.__init__(self, *args, **kwargs)
self.queue = q
@pyqtSlot()
def run(self):
while True:
o = self.queue.get()
# noinspection PyUnresolvedReferences
self.s_tqdm_object_received_signal.emit(o)
class QTQDMProgressBar(QProgressBar):
def __init__(self, parent, tqdm_signal: pyqtSignal):
super(QTQDMProgressBar, self).__init__(parent)
self.setAlignment(Qt.AlignCenter)
self.setVisible(False)
# noinspection PyUnresolvedReferences
tqdm_signal.connect(self.do_it)
def do_it(self, e):
if not isinstance(e, dict):
return
do_reset = e.get("do_reset", False) # different from close, because we want visible=true
initial = e.get("initial", 0)
total = e.get("total", None)
n = e.get("n", None)
desc = e.get("prefix", None)
text = e.get("text", None)
do_close = e.get("close", False) # different from do_reset, we want visible=false
if do_reset:
self.reset()
if do_close:
self.reset()
self.setVisible(not do_close)
if initial:
self.setMinimum(initial)
else:
self.setMinimum(0)
if total:
self.setMaximum(total)
else:
self.setMaximum(0)
if n:
self.setValue(n)
if desc:
self.setFormat(f"{desc} %v/%m | %p %")
elif text:
self.setFormat(text)
else:
self.setFormat("%v/%m | %p")
def long_procedure():
# emulate late import of modules
from tqdm.auto import tqdm # don't import before patch !
__logger = logging.getLogger('long_procedure')
__logger.setLevel(logging.DEBUG)
tqdm_object = tqdm(range(10), unit_scale=True, dynamic_ncols=True)
tqdm_object.set_description("My progress bar description")
from tqdm.contrib.logging import logging_redirect_tqdm # don't import before patch !
with logging_redirect_tqdm():
for i in tqdm_object:
QtTest.QTest.qWait(200)
__logger.info(f'foo {i}')
class QtLoggingHelper(ABC):
@abstractmethod
def transform(self, msg: str):
raise NotImplementedError()
class QtLoggingBasic(QtLoggingHelper):
def transform(self, msg: str):
return msg
class QtLoggingColoredLogs(QtLoggingHelper):
def __init__(self):
# offensive programming: crash if necessary if import is not present
pass
def transform(self, msg: str):
import coloredlogs.converter
msg_html = coloredlogs.converter.convert(msg)
return msg_html
class QTextEditLogger(logging.Handler, QObject):
appendText = pyqtSignal(str)
def __init__(self,
logger_: logging.Logger,
formatter: logging.Formatter,
text_widget: QPlainTextEdit,
# table_widget: QTableWidget,
parent: QWidget):
super(QTextEditLogger, self).__init__()
super(QObject, self).__init__(parent=parent)
self.text_widget = text_widget
self.text_widget.setReadOnly(True)
# self.table_widget = table_widget
try:
self.helper = QtLoggingColoredLogs()
self.appendText.connect(self.text_widget.appendHtml)
logger_.info("Using QtLoggingColoredLogs")
except ImportError:
self.helper = QtLoggingBasic()
self.appendText.connect(self.text_widget.appendPlainText)
logger_.warning("Using QtLoggingBasic")
# logTextBox = QTextEditLogger(self)
# You can format what is printed to text box
self.setFormatter(formatter)
logger_.addHandler(self)
# You can control the logging level
self.setLevel(logging.DEBUG)
def emit(self, record: logging.LogRecord):
msg = self.format(record)
display_msg = self.helper.transform(msg=msg)
self.appendText.emit(display_msg)
# self.add_row(record)
class MainApp(QWidget):
def __init__(self):
super().__init__()
self.__logger = logging.getLogger(self.__class__.__name__)
self.__logger.setLevel(logging.DEBUG)
layout = QVBoxLayout()
self.setMinimumWidth(500)
self.btn_perform_actions = QToolButton(self)
self.btn_perform_actions.setText('Launch long processing')
self.btn_perform_actions.clicked.connect(self._btn_go_clicked)
self.thread_initialize = QThread()
self.init_procedure_object = LongProcedureWorker(self)
self.thread_tqdm_update_queue_listener = QThread()
# must be done before any TQDM import
self.tqdm_update_receiver = setup_streams_redirection()
self.tqdm_update_receiver.moveToThread(self.thread_tqdm_update_queue_listener)
self.thread_tqdm_update_queue_listener.started.connect(self.tqdm_update_receiver.run)
self.pb_tqdm = QTQDMProgressBar(self, tqdm_signal=self.tqdm_update_receiver.s_tqdm_object_received_signal)
layout.addWidget(self.pb_tqdm)
self.thread_tqdm_update_queue_listener.start()
self.plain_text_edit_logger = QPlainTextEdit(self)
LOG_FMT = "{asctime} | {levelname:10s} | {message}"
try:
import coloredlogs
FORMATTER = coloredlogs.ColoredFormatter(fmt=LOG_FMT, style="{")
except ImportError:
FORMATTER = logging.Formatter(fmt=LOG_FMT, style="{")
self.logging_ = QTextEditLogger(logger_=logging.getLogger(), # root logger, to intercept every log of app
formatter=FORMATTER,
text_widget=self.plain_text_edit_logger,
parent=self)
layout.addWidget(self.plain_text_edit_logger)
layout.addWidget(self.btn_perform_actions)
self.setLayout(layout)
import tqdm
self.__logger.info(f"tqdm {tqdm.__version__}")
self.__logger.info(f"Qt={QT_VERSION_STR}; PyQt={PYQT_VERSION_STR}")
with contextlib.suppress(ImportError):
import coloredlogs
self.__logger.info(f"coloredlogs {coloredlogs.__version__}")
# prepare thread for long operation
self.init_procedure_object.moveToThread(self.thread_initialize)
self.thread_initialize.started.connect(self.init_procedure_object.run)
self.init_procedure_object.finished.connect(self._init_procedure_finished)
self.init_procedure_object.finished.connect(self.thread_initialize.quit)
self.show()
@pyqtSlot()
def _btn_go_clicked(self):
# start thread
self.btn_perform_actions.setEnabled(False)
self.__logger.info("Launch Thread")
self.thread_initialize.start()
def _init_procedure_finished(self):
self.btn_perform_actions.setEnabled(True)
class LongProcedureWorker(QObject):
finished = pyqtSignal()
def __init__(self, main_app: MainApp):
super(LongProcedureWorker, self).__init__()
self._main_app = main_app
@pyqtSlot()
def run(self):
long_procedure()
self.finished.emit()
if __name__ == '__main__':
app = QApplication(sys.argv)
app.setStyle('Fusion')
ex = MainApp()
sys.exit(app.exec_())