Change logging "print" function to "tqdm.write" so logging doesn't interfere with progress bars
Question:
I have a simple question: How do I change the built-in Python logger’s print
function to tqdm.write
such that logging messages do not interfere with tqdm’s progress bars? Thanks!
Answers:
You need a custom logging handler:
import logging
import tqdm
class TqdmLoggingHandler(logging.Handler):
def __init__(self, level=logging.NOTSET):
super().__init__(level)
def emit(self, record):
try:
msg = self.format(record)
tqdm.tqdm.write(msg)
self.flush()
except Exception:
self.handleError(record)
and then add this to the logging chain:
import time
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
log.addHandler(TqdmLoggingHandler())
for i in tqdm.tqdm(range(100)):
if i == 50:
log.info("Half-way there!")
time.sleep(0.1)
Edit: Fixed a bug in the call to super TqdmLoggingHandler’s init method, that was pointed out by diligent reader @BlaineRogers in the comments. (If anyone wants to read further about this murky area of Python, I recommend https://fuhm.net/super-harmful/)
One easy solution, not so elegant, is convert the tqdm object to string. After that yo can log the message (or handle it as you want). The "format_dict" attribute may be useful as well:
from tqdm import tqdm
import time
#loop with progressbar:
it=tqdm(total=10)
for i in range(10):
time.sleep(0.1)
it.update(1)
it.close()
print("n","--"*10)
# Convert tdqm object last output to sting
str_bar_msg = str(it)
print(str_bar_msg)
# See attributes:
print(it.format_dict)
output:
100%|██████████| 10/10 [00:01<00:00, 8.99it/s]
--------------------
100%|██████████| 10/10 [00:01<00:00, 8.98it/s]
{'n': 10, 'total': 10, 'elapsed': 1.1145293712615967, 'ncols': None, 'nrows': None, 'prefix': '', 'ascii': False, 'unit': 'it', 'unit_scale': False, 'rate': None, 'bar_format': None, 'postfix': None, 'unit_divisor': 1000, 'initial': 0, 'colour': None}
Best regards
the easiest way is change stream of StreamHandler
object e.g.:
import logging
from tqdm import tqdm, trange
import time
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
handler.setStream(tqdm) # <-- important
handler = log.addHandler(handler)
for i in trange(100):
if i == 50:
log.info("Half-way there!")
time.sleep(0.1)
Based on RolKau’s answer above, simplified:
import logging
from tqdm import tqdm
class TqdmLoggingHandler(logging.StreamHandler):
"""Avoid tqdm progress bar interruption by logger's output to console"""
# see logging.StreamHandler.eval method:
# https://github.com/python/cpython/blob/d2e2534751fd675c4d5d3adc208bf4fc984da7bf/Lib/logging/__init__.py#L1082-L1091
# and tqdm.write method:
# https://github.com/tqdm/tqdm/blob/f86104a1f30c38e6f80bfd8fb16d5fcde1e7749f/tqdm/std.py#L614-L620
def emit(self, record):
try:
msg = self.format(record)
tqdm.write(msg, end=self.terminator)
except RecursionError:
raise
except Exception:
self.handleError(record)
Testing:
import time
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
log.addHandler(TqdmLoggingHandler())
# ^-- Assumes this will be the unique handler emitting messages to sys.stdout.
# If other handlers output to sys.stdout (without tqdm.write),
# progress bar will be interrupted by those outputs
for i in tqdm(range(20)):
log.info(f"Looping {i}")
time.sleep(0.1)
Caveat: if you’re working in a Jupyter notebook, progress bar will be interrupted and AFAIK there’s no way to avoid it.
tqdm
now has a built-in contextmanager for redirecting the logger:
import logging
from tqdm import trange
from tqdm.contrib.logging import logging_redirect_tqdm
LOG = logging.getLogger(__name__)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
with logging_redirect_tqdm():
for i in trange(9):
if i == 4:
LOG.info("console logging redirected to `tqdm.write()`")
# logging restored
Copied from tqdm documentation
The new io handler is useful!
class TqdmToLogger(io.StringIO):
logger = None
level = None
buf = ""
def __init__(self, logger, level=None, mininterval=5):
super(TqdmToLogger, self).__init__()
self.logger = logger
self.level = level or logging.INFO
self.mininterval = mininterval
self.last_time = 0
def write(self, buf):
self.buf = buf.strip("rnt ")
def flush(self):
if len(self.buf) > 0 and time.time() - self.last_time > self.mininterval:
self.logger.log(self.level, self.buf)
self.last_time = time.time()```
# before this line, you need to create logger with file handler
tqdm_out = TqdmToLogger(logger)
tbar = tqdm(sample, total=len(sample), file=tqdm_out)
logger.info("Model Inference.")
for it, batch_data in enumerate(tbar):
pass
```
I have a simple question: How do I change the built-in Python logger’s print
function to tqdm.write
such that logging messages do not interfere with tqdm’s progress bars? Thanks!
You need a custom logging handler:
import logging
import tqdm
class TqdmLoggingHandler(logging.Handler):
def __init__(self, level=logging.NOTSET):
super().__init__(level)
def emit(self, record):
try:
msg = self.format(record)
tqdm.tqdm.write(msg)
self.flush()
except Exception:
self.handleError(record)
and then add this to the logging chain:
import time
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
log.addHandler(TqdmLoggingHandler())
for i in tqdm.tqdm(range(100)):
if i == 50:
log.info("Half-way there!")
time.sleep(0.1)
Edit: Fixed a bug in the call to super TqdmLoggingHandler’s init method, that was pointed out by diligent reader @BlaineRogers in the comments. (If anyone wants to read further about this murky area of Python, I recommend https://fuhm.net/super-harmful/)
One easy solution, not so elegant, is convert the tqdm object to string. After that yo can log the message (or handle it as you want). The "format_dict" attribute may be useful as well:
from tqdm import tqdm
import time
#loop with progressbar:
it=tqdm(total=10)
for i in range(10):
time.sleep(0.1)
it.update(1)
it.close()
print("n","--"*10)
# Convert tdqm object last output to sting
str_bar_msg = str(it)
print(str_bar_msg)
# See attributes:
print(it.format_dict)
output:
100%|██████████| 10/10 [00:01<00:00, 8.99it/s]
--------------------
100%|██████████| 10/10 [00:01<00:00, 8.98it/s]
{'n': 10, 'total': 10, 'elapsed': 1.1145293712615967, 'ncols': None, 'nrows': None, 'prefix': '', 'ascii': False, 'unit': 'it', 'unit_scale': False, 'rate': None, 'bar_format': None, 'postfix': None, 'unit_divisor': 1000, 'initial': 0, 'colour': None}
Best regards
the easiest way is change stream of StreamHandler
object e.g.:
import logging
from tqdm import tqdm, trange
import time
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
handler.setStream(tqdm) # <-- important
handler = log.addHandler(handler)
for i in trange(100):
if i == 50:
log.info("Half-way there!")
time.sleep(0.1)
Based on RolKau’s answer above, simplified:
import logging
from tqdm import tqdm
class TqdmLoggingHandler(logging.StreamHandler):
"""Avoid tqdm progress bar interruption by logger's output to console"""
# see logging.StreamHandler.eval method:
# https://github.com/python/cpython/blob/d2e2534751fd675c4d5d3adc208bf4fc984da7bf/Lib/logging/__init__.py#L1082-L1091
# and tqdm.write method:
# https://github.com/tqdm/tqdm/blob/f86104a1f30c38e6f80bfd8fb16d5fcde1e7749f/tqdm/std.py#L614-L620
def emit(self, record):
try:
msg = self.format(record)
tqdm.write(msg, end=self.terminator)
except RecursionError:
raise
except Exception:
self.handleError(record)
Testing:
import time
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
log.addHandler(TqdmLoggingHandler())
# ^-- Assumes this will be the unique handler emitting messages to sys.stdout.
# If other handlers output to sys.stdout (without tqdm.write),
# progress bar will be interrupted by those outputs
for i in tqdm(range(20)):
log.info(f"Looping {i}")
time.sleep(0.1)
Caveat: if you’re working in a Jupyter notebook, progress bar will be interrupted and AFAIK there’s no way to avoid it.
tqdm
now has a built-in contextmanager for redirecting the logger:
import logging
from tqdm import trange
from tqdm.contrib.logging import logging_redirect_tqdm
LOG = logging.getLogger(__name__)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
with logging_redirect_tqdm():
for i in trange(9):
if i == 4:
LOG.info("console logging redirected to `tqdm.write()`")
# logging restored
Copied from tqdm documentation
The new io handler is useful!
class TqdmToLogger(io.StringIO):
logger = None
level = None
buf = ""
def __init__(self, logger, level=None, mininterval=5):
super(TqdmToLogger, self).__init__()
self.logger = logger
self.level = level or logging.INFO
self.mininterval = mininterval
self.last_time = 0
def write(self, buf):
self.buf = buf.strip("rnt ")
def flush(self):
if len(self.buf) > 0 and time.time() - self.last_time > self.mininterval:
self.logger.log(self.level, self.buf)
self.last_time = time.time()```
# before this line, you need to create logger with file handler
tqdm_out = TqdmToLogger(logger)
tbar = tqdm(sample, total=len(sample), file=tqdm_out)
logger.info("Model Inference.")
for it, batch_data in enumerate(tbar):
pass
```