From 52d78121b1c6499b1a477c8f1f3a69cd896c6c55 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 5 Aug 2023 22:14:43 +0100 Subject: [PATCH] env var override support - fixes #370, fixes #612, fixes #619, fixes #1318 - closes #950, closes #1061 --- tqdm/std.py | 234 +++++++++++++++++++++++++------------------------- tqdm/utils.py | 58 ++++++++++++- 2 files changed, 175 insertions(+), 117 deletions(-) diff --git a/tqdm/std.py b/tqdm/std.py index 1163137e..fa9777a7 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -19,7 +19,8 @@ from weakref import WeakSet from ._monitor import TMonitor from .utils import ( CallbackIOWrapper, Comparable, DisableOnWriteError, FormatReplace, SimpleTextIOWrapper, - _is_ascii, _screen_shape_wrapper, _supports_unicode, _term_move_up, disp_len, disp_trim) + _is_ascii, _screen_shape_wrapper, _supports_unicode, _term_move_up, disp_len, disp_trim, + envwrap) __author__ = "https://github.com/tqdm/tqdm#contributions" __all__ = ['tqdm', 'trange', @@ -246,6 +247,120 @@ class tqdm(Comparable): Decorate an iterable object, returning an iterator which acts exactly like the original iterable, but prints a dynamically updating progressbar every time a value is requested. + + Parameters + ---------- + iterable : iterable, optional + Iterable to decorate with a progressbar. + Leave blank to manually manage the updates. + desc : str, optional + Prefix for the progressbar. + total : int or float, optional + The number of expected iterations. If unspecified, + len(iterable) is used if possible. If float("inf") or as a last + resort, only basic progress statistics are displayed + (no ETA, no progressbar). + If `gui` is True and this parameter needs subsequent updating, + specify an initial arbitrary large positive number, + e.g. 9e9. + leave : bool, optional + If [default: True], keeps all traces of the progressbar + upon termination of iteration. + If `None`, will leave only if `position` is `0`. + file : `io.TextIOWrapper` or `io.StringIO`, optional + Specifies where to output the progress messages + (default: sys.stderr). Uses `file.write(str)` and `file.flush()` + methods. For encoding, see `write_bytes`. + ncols : int, optional + The width of the entire output message. If specified, + dynamically resizes the progressbar to stay within this bound. + If unspecified, attempts to use environment width. The + fallback is a meter width of 10 and no limit for the counter and + statistics. If 0, will not print any meter (only stats). + mininterval : float, optional + Minimum progress display update interval [default: 0.1] seconds. + maxinterval : float, optional + Maximum progress display update interval [default: 10] seconds. + Automatically adjusts `miniters` to correspond to `mininterval` + after long display update lag. Only works if `dynamic_miniters` + or monitor thread is enabled. + miniters : int or float, optional + Minimum progress display update interval, in iterations. + If 0 and `dynamic_miniters`, will automatically adjust to equal + `mininterval` (more CPU efficient, good for tight loops). + If > 0, will skip display of specified number of iterations. + Tweak this and `mininterval` to get very efficient loops. + If your progress is erratic with both fast and slow iterations + (network, skipping items, etc) you should set miniters=1. + ascii : bool or str, optional + If unspecified or False, use unicode (smooth blocks) to fill + the meter. The fallback is to use ASCII characters " 123456789#". + disable : bool, optional + Whether to disable the entire progressbar wrapper + [default: False]. If set to None, disable on non-TTY. + unit : str, optional + String that will be used to define the unit of each iteration + [default: it]. + unit_scale : bool or int or float, optional + If 1 or True, the number of iterations will be reduced/scaled + automatically and a metric prefix following the + International System of Units standard will be added + (kilo, mega, etc.) [default: False]. If any other non-zero + number, will scale `total` and `n`. + dynamic_ncols : bool, optional + If set, constantly alters `ncols` and `nrows` to the + environment (allowing for window resizes) [default: False]. + smoothing : float, optional + Exponential moving average smoothing factor for speed estimates + (ignored in GUI mode). Ranges from 0 (average speed) to 1 + (current/instantaneous speed) [default: 0.3]. + bar_format : str, optional + Specify a custom bar string formatting. May impact performance. + [default: '{l_bar}{bar}{r_bar}'], where + l_bar='{desc}: {percentage:3.0f}%|' and + r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' + '{rate_fmt}{postfix}]' + Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, + percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, + rate, rate_fmt, rate_noinv, rate_noinv_fmt, + rate_inv, rate_inv_fmt, postfix, unit_divisor, + remaining, remaining_s, eta. + Note that a trailing ": " is automatically removed after {desc} + if the latter is empty. + initial : int or float, optional + The initial counter value. Useful when restarting a progress + bar [default: 0]. If using float, consider specifying `{n:.3f}` + or similar in `bar_format`, or specifying `unit_scale`. + position : int, optional + Specify the line offset to print this bar (starting from 0) + Automatic if unspecified. + Useful to manage multiple bars at once (eg, from threads). + postfix : dict or *, optional + Specify additional stats to display at the end of the bar. + Calls `set_postfix(**postfix)` if possible (dict). + unit_divisor : float, optional + [default: 1000], ignored unless `unit_scale` is True. + write_bytes : bool, optional + Whether to write bytes. If (default: False) will write unicode. + lock_args : tuple, optional + Passed to `refresh` for intermediate output + (initialisation, iterating, and updating). + nrows : int, optional + The screen height. If specified, hides nested bars outside this + bound. If unspecified, attempts to use environment height. + The fallback is 20. + colour : str, optional + Bar colour (e.g. 'green', '#00ff00'). + delay : float, optional + Don't display until [default: 0] seconds have elapsed. + gui : bool, optional + WARNING: internal parameter - do not use. + Use tqdm.gui.tqdm(...) instead. If set, will attempt to use + matplotlib animations for a graphical output [default: False]. + + Returns + ------- + out : decorated iterator. """ monitor_interval = 10 # set to 0 to disable the thread @@ -834,6 +949,7 @@ class tqdm(Comparable): elif _Rolling_and_Expanding is not None: _Rolling_and_Expanding.progress_apply = inner_generator() + @envwrap("TQDM_", is_method=True) def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None, ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None, ascii=None, disable=False, unit='it', unit_scale=False, @@ -841,121 +957,7 @@ class tqdm(Comparable): position=None, postfix=None, unit_divisor=1000, write_bytes=False, lock_args=None, nrows=None, colour=None, delay=0, gui=False, **kwargs): - """ - Parameters - ---------- - iterable : iterable, optional - Iterable to decorate with a progressbar. - Leave blank to manually manage the updates. - desc : str, optional - Prefix for the progressbar. - total : int or float, optional - The number of expected iterations. If unspecified, - len(iterable) is used if possible. If float("inf") or as a last - resort, only basic progress statistics are displayed - (no ETA, no progressbar). - If `gui` is True and this parameter needs subsequent updating, - specify an initial arbitrary large positive number, - e.g. 9e9. - leave : bool, optional - If [default: True], keeps all traces of the progressbar - upon termination of iteration. - If `None`, will leave only if `position` is `0`. - file : `io.TextIOWrapper` or `io.StringIO`, optional - Specifies where to output the progress messages - (default: sys.stderr). Uses `file.write(str)` and `file.flush()` - methods. For encoding, see `write_bytes`. - ncols : int, optional - The width of the entire output message. If specified, - dynamically resizes the progressbar to stay within this bound. - If unspecified, attempts to use environment width. The - fallback is a meter width of 10 and no limit for the counter and - statistics. If 0, will not print any meter (only stats). - mininterval : float, optional - Minimum progress display update interval [default: 0.1] seconds. - maxinterval : float, optional - Maximum progress display update interval [default: 10] seconds. - Automatically adjusts `miniters` to correspond to `mininterval` - after long display update lag. Only works if `dynamic_miniters` - or monitor thread is enabled. - miniters : int or float, optional - Minimum progress display update interval, in iterations. - If 0 and `dynamic_miniters`, will automatically adjust to equal - `mininterval` (more CPU efficient, good for tight loops). - If > 0, will skip display of specified number of iterations. - Tweak this and `mininterval` to get very efficient loops. - If your progress is erratic with both fast and slow iterations - (network, skipping items, etc) you should set miniters=1. - ascii : bool or str, optional - If unspecified or False, use unicode (smooth blocks) to fill - the meter. The fallback is to use ASCII characters " 123456789#". - disable : bool, optional - Whether to disable the entire progressbar wrapper - [default: False]. If set to None, disable on non-TTY. - unit : str, optional - String that will be used to define the unit of each iteration - [default: it]. - unit_scale : bool or int or float, optional - If 1 or True, the number of iterations will be reduced/scaled - automatically and a metric prefix following the - International System of Units standard will be added - (kilo, mega, etc.) [default: False]. If any other non-zero - number, will scale `total` and `n`. - dynamic_ncols : bool, optional - If set, constantly alters `ncols` and `nrows` to the - environment (allowing for window resizes) [default: False]. - smoothing : float, optional - Exponential moving average smoothing factor for speed estimates - (ignored in GUI mode). Ranges from 0 (average speed) to 1 - (current/instantaneous speed) [default: 0.3]. - bar_format : str, optional - Specify a custom bar string formatting. May impact performance. - [default: '{l_bar}{bar}{r_bar}'], where - l_bar='{desc}: {percentage:3.0f}%|' and - r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' - '{rate_fmt}{postfix}]' - Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, - percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, - rate, rate_fmt, rate_noinv, rate_noinv_fmt, - rate_inv, rate_inv_fmt, postfix, unit_divisor, - remaining, remaining_s, eta. - Note that a trailing ": " is automatically removed after {desc} - if the latter is empty. - initial : int or float, optional - The initial counter value. Useful when restarting a progress - bar [default: 0]. If using float, consider specifying `{n:.3f}` - or similar in `bar_format`, or specifying `unit_scale`. - position : int, optional - Specify the line offset to print this bar (starting from 0) - Automatic if unspecified. - Useful to manage multiple bars at once (eg, from threads). - postfix : dict or *, optional - Specify additional stats to display at the end of the bar. - Calls `set_postfix(**postfix)` if possible (dict). - unit_divisor : float, optional - [default: 1000], ignored unless `unit_scale` is True. - write_bytes : bool, optional - Whether to write bytes. If (default: False) will write unicode. - lock_args : tuple, optional - Passed to `refresh` for intermediate output - (initialisation, iterating, and updating). - nrows : int, optional - The screen height. If specified, hides nested bars outside this - bound. If unspecified, attempts to use environment height. - The fallback is 20. - colour : str, optional - Bar colour (e.g. 'green', '#00ff00'). - delay : float, optional - Don't display until [default: 0] seconds have elapsed. - gui : bool, optional - WARNING: internal parameter - do not use. - Use tqdm.gui.tqdm(...) instead. If set, will attempt to use - matplotlib animations for a graphical output [default: False]. - - Returns - ------- - out : decorated iterator. - """ + """see tqdm.tqdm for arguments""" if file is None: file = sys.stderr diff --git a/tqdm/utils.py b/tqdm/utils.py index c5d3a6e3..93f635ad 100644 --- a/tqdm/utils.py +++ b/tqdm/utils.py @@ -4,7 +4,9 @@ General helpers required for `tqdm.std`. import os import re import sys -from functools import wraps +from ast import literal_eval as safe_eval +from functools import partial, partialmethod, wraps +from inspect import signature # TODO consider using wcswidth third-party package for 0-width characters from unicodedata import east_asian_width from warnings import warn @@ -30,6 +32,60 @@ else: colorama.init() +def envwrap(prefix, case_sensitive=False, literal_eval=False, is_method=False): + """ + Override parameter defaults via `os.environ[prefix + param_name]`. + Precedence (highest first): + - call (`foo(a=3)`) + - environ (`FOO_A=2`) + - signature (`def foo(a=1)`) + + Parameters + ---------- + prefix : str + Env var prefix, e.g. "FOO_" + case_sensitive : bool, optional + If (default: False), treat env var "FOO_Some_ARG" as "FOO_some_arg". + literal_eval : bool, optional + Whether to `ast.literal_eval` the detected env var overrides. + Otherwise if (default: False), infer types from function signature. + is_method : bool, optional + Whether to use `functools.partialmethod`. If (default: False) use `functools.partial`. + + Examples: + ``` + $ cat foo.py + from tqdm.utils import envwrap + @envwrap("FOO_") + def test(a=1, b=2, c=3): + print(f"received: a={a}, b={b}, c={c}") + + $ FOO_A=42 FOO_C=1337 python -c 'import foo; foo.test(c=99)' + received: a=42, b=2, c=99 + ``` + """ + i = len(prefix) + overrides = {k[i:] if case_sensitive else k[i:].lower(): v + for k, v in os.environ.items() if k.startswith(prefix)} + part = partialmethod if is_method else partial + + def wrap(func): + if literal_eval: + return part(func, **{k: safe_eval(v) for k, v in overrides.items()}) + # use `func` signature to infer env override `type` (fallback to `str`) + params = signature(func).parameters + for k in overrides: + param = params[k] + if param.annotation is not param.empty: + typ = param.annotation + # TODO: parse type in {Union, Any, Optional, ...} + else: + typ = str if param.default is None else type(param.default) + overrides[k] = typ(overrides[k]) + return part(func, **overrides) + return wrap + + class FormatReplace(object): """ >>> a = FormatReplace('something')