tqdm/examples/parallel_bars.py

from __future__ import print_function

import sys
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from multiprocessing import Pool, RLock, freeze_support
from random import random
from threading import RLock as TRLock
from time import sleep

from tqdm.auto import tqdm, trange
from tqdm.contrib.concurrent import process_map, thread_map

NUM_SUBITERS = 9
PY2 = sys.version_info[:1] <= (2,)


def progresser(n, auto_position=True, write_safe=False, blocking=True, progress=False):
    interval = random() * 0.002 / (NUM_SUBITERS - n + 2)  # nosec
    total = 5000
    text = "#{0}, est. {1:<04.2}s".format(n, interval * total)
    for _ in trange(total, desc=text, disable=not progress,
                    lock_args=None if blocking else (False,),
                    position=None if auto_position else n):
        sleep(interval)
    # NB: may not clear instances with higher `position` upon completion
    # since this worker may not know about other bars #796
    if write_safe:
        # we think we know about other bars (currently only py3 threading)
        if n == 6:
            tqdm.write("n == 6 completed")
    return n + 1


if __name__ == '__main__':
    freeze_support()  # for Windows support
    L = list(range(NUM_SUBITERS))[::-1]

    print("Simple thread mapping")
    thread_map(partial(progresser, write_safe=not PY2), L, max_workers=4)

    print("Simple process mapping")
    process_map(partial(progresser), L, max_workers=4)

    print("Manual nesting")
    for i in trange(16, desc="1"):
        for _ in trange(16, desc="2 @ %d" % i, leave=i % 2):
            sleep(0.01)

    print("Multi-processing")
    tqdm.set_lock(RLock())
    p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),))
    p.map(partial(progresser, progress=True), L)

    print("Multi-threading")
    tqdm.set_lock(TRLock())
    pool_args = {}
    if not PY2:
        pool_args.update(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),))
    with ThreadPoolExecutor(**pool_args) as p:
        p.map(partial(progresser, progress=True, write_safe=not PY2, blocking=False), L)