From 0c922ed4dfe845b57eb42c7b78553d99bedc9070 Mon Sep 17 00:00:00 2001 From: Joshua Coales Date: Tue, 2 Mar 2021 09:56:35 +0000 Subject: [PATCH 1/5] Adding an asyncio.gather() replacement for tqdm --- tqdm/asyncio.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tqdm/asyncio.py b/tqdm/asyncio.py index e8ac9032..3c643e41 100644 --- a/tqdm/asyncio.py +++ b/tqdm/asyncio.py @@ -8,12 +8,15 @@ Usage: ... ... """ import asyncio +from typing import Awaitable, TypeVar, List from .std import tqdm as std_tqdm __author__ = {"github.com/": ["casperdcl"]} __all__ = ['tqdm_asyncio', 'tarange', 'tqdm', 'trange'] +T = TypeVar("T") + class tqdm_asyncio(std_tqdm): """ @@ -63,6 +66,36 @@ class tqdm_asyncio(std_tqdm): yield from cls(asyncio.as_completed(fs, loop=loop, timeout=timeout), total=total, **tqdm_kwargs) + @classmethod + def gather( + cls, + fs: List[Awaitable[T]], + *, + loop=None, + timeout=None, + total=None, + **tqdm_kwargs + ) -> List[T]: + """ + Re-creating the functionality of asyncio.gather, giving a progress bar like + tqdm.as_completed(), but returning the results in original order. + """ + async def wrap_awaitable(number: int, awaitable: Awaitable[T]): + return number, await awaitable + if total is None: + total = len(fs) + + numbered_awaitables = [wrap_awaitable(idx, fs[idx]) for idx in range(len(fs))] + + numbered_results = [ + await f for f in cls.as_completed( + numbered_awaitables, total=total, loop=loop, timeout=timeout, **tqdm_kwargs + ) + ] + + results = [result_tuple[1] for result_tuple in sorted(numbered_results)] + return results + def tarange(*args, **kwargs): """ From 644fd81bce6f6cca1df3cc4ce217a02eaa58c85a Mon Sep 17 00:00:00 2001 From: Joshua Coales Date: Tue, 2 Mar 2021 10:05:52 +0000 Subject: [PATCH 2/5] Adding a test case for tqdm_asyncio.gather, mirroring tqdm_asyncio.as_completed --- tests/py37_asyncio.py | 20 ++++++++++++++++++++ tqdm/asyncio.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/py37_asyncio.py b/tests/py37_asyncio.py index be1903f2..1e051178 100644 --- a/tests/py37_asyncio.py +++ b/tests/py37_asyncio.py @@ -10,6 +10,7 @@ from .tests_tqdm import StringIO, closing, mark tqdm = partial(tqdm_asyncio, miniters=0, mininterval=0) trange = partial(tarange, miniters=0, mininterval=0) as_completed = partial(tqdm_asyncio.as_completed, miniters=0, mininterval=0) +gather = partial(tqdm_asyncio.gather, miniters=0, mininterval=0) def count(start=0, step=1): @@ -112,3 +113,22 @@ async def test_as_completed(capsys, tol): except AssertionError: if retry == 2: raise + + +@mark.slow +@mark.asyncio +@mark.parametrize("tol", [0.2 if platform.startswith("darwin") else 0.1]) +async def test_gather(capsys, tol): + """Test asyncio gather""" + for retry in range(3): + t = time() + skew = time() - t + await gather([asyncio.sleep(0.01 * i) for i in range(30, 0, -1)]) + t = time() - t - 2 * skew + try: + assert 0.3 * (1 - tol) < t < 0.3 * (1 + tol), t + _, err = capsys.readouterr() + assert '30/30' in err + except AssertionError: + if retry == 2: + raise diff --git a/tqdm/asyncio.py b/tqdm/asyncio.py index 3c643e41..0d527916 100644 --- a/tqdm/asyncio.py +++ b/tqdm/asyncio.py @@ -67,7 +67,7 @@ class tqdm_asyncio(std_tqdm): total=total, **tqdm_kwargs) @classmethod - def gather( + async def gather( cls, fs: List[Awaitable[T]], *, From 82e0851f6306de59ab8313ebe6e538e9631e375e Mon Sep 17 00:00:00 2001 From: Joshua Coales Date: Tue, 2 Mar 2021 10:18:53 +0000 Subject: [PATCH 3/5] Fix formatting - reorder imports - fix line length --- tqdm/asyncio.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tqdm/asyncio.py b/tqdm/asyncio.py index 0d527916..5d25749a 100644 --- a/tqdm/asyncio.py +++ b/tqdm/asyncio.py @@ -8,7 +8,7 @@ Usage: ... ... """ import asyncio -from typing import Awaitable, TypeVar, List +from typing import Awaitable, List, TypeVar from .std import tqdm as std_tqdm @@ -89,7 +89,11 @@ class tqdm_asyncio(std_tqdm): numbered_results = [ await f for f in cls.as_completed( - numbered_awaitables, total=total, loop=loop, timeout=timeout, **tqdm_kwargs + numbered_awaitables, + total=total, + loop=loop, + timeout=timeout, + **tqdm_kwargs ) ] From 8fdcddb446088241b51da0cd7667c7ad452c0222 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 3 Mar 2021 11:41:59 +0000 Subject: [PATCH 4/5] asyncio: fix, tidy & update `gather` & tests --- tests/py37_asyncio.py | 24 +++++++++--------------- tqdm/asyncio.py | 40 ++++++++-------------------------------- 2 files changed, 17 insertions(+), 47 deletions(-) diff --git a/tests/py37_asyncio.py b/tests/py37_asyncio.py index 1e051178..18997ca7 100644 --- a/tests/py37_asyncio.py +++ b/tests/py37_asyncio.py @@ -115,20 +115,14 @@ async def test_as_completed(capsys, tol): raise -@mark.slow +async def double(i): + return i * 2 + + @mark.asyncio -@mark.parametrize("tol", [0.2 if platform.startswith("darwin") else 0.1]) -async def test_gather(capsys, tol): +async def test_gather(capsys): """Test asyncio gather""" - for retry in range(3): - t = time() - skew = time() - t - await gather([asyncio.sleep(0.01 * i) for i in range(30, 0, -1)]) - t = time() - t - 2 * skew - try: - assert 0.3 * (1 - tol) < t < 0.3 * (1 + tol), t - _, err = capsys.readouterr() - assert '30/30' in err - except AssertionError: - if retry == 2: - raise + res = await gather(list(map(double, range(30)))) + _, err = capsys.readouterr() + assert '30/30' in err + assert res == list(range(0, 30 * 2, 2)) diff --git a/tqdm/asyncio.py b/tqdm/asyncio.py index 5d25749a..a61d28b3 100644 --- a/tqdm/asyncio.py +++ b/tqdm/asyncio.py @@ -8,15 +8,12 @@ Usage: ... ... """ import asyncio -from typing import Awaitable, List, TypeVar from .std import tqdm as std_tqdm __author__ = {"github.com/": ["casperdcl"]} __all__ = ['tqdm_asyncio', 'tarange', 'tqdm', 'trange'] -T = TypeVar("T") - class tqdm_asyncio(std_tqdm): """ @@ -67,38 +64,17 @@ class tqdm_asyncio(std_tqdm): total=total, **tqdm_kwargs) @classmethod - async def gather( - cls, - fs: List[Awaitable[T]], - *, - loop=None, - timeout=None, - total=None, - **tqdm_kwargs - ) -> List[T]: + async def gather(cls, fs, *, loop=None, timeout=None, total=None, **tqdm_kwargs): """ - Re-creating the functionality of asyncio.gather, giving a progress bar like - tqdm.as_completed(), but returning the results in original order. + Wrapper for `asyncio.gather`. """ - async def wrap_awaitable(number: int, awaitable: Awaitable[T]): - return number, await awaitable - if total is None: - total = len(fs) + async def wrap_awaitable(i, f): + return i, await f - numbered_awaitables = [wrap_awaitable(idx, fs[idx]) for idx in range(len(fs))] - - numbered_results = [ - await f for f in cls.as_completed( - numbered_awaitables, - total=total, - loop=loop, - timeout=timeout, - **tqdm_kwargs - ) - ] - - results = [result_tuple[1] for result_tuple in sorted(numbered_results)] - return results + ifs = [wrap_awaitable(i, f) for i, f in enumerate(fs)] + res = [await f for f in cls.as_completed(ifs, loop=loop, timeout=timeout, + total=total, **tqdm_kwargs)] + return [i for _, i in sorted(res)] def tarange(*args, **kwargs): From 94009b5adb6f38fd5eea688385042033bed73278 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 3 Mar 2021 11:55:21 +0000 Subject: [PATCH 5/5] asyncio: require py>=3.6 for awaitable comprehensions --- tqdm/asyncio.py | 2 +- tqdm/auto.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tqdm/asyncio.py b/tqdm/asyncio.py index a61d28b3..0d3ba747 100644 --- a/tqdm/asyncio.py +++ b/tqdm/asyncio.py @@ -17,7 +17,7 @@ __all__ = ['tqdm_asyncio', 'tarange', 'tqdm', 'trange'] class tqdm_asyncio(std_tqdm): """ - Asynchronous-friendly version of tqdm (Python 3.5+). + Asynchronous-friendly version of tqdm (Python 3.6+). """ def __init__(self, iterable=None, *args, **kwargs): super(tqdm_asyncio, self).__init__(iterable, *args, **kwargs) diff --git a/tqdm/auto.py b/tqdm/auto.py index 28bc722d..cffca206 100644 --- a/tqdm/auto.py +++ b/tqdm/auto.py @@ -4,7 +4,7 @@ Enables multiple commonly used features. Method resolution order: - `tqdm.autonotebook` without import warnings -- `tqdm.asyncio` on Python3.5+ +- `tqdm.asyncio` on Python3.6+ - `tqdm.std` base class Usage: @@ -22,10 +22,10 @@ with warnings.catch_warnings(): from .autonotebook import tqdm as notebook_tqdm from .autonotebook import trange as notebook_trange -if sys.version_info[:2] < (3, 5): +if sys.version_info[:2] < (3, 6): tqdm = notebook_tqdm trange = notebook_trange -else: # Python3.5+ +else: # Python3.6+ from .asyncio import tqdm as asyncio_tqdm from .std import tqdm as std_tqdm