From b4e3fd866a5ea13a02a252ff266da240b4c66975 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Fri, 24 Jul 2020 14:16:07 +0100 Subject: [PATCH 1/6] Allow querying non-host python intepreters This way you can install once (for example via pipx) and reuse it for any existing python environment. Signed-off-by: Bernat Gabor --- dev-requirements.txt | 8 +++++--- pipdeptree.py | 39 ++++++++++++++++++++++++++++++++++++++- tests/test_pipdeptree.py | 23 +++++++++++++++++++++++ tox.ini | 1 + 4 files changed, 67 insertions(+), 4 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 95ee2a7..d214e59 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,8 @@ +flake8 +graphviz +pip>=8.0.2 pytest pytest-cov -jinja2 -ipython -flake8 +tox>=3 +virtualenv>=20,<21 mock;python_version<"3" diff --git a/pipdeptree.py b/pipdeptree.py index 9c313a5..f2f6a09 100644 --- a/pipdeptree.py +++ b/pipdeptree.py @@ -1,12 +1,15 @@ from __future__ import print_function import os +import inspect import sys +import subprocess from itertools import chain from collections import defaultdict, deque import argparse from operator import attrgetter import json from importlib import import_module +import tempfile try: from collections import OrderedDict @@ -711,6 +714,9 @@ def get_parser(): version='{0}'.format(__version__)) parser.add_argument('-f', '--freeze', action='store_true', help='Print names so as to write freeze files') + parser.add_argument('--python', default=sys.executable, + help='Python to use to look for packages in it (default: where' + ' installed)') parser.add_argument('-a', '--all', action='store_true', help='list all deps at top level') parser.add_argument('-l', '--local-only', @@ -775,9 +781,40 @@ def _get_args(): return parser.parse_args() +def handle_non_host_target(args): + of_python = os.path.abspath(args.python) + # if target is not current python re-invoke it under the actual host + if of_python != os.path.abspath(sys.executable): + # there's no way to guarantee that graphviz is available, so refuse + if args.output_format: + print("graphviz functionality is not supported when querying" + " non-host python", file=sys.stderr) + raise SystemExit(1) + argv = sys.argv[1:] # remove current python executable + py_at = argv.index('--python') # plus the new python target + del argv[py_at] + del argv[py_at] + # feed the file as argument, instead of file + # to avoid adding the file path to sys.path, that can affect result + file_path = inspect.getsourcefile(sys.modules[__name__]) + with open(file_path, 'rt') as file_handler: + content = file_handler.read() + cmd = [of_python, "-c", content] + cmd.extend(argv) + # invoke from an empty folder to avoid cwd altering sys.path + cwd = tempfile.mkdtemp() + try: + return subprocess.call(cmd, cwd=cwd) + finally: + os.removedirs(cwd) + return None + + def main(): args = _get_args() - + result = handle_non_host_target(args) + if result is not None: + return result pkgs = get_installed_distributions(local_only=args.local_only, user_only=args.user_only) diff --git a/tests/test_pipdeptree.py b/tests/test_pipdeptree.py index 99302a2..b78bda0 100644 --- a/tests/test_pipdeptree.py +++ b/tests/test_pipdeptree.py @@ -7,6 +7,7 @@ except ImportError: import mock import pytest +import virtualenv import pipdeptree as p @@ -520,3 +521,25 @@ def test_parser_svg(): args = parser.parse_args(['--graph-output', 'svg']) assert args.output_format == 'svg' assert not args.json + + +def test_custom_interpreter(tmp_path, monkeypatch, capfd): + result = virtualenv.cli_run([str(tmp_path), '--activators', '']) + cmd = [sys.executable, '--python', str(result.creator.exe)] + monkeypatch.setattr(sys, 'argv', cmd) + p.main() + out, _ = capfd.readouterr() + found = {i.split('==')[0] for i in out.splitlines()} + assert found == {'pip', 'setuptools', 'wheel'}, out + + monkeypatch.setattr(sys, 'argv', cmd + ['--graph-output', 'something']) + with pytest.raises(SystemExit) as context: + p.main() + out, err = capfd.readouterr() + assert context.value.code == 1 + assert not out + assert err == 'graphviz functionality is not supported when querying' \ + ' non-host python\n' + + + diff --git a/tox.ini b/tox.ini index 65203d9..4192201 100644 --- a/tox.ini +++ b/tox.ini @@ -15,3 +15,4 @@ deps = pip>=8.0.2 pytest pytest-cov + virtualenv>=20 From c8fbbba497c4d8b2700604637cab26793e1421c5 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 21 Sep 2020 08:22:55 +0100 Subject: [PATCH 2/6] Also support --python= arg format Signed-off-by: Bernat Gabor --- pipdeptree.py | 9 ++++++--- tests/test_pipdeptree.py | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pipdeptree.py b/pipdeptree.py index f2f6a09..540ca7a 100644 --- a/pipdeptree.py +++ b/pipdeptree.py @@ -791,9 +791,12 @@ def handle_non_host_target(args): " non-host python", file=sys.stderr) raise SystemExit(1) argv = sys.argv[1:] # remove current python executable - py_at = argv.index('--python') # plus the new python target - del argv[py_at] - del argv[py_at] + for py_at, value in enumerate(argv): + if value == "--python": + del argv[py_at] + del argv[py_at] + elif value.startswith("--python"): + del argv[py_at] # feed the file as argument, instead of file # to avoid adding the file path to sys.path, that can affect result file_path = inspect.getsourcefile(sys.modules[__name__]) diff --git a/tests/test_pipdeptree.py b/tests/test_pipdeptree.py index b78bda0..610a6f5 100644 --- a/tests/test_pipdeptree.py +++ b/tests/test_pipdeptree.py @@ -523,9 +523,11 @@ def test_parser_svg(): assert not args.json -def test_custom_interpreter(tmp_path, monkeypatch, capfd): +@pytest.mark.parametrize('args_joined', [True, False]) +def test_custom_interpreter(tmp_path, monkeypatch, capfd, args_joined): result = virtualenv.cli_run([str(tmp_path), '--activators', '']) - cmd = [sys.executable, '--python', str(result.creator.exe)] + cmd = [sys.executable] + cmd += ['--python={}'.format(result.creator.exe)] if args_joined else ['--python', str(result.creator.exe)] monkeypatch.setattr(sys, 'argv', cmd) p.main() out, _ = capfd.readouterr() From 785cd4e828bf691f184fe91e224329831bb9e15e Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 21 Sep 2020 13:46:28 +0100 Subject: [PATCH 3/6] Specify we need virtualenv>20,<21 Signed-off-by: Bernat Gabor --- tox.ini | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tox.ini b/tox.ini index 4192201..03e304e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,24 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - +# http://tox.readthedocs.org/ - sets up and runs the test suite based on a declarative configuration [tox] -envlist = py27, py34, py35, py36, py37, py38 +envlist = + py39 + py38 + py37 + py36 + py35 + py34 + py27 + pypy3 + pypy2 [testenv] +description = run test suite under {basepython} commands = - pytest {posargs:-vv} + pytest {posargs:-vv} deps = - tox>=3.0.0 - graphviz - pip>=8.0.2 - pytest - pytest-cov - virtualenv>=20 + graphviz + pip>=8.0.2 + pytest + pytest-cov + virtualenv>=20,<21 + mock;python_version<"3" From 8a7df561126c2b992c707a6ef3e43fe33a601034 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 21 Sep 2020 13:53:53 +0100 Subject: [PATCH 4/6] Use tox to run the test suite in CI for isolation Signed-off-by: Bernat Gabor --- .github/workflows/check.yml | 24 +++++++++++++++++------- dev-requirements.txt | 8 -------- tox.ini | 2 ++ 3 files changed, 19 insertions(+), 15 deletions(-) delete mode 100644 dev-requirements.txt diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b349947..538a204 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -25,16 +25,26 @@ jobs: - pypy3 - pypy2 steps: + - name: Setup graphviz + uses: ts-graphviz/setup-graphviz@v1 + - name: Setup python for tox + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install tox + run: python -m pip install tox - name: Setup python for test ${{ matrix.py }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.py }} - uses: actions/checkout@v2 - - name: Setup graphviz - uses: ts-graphviz/setup-graphviz@v1 - - name: Install dev requirements - run: pip install -r dev-requirements.txt - - name: Install project - run: pip install .[graphviz] + - name: Pick tox environment to run + run: | + import subprocess; import json + major, minor, impl = json.loads(subprocess.check_output(["python", "-c", "import json; import sys; import platform; print(json.dumps([sys.version_info[0], sys.version_info[1], platform.python_implementation()]));"], universal_newlines=True)) + print('::set-env name=TOXENV::' + ("py" if impl == "CPython" else "pypy") + ("{}{}".format(major, minor) if impl == "CPython" else ("3" if major == 3 else ""))) + shell: python + - name: Setup test suite + run: tox -vv --notest - name: Run test suite - run: pytest -v tests + run: tox --skip-pkg-install diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index d214e59..0000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -flake8 -graphviz -pip>=8.0.2 -pytest -pytest-cov -tox>=3 -virtualenv>=20,<21 -mock;python_version<"3" diff --git a/tox.ini b/tox.ini index 03e304e..62958cb 100644 --- a/tox.ini +++ b/tox.ini @@ -22,3 +22,5 @@ deps = pytest-cov virtualenv>=20,<21 mock;python_version<"3" +extras = + graphviz From 360ceb7a0f3497d11e2d1d74cc583e703dc16059 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 21 Sep 2020 13:59:20 +0100 Subject: [PATCH 5/6] Handle PyPy different starting dependencies Signed-off-by: Bernat Gabor --- tests/test_pipdeptree.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/test_pipdeptree.py b/tests/test_pipdeptree.py index 610a6f5..af854ab 100644 --- a/tests/test_pipdeptree.py +++ b/tests/test_pipdeptree.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +import platform import sys from tempfile import NamedTemporaryFile try: @@ -523,25 +524,31 @@ def test_parser_svg(): assert not args.json -@pytest.mark.parametrize('args_joined', [True, False]) +@pytest.mark.parametrize("args_joined", [True, False]) def test_custom_interpreter(tmp_path, monkeypatch, capfd, args_joined): - result = virtualenv.cli_run([str(tmp_path), '--activators', '']) + result = virtualenv.cli_run([str(tmp_path), "--activators", ""]) cmd = [sys.executable] - cmd += ['--python={}'.format(result.creator.exe)] if args_joined else ['--python', str(result.creator.exe)] - monkeypatch.setattr(sys, 'argv', cmd) + cmd += ["--python={}".format(result.creator.exe)] if args_joined else ["--python", str(result.creator.exe)] + monkeypatch.setattr(sys, "argv", cmd) p.main() out, _ = capfd.readouterr() - found = {i.split('==')[0] for i in out.splitlines()} - assert found == {'pip', 'setuptools', 'wheel'}, out + found = {i.split("==")[0] for i in out.splitlines()} + implementation = platform.python_implementation() + if implementation == "CPython": + expected = {"pip", "setuptools", "wheel"} + elif implementation == "PyPy": + expected = {"cffi", "greenlet", "pip", "readline", "setuptools", "wheel"} + else: + raise ValueError(implementation) + assert found == expected, out - monkeypatch.setattr(sys, 'argv', cmd + ['--graph-output', 'something']) + monkeypatch.setattr(sys, "argv", cmd + ["--graph-output", "something"]) with pytest.raises(SystemExit) as context: p.main() out, err = capfd.readouterr() assert context.value.code == 1 assert not out - assert err == 'graphviz functionality is not supported when querying' \ - ' non-host python\n' + assert err == "graphviz functionality is not supported when querying" " non-host python\n" From e273087052d6124c65a80b666c821b69008eb6ae Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Tue, 22 Sep 2020 10:17:56 +0100 Subject: [PATCH 6/6] Simplify Github check Signed-off-by: Bernat Gabor --- .github/workflows/check.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 538a204..4c5e94f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -40,8 +40,9 @@ jobs: - uses: actions/checkout@v2 - name: Pick tox environment to run run: | - import subprocess; import json - major, minor, impl = json.loads(subprocess.check_output(["python", "-c", "import json; import sys; import platform; print(json.dumps([sys.version_info[0], sys.version_info[1], platform.python_implementation()]));"], universal_newlines=True)) + import platform + import sys + major, minor, impl = sys.version_info[0], sys.version_info[1], platform.python_implementation() print('::set-env name=TOXENV::' + ("py" if impl == "CPython" else "pypy") + ("{}{}".format(major, minor) if impl == "CPython" else ("3" if major == 3 else ""))) shell: python - name: Setup test suite