From ec1ee96963269b311155ea950a34ca65be683c86 Mon Sep 17 00:00:00 2001 From: Vineet Naik Date: Sun, 12 Apr 2020 11:40:33 +0530 Subject: [PATCH] Write new style integration tests The earlier approach for integration tests was unreliable as it used pickled objects from virtualenvs, which didn't work with all the pip internal code and would often require tests to be fixed. In this new approach, a combination of bash and python scripts drive the tests. Test virtualenvs are setup on demand with support for varying python and pip versions. As a result, the test environment (python 3.6) can be completely different from the test environment ie. the env in which pipdeptree is tested. TODO: Get the new approach working with travis and remove old integration tests. --- dev-requirements.txt | 4 ++ tests/e2e-tests | 42 +++++++++++++ tests/e2e_tests.py | 61 +++++++++++++++++++ tests/profiles/conflicting/.gitignore | 1 + tests/profiles/conflicting/default.err | 4 ++ tests/profiles/conflicting/default.out | 9 +++ tests/profiles/conflicting/requirements.txt | 6 ++ tests/profiles/conflicting/reverse.out | 11 ++++ tests/profiles/conflicting/test_spec.json | 16 +++++ tests/profiles/cyclic/.gitignore | 1 + tests/profiles/cyclic/default.err | 4 ++ tests/profiles/cyclic/default.out | 4 ++ tests/profiles/cyclic/requirements.txt | 2 + tests/profiles/cyclic/test_spec.json | 9 +++ tests/profiles/webapp/.gitignore | 1 + tests/profiles/webapp/all_flag.out | 65 +++++++++++++++++++++ tests/profiles/webapp/default.out | 31 ++++++++++ tests/profiles/webapp/packages_opt.out | 17 ++++++ tests/profiles/webapp/requirements.txt | 7 +++ tests/profiles/webapp/reverse.out | 49 ++++++++++++++++ tests/profiles/webapp/test_spec.json | 30 ++++++++++ 21 files changed, 374 insertions(+) create mode 100644 dev-requirements.txt create mode 100755 tests/e2e-tests create mode 100644 tests/e2e_tests.py create mode 100644 tests/profiles/conflicting/.gitignore create mode 100644 tests/profiles/conflicting/default.err create mode 100644 tests/profiles/conflicting/default.out create mode 100644 tests/profiles/conflicting/requirements.txt create mode 100644 tests/profiles/conflicting/reverse.out create mode 100644 tests/profiles/conflicting/test_spec.json create mode 100644 tests/profiles/cyclic/.gitignore create mode 100644 tests/profiles/cyclic/default.err create mode 100644 tests/profiles/cyclic/default.out create mode 100644 tests/profiles/cyclic/requirements.txt create mode 100644 tests/profiles/cyclic/test_spec.json create mode 100644 tests/profiles/webapp/.gitignore create mode 100644 tests/profiles/webapp/all_flag.out create mode 100644 tests/profiles/webapp/default.out create mode 100644 tests/profiles/webapp/packages_opt.out create mode 100644 tests/profiles/webapp/requirements.txt create mode 100644 tests/profiles/webapp/reverse.out create mode 100644 tests/profiles/webapp/test_spec.json diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..5766cde --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,4 @@ +pytest +jinja2 +ipython +flake8 diff --git a/tests/e2e-tests b/tests/e2e-tests new file mode 100755 index 0000000..2190f91 --- /dev/null +++ b/tests/e2e-tests @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -e + +PROFILE=$1 +PYTHON_EXE=${PYTHON_EXE:-python3.6} +PIP_VERSION=latest + +cd profiles/$PROFILE + +echo "Profile dir: $(pwd)" + +env_dir=".env_$(basename $PYTHON_EXE)_pip-${PIP_VERSION}" + +echo "Profile env: $env_dir" + +if [ ! -d $env_dir ]; then + virtualenv -p $PYTHON_EXE $env_dir +fi + +pip=$env_dir/bin/pip + +if [ "$PIP_VERSION" == "latest" ]; then + $pip install -U pip +fi + +# Install requirements +$pip install -r requirements.txt + +# Install pipdeptree +$pip install -e ../../../ + +pip_deptree=$env_dir/bin/pipdeptree + +export TEST_PROFILE_DIR="profiles/$PROFILE" +export PIPDEPTREE_EXE=$TEST_PROFILE_DIR/$pip_deptree + +cd - + +py.test e2e_tests.py + + diff --git a/tests/e2e_tests.py b/tests/e2e_tests.py new file mode 100644 index 0000000..e35e2fd --- /dev/null +++ b/tests/e2e_tests.py @@ -0,0 +1,61 @@ +import json +import os +import shlex +import subprocess + +from jinja2 import Environment, BaseLoader +import pytest + + +## Uncomment following lines for running in shell +# os.environ['TEST_PROFILE_DIR'] = 'profiles/webapp' +# os.environ['PIPDEPTREE_EXE'] = 'profiles/webapp/.env_python3.6_pip-latest/bin/pipdeptree' + + +test_profile_dir = os.environ['TEST_PROFILE_DIR'] +pipdeptree_path = os.environ['PIPDEPTREE_EXE'] + + +def load_test_spec(): + test_spec_path = os.path.join(test_profile_dir, 'test_spec.json') + with open(test_spec_path) as f: + return json.load(f) + + +test_spec = load_test_spec() + + +def final_command(s): + tmpl = Environment(loader=BaseLoader).from_string(s) + return tmpl.render(pipdeptree=pipdeptree_path) + + +def _test_cmp_with_file_contents(spec): + p = subprocess.Popen(shlex.split(spec['command']), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + if spec['expected_output_file'] is not None: + exp_output_file = os.path.join(test_profile_dir, + spec['expected_output_file']) + with open(exp_output_file, 'rb') as f: + expected_output = f.read() + assert expected_output == out + else: + assert out == b'' + + if spec['expected_err_file'] is not None: + exp_err_file = os.path.join(test_profile_dir, + spec['expected_err_file']) + with open(exp_err_file, 'rb') as f: + expected_err = f.read() + assert expected_err == err + else: + assert err == b'' + + +@pytest.mark.parametrize('spec', test_spec) +def test_all_tests_in_profile(spec): + spec['command'] = final_command(spec['command']) + if spec['method'] == 'cmp_with_file_contents': + _test_cmp_with_file_contents(spec) diff --git a/tests/profiles/conflicting/.gitignore b/tests/profiles/conflicting/.gitignore new file mode 100644 index 0000000..e6905a2 --- /dev/null +++ b/tests/profiles/conflicting/.gitignore @@ -0,0 +1 @@ +.env* \ No newline at end of file diff --git a/tests/profiles/conflicting/default.err b/tests/profiles/conflicting/default.err new file mode 100644 index 0000000..1786b7a --- /dev/null +++ b/tests/profiles/conflicting/default.err @@ -0,0 +1,4 @@ +Warning!!! Possibly conflicting dependencies found: +* Jinja2==2.11.1 + - MarkupSafe [required: >=0.23, installed: 0.22] +------------------------------------------------------------------------ diff --git a/tests/profiles/conflicting/default.out b/tests/profiles/conflicting/default.out new file mode 100644 index 0000000..29b7d2f --- /dev/null +++ b/tests/profiles/conflicting/default.out @@ -0,0 +1,9 @@ +Flask==0.10.1 + - itsdangerous [required: >=0.21, installed: 0.24] + - Jinja2 [required: >=2.4, installed: 2.11.1] + - MarkupSafe [required: >=0.23, installed: 0.22] + - Werkzeug [required: >=0.7, installed: 0.11.2] +pipdeptree==2.0.0b1 + - pip [required: >=6.0.0, installed: 20.0.2] +setuptools==46.1.3 +wheel==0.34.2 diff --git a/tests/profiles/conflicting/requirements.txt b/tests/profiles/conflicting/requirements.txt new file mode 100644 index 0000000..bc5fabc --- /dev/null +++ b/tests/profiles/conflicting/requirements.txt @@ -0,0 +1,6 @@ +Flask==0.10.1 +itsdangerous==0.24 +Jinja2 +MarkupSafe==0.22 +Werkzeug==0.11.2 +argparse diff --git a/tests/profiles/conflicting/reverse.out b/tests/profiles/conflicting/reverse.out new file mode 100644 index 0000000..8a28ade --- /dev/null +++ b/tests/profiles/conflicting/reverse.out @@ -0,0 +1,11 @@ +itsdangerous==0.24 + - Flask==0.10.1 [requires: itsdangerous>=0.21] +MarkupSafe==0.22 + - Jinja2==2.11.1 [requires: MarkupSafe>=0.23] + - Flask==0.10.1 [requires: Jinja2>=2.4] +pip==20.0.2 + - pipdeptree==2.0.0b1 [requires: pip>=6.0.0] +setuptools==46.1.3 +Werkzeug==0.11.2 + - Flask==0.10.1 [requires: Werkzeug>=0.7] +wheel==0.34.2 diff --git a/tests/profiles/conflicting/test_spec.json b/tests/profiles/conflicting/test_spec.json new file mode 100644 index 0000000..3222170 --- /dev/null +++ b/tests/profiles/conflicting/test_spec.json @@ -0,0 +1,16 @@ +[ + { + "id": "default_output", + "method": "cmp_with_file_contents", + "command": "{{pipdeptree}}", + "expected_output_file": "default.out", + "expected_err_file": "default.err" + }, + { + "id": "reverse_output", + "method": "cmp_with_file_contents", + "command": "{{pipdeptree}} -r", + "expected_output_file": "reverse.out", + "expected_err_file": "default.err" + } +] diff --git a/tests/profiles/cyclic/.gitignore b/tests/profiles/cyclic/.gitignore new file mode 100644 index 0000000..e6905a2 --- /dev/null +++ b/tests/profiles/cyclic/.gitignore @@ -0,0 +1 @@ +.env* \ No newline at end of file diff --git a/tests/profiles/cyclic/default.err b/tests/profiles/cyclic/default.err new file mode 100644 index 0000000..857656a --- /dev/null +++ b/tests/profiles/cyclic/default.err @@ -0,0 +1,4 @@ +Warning!! Cyclic dependencies found: +* CircularDependencyB => CircularDependencyA => CircularDependencyB +* CircularDependencyA => CircularDependencyB => CircularDependencyA +------------------------------------------------------------------------ diff --git a/tests/profiles/cyclic/default.out b/tests/profiles/cyclic/default.out new file mode 100644 index 0000000..d9c9c41 --- /dev/null +++ b/tests/profiles/cyclic/default.out @@ -0,0 +1,4 @@ +pipdeptree==2.0.0b1 + - pip [required: >=6.0.0, installed: 20.0.2] +setuptools==46.1.3 +wheel==0.34.2 diff --git a/tests/profiles/cyclic/requirements.txt b/tests/profiles/cyclic/requirements.txt new file mode 100644 index 0000000..64644a8 --- /dev/null +++ b/tests/profiles/cyclic/requirements.txt @@ -0,0 +1,2 @@ +CircularDependencyA +CircularDependencyB diff --git a/tests/profiles/cyclic/test_spec.json b/tests/profiles/cyclic/test_spec.json new file mode 100644 index 0000000..30fd876 --- /dev/null +++ b/tests/profiles/cyclic/test_spec.json @@ -0,0 +1,9 @@ +[ + { + "id": "default_output", + "method": "cmp_with_file_contents", + "command": "{{pipdeptree}}", + "expected_output_file": "default.out", + "expected_err_file": "default.err" + } +] diff --git a/tests/profiles/webapp/.gitignore b/tests/profiles/webapp/.gitignore new file mode 100644 index 0000000..e6905a2 --- /dev/null +++ b/tests/profiles/webapp/.gitignore @@ -0,0 +1 @@ +.env* \ No newline at end of file diff --git a/tests/profiles/webapp/all_flag.out b/tests/profiles/webapp/all_flag.out new file mode 100644 index 0000000..72c115d --- /dev/null +++ b/tests/profiles/webapp/all_flag.out @@ -0,0 +1,65 @@ +appnope==0.1.0 +backcall==0.1.0 +click==7.1.1 +decorator==4.4.2 +Flask==1.1.2 + - click [required: >=5.1, installed: 7.1.1] + - itsdangerous [required: >=0.24, installed: 1.1.0] + - Jinja2 [required: >=2.10.1, installed: 2.11.1] + - MarkupSafe [required: >=0.23, installed: 1.1.1] + - Werkzeug [required: >=0.15, installed: 1.0.1] +Flask-Script==2.0.6 + - Flask [required: Any, installed: 1.1.2] + - click [required: >=5.1, installed: 7.1.1] + - itsdangerous [required: >=0.24, installed: 1.1.0] + - Jinja2 [required: >=2.10.1, installed: 2.11.1] + - MarkupSafe [required: >=0.23, installed: 1.1.1] + - Werkzeug [required: >=0.15, installed: 1.0.1] +gnureadline==8.0.0 +ipython==7.13.0 + - appnope [required: Any, installed: 0.1.0] + - backcall [required: Any, installed: 0.1.0] + - decorator [required: Any, installed: 4.4.2] + - jedi [required: >=0.10, installed: 0.16.0] + - parso [required: >=0.5.2, installed: 0.6.2] + - pexpect [required: Any, installed: 4.8.0] + - ptyprocess [required: >=0.5, installed: 0.6.0] + - pickleshare [required: Any, installed: 0.7.5] + - prompt-toolkit [required: >=2.0.0,<3.1.0,!=3.0.1,!=3.0.0, installed: 3.0.5] + - wcwidth [required: Any, installed: 0.1.9] + - pygments [required: Any, installed: 2.6.1] + - setuptools [required: >=18.5, installed: 46.1.3] + - traitlets [required: >=4.2, installed: 4.3.3] + - decorator [required: Any, installed: 4.4.2] + - ipython-genutils [required: Any, installed: 0.2.0] + - six [required: Any, installed: 1.14.0] +ipython-genutils==0.2.0 +itsdangerous==1.1.0 +jedi==0.16.0 + - parso [required: >=0.5.2, installed: 0.6.2] +Jinja2==2.11.1 + - MarkupSafe [required: >=0.23, installed: 1.1.1] +MarkupSafe==1.1.1 +parso==0.6.2 +pexpect==4.8.0 + - ptyprocess [required: >=0.5, installed: 0.6.0] +pickleshare==0.7.5 +pip==20.0.2 +pipdeptree==2.0.0b1 + - pip [required: >=6.0.0, installed: 20.0.2] +prompt-toolkit==3.0.5 + - wcwidth [required: Any, installed: 0.1.9] +ptyprocess==0.6.0 +Pygments==2.6.1 +pymongo==3.10.1 +redis==3.4.1 +setuptools==46.1.3 +six==1.14.0 +slugify==0.0.1 +traitlets==4.3.3 + - decorator [required: Any, installed: 4.4.2] + - ipython-genutils [required: Any, installed: 0.2.0] + - six [required: Any, installed: 1.14.0] +wcwidth==0.1.9 +Werkzeug==1.0.1 +wheel==0.34.2 diff --git a/tests/profiles/webapp/default.out b/tests/profiles/webapp/default.out new file mode 100644 index 0000000..5dc3a26 --- /dev/null +++ b/tests/profiles/webapp/default.out @@ -0,0 +1,31 @@ +Flask-Script==2.0.6 + - Flask [required: Any, installed: 1.1.2] + - click [required: >=5.1, installed: 7.1.1] + - itsdangerous [required: >=0.24, installed: 1.1.0] + - Jinja2 [required: >=2.10.1, installed: 2.11.1] + - MarkupSafe [required: >=0.23, installed: 1.1.1] + - Werkzeug [required: >=0.15, installed: 1.0.1] +gnureadline==8.0.0 +ipython==7.13.0 + - appnope [required: Any, installed: 0.1.0] + - backcall [required: Any, installed: 0.1.0] + - decorator [required: Any, installed: 4.4.2] + - jedi [required: >=0.10, installed: 0.16.0] + - parso [required: >=0.5.2, installed: 0.6.2] + - pexpect [required: Any, installed: 4.8.0] + - ptyprocess [required: >=0.5, installed: 0.6.0] + - pickleshare [required: Any, installed: 0.7.5] + - prompt-toolkit [required: >=2.0.0,<3.1.0,!=3.0.1,!=3.0.0, installed: 3.0.5] + - wcwidth [required: Any, installed: 0.1.9] + - pygments [required: Any, installed: 2.6.1] + - setuptools [required: >=18.5, installed: 46.1.3] + - traitlets [required: >=4.2, installed: 4.3.3] + - decorator [required: Any, installed: 4.4.2] + - ipython-genutils [required: Any, installed: 0.2.0] + - six [required: Any, installed: 1.14.0] +pipdeptree==2.0.0b1 + - pip [required: >=6.0.0, installed: 20.0.2] +pymongo==3.10.1 +redis==3.4.1 +slugify==0.0.1 +wheel==0.34.2 diff --git a/tests/profiles/webapp/packages_opt.out b/tests/profiles/webapp/packages_opt.out new file mode 100644 index 0000000..cb42cc0 --- /dev/null +++ b/tests/profiles/webapp/packages_opt.out @@ -0,0 +1,17 @@ +ipython==7.13.0 + - appnope [required: Any, installed: 0.1.0] + - backcall [required: Any, installed: 0.1.0] + - decorator [required: Any, installed: 4.4.2] + - jedi [required: >=0.10, installed: 0.16.0] + - parso [required: >=0.5.2, installed: 0.6.2] + - pexpect [required: Any, installed: 4.8.0] + - ptyprocess [required: >=0.5, installed: 0.6.0] + - pickleshare [required: Any, installed: 0.7.5] + - prompt-toolkit [required: >=2.0.0,<3.1.0,!=3.0.1,!=3.0.0, installed: 3.0.5] + - wcwidth [required: Any, installed: 0.1.9] + - pygments [required: Any, installed: 2.6.1] + - setuptools [required: >=18.5, installed: 46.1.3] + - traitlets [required: >=4.2, installed: 4.3.3] + - decorator [required: Any, installed: 4.4.2] + - ipython-genutils [required: Any, installed: 0.2.0] + - six [required: Any, installed: 1.14.0] diff --git a/tests/profiles/webapp/requirements.txt b/tests/profiles/webapp/requirements.txt new file mode 100644 index 0000000..3a740b2 --- /dev/null +++ b/tests/profiles/webapp/requirements.txt @@ -0,0 +1,7 @@ +Flask==1.1.2 +Flask-Script==2.0.6 +gnureadline==8.0.0 +pymongo==3.10.1 +redis==3.4.1 +slugify==0.0.1 +ipython==7.13.0 diff --git a/tests/profiles/webapp/reverse.out b/tests/profiles/webapp/reverse.out new file mode 100644 index 0000000..ccc738f --- /dev/null +++ b/tests/profiles/webapp/reverse.out @@ -0,0 +1,49 @@ +appnope==0.1.0 + - ipython==7.13.0 [requires: appnope] +backcall==0.1.0 + - ipython==7.13.0 [requires: backcall] +click==7.1.1 + - Flask==1.1.2 [requires: click>=5.1] + - Flask-Script==2.0.6 [requires: Flask] +decorator==4.4.2 + - ipython==7.13.0 [requires: decorator] + - traitlets==4.3.3 [requires: decorator] + - ipython==7.13.0 [requires: traitlets>=4.2] +gnureadline==8.0.0 +ipython-genutils==0.2.0 + - traitlets==4.3.3 [requires: ipython-genutils] + - ipython==7.13.0 [requires: traitlets>=4.2] +itsdangerous==1.1.0 + - Flask==1.1.2 [requires: itsdangerous>=0.24] + - Flask-Script==2.0.6 [requires: Flask] +MarkupSafe==1.1.1 + - Jinja2==2.11.1 [requires: MarkupSafe>=0.23] + - Flask==1.1.2 [requires: Jinja2>=2.10.1] + - Flask-Script==2.0.6 [requires: Flask] +parso==0.6.2 + - jedi==0.16.0 [requires: parso>=0.5.2] + - ipython==7.13.0 [requires: jedi>=0.10] +pickleshare==0.7.5 + - ipython==7.13.0 [requires: pickleshare] +pip==20.0.2 + - pipdeptree==2.0.0b1 [requires: pip>=6.0.0] +ptyprocess==0.6.0 + - pexpect==4.8.0 [requires: ptyprocess>=0.5] + - ipython==7.13.0 [requires: pexpect] +pygments==2.6.1 + - ipython==7.13.0 [requires: pygments] +pymongo==3.10.1 +redis==3.4.1 +setuptools==46.1.3 + - ipython==7.13.0 [requires: setuptools>=18.5] +six==1.14.0 + - traitlets==4.3.3 [requires: six] + - ipython==7.13.0 [requires: traitlets>=4.2] +slugify==0.0.1 +wcwidth==0.1.9 + - prompt-toolkit==3.0.5 [requires: wcwidth] + - ipython==7.13.0 [requires: prompt-toolkit>=2.0.0,<3.1.0,!=3.0.1,!=3.0.0] +Werkzeug==1.0.1 + - Flask==1.1.2 [requires: Werkzeug>=0.15] + - Flask-Script==2.0.6 [requires: Flask] +wheel==0.34.2 diff --git a/tests/profiles/webapp/test_spec.json b/tests/profiles/webapp/test_spec.json new file mode 100644 index 0000000..a47694b --- /dev/null +++ b/tests/profiles/webapp/test_spec.json @@ -0,0 +1,30 @@ +[ + { + "id": "default_output", + "method": "cmp_with_file_contents", + "command": "{{pipdeptree}}", + "expected_output_file": "default.out", + "expected_err_file": null + }, + { + "id": "reverse_output", + "method": "cmp_with_file_contents", + "command": "{{pipdeptree}} -r", + "expected_output_file": "reverse.out", + "expected_err_file": null + }, + { + "id": "--all_output", + "method": "cmp_with_file_contents", + "command": "{{pipdeptree}} --all", + "expected_output_file": "all_flag.out", + "expected_err_file": null + }, + { + "id": "--packages_output", + "method": "cmp_with_file_contents", + "command": "{{pipdeptree}} --packages pexpect,ipython", + "expected_output_file": "packages_opt.out", + "expected_err_file": null + } +]