diff --git a/CHANGES.md b/CHANGES.md index e8ecd80..94c2806 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,24 @@ Changelog ========= +0.8.0 +----- + +* Use pip's list of excluded default packages. This means that the + ``pipdeptree`` package itself is no longer excluded and will appear + in the output tree. + +* Fix the bug that caused a package to appear in conflicting deps + although it's installed version could be guessed. + + +0.7.0 +----- + +* Fix for a bug in reverse mode. +* Alphabetical sorting of packages in the output. +* Fallback to guess installed version of packages "skipped" by pip. + 0.6.0 ----- diff --git a/pipdeptree.py b/pipdeptree.py index 6151b33..9af64a5 100644 --- a/pipdeptree.py +++ b/pipdeptree.py @@ -1,9 +1,16 @@ from __future__ import print_function import sys -from itertools import chain, tee +from itertools import chain from collections import defaultdict import argparse +from operator import attrgetter import json +from importlib import import_module + +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict import pip import pkg_resources @@ -11,7 +18,7 @@ import pkg_resources # import networkx -__version__ = '0.6.0' +__version__ = '0.8.0' flatten = chain.from_iterable @@ -44,6 +51,38 @@ def construct_tree(index): for p in index.values()) +def sorted_tree(tree): + """Sorts the dict representation of the tree + + The root packages as well as the intermediate packages are sorted + in the alphabetical order of the package names. + + :param dict tree: the pkg dependency tree obtained by calling + `construct_tree` function + :returns: sorted tree + :rtype: collections.OrderedDict + + """ + return OrderedDict(sorted([(k, sorted(v, key=attrgetter('key'))) + for k, v in tree.items()], + key=lambda kv: kv[0].key)) + + +def find_tree_root(tree, key): + """Find a root in a tree by it's key + + :param dict tree: the pkg dependency tree obtained by calling + `construct_tree` function + :param str key: key of the root node to find + :returns: a root node if found else None + :rtype: mixed + + """ + result = [p for p in tree.keys() if p.key == key] + assert len(result) in [0, 1] + return None if len(result) == 0 else result[0] + + def reverse_tree(tree): """Reverse the dependency tree. @@ -56,20 +95,34 @@ def reverse_tree(tree): :rtype: dict """ - rtree = {} - visited = set() + rtree = defaultdict(list) child_keys = set(c.key for c in flatten(tree.values())) for k, vs in tree.items(): for v in vs: - if v not in rtree: - rtree[v] = [] - rtree[v].append(k) - visited.add(v.key) + node = find_tree_root(rtree, v.key) or v + rtree[node].append(k.as_required_by(v)) if k.key not in child_keys: rtree[k.as_requirement()] = [] return rtree +def guess_version(pkg_key, default='?'): + """Guess the version of a pkg when pip doesn't provide it + + :param str pkg_key: key of the package + :param str default: default version to return if unable to find + :returns: version + :rtype: string + + """ + try: + m = import_module(pkg_key) + except ImportError: + return default + else: + return getattr(m, '__version__', default) + + class Package(object): """Abstract class for wrappers around objects that pip returns. @@ -82,29 +135,23 @@ class Package(object): self._obj = obj self.project_name = obj.project_name self.key = obj.key - # an instance of every subclass of Package will have a - # DistPackage object associated with it. In case of - # DistPackage class, it will be the object itself. - self.dist = None def render_as_root(self, frozen): return NotImplementedError - def render_as_branch(self, parent, frozen): + def render_as_branch(self, frozen): return NotImplementedError def render(self, parent=None, frozen=False): if not parent: return self.render_as_root(frozen) else: - return self.render_as_branch(parent, frozen) + return self.render_as_branch(frozen) - def frozen_repr(self): - if self.dist: - fr = pip.FrozenRequirement.from_dist(self.dist._obj, []) - return str(fr).strip() - else: - return self.project_name + @staticmethod + def frozen_repr(obj): + fr = pip.FrozenRequirement.from_dist(obj, []) + return str(fr).strip() def __getattr__(self, key): return getattr(self._obj, key) @@ -114,24 +161,30 @@ class Package(object): class DistPackage(Package): - """Wrapper class for pkg_resources.Distribution instances""" + """Wrapper class for pkg_resources.Distribution instances - def __init__(self, obj): + :param obj: pkg_resources.Distribution to wrap over + :param req: optional ReqPackage object to associate this + DistPackage with. This is useful for displaying the + tree in reverse + """ + + def __init__(self, obj, req=None): super(DistPackage, self).__init__(obj) - # this itself is the associated dist package obj - self.dist = self self.version_spec = None + self.req = req def render_as_root(self, frozen): if not frozen: return '{0}=={1}'.format(self.project_name, self.version) else: - return self.frozen_repr() + return self.__class__.frozen_repr(self._obj) - def render_as_branch(self, parent, frozen): + def render_as_branch(self, frozen): + assert self.req is not None if not frozen: - parent_ver_spec = parent.version_spec - parent_str = parent.project_name + parent_ver_spec = self.req.version_spec + parent_str = self.req.project_name if parent_ver_spec: parent_str += parent_ver_spec return ( @@ -141,8 +194,21 @@ class DistPackage(Package): return self.render_as_root(frozen) def as_requirement(self): + """Return a ReqPackage representation of this DistPackage""" return ReqPackage(self._obj.as_requirement(), dist=self) + def as_required_by(self, req): + """Return a DistPackage instance associated to a requirement + + This association is necessary for displaying the tree in + reverse. + + :param ReqPackage req: the requirement to associate with + :returns: DistPackage instance + + """ + return self.__class__(self._obj, req) + def as_dict(self): return {'key': self.key, 'package_name': self.project_name, @@ -150,7 +216,14 @@ class DistPackage(Package): class ReqPackage(Package): - """Wrapper class for Requirements instance""" + """Wrapper class for Requirements instance + + :param obj: The `Requirements` instance to wrap over + :param dist: optional `pkg_resources.Distribution` instance for + this requirement + """ + + UNKNOWN_VERSION = '?' def __init__(self, obj, dist=None): super(ReqPackage, self).__init__(obj) @@ -163,27 +236,34 @@ class ReqPackage(Package): @property def installed_version(self): - # if the dist is None as in some cases, we don't know the - # installed version - return self.dist.version if self.dist else '?' + if not self.dist: + return guess_version(self.key, self.UNKNOWN_VERSION) + return self.dist.version + + def is_conflicting(self): + """If installed version conflicts with required version""" + # unknown installed version is also considered conflicting + if self.installed_version == self.UNKNOWN_VERSION: + return True + ver_spec = (self.version_spec if self.version_spec else '') + req_version_str = '{0}{1}'.format(self.project_name, ver_spec) + req_obj = pkg_resources.Requirement.parse(req_version_str) + return self.installed_version not in req_obj def render_as_root(self, frozen): if not frozen: return '{0}=={1}'.format(self.project_name, self.installed_version) + elif self.dist: + return self.__class__.frozen_repr(self.dist._obj) else: - return self.frozen_repr() + return self.project_name - def render_as_branch(self, _parent, frozen): + def render_as_branch(self, frozen): if not frozen: - vers = [] - if self.version_spec: - vers.append(('required', self.version_spec)) - if self.dist: - vers.append(('installed', self.installed_version)) - if not vers: - return self.key - ver_str = ', '.join(['{0}: {1}'.format(k, v) for k, v in vers]) - return '{0} [{1}]'.format(self.project_name, ver_str) + return ( + '{0} [required: {1}, installed: {2}]' + ).format(self.project_name, self.version_spec, + self.installed_version) else: return self.render_as_root(frozen) @@ -209,14 +289,13 @@ def render_tree(tree, list_all=True, show_only=None, frozen=False): :rtype: str """ + tree = sorted_tree(tree) branch_keys = set(r.key for r in flatten(tree.values())) nodes = tree.keys() use_bullets = not frozen key_tree = dict((k.key, v) for k, v in tree.items()) - - def get_children(n): - return key_tree[n.key] + get_children = lambda n: key_tree.get(n.key, []) if show_only: nodes = [p for p in nodes @@ -229,20 +308,14 @@ def render_tree(tree, list_all=True, show_only=None, frozen=False): chain = [node.project_name] node_str = node.render(parent, frozen) if parent: - prefix = ' '*indent + ('-' if use_bullets else ' ') + ' ' + prefix = ' '*indent + ('- ' if use_bullets else '') node_str = prefix + node_str result = [node_str] - - # the dist attr for some ReqPackage could be None - # eg. testresources, setuptools which is a dependencies of - # some pkg but doesn't get listed in the result of - # pip.get_installed_distributions. - if node.dist: - children = [aux(c, node, indent=indent+2, - chain=chain+[c.project_name]) - for c in get_children(node) - if c.project_name not in chain] - result += list(flatten(children)) + children = [aux(c, node, indent=indent+2, + chain=chain+[c.project_name]) + for c in get_children(node) + if c.project_name not in chain] + result += list(flatten(children)) return result lines = flatten([aux(p) for p in nodes]) @@ -307,63 +380,30 @@ def conflicting_deps(tree): """ conflicting = defaultdict(list) - req_parse = pkg_resources.Requirement.parse for p, rs in tree.items(): for req in rs: - if not req.dist: + if req.is_conflicting(): conflicting[p].append(req) - else: - req_version_str = '%s%s' % (req.project_name, (req.version_spec if req.version_spec else '')) - if req.installed_version not in req_parse(req_version_str): - conflicting[p].append(req) return conflicting def cyclic_deps(tree): - """Generator that produces cyclic dependencies + """Return cyclic dependencies as list of tuples :param list pkgs: pkg_resources.Distribution instances :param dict pkg_index: mapping of pkgs with their respective keys - :returns: generator that yields str representation of cyclic - dependencies + :returns: list of tuples representing cyclic dependencies :rtype: generator """ - nodes = tree.keys() key_tree = dict((k.key, v) for k, v in tree.items()) - - def get_children(n): - return key_tree[n.key] - - def aux(node, chain): - if node.dist: - for c in get_children(node): - if c.project_name in chain: - yield ' => '.join([str(p) for p in chain] + [str(c)]) - else: - for cycle in aux(c, chain=chain+[c.project_name]): - yield cycle - - for cycle in flatten([aux(n, chain=[]) for n in nodes]): - yield cycle - - -def peek_into(iterator): - """Peeks into an iterator to check if it's empty - - :param iterator: an iterator - :returns: tuple of boolean representing whether the iterator is - empty or not and the iterator itself. - :rtype: tuple - - """ - a, b = tee(iterator) - is_empty = False - try: - next(a) - except StopIteration: - is_empty = True - return is_empty, b + get_children = lambda n: key_tree.get(n.key, []) + cyclic = [] + for p, rs in tree.items(): + for req in rs: + if p.key in map(attrgetter('key'), get_children(req)): + cyclic.append((p, req, p)) + return cyclic def main(): @@ -411,10 +451,7 @@ def main(): help='Print dependency tree as GraphViz dot code.') args = parser.parse_args() - default_skip = ['setuptools', 'pip', 'python', 'distribute'] - skip = default_skip + ['pipdeptree'] - pkgs = pip.get_installed_distributions(local_only=args.local_only, - skip=skip) + pkgs = pip.get_installed_distributions(local_only=args.local_only) dist_index = build_dist_index(pkgs) tree = construct_tree(dist_index) @@ -439,24 +476,21 @@ def main(): pkg = p.render_as_root(False) print('* %s' % pkg, file=sys.stderr) for req in reqs: - if not req.dist: - req_str = ( - '{0} [required: {1}, ' - 'installed: ]' - ).format(req.project_name, req.version_spec) - else: - req_str = req.render_as_branch(p, False) + req_str = req.render_as_branch(False) print(' - %s' % req_str, file=sys.stderr) print('-'*72, file=sys.stderr) - is_empty, cyclic = peek_into(cyclic_deps(tree)) - if not is_empty: - print('Warning!!! Cyclic dependencies found:', file=sys.stderr) - for xs in cyclic: - print('- {0}'.format(xs), file=sys.stderr) + cyclic = cyclic_deps(tree) + if cyclic: + print('Warning!! Cyclic dependencies found:', file=sys.stderr) + for a, b, c in cyclic: + print('* {0} => {1} => {2}'.format(a.project_name, + b.project_name, + c.project_name), + file=sys.stderr) print('-'*72, file=sys.stderr) - if args.warn == 'fail' and (conflicting or not is_empty): + if args.warn == 'fail' and (conflicting or cyclic): return_code = 1 show_only = set(args.packages.split(',')) if args.packages else None diff --git a/setup.py b/setup.py index 815cff7..4de03de 100644 --- a/setup.py +++ b/setup.py @@ -16,9 +16,11 @@ with open('./README.rst') as f: long_desc = f.read() -install_requires = ["pip >= 1.4.1"] +install_requires = ["pip >= 6.0.0"] if sys.version_info < (2, 7): install_requires.append('argparse') + install_requires.append('ordereddict') + install_requires.append('importlib') setup( diff --git a/tests/test_pipdeptree.py b/tests/test_pipdeptree.py index 73ce72b..f6b8c7d 100644 --- a/tests/test_pipdeptree.py +++ b/tests/test_pipdeptree.py @@ -1,8 +1,9 @@ import pickle +from operator import itemgetter, attrgetter -from pipdeptree import (build_dist_index, construct_tree, peek_into, +from pipdeptree import (build_dist_index, construct_tree, DistPackage, ReqPackage, render_tree, - reverse_tree, conflicting_deps) + reverse_tree, cyclic_deps, conflicting_deps) def venv_fixture(pickle_file): @@ -47,8 +48,10 @@ def test_tree(): def test_reverse_tree(): rtree = reverse_tree(tree) - assert all((isinstance(k, ReqPackage) and - all(isinstance(v, DistPackage) for v in vs)) + assert all(isinstance(k, ReqPackage) for k, vs in rtree.items()) + assert all(all(isinstance(v, DistPackage) for v in vs) + for k, vs in rtree.items()) + assert all(all(v.req is not None for v in vs) for k, vs in rtree.items()) @@ -60,15 +63,15 @@ def test_DistPackage_render_as_root(): def test_DistPackage_render_as_branch(): - alembic = find_dist('alembic') + sqlalchemy = find_req('sqlalchemy', 'alembic') + alembic = find_dist('alembic').as_required_by(sqlalchemy) assert alembic.project_name == 'alembic' assert alembic.version == '0.6.2' - sqlalchemy = find_req('sqlalchemy', 'alembic') assert sqlalchemy.project_name == 'SQLAlchemy' assert sqlalchemy.version_spec == '>=0.7.3' assert sqlalchemy.installed_version == '0.9.1' - result_1 = alembic.render_as_branch(sqlalchemy, False) - result_2 = alembic.render_as_branch(sqlalchemy, False) + result_1 = alembic.render_as_branch(False) + result_2 = alembic.render_as_branch(False) assert result_1 == result_2 == 'alembic==0.6.2 [requires: SQLAlchemy>=0.7.3]' @@ -81,19 +84,17 @@ def test_ReqPackage_render_as_root(): def test_ReqPackage_render_as_branch(): mks1 = find_req('markupsafe', 'jinja2') - jinja = find_dist('jinja2') assert mks1.project_name == 'markupsafe' assert mks1.installed_version == '0.18' assert mks1.version_spec is None - assert mks1.render_as_branch(jinja, False) == 'markupsafe [installed: 0.18]' - assert mks1.render_as_branch(jinja, True) == 'MarkupSafe==0.18' + assert mks1.render_as_branch(False) == 'markupsafe [required: None, installed: 0.18]' + assert mks1.render_as_branch(True) == 'MarkupSafe==0.18' mks2 = find_req('markupsafe', 'mako') - mako = find_dist('mako') assert mks2.project_name == 'MarkupSafe' assert mks2.installed_version == '0.18' assert mks2.version_spec == '>=0.9.2' - assert mks2.render_as_branch(mako, False) == 'MarkupSafe [required: >=0.9.2, installed: 0.18]' - assert mks2.render_as_branch(mako, True) == 'MarkupSafe==0.18' + assert mks2.render_as_branch(False) == 'MarkupSafe [required: >=0.9.2, installed: 0.18]' + assert mks2.render_as_branch(True) == 'MarkupSafe==0.18' def test_render_tree_only_top(): @@ -125,20 +126,30 @@ def test_render_tree_freeze(): line = line.replace('origin/HEAD', 'master') lines.add(line) assert 'Flask-Script==0.6.6' in lines - assert ' SQLAlchemy==0.9.1' in lines + assert ' SQLAlchemy==0.9.1' in lines # TODO! Fix the following failing test # assert '-e git+https://github.com/naiquevin/lookupy.git@cdbe30c160e1c29802df75e145ea4ad903c05386#egg=Lookupy-master' in lines assert 'itsdangerous==0.23' not in lines +def test_cyclic_dependencies(): + cyclic_pkgs, dist_index, tree = venv_fixture('tests/virtualenvs/cyclicenv.pickle') + cyclic = [map(attrgetter('key'), cs) for cs in cyclic_deps(tree)] + assert len(cyclic) == 2 + a, b, c = cyclic[0] + x, y, z = cyclic[1] + assert a == c == y + assert x == z == b + + def test_render_tree_cyclic_dependency(): cyclic_pkgs, dist_index, tree = venv_fixture('tests/virtualenvs/cyclicenv.pickle') tree_str = render_tree(tree, list_all=True) lines = set(tree_str.split('\n')) assert 'CircularDependencyA==0.0.0' in lines - assert ' - CircularDependencyB [installed: 0.0.0]' in lines + assert ' - CircularDependencyB [required: None, installed: 0.0.0]' in lines assert 'CircularDependencyB==0.0.0' in lines - assert ' - CircularDependencyA [installed: 0.0.0]' in lines + assert ' - CircularDependencyA [required: None, installed: 0.0.0]' in lines def test_render_tree_freeze_cyclic_dependency(): @@ -146,18 +157,9 @@ def test_render_tree_freeze_cyclic_dependency(): tree_str = render_tree(tree, list_all=True, frozen=True) lines = set(tree_str.split('\n')) assert 'CircularDependencyA==0.0.0' in lines - assert ' CircularDependencyB==0.0.0' in lines + assert ' CircularDependencyB==0.0.0' in lines assert 'CircularDependencyB==0.0.0' in lines - assert ' CircularDependencyA==0.0.0' in lines - - -def test_peek_into(): - r1, g1 = peek_into(i for i in []) - assert r1 - assert len(list(g1)) == 0 - r2, g2 = peek_into(i for i in range(100)) - assert not r2 - assert len(list(g2)) == 100 + assert ' CircularDependencyA==0.0.0' in lines def test_conflicting_deps():