PEP8ify + pyflakes

Changelog

@@ -42,7 +42,7 @@
   can be used to check if the connection instance has established
   a connection.
 
-* ``ConnectionPool.acquire_channel` now returns the connections
+* ``ConnectionPool.acquire_channel`` now returns the connections
   default channel rather than establising a new channel that
   must be manually handled.
 

contrib/release/flakeplus.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import with_statement
+
+import os
+import re
+import sys
+
+from collections import defaultdict
+from unipath import Path
+
+RE_COMMENT = r'^\s*\#'
+RE_NOQA = r'.+?\#\s+noqa+'
+RE_MULTILINE_COMMENT_O = r'^\s*(?:\'\'\'|""").+?(?:\'\'\'|""")'
+RE_MULTILINE_COMMENT_S = r'^\s*(?:\'\'\'|""")'
+RE_MULTILINE_COMMENT_E = r'(?:^|.+?)(?:\'\'\'|""")'
+RE_WITH = r'(?:^|\s+)with\s+'
+RE_WITH_IMPORT = r'''from\s+ __future__\s+ import\s+ with_statement'''
+RE_PRINT = r'''(?:^|\s+)print\((?:"|')(?:\W+?)?[A-Z0-9:]{2,}'''
+RE_ABS_IMPORT = r'''from\s+ __future__\s+ import\s+ absolute_import'''
+
+acc = defaultdict(lambda: {"abs": False, "print": False})
+
+
+def compile(regex):
+    return re.compile(regex, re.VERBOSE)
+
+
+class FlakePP(object):
+    re_comment = compile(RE_COMMENT)
+    re_ml_comment_o = compile(RE_MULTILINE_COMMENT_O)
+    re_ml_comment_s = compile(RE_MULTILINE_COMMENT_S)
+    re_ml_comment_e = compile(RE_MULTILINE_COMMENT_E)
+    re_abs_import = compile(RE_ABS_IMPORT)
+    re_print = compile(RE_PRINT)
+    re_with_import = compile(RE_WITH_IMPORT)
+    re_with = compile(RE_WITH)
+    re_noqa = compile(RE_NOQA)
+    map = {"abs": True, "print": False,
+            "with": False, "with-used": False}
+
+    def __init__(self, verbose=False):
+        self.verbose = verbose
+        self.steps = (("abs", self.re_abs_import),
+                      ("with", self.re_with_import),
+                      ("with-used", self.re_with),
+                      ("print", self.re_print))
+
+    def analyze_fh(self, fh):
+        steps = self.steps
+        filename = fh.name
+        acc = dict(self.map)
+        index = 0
+        errors = [0]
+
+        def error(fmt, **kwargs):
+            errors[0] += 1
+            self.announce(fmt, **dict(kwargs, filename=filename))
+
+        for index, line in enumerate(self.strip_comments(fh)):
+            for key, pattern in steps:
+                if pattern.match(line):
+                    acc[key] = True
+        if index:
+            if not acc["abs"]:
+                error("%(filename)s: missing abs import")
+            if acc["with-used"] and not acc["with"]:
+                error("%(filename)s: missing with import")
+            if acc["print"]:
+                error("%(filename)s: left over print statement")
+
+        return filename, errors[0], acc
+
+    def analyze_file(self, filename):
+        with open(filename) as fh:
+            return self.analyze_fh(fh)
+
+    def analyze_tree(self, dir):
+        for dirpath, _, filenames in os.walk(dir):
+            for path in (Path(dirpath, f) for f in filenames):
+                if path.endswith(".py"):
+                    yield self.analyze_file(path)
+
+    def analyze(self, *paths):
+        for path in map(Path, paths):
+            if path.isdir():
+                for res in self.analyze_tree(path):
+                    yield res
+            else:
+                yield self.analyze_file(path)
+
+    def strip_comments(self, fh):
+        re_comment = self.re_comment
+        re_ml_comment_o = self.re_ml_comment_o
+        re_ml_comment_s = self.re_ml_comment_s
+        re_ml_comment_e = self.re_ml_comment_e
+        re_noqa = self.re_noqa
+        in_ml = False
+
+        for line in fh.readlines():
+            if in_ml:
+                if re_ml_comment_e.match(line):
+                    in_ml = False
+            else:
+                if re_noqa.match(line) or re_ml_comment_o.match(line):
+                    pass
+                elif re_ml_comment_s.match(line):
+                    in_ml = True
+                elif re_comment.match(line):
+                    pass
+                else:
+                    yield line
+
+    def announce(self, fmt, **kwargs):
+        sys.stderr.write((fmt + "\n") % kwargs)
+
+
+def main(argv=sys.argv, exitcode=0):
+    for _, errors, _ in FlakePP(verbose=True).analyze(*argv[1:]):
+        if errors:
+            exitcode = 1
+    return exitcode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docs/reference/index.rst

@@ -13,6 +13,7 @@
     kombu.messaging
     kombu.entity
     kombu.common
+    kombu.mixins
     kombu.clocks
     kombu.compat
     kombu.pidbox
@@ -37,6 +38,7 @@
     kombu.abstract
     kombu.syn
     kombu.utils
+    kombu.utils.limits
     kombu.utils.compat
     kombu.utils.debug
     kombu.utils.encoding

docs/reference/kombu.mixins.rst

@@ -0,0 +1,11 @@
+==========================================================
+ Mixin Classes - kombu.mixins
+==========================================================
+
+.. contents::
+    :local:
+.. currentmodule:: kombu.mixins
+
+.. automodule:: kombu.mixins
+    :members:
+    :undoc-members:

docs/reference/kombu.utils.limits.rst

@@ -0,0 +1,11 @@
+==========================================================
+ Rate limiting - kombu.utils.limits
+==========================================================
+
+.. contents::
+    :local:
+.. currentmodule:: kombu.utils.limits
+
+.. automodule:: kombu.utils.limits
+    :members:
+    :undoc-members:

docs/userguide/examples.rst

@@ -1,3 +1,11 @@
+.. _examples:
+
+========================
+ Examples
+========================
+
+.. _task-queue-example:
+
 Task Queue Example
 ==================
 

docs/userguide/index.rst

@@ -9,7 +9,7 @@
     :maxdepth: 2
 
     connections
-    simple
     examples
+    simple
     pools
     serialization

examples/simple_task_queue/client.py

@@ -9,6 +9,7 @@ priority_to_routing_key = {"high": "hipri",
                            "mid": "midpri",
                            "low": "lopri"}
 
+
 def send_as_task(connection, fun, args, kwargs, priority="mid"):
     payload = {"fun": fun, "args": args, "kwargs": kwargs}
     routing_key = priority_to_routing_key[priority]
@@ -23,6 +24,6 @@ if __name__ == "__main__":
     from kombu import BrokerConnection
     from .tasks import hello_task
 
-    conection = BrokerConnection("amqp://guest:guest@localhost:5672//")
+    connection = BrokerConnection("amqp://guest:guest@localhost:5672//")
     send_as_task(connection, fun=hello_task, args=("Kombu", ),
                  priority="high")

examples/simple_task_queue/queues.py

@@ -4,4 +4,3 @@ task_exchange = Exchange("tasks", type="direct")
 task_queues = [Queue("hipri", task_exchange, routing_key="hipri"),
                Queue("midpri", task_exchange, routing_key="midpri"),
                Queue("lopri", task_exchange, routing_key="lopri")]
-

examples/simple_task_queue/tasks.py

@@ -1,4 +1,2 @@
 def hello_task(who="world"):
     print("Hello %s" % (who, ))
-
-

examples/simple_task_queue/worker.py

@@ -1,11 +1,11 @@
 from __future__ import with_statement
 
-from kombu import Exchange, Queue
 from kombu.mixins import ConsumerMixin
 from kombu.utils import kwdict
 
 from queues import task_queues
 
+
 class Worker(ConsumerMixin):
 
     def get_consumers(self, Consumer, channel):
@@ -26,5 +26,3 @@ if __name__ == "__main__":
 
     with BrokerConnection("amqp://guest:guest@localhost:5672//") as conn:
         Worker(conn).run()
-
-

kombu/clocks.py

@@ -33,8 +33,12 @@ class LamportClock(object):
 
     .. seealso::
 
-        http://en.wikipedia.org/wiki/Lamport_timestamps
-        http://en.wikipedia.org/wiki/Lamport's_Distributed_Mutual_Exclusion_Algorithm
+        * `Lamport timestamps`_
+
+        * `Lamports distributed mutex`_
+
+    .. _`Lamport Timestamps`: http://en.wikipedia.org/wiki/Lamport_timestamps
+    .. _`Lamports distributed mutex`: http://bit.ly/p99ybE
 
     *Usage*
 

kombu/connection.py

@@ -19,7 +19,7 @@ from urlparse import urlparse
 try:
     from urlparse import parse_qsl
 except ImportError:
-    from cgi import parse_qsl
+    from cgi import parse_qsl  # noqa
 
 from kombu import exceptions
 from kombu.transport import get_transport_cls

kombu/mixins.py

@@ -12,7 +12,6 @@ from __future__ import absolute_import
 from __future__ import with_statement
 
 import socket
-import sys
 
 from contextlib import nested, contextmanager
 from functools import partial

kombu/utils/compat.py

@@ -280,9 +280,10 @@ class LifoQueue(Queue):
 try:
     from collections import defaultdict
 except ImportError:
+
     # Written by Jason Kirtland, taken from Python Cookbook:
     # <http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/523034>
-    class defaultdict(dict):
+    class defaultdict(dict):  # noqa
 
         def __init__(self, default_factory=None, *args, **kwargs):
             dict.__init__(self, *args, **kwargs)
@@ -321,4 +322,3 @@ except ImportError:
                                             dict.__repr__(self))
     import collections
     collections.defaultdict = defaultdict               # Pickle needs this.
-

kombu/utils/encoding.py

@@ -13,6 +13,7 @@ import traceback
 
 
 if sys.version_info >= (3, 0):
+
     def str_to_bytes(s):
         if isinstance(s, str):
             return s.encode()
@@ -23,17 +24,23 @@ if sys.version_info >= (3, 0):
             return s.decode()
         return s
 
-else:  # noqa
-    def str_to_bytes(s):
+else:
+
+    def str_to_bytes(s):  # noqa
         return s
 
-    def bytes_to_str(s):
+    def bytes_to_str(s):  # noqa
         return s
 
 
-def default_encoding():
-    if sys.platform.startswith("java"):
+if sys.platform.startswith("java"):
+
+    def default_encoding():
         return "utf-8"
+
+else:
+
+    def default_encoding():  # noqa
         return sys.getfilesystemencoding()
 
 

pavement.py

@@ -67,11 +67,6 @@ def verifyindex(options):
     sh("contrib/release/verify-reference-index.sh")
 
 
-@task
-def flakes(options):
-    sh("find kombu funtests examples -name '*.py' | xargs pyflakes")
-
-
 @task
 def clean_readme(options):
     path("README").unlink()
@@ -122,6 +117,25 @@ def flake8(options):
         '""" % (complexity, ), ignore_error=noerror)
 
 
+@task
+@cmdopts([
+    ("noerror", "E", "Ignore errors"),
+])
+def flakeplus(options):
+    noerror = getattr(options, "noerror", False)
+    sh("python contrib/release/flakeplus.py kombu",
+       ignore_error=noerror)
+
+
+@task
+@cmdopts([
+    ("noerror", "E", "Ignore errors"),
+])
+def flakes(options):
+    flake8(options)
+    flakeplus(options)
+
+
 @task
 @cmdopts([
     ("noerror", "E", "Ignore errors"),
@@ -150,7 +164,7 @@ def gitcleanforce(options):
 
 
 @task
-@needs("flake8", "autodoc", "verifyindex", "test", "gitclean")
+@needs("flakes", "autodoc", "verifyindex", "test", "gitclean")
 def releaseok(options):
     pass