Reformat and improve kombu transport reference docs (#1264)

docs/reference/index.rst

@@ -52,7 +52,6 @@ Kombu Transports
     kombu.transport.zookeeper
     kombu.transport.filesystem
     kombu.transport.sqlalchemy
-    kombu.transport.sqlalchemy.models
     kombu.transport.SQS
     kombu.transport.SLMQ
     kombu.transport.pyro

docs/reference/kombu.transport.sqlalchemy.models.rst

@@ -1,32 +0,0 @@
-=====================================================================
- SQLAlchemy Transport Model - ``kombu.transport.sqlalchemy.models``
-=====================================================================
-
-
-.. currentmodule:: kombu.transport.sqlalchemy.models
-
-.. automodule:: kombu.transport.sqlalchemy.models
-
-    .. contents::
-        :local:
-
-    Models
-    ------
-
-    .. autoclass:: Queue
-
-        .. autoattribute:: Queue.id
-
-        .. autoattribute:: Queue.name
-
-    .. autoclass:: Message
-
-        .. autoattribute:: Message.id
-
-        .. autoattribute:: Message.visible
-
-        .. autoattribute:: Message.sent_at
-
-        .. autoattribute:: Message.payload
-
-        .. autoattribute:: Message.version

docs/reference/kombu.transport.sqlalchemy.rst

@@ -1,6 +1,6 @@
-===========================================================
- SQLAlchemy Transport Model - kombu.transport.sqlalchemy
-===========================================================
+=============================================================
+ SQLAlchemy Transport Model - ``kombu.transport.sqlalchemy``
+=============================================================
 
 
 .. currentmodule:: kombu.transport.sqlalchemy
@@ -23,3 +23,36 @@
     .. autoclass:: Channel
         :members:
         :undoc-members:
+
+=====================================================================
+ SQLAlchemy Transport Model - ``kombu.transport.sqlalchemy.models``
+=====================================================================
+
+
+.. currentmodule:: kombu.transport.sqlalchemy.models
+
+.. automodule:: kombu.transport.sqlalchemy.models
+
+    .. contents::
+        :local:
+
+    Models
+    ------
+
+    .. autoclass:: Queue
+
+        .. autoattribute:: Queue.id
+
+        .. autoattribute:: Queue.name
+
+    .. autoclass:: Message
+
+        .. autoattribute:: Message.id
+
+        .. autoattribute:: Message.visible
+
+        .. autoattribute:: Message.sent_at
+
+        .. autoattribute:: Message.payload
+
+        .. autoattribute:: Message.version

kombu/transport/SLMQ.py

@@ -1,4 +1,22 @@
-"""SoftLayer Message Queue transport."""
+"""SoftLayer Message Queue transport module for kombu.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: No
+* Supports TTL: No
+
+Connection String
+=================
+ *Unreviewed*
+
+Transport Options
+=================
+ *Unreviewed*
+"""
 
 import socket
 import string

kombu/transport/SQS.py

@@ -81,6 +81,15 @@ as follows:
 
 For a complete list of settings you can adjust using this option see
 https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: No
+* Supports TTL: No
 """  # noqa: E501
 
 

kombu/transport/azureservicebus.py

@@ -1,10 +1,4 @@
-"""Azure Service Bus Message Queue transport.
-
-The transport can be enabled by setting the CELERY_BROKER_URL to:
-
-```
-azureservicebus://{SAS policy name}:{SAS key}@{Service Bus Namespace}
-```
+"""Azure Service Bus Message Queue transport module for kombu.
 
 Note that the Shared Access Policy used to connect to Azure Service Bus
 requires Manage, Send and Listen claims since the broker will create new
@@ -16,6 +10,31 @@ have to be regenerated before it can be used in the connection URL.
 More information about Azure Service Bus:
 https://azure.microsoft.com/en-us/services/service-bus/
 
+Features
+========
+* Type: Virtual
+* Supports Direct: *Unreviewed*
+* Supports Topic: *Unreviewed*
+* Supports Fanout: *Unreviewed*
+* Supports Priority: *Unreviewed*
+* Supports TTL: *Unreviewed*
+
+Connection String
+=================
+
+Connection string has the following format:
+
+.. code-block::
+
+    azureservicebus://SAS_POLICY_NAME:SAS_KEY@SERVICE_BUSNAMESPACE
+
+Transport Options
+=================
+
+* ``visibility_timeout``
+* ``queue_name_prefix``
+* ``wait_time_seconds``
+* ``peek_lock``
 """
 
 import string

kombu/transport/azurestoragequeues.py

@@ -1,17 +1,33 @@
-"""Azure Storage Queues transport.
-
-The transport can be enabled by setting the CELERY_BROKER_URL to:
-
-```
-azurestoragequeues://:{Storage Account Access Key}@{Storage Account Name}
-```
-
-Note that if the access key for the storage account contains a slash, it will
-have to be regenerated before it can be used in the connection URL.
+"""Azure Storage Queues transport module for kombu.
 
 More information about Azure Storage Queues:
 https://azure.microsoft.com/en-us/services/storage/queues/
 
+Features
+========
+* Type: Virtual
+* Supports Direct: *Unreviewed*
+* Supports Topic: *Unreviewed*
+* Supports Fanout: *Unreviewed*
+* Supports Priority: *Unreviewed*
+* Supports TTL: *Unreviewed*
+
+Connection String
+=================
+
+Connection string has the following format:
+
+.. code-block::
+
+    azurestoragequeues://:STORAGE_ACCOUNT_ACCESS kEY@STORAGE_ACCOUNT_NAME
+
+Note that if the access key for the storage account contains a slash, it will
+have to be regenerated before it can be used in the connection URL.
+
+Transport Options
+=================
+
+* ``queue_name_prefix``
 """
 
 from queue import Empty

kombu/transport/consul.py

@@ -1,8 +1,30 @@
-"""Consul Transport.
+"""Consul Transport module for Kombu.
+
+Features
+========
 
 It uses Consul.io's Key/Value store to transport messages in Queues
 
 It uses python-consul for talking to Consul's HTTP API
+
+Features
+========
+* Type: Native
+* Supports Direct: Yes
+* Supports Topic: *Unreviewed*
+* Supports Fanout: *Unreviewed*
+* Supports Priority: *Unreviewed*
+* Supports TTL: *Unreviewed*
+
+Connection String
+=================
+
+Connection string has the following format:
+
+.. code-block::
+
+    consul://CONSUL_ADDRESS[:PORT]
+
 """
 
 import uuid

kombu/transport/etcd.py

@@ -1,8 +1,27 @@
-"""Etcd Transport.
+"""Etcd Transport module for Kombu.
 
 It uses Etcd as a store to transport messages in Queues
 
 It uses python-etcd for talking to Etcd's HTTP API
+
+Features
+========
+* Type: Virtual
+* Supports Direct: *Unreviewed*
+* Supports Topic: *Unreviewed*
+* Supports Fanout: *Unreviewed*
+* Supports Priority: *Unreviewed*
+* Supports TTL: *Unreviewed*
+
+Connection String
+=================
+
+Connection string has the following format:
+
+.. code-block::
+
+    'etcd'://SERVER:PORT
+
 """
 
 import os

kombu/transport/filesystem.py

@@ -1,6 +1,91 @@
-"""File-system Transport.
+"""File-system Transport module for kombu.
 
-Transport using the file-system as the message store.
+Transport using the file-system as the message store. Messages written to the
+queue are stored in `data_folder_in` directory and
+messages read from the queue are read from `data_folder_out` directory. Both
+directories must be created manually. Simple example:
+
+* Producer:
+
+.. code-block:: python
+
+    import kombu
+
+    conn = kombu.Connection(
+        'filesystem://', transport_options={
+            'data_folder_in': 'data_in', 'data_folder_out': 'data_out'
+        }
+    )
+    conn.connect()
+
+    test_queue = kombu.Queue('test', routing_key='test')
+
+    with conn as conn:
+        with conn.default_channel as channel:
+            producer = kombu.Producer(channel)
+            producer.publish(
+                        {'hello': 'world'},
+                        retry=True,
+                        exchange=test_queue.exchange,
+                        routing_key=test_queue.routing_key,
+                        declare=[test_queue],
+                        serializer='pickle'
+            )
+
+* Consumer:
+
+.. code-block:: python
+
+    import kombu
+
+    conn = kombu.Connection(
+        'filesystem://', transport_options={
+            'data_folder_in': 'data_out', 'data_folder_out': 'data_in'
+        }
+    )
+    conn.connect()
+
+    def callback(body, message):
+        print(body, message)
+        message.ack()
+
+    test_queue = kombu.Queue('test', routing_key='test')
+
+    with conn as conn:
+        with conn.default_channel as channel:
+            consumer = kombu.Consumer(
+                conn, [test_queue], accept=['pickle']
+            )
+            consumer.register_callback(callback)
+            with consumer:
+                conn.drain_events(timeout=1)
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: No
+* Supports TTL: No
+
+Connection String
+=================
+Connection string is in the following format:
+
+.. code-block::
+
+    filesystem://
+
+Transport Options
+=================
+* ``data_folder_in`` - directory where are messages stored when written
+  to queue.
+* ``data_folder_out`` - directory from which are messages read when read from
+  queue.
+* ``store_processed`` - if set to True, all processed messages are backed up to
+  ``processed_folder``.
+* ``processed_folder`` - directory where are backed up processed files.
 """
 
 import os

kombu/transport/memory.py

@@ -1,4 +1,26 @@
-"""In-memory transport."""
+"""In-memory transport module for Kombu.
+
+Simple transport using memory for storing messages.
+Messages can be passed only between threads.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: No
+* Supports TTL: Yes
+
+Connection String
+=================
+Connection string is in the following format:
+
+.. code-block::
+
+    memory://
+
+"""
 
 from queue import Queue
 

kombu/transport/mongodb.py

@@ -1,7 +1,36 @@
-"""MongoDB transport.
+# copyright: (c) 2010 - 2013 by Flavio Percoco Premoli.
+# license: BSD, see LICENSE for more details.
 
-:copyright: (c) 2010 - 2013 by Flavio Percoco Premoli.
-:license: BSD, see LICENSE for more details.
+"""MongoDB transport module for kombu.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: Yes
+* Supports TTL: Yes
+
+Connection String
+=================
+ *Unreviewed*
+
+Transport Options
+=================
+
+* ``connect_timeout``,
+* ``ssl``,
+* ``ttl``,
+* ``capped_queue_size``,
+* ``default_hostname``,
+* ``default_port``,
+* ``default_database``,
+* ``messages_collection``,
+* ``routing_collection``,
+* ``broadcast_collection``,
+* ``queues_collection``,
+* ``calc_queue_size``,
 """
 
 import datetime

kombu/transport/pyamqp.py

@@ -1,4 +1,38 @@
-"""Pure-Python amqp transport."""
+"""pyamqp transport module for Kombu.
+
+Pure-Python amqp transport using py-amqp library.
+
+Features
+========
+* Type: Native
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: Yes
+* Supports TTL: Yes
+
+Connection String
+=================
+Connection string can have the following formats:
+
+.. code-block::
+
+    amqp://[USER:PASSWORD@]BROKER_ADDRESS[:PORT][/VIRTUALHOST]
+    [USER:PASSWORD@]BROKER_ADDRESS[:PORT][/VIRTUALHOST]
+    amqp://
+
+For TLS encryption use:
+
+.. code-block::
+
+    amqps://[USER:PASSWORD@]BROKER_ADDRESS[:PORT][/VIRTUALHOST]
+
+Transport Options
+=================
+Transport Options are passed to constructor of underlying py-amqp
+:class:`~kombu.connection.Connection` class.
+"""
+
 
 import amqp
 

kombu/transport/pyro.py

@@ -1,14 +1,34 @@
-"""Pyro transport, and Kombu Broker daemon.
+"""Pyro transport module for kombu.
+
+Pyro transport, and Kombu Broker daemon.
 
 Requires the :mod:`Pyro4` library to be installed.
 
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: No
+* Supports TTL: No
+
+Connection String
+=================
+
 To use the Pyro transport with Kombu, use an url of the form:
-``pyro://localhost/kombu.broker``
+
+.. code-block::
+
+    pyro://localhost/kombu.broker
 
 The hostname is where the transport will be looking for a Pyro name server,
 which is used in turn to locate the kombu.broker Pyro service.
 This broker can be launched by simply executing this transport module directly,
 with the command: ``python -m kombu.transport.pyro``
+
+Transport Options
+=================
 """
 
 

kombu/transport/qpid.py

@@ -1,4 +1,4 @@
-"""Qpid Transport.
+"""Qpid Transport module for kombu.
 
 `Qpid`_ transport using `qpid-python`_ as the client and `qpid-tools`_ for
 broker management.
@@ -27,6 +27,15 @@ or to install the requirements manually:
 .. _`qpid-python`: https://pypi.org/project/qpid-python/
 .. _`qpid-tools`: https://pypi.org/project/qpid-tools/
 
+Features
+========
+* Type: Native
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: Yes
+* Supports TTL: Yes
+
 Authentication
 ==============
 

kombu/transport/redis.py

@@ -1,4 +1,53 @@
-"""Redis transport."""
+"""Redis transport module for Kombu.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: Yes
+* Supports Priority: Yes
+* Supports TTL: No
+
+Connection String
+=================
+Connection string has the following format:
+
+.. code-block::
+
+    redis://REDIS_ADDRESS[:PORT][/VIRTUALHOST]
+    rediss://REDIS_ADDRESS[:PORT][/VIRTUALHOST]
+
+To use sentinel for dynamic Redis discovery,
+the connection string has following format:
+
+.. code-block::
+
+    sentinel://SENTINEL_ADDRESS[:PORT]
+
+Transport Options
+=================
+* ``sep``
+* ``ack_emulation``: (bool) If set to True transport will
+  simulate Acknowledge of AMQP protocol.
+* ``unacked_key``
+* ``unacked_index_key``
+* ``unacked_mutex_key``
+* ``unacked_mutex_expire``
+* ``visibility_timeout``
+* ``unacked_restore_limit``
+* ``fanout_prefix``
+* ``fanout_patterns``
+* ``socket_timeout``
+* ``socket_connect_timeout``
+* ``socket_keepalive``
+* ``socket_keepalive_options``
+* ``queue_order_strategy``
+* ``max_connections``
+* ``health_check_interval``
+* ``retry_on_timeout``
+* ``priority_steps``
+"""
 
 import numbers
 import socket

kombu/transport/sqlalchemy/__init__.py

@@ -1,4 +1,55 @@
-"""Kombu transport using SQLAlchemy as the message store."""
+"""SQLAlchemy Transport module for kombu.
+
+Kombu transport using SQL Database as the message store.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: yes
+* Supports Topic: yes
+* Supports Fanout: no
+* Supports Priority: no
+* Supports TTL: no
+
+Connection String
+=================
+
+.. code-block::
+
+    sqla+SQL_ALCHEMY_CONNECTION_STRING
+    sqlalchemy+SQL_ALCHEMY_CONNECTION_STRING
+
+For details about ``SQL_ALCHEMY_CONNECTION_STRING`` see SQLAlchemy Engine Configuration documentation.
+Examples:
+
+.. code-block::
+
+    # PostgreSQL with default driver
+    sqla+postgresql://scott:tiger@localhost/mydatabase
+
+    # PostgreSQL with psycopg2 driver
+    sqla+postgresql+psycopg2://scott:tiger@localhost/mydatabase
+
+    # PostgreSQL with pg8000 driver
+    sqla+postgresql+pg8000://scott:tiger@localhost/mydatabase
+
+    # MySQL with default driver
+    sqla+mysql://scott:tiger@localhost/foo
+
+    # MySQL with mysqlclient driver (a maintained fork of MySQL-Python)
+    sqla+mysql+mysqldb://scott:tiger@localhost/foo
+
+    # MySQL with PyMySQL driver
+    sqla+mysql+pymysql://scott:tiger@localhost/foo
+
+Transport Options
+=================
+
+* ``queue_tablename``: Name of table storing queues.
+* ``message_tablename``: Name of table storing messages.
+
+Moreover parameters of :func:`sqlalchemy.create_engine()` function can be passed as transport options.
+"""
 # SQLAlchemy overrides != False to have special meaning and pep8 complains
 # flake8: noqa
 

kombu/transport/zookeeper.py

@@ -1,25 +1,45 @@
-"""Zookeeper transport.
+# copyright: (c) 2010 - 2013 by Mahendra M.
+# license: BSD, see LICENSE for more details.
 
-:copyright: (c) 2010 - 2013 by Mahendra M.
-:license: BSD, see LICENSE for more details.
+"""Zookeeper transport module for kombu.
 
-**Synopsis**
-
-Connects to a zookeeper node as <server>:<port>/<vhost>
-The <vhost> becomes the base for all the other znodes.  So we can use
-it like a vhost.
-
-This uses the built-in kazoo recipe for queues
+Zookeeper based transport. This transport uses the built-in kazoo Zookeeper
+based queue implementation.
 
 **References**
 
-- https://zookeeper.apache.org/doc/trunk/recipes.html#sc_recipes_Queues
+- https://zookeeper.apache.org/doc/current/recipes.html#sc_recipes_Queues
 - https://kazoo.readthedocs.io/en/latest/api/recipe/queue.html
 
 **Limitations**
 This queue does not offer reliable consumption.  An entry is removed from
 the queue prior to being processed.  So if an error occurs, the consumer
 has to re-queue the item or it will be lost.
+
+Features
+========
+* Type: Virtual
+* Supports Direct: Yes
+* Supports Topic: Yes
+* Supports Fanout: No
+* Supports Priority: Yes
+* Supports TTL: No
+
+Connection String
+=================
+Connects to a zookeeper node as:
+
+.. code-block::
+
+    zookeeper://SERVER:PORT/VHOST
+
+The <vhost> becomes the base for all the other znodes.  So we can use
+it like a vhost.
+
+
+Transport Options
+=================
+
 """
 
 import os