From 493da6120543b8f13de86a78cc818a94fce76921 Mon Sep 17 00:00:00 2001 From: Ask Solem Date: Fri, 4 Oct 2013 16:26:17 +0100 Subject: [PATCH] Improve docs about pickle serailization and disabled by defaualt --- Changelog | 52 ++++++++++++++++++++++++++++++-- docs/userguide/consumers.rst | 24 +++++++++++---- docs/userguide/serialization.rst | 50 ++++++++++++++---------------- 3 files changed, 90 insertions(+), 36 deletions(-) diff --git a/Changelog b/Changelog index 01687463..d3b1457c 100644 --- a/Changelog +++ b/Changelog @@ -18,11 +18,57 @@ - pickle, yaml and msgpack deserialization is now disabled by default. - To enable insecure serializers you have to call: + This means that Kombu will by default refuse to handle any content type other + than json. + + Pickle is known to be a security concern as it will happily + load any object that is embedded in a pickle payload, and payloads + can be crafted to do almost anything you want. The default + serializer in Kombu is json but it also supports a number + of other serialization formats that it will evaluate if received: + including pickle. + + It was always assumed that users were educated about the security + implications of pickle, but in hindsight we don't think users + should be expected to secure their services if we have the ability to + be secure by default. + + By disabling any content type that the user did not explicitly + want enabled we ensure that the user must be conscious when they + add pickle as a serialization format to support. + + The other built-in serializers (yaml and msgpack) are also disabled + even though they aren't considered insecure [#f1]_ at this point. + Instead they're disabled so that if a security flaw is found in one of these + libraries in the future, you will only be affected if you have + explicitly enabled them. + + To have your consumer accept formats other than json you have to + explicitly add the wanted formats to a white-list of accepted + content types:: + + >>> c = Consumer(conn, accept=['json', 'pickle', 'msgpack']) + + or when using synchronous access:: + + >>> msg = queue.get(accept=['json', 'pickle', 'msgpack']) + + The ``accept`` argument was first supported for consumers in version + 2.5.10, and first supported by ``Queue.get`` in version 2.5.15 + so to stay compatible with previous versions you can enable + the previous behavior: >>> from kombu import enable_insecure_serializers >>> enable_insecure_serializers() + But note that this has global effect, so be very careful should you use it. + + .. rubric:: Footnotes + + .. [#f1] The PyYAML library has a :func:`yaml.load` function with some of the + same security implications as pickle, but Kombu uses the + :func:`yaml.safe_load` function which is not known to be affected. + - kombu.async: Experimental event loop implementation. This code was previously in Celery but was moved here @@ -38,7 +84,7 @@ Contributed by Mark Lavin. - ``StdConnectionError`` and ``StdChannelError`` is removed - and ``:exc:`amqp.ConnectionError` and :exc:`amqp.ChannelError` is used + and :exc:`amqp.ConnectionError` and :exc:`amqp.ChannelError` is used instead. - Message object implementation has moved to :class:`kombu.message.Message`. @@ -92,7 +138,7 @@ Contributed by Kevin McDonald -.. _`SoftLayer MQ`_: http://www.softlayer.com/services/additional/message-queue +.. _`SoftLayer MQ`: http://www.softlayer.com/services/additional/message-queue - Eventio: Kqueue breaks in subtle ways so select is now used instead. diff --git a/docs/userguide/consumers.rst b/docs/userguide/consumers.rst index 5e9d3d61..76c53638 100644 --- a/docs/userguide/consumers.rst +++ b/docs/userguide/consumers.rst @@ -14,12 +14,20 @@ consume from. Several consumers can be mixed to consume from different channels, as they all bind to the same connection, and ``drain_events`` will drain events from all channels on that connection. +.. note:: + + Kombu since 3.0 will only accept json/binary or text messages by default, + to allow deserialization of other formats you have to specify them + in the ``accept`` argument:: + + Consumer(conn, accept=['json', 'pickle', 'msgpack', 'yaml']) + Draining events from a single consumer: .. code-block:: python - with Consumer(connection, queues): + with Consumer(connection, queues, accept=['json']): connection.drain_events(timeout=1) @@ -30,8 +38,8 @@ Draining events from several consumers: from kombu.utils import nested with connection.channel(), connection.channel() as (channel1, channel2): - consumers = [Consumer(channel1, queues1), - Consumer(channel2, queues2)] + consumers = [Consumer(channel1, queues1, accept=['json']), + Consumer(channel2, queues2, accept=['json'])] with nested(\*consumers): connection.drain_events(timeout=1) @@ -48,7 +56,9 @@ Or using :class:`~kombu.mixins.ConsumerMixin`: self.connection = connection def get_consumers(self, Consumer, channel): - return [Consumer(queues, callbacks=[self.on_message])] + return [ + Consumer(queues, callbacks=[self.on_message], accept=['json']), + ] def on_message(self, body, message): print("RECEIVED MESSAGE: %r" % (body, )) @@ -73,9 +83,11 @@ and with multiple channels again: def get_consumers(self, _, default_channel): self.channel2 = default_channel.connection.channel() return [Consumer(default_channel, queues1, - callbacks=[self.on_message]), + callbacks=[self.on_message], + accept=['json']), Consumer(self.channel2, queues2, - callbacks=[self.on_special_message])] + callbacks=[self.on_special_message], + accept=['json'])] def on_consumer_end(self, connection, default_channel): if self.channel2: diff --git a/docs/userguide/serialization.rst b/docs/userguide/serialization.rst index 7a8ec0e4..479d7c7d 100644 --- a/docs/userguide/serialization.rst +++ b/docs/userguide/serialization.rst @@ -15,6 +15,17 @@ Python data structures like dictionaries and lists works. and if needed you can register any custom serialization scheme you want to use. + +By default Kombu will only load JSON messages, so if you want +to use other serialization format you must explicitly enable +them in your consumer by using the ``accept`` argument: + +.. code-block:: python + + Consumer(conn, [queue], accept=['json', 'pickle', 'msgpack']) + +The accept argument can also include MIME-types. + .. _`JSON`: http://www.json.org/ .. _`YAML`: http://yaml.org/ .. _`msgpack`: http://msgpack.sourceforge.net/ @@ -44,6 +55,18 @@ Each option has its advantages and disadvantages. smaller messages when sending binary files, and a slight speedup over `JSON` processing. + .. admonition:: Pickle and Security + + The pickle format is very convenient as it can serialize + and deserialize almost any object, but this is also a concern + for security. + + Carefully crafted pickle payloads can do almost anything + a regular Python program can do, so if you let your consumer + automatically decode pickled objects you must make sure + to limit access to the broker so that untrusted + parties do not have the ability to send messages! + By default Kombu uses pickle protocol 2, but this can be changed using the :envvar:`PICKLE_PROTOCOL` environment variable or by changing the global :data:`kombu.serialization.pickle_protocol` flag. @@ -77,33 +100,6 @@ Note that a `Consumer` do not need the serialization method specified. They can auto-detect the serialization method as the content-type is sent as a message header. -.. _disable-untrusted-serializers: - -Disabling Insecure Serializers ------------------------------- - -.. versionadded:: 2.5.10 - -Deserializing pickle and yaml from untrusted sources is not safe, -as both pickle and yaml have the ability to execute arbitrary code. - -If you are not using these formats you should disable them -by calling :func:`kombu.disable_insecure_serializers`:: - - >>> import kombu - >>> kombu.disable_insecure_serializers() - -Or you can specify the content types your consumers should -accept by using the ``accept`` argument:: - - >>> Consumer(accept=['json', 'pickle']) - >>> Consumer(accept=['application/json']) - -.. note:: - - Insecure serializers will be disabled by default - in the next major version (Kombu 3.0) - .. _sending-raw-data: Sending raw data without Serialization