Rooba
/
kombu
mirror of https://github.com/celery/kombu.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity
kombu/t/unit/test_serialization.py

318 lines
11 KiB
Python
Raw Blame History

#!/usr/bin/python
from __future__ import annotations
from base64 import b64decode
from unittest.mock import call, patch
import pytest
import t.skip
from kombu.exceptions import ContentDisallowed, DecodeError, EncodeError
from kombu.serialization import (SerializerNotInstalled,
disable_insecure_serializers, dumps,
enable_insecure_serializers, loads, pickle,
pickle_protocol, prepare_accept_content,
raw_encode, register, register_msgpack,
register_pickle, register_yaml, registry,
unregister)
from kombu.utils.encoding import str_to_bytes
# For content_encoding tests
unicode_string = 'abcdé\u8463'
unicode_string_as_utf8 = unicode_string.encode('utf-8')
latin_string = 'abcdé'
latin_string_as_latin1 = latin_string.encode('latin-1')
latin_string_as_utf8 = latin_string.encode('utf-8')
# For serialization tests
py_data = {
'string': 'The quick brown fox jumps over the lazy dog',
'int': 10,
'float': 3.14159265,
'unicode': 'Thé quick brown fox jumps over thé lazy dog',
'list': ['george', 'jerry', 'elaine', 'cosmo'],
}
# JSON serialization tests
json_data = """\
{"int": 10, "float": 3.1415926500000002, \
"list": ["george", "jerry", "elaine", "cosmo"], \
"string": "The quick brown fox jumps over the lazy \
dog", "unicode": "Th\\u00e9 quick brown fox jumps over \
th\\u00e9 lazy dog"}\
"""
# Pickle serialization tests
pickle_data = pickle.dumps(py_data, protocol=pickle_protocol)
# YAML serialization tests
yaml_data = """\
float: 3.1415926500000002
int: 10
list: [george, jerry, elaine, cosmo]
string: The quick brown fox jumps over the lazy dog
unicode: "Th\\xE9 quick brown fox jumps over th\\xE9 lazy dog"
"""
msgpack_py_data = dict(py_data)
msgpack_py_data['unicode'] = 'Th quick brown fox jumps over th lazy dog'
# Unicode chars are lost in transmit :(
msgpack_data = b64decode(str_to_bytes("""\
haNpbnQKpWZsb2F0y0AJIftTyNTxpGxpc3SUpmdlb3JnZaVqZXJyeaZlbGFpbmWlY29zbW+mc3Rya\
W5n2gArVGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZ6d1bmljb2Rl2g\
ApVGggcXVpY2sgYnJvd24gZm94IGp1bXBzIG92ZXIgdGggbGF6eSBkb2c=\
"""))
registry.register('testS', lambda s: s, lambda s: 'decoded',
'application/testS', 'utf-8')
class test_Serialization:
def test_disable(self):
disabled = registry._disabled_content_types
try:
registry.disable('testS')
assert 'application/testS' in disabled
disabled.clear()
registry.disable('application/testS')
assert 'application/testS' in disabled
finally:
disabled.clear()
def test_enable(self):
registry._disabled_content_types.add('application/json')
registry.enable('json')
assert 'application/json' not in registry._disabled_content_types
registry._disabled_content_types.add('application/json')
registry.enable('application/json')
assert 'application/json' not in registry._disabled_content_types
def test_loads_when_disabled(self):
disabled = registry._disabled_content_types
try:
registry.disable('testS')
with pytest.raises(SerializerNotInstalled):
loads('xxd', 'application/testS', 'utf-8', force=False)
ret = loads('xxd', 'application/testS', 'utf-8', force=True)
assert ret == 'decoded'
finally:
disabled.clear()
def test_loads_when_data_is_None(self):
loads(None, 'application/testS', 'utf-8')
def test_content_type_decoding(self):
assert loads(
unicode_string_as_utf8,
content_type='plain/text',
content_encoding='utf-8') == unicode_string
assert loads(
latin_string_as_latin1,
content_type='application/data',
content_encoding='latin-1') == latin_string
def test_content_type_binary(self):
assert isinstance(
loads(unicode_string_as_utf8,
content_type='application/data', content_encoding='binary'),
bytes)
assert loads(
unicode_string_as_utf8,
content_type='application/data',
content_encoding='binary') == unicode_string_as_utf8
def test_content_type_encoding(self):
# Using the 'raw' serializer
assert (dumps(unicode_string, serializer='raw')[-1] ==
unicode_string_as_utf8)
assert (dumps(latin_string, serializer='raw')[-1] ==
latin_string_as_utf8)
# And again w/o a specific serializer to check the
# code where we force unicode objects into a string.
assert dumps(unicode_string)[-1] == unicode_string_as_utf8
assert dumps(latin_string)[-1] == latin_string_as_utf8
def test_enable_insecure_serializers(self):
with patch('kombu.serialization.registry') as registry:
enable_insecure_serializers()
registry.assert_has_calls([
call.enable('pickle'), call.enable('yaml'),
call.enable('msgpack'),
])
registry.enable.side_effect = KeyError()
enable_insecure_serializers()
with patch('kombu.serialization.registry') as registry:
enable_insecure_serializers(['msgpack'])
registry.assert_has_calls([call.enable('msgpack')])
def test_disable_insecure_serializers(self):
with patch('kombu.serialization.registry') as registry:
registry._decoders = ['pickle', 'yaml', 'doomsday']
disable_insecure_serializers(allowed=['doomsday'])
registry.disable.assert_has_calls([call('pickle'), call('yaml')])
registry.enable.assert_has_calls([call('doomsday')])
disable_insecure_serializers(allowed=None)
registry.disable.assert_has_calls([
call('pickle'), call('yaml'), call('doomsday')
])
def test_reraises_EncodeError(self):
with pytest.raises(EncodeError):
dumps([object()], serializer='json')
def test_reraises_DecodeError(self):
with pytest.raises(DecodeError):
loads(object(), content_type='application/json',
content_encoding='utf-8')
def test_json_loads(self):
assert loads(json_data,
content_type='application/json',
content_encoding='utf-8') == py_data
def test_json_dumps(self):
a = loads(
dumps(py_data, serializer='json')[-1],
content_type='application/json',
content_encoding='utf-8',
)
b = loads(
json_data,
content_type='application/json',
content_encoding='utf-8',
)
assert a == b
@t.skip.if_pypy
def test_msgpack_loads(self):
register_msgpack()
pytest.importorskip('msgpack')
res = loads(msgpack_data,
content_type='application/x-msgpack',
content_encoding='binary')
assert res == msgpack_py_data
@t.skip.if_pypy
def test_msgpack_dumps(self):
pytest.importorskip('msgpack')
register_msgpack()
a = loads(
dumps(msgpack_py_data, serializer='msgpack')[-1],
content_type='application/x-msgpack',
content_encoding='binary',
)
b = loads(
msgpack_data,
content_type='application/x-msgpack',
content_encoding='binary',
)
assert a == b
def test_yaml_loads(self):
pytest.importorskip('yaml')
register_yaml()
assert loads(
yaml_data,
content_type='application/x-yaml',
content_encoding='utf-8') == py_data
def test_yaml_dumps(self):
pytest.importorskip('yaml')
register_yaml()
a = loads(
dumps(py_data, serializer='yaml')[-1],
content_type='application/x-yaml',
content_encoding='utf-8',
)
b = loads(
yaml_data,
content_type='application/x-yaml',
content_encoding='utf-8',
)
assert a == b
def test_pickle_loads(self):
assert loads(
pickle_data,
content_type='application/x-python-serialize',
content_encoding='binary') == py_data
def test_pickle_dumps(self):
a = pickle.loads(pickle_data),
b = pickle.loads(dumps(py_data, serializer='pickle')[-1]),
assert a == b
def test_register(self):
register(None, None, None, None)
def test_unregister(self):
with pytest.raises(SerializerNotInstalled):
unregister('nonexisting')
dumps('foo', serializer='pickle')
unregister('pickle')
with pytest.raises(SerializerNotInstalled):
dumps('foo', serializer='pickle')
register_pickle()
def test_set_default_serializer_missing(self):
with pytest.raises(SerializerNotInstalled):
registry._set_default_serializer('nonexisting')
def test_dumps_missing(self):
with pytest.raises(SerializerNotInstalled):
dumps('foo', serializer='nonexisting')
def test_dumps__no_serializer(self):
ctyp, cenc, data = dumps(str_to_bytes('foo'))
assert ctyp == 'application/data'
assert cenc == 'binary'
def test_loads__trusted_content(self):
loads('tainted', 'application/data', 'binary', accept=[])
loads('tainted', 'application/text', 'utf-8', accept=[])
def test_loads__not_accepted(self):
with pytest.raises(ContentDisallowed):
loads('tainted', 'application/x-evil', 'binary', accept=[])
with pytest.raises(ContentDisallowed):
loads('tainted', 'application/x-evil', 'binary',
accept=['application/x-json'])
assert loads('tainted', 'application/x-doomsday', 'binary',
accept=['application/x-doomsday'])
def test_raw_encode(self):
assert raw_encode(b'foo') == (
'application/data', 'binary', b'foo',
)
@pytest.mark.masked_modules('yaml')
def test_register_yaml__no_yaml(self, mask_modules):
register_yaml()
with pytest.raises(SerializerNotInstalled):
loads('foo', 'application/x-yaml', 'utf-8')
@pytest.mark.masked_modules('msgpack')
def test_register_msgpack__no_msgpack(self, mask_modules):
register_msgpack()
with pytest.raises(SerializerNotInstalled):
loads('foo', 'application/x-msgpack', 'utf-8')
def test_prepare_accept_content(self):
assert {'application/json'} == prepare_accept_content(['json'])
assert {'application/json'} == prepare_accept_content(
['application/json'])
def test_prepare_accept_content_bad_serializer(self):
with pytest.raises(SerializerNotInstalled):
prepare_accept_content(['bad_serializer'])
Reference in New Issue View Git Blame Copy Permalink