Rooba
/
attrs
mirror of https://github.com/python-attrs/attrs.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add hooks for field transformation and for asdict serialization (#653)

This commit is contained in:
Stefan Scherfke 2020-10-15 09:33:59 +02:00 committed by GitHub
parent 0343927ecd
commit 0eae613ce1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 432 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
changelog.d/653.change.rst Normal file
View File

@ -0,0 +1,3 @@
``attr.s()`` now has a *field_transformer* hook that is called for all ``Attribute``\ s and returns a (modified or updated) list of ``Attribute`` instances.
``attr.asdict()`` has a *value_serializer* hook that can change the way values are converted.
Both hooks are meant to help with data (de-)serialization workflows.

1
conftest.py
View File

@ -18,6 +18,7 @@ if sys.version_info[:2] < (3, 6):
collect_ignore.extend(
[
"tests/test_annotations.py",
"tests/test_hooks.py",
"tests/test_init_subclass.py",
"tests/test_next_gen.py",
]

8
docs/api.rst
View File

@ -24,7 +24,7 @@ Core
.. autodata:: attr.NOTHING
.. autofunction:: attr.s(these=None, repr_ns=None, repr=None, cmp=None, hash=None, init=None, slots=False, frozen=False, weakref_slot=True, str=False, auto_attribs=False, kw_only=False, cache_hash=False, auto_exc=False, eq=None, order=None, auto_detect=False, collect_by_mro=False, getstate_setstate=None, on_setattr=None)
.. autofunction:: attr.s(these=None, repr_ns=None, repr=None, cmp=None, hash=None, init=None, slots=False, frozen=False, weakref_slot=True, str=False, auto_attribs=False, kw_only=False, cache_hash=False, auto_exc=False, eq=None, order=None, auto_detect=False, collect_by_mro=False, getstate_setstate=None, on_setattr=None, field_transformer=None)
.. note::
@ -93,11 +93,7 @@ Core
ValueError: x must be positive
.. autoclass:: attr.Attribute
Instances of this class are frequently used for introspection purposes like:
- `fields` returns a tuple of them.
- Validators get them passed as the first argument.
:members: assoc
.. warning::

113
docs/extending.rst
View File

@ -159,3 +159,116 @@ Here are some tips for effective use of metadata:
... x = typed(int, default=1, init=False)
>>> attr.fields(C).x.metadata[MY_TYPE_METADATA]
<class 'int'>
.. _transform-fields:
Automatic Field Transformation and Modification
-----------------------------------------------
Attrs allows you to automatically modify or transform the class' fields while the class is being created.
You do this by passing a *field_transformer* hook to `attr.define` (and its friends).
Its main purpose is to automatically add converters to attributes based on their type to aid the development of API clients and other typed data loaders.
This hook must have the following signature:
.. function:: your_hook(cls: type, fields: List[attr.Attribute]) -> List[attr.Attribute]
:noindex:
- *cls* is your class right *before* it is being converted into an attrs class.
This means it does not yet have the ``__attrs_attrs__`` attribute.
- *fields* is a list of all :class:`attr.Attribute` instances that will later be set to ``__attrs_attrs__``.
You can modify these attributes any way you want:
You can add converters, change types, and even remove attributes completely or create new ones!
For example, let's assume that you really don't like floats:
.. doctest::
>>> def drop_floats(cls, fields):
... return [f for f in fields if f.type not in {float, 'float'}]
...
>>> @attr.frozen(field_transformer=drop_floats)
... class Data:
... a: int
... b: float
... c: str
...
>>> Data(42, "spam")
Data(a=42, c='spam')
A more realistic example would be to automatically convert data that you, e.g., load from JSON:
.. doctest::
>>> from datetime import datetime
>>>
>>> def auto_convert(cls, fields):
... results = []
... for field in fields:
... if field.converter is not None:
... results.append(field)
... continue
... if field.type in {datetime, 'datetime'}:
... converter = (lambda d: datetime.fromisoformat(d) if isinstance(d, str) else d)
... else:
... converter = None
... results.append(field.assoc(converter=converter))
... return results
...
>>> @attr.frozen(field_transformer=auto_convert)
... class Data:
... a: int
... b: str
... c: datetime
...
>>> from_json = {"a": 3, "b": "spam", "c": "2020-05-04T13:37:00"}
>>> Data(**from_json) # ****
Data(a=3, b='spam', c=datetime.datetime(2020, 5, 4, 13, 37))
Customize Value Serialization in ``asdict()``
---------------------------------------------
``attrs`` allows you to serialize instances of ``attrs`` classes to dicts using the `attr.asdict` function.
However, the result can not always be serialized since most data types will remain as they are:
.. doctest::
>>> import json
>>> import datetime
>>>
>>> @attr.frozen
... class Data:
... dt: datetime.datetime
...
>>> data = attr.asdict(Data(datetime.datetime(2020, 5, 4, 13, 37)))
>>> data
{'dt': datetime.datetime(2020, 5, 4, 13, 37)}
>>> json.dumps(data)
Traceback (most recent call last):
...
TypeError: Object of type datetime is not JSON serializable
To help you with this, `attr.asdict` allows you to pass a *value_serializer* hook.
It has the signature
.. function:: your_hook(inst: type, field: attr.Attribute, value: typing.Any) -> typing.Any
:noindex:
.. doctest::
>>> def serialize(inst, field, value):
... if isinstance(value, datetime.datetime):
... return value.isoformat()
... return value
...
>>> data = attr.asdict(
... Data(datetime.datetime(2020, 5, 4, 13, 37)),
... value_serializer=serialize,
... )
>>> data
{'dt': '2020-05-04T13:37:00'}
>>> json.dumps(data)
'{"dt": "2020-05-04T13:37:00"}'

7
src/attr/__init__.pyi
View File

@ -46,6 +46,7 @@ _OnSetAttrType = Callable[[Any, Attribute[Any], Any], Any]
_OnSetAttrArgType = Union[
_OnSetAttrType, List[_OnSetAttrType], setters._NoOpType
]
_FieldTransformer = Callable[[type, List[Attribute]], List[Attribute]]
# FIXME: in reality, if multiple validators are passed they must be in a list
# or tuple, but those are invariant and so would prevent subtypes of
# _ValidatorType from working when passed in a list or tuple.
@ -274,6 +275,7 @@ def attrs(
auto_detect: bool = ...,
getstate_setstate: Optional[bool] = ...,
on_setattr: Optional[_OnSetAttrArgType] = ...,
field_transformer: Optional[_FieldTransformer] = ...,
) -> _C: ...
@overload
def attrs(
@ -297,6 +299,7 @@ def attrs(
auto_detect: bool = ...,
getstate_setstate: Optional[bool] = ...,
on_setattr: Optional[_OnSetAttrArgType] = ...,
field_transformer: Optional[_FieldTransformer] = ...,
) -> Callable[[_C], _C]: ...
@overload
def define(
@ -319,6 +322,7 @@ def define(
auto_detect: bool = ...,
getstate_setstate: Optional[bool] = ...,
on_setattr: Optional[_OnSetAttrArgType] = ...,
field_transformer: Optional[_FieldTransformer] = ...,
) -> _C: ...
@overload
def define(
@ -341,6 +345,7 @@ def define(
auto_detect: bool = ...,
getstate_setstate: Optional[bool] = ...,
on_setattr: Optional[_OnSetAttrArgType] = ...,
field_transformer: Optional[_FieldTransformer] = ...,
) -> Callable[[_C], _C]: ...
mutable = define
@ -382,6 +387,7 @@ def make_class(
eq: Optional[bool] = ...,
order: Optional[bool] = ...,
on_setattr: Optional[_OnSetAttrArgType] = ...,
field_transformer: Optional[_FieldTransformer] = ...,
) -> type: ...
# _funcs --
@ -397,6 +403,7 @@ def asdict(
filter: Optional[_FilterType[Any]] = ...,
dict_factory: Type[Mapping[Any, Any]] = ...,
retain_collection_types: bool = ...,
value_serializer: Optional[Callable[[type, Attribute, Any], Any]] = ...,
) -> Dict[str, Any]: ...
# TODO: add support for returning NamedTuple from the mypy plugin

66
src/attr/_funcs.py
View File

@ -13,6 +13,7 @@ def asdict(
filter=None,
dict_factory=dict,
retain_collection_types=False,
value_serializer=None,
):
"""
Return the ``attrs`` attribute values of *inst* as a dict.
@ -32,6 +33,10 @@ def asdict(
:param bool retain_collection_types: Do not convert to ``list`` when
encountering an attribute whose type is ``tuple`` or ``set``. Only
meaningful if ``recurse`` is ``True``.
:param Optional[callable] value_serializer: A hook that is called for every
attribute or dict key/value. It receives the current instance, field
and value and must return the (updated) value. The hook is run *after*
the optional *filter* has been applied.
:rtype: return type of *dict_factory*
@ -40,6 +45,7 @@ def asdict(
.. versionadded:: 16.0.0 *dict_factory*
.. versionadded:: 16.1.0 *retain_collection_types*
.. versionadded:: 20.3.0 *value_serializer*
"""
attrs = fields(inst.__class__)
rv = dict_factory()
@ -47,17 +53,28 @@ def asdict(
v = getattr(inst, a.name)
if filter is not None and not filter(a, v):
continue
if value_serializer is not None:
v = value_serializer(inst, a, v)
if recurse is True:
if has(v.__class__):
rv[a.name] = asdict(
v, True, filter, dict_factory, retain_collection_types
v,
True,
filter,
dict_factory,
retain_collection_types,
value_serializer,
)
elif isinstance(v, (tuple, list, set)):
cf = v.__class__ if retain_collection_types is True else list
rv[a.name] = cf(
[
_asdict_anything(
i, filter, dict_factory, retain_collection_types
i,
filter,
dict_factory,
retain_collection_types,
value_serializer,
)
for i in v
]
@ -67,10 +84,18 @@ def asdict(
rv[a.name] = df(
(
_asdict_anything(
kk, filter, df, retain_collection_types
kk,
filter,
df,
retain_collection_types,
value_serializer,
),
_asdict_anything(
vv, filter, df, retain_collection_types
vv,
filter,
df,
retain_collection_types,
value_serializer,
),
)
for kk, vv in iteritems(v)
@ -82,19 +107,36 @@ def asdict(
return rv
def _asdict_anything(val, filter, dict_factory, retain_collection_types):
def _asdict_anything(
val,
filter,
dict_factory,
retain_collection_types,
value_serializer,
):
"""
``asdict`` only works on attrs instances, this works on anything.
"""
if getattr(val.__class__, "__attrs_attrs__", None) is not None:
# Attrs class.
rv = asdict(val, True, filter, dict_factory, retain_collection_types)
rv = asdict(
val,
True,
filter,
dict_factory,
retain_collection_types,
value_serializer,
)
elif isinstance(val, (tuple, list, set)):
cf = val.__class__ if retain_collection_types is True else list
rv = cf(
[
_asdict_anything(
i, filter, dict_factory, retain_collection_types
i,
filter,
dict_factory,
retain_collection_types,
value_serializer,
)
for i in val
]
@ -103,13 +145,19 @@ def _asdict_anything(val, filter, dict_factory, retain_collection_types):
df = dict_factory
rv = df(
(
_asdict_anything(kk, filter, df, retain_collection_types),
_asdict_anything(vv, filter, df, retain_collection_types),
_asdict_anything(
kk, filter, df, retain_collection_types, value_serializer
),
_asdict_anything(
vv, filter, df, retain_collection_types, value_serializer
),
)
for kk, vv in iteritems(val)
)
else:
rv = val
if value_serializer is not None:
rv = value_serializer(None, None, rv)
return rv

43
src/attr/_make.py
View File

@ -373,7 +373,7 @@ def _collect_base_attrs(cls, taken_attr_names):
if a.inherited or a.name in taken_attr_names:
continue
a = a._assoc(inherited=True)
a = a.assoc(inherited=True)
base_attrs.append(a)
base_attr_map[a.name] = base_cls
@ -411,7 +411,7 @@ def _collect_base_attrs_broken(cls, taken_attr_names):
if a.name in taken_attr_names:
continue
a = a._assoc(inherited=True)
a = a.assoc(inherited=True)
taken_attr_names.add(a.name)
base_attrs.append(a)
base_attr_map[a.name] = base_cls
@ -419,7 +419,9 @@ def _collect_base_attrs_broken(cls, taken_attr_names):
return base_attrs, base_attr_map
def _transform_attrs(cls, these, auto_attribs, kw_only, collect_by_mro):
def _transform_attrs(
cls, these, auto_attribs, kw_only, collect_by_mro, field_transformer
):
"""
Transform all `_CountingAttr`s on a class into `Attribute`s.
@ -451,6 +453,7 @@ def _transform_attrs(cls, these, auto_attribs, kw_only, collect_by_mro):
continue
annot_names.add(attr_name)
a = cd.get(attr_name, NOTHING)
if not isinstance(a, _CountingAttr):
if a is NOTHING:
a = attrib()
@ -498,8 +501,8 @@ def _transform_attrs(cls, these, auto_attribs, kw_only, collect_by_mro):
AttrsClass = _make_attr_tuple_class(cls.__name__, attr_names)
if kw_only:
own_attrs = [a._assoc(kw_only=True) for a in own_attrs]
base_attrs = [a._assoc(kw_only=True) for a in base_attrs]
own_attrs = [a.assoc(kw_only=True) for a in own_attrs]
base_attrs = [a.assoc(kw_only=True) for a in base_attrs]
attrs = AttrsClass(base_attrs + own_attrs)
@ -518,6 +521,8 @@ def _transform_attrs(cls, these, auto_attribs, kw_only, collect_by_mro):
if had_default is False and a.default is not NOTHING:
had_default = True
if field_transformer is not None:
attrs = field_transformer(cls, attrs)
return _Attributes((attrs, base_attrs, base_attr_map))
@ -574,9 +579,15 @@ class _ClassBuilder(object):
collect_by_mro,
on_setattr,
has_custom_setattr,
field_transformer,
):
attrs, base_attrs, base_map = _transform_attrs(
cls, these, auto_attribs, kw_only, collect_by_mro
cls,
these,
auto_attribs,
kw_only,
collect_by_mro,
field_transformer,
)
self._cls = cls
@ -1001,6 +1012,7 @@ def attrs(
collect_by_mro=False,
getstate_setstate=None,
on_setattr=None,
field_transformer=None,
):
r"""
A class decorator that adds `dunder
@ -1196,6 +1208,11 @@ def attrs(
If a list of callables is passed, they're automatically wrapped in an
`attr.setters.pipe`.
:param Optional[callable] field_transformer:
A function that is called with the original class object and all
fields right before ``attrs`` finalizes the class. You can use
this, e.g., to automatically add converters or validators to
fields based on their types. See `transform-fields` for more details.
.. versionadded:: 16.0.0 *slots*
.. versionadded:: 16.1.0 *frozen*
@ -1225,6 +1242,7 @@ def attrs(
.. versionadded:: 20.1.0 *collect_by_mro*
.. versionadded:: 20.1.0 *getstate_setstate*
.. versionadded:: 20.1.0 *on_setattr*
.. versionadded:: 20.3.0 *field_transformer*
"""
if auto_detect and PY2:
raise PythonTooOldError(
@ -1271,6 +1289,7 @@ def attrs(
collect_by_mro,
on_setattr,
has_own_setattr,
field_transformer,
)
if _determine_whether_to_implement(
cls, repr, auto_detect, ("__repr__",)
@ -2183,6 +2202,13 @@ class Attribute(object):
"""
*Read-only* representation of an attribute.
Instances of this class are frequently used for introspection purposes
like:
- `fields` returns a tuple of them.
- Validators get them passed as the first argument.
- The *field transformer* hook receives a list of them.
:attribute name: The name of the attribute.
:attribute inherited: Whether or not that attribute has been inherited from
a base class.
@ -2306,9 +2332,12 @@ class Attribute(object):
return self.eq and self.order
# Don't use attr.assoc since fields(Attribute) doesn't work
def _assoc(self, **changes):
def assoc(self, **changes):
"""
Copy *self* and apply *changes*.
This works similarly to `attr.evolve` but that function does not work
with ``Attribute``.
"""
new = copy.copy(self)

2
src/attr/_next_gen.py
View File

@ -33,6 +33,7 @@ def define(
auto_detect=True,
getstate_setstate=None,
on_setattr=None,
field_transformer=None,
):
r"""
The only behavioral differences are the handling of the *auto_attribs*
@ -72,6 +73,7 @@ def define(
collect_by_mro=True,
getstate_setstate=getstate_setstate,
on_setattr=on_setattr,
field_transformer=field_transformer,
)
def wrap(cls):

187
tests/test_hooks.py Normal file
View File

@ -0,0 +1,187 @@
from datetime import datetime
from typing import Dict, List
import attr
class TestTransformHook:
"""
Tests for `attrs(tranform_value_serializer=func)`
"""
def test_hook_applied(self):
"""
The transform hook is applied to all attributes. Types can be missing,
explicitly set, or annotated.
"""
results = []
def hook(cls, attribs):
results[:] = [(a.name, a.type) for a in attribs]
return attribs
@attr.s(field_transformer=hook)
class C:
x = attr.ib()
y = attr.ib(type=int)
z: float = attr.ib()
assert results == [("x", None), ("y", int), ("z", float)]
def test_hook_applied_auto_attrib(self):
"""
The transform hook is applied to all attributes and type annotations
are detected.
"""
results = []
def hook(cls, attribs):
results[:] = [(a.name, a.type) for a in attribs]
return attribs
@attr.s(auto_attribs=True, field_transformer=hook)
class C:
x: int
y: str = attr.ib()
assert results == [("x", int), ("y", str)]
def test_hook_applied_modify_attrib(self):
"""
The transform hook can modify attributes.
"""
def hook(cls, attribs):
return [a.assoc(converter=a.type) for a in attribs]
@attr.s(auto_attribs=True, field_transformer=hook)
class C:
x: int = attr.ib(converter=int)
y: float
c = C(x="3", y="3.14")
assert c == C(x=3, y=3.14)
def test_hook_remove_field(self):
"""
It is possible to remove fields via the hook.
"""
def hook(cls, attribs):
return [a for a in attribs if a.type is not int]
@attr.s(auto_attribs=True, field_transformer=hook)
class C:
x: int
y: float
assert attr.asdict(C(2.7)) == {"y": 2.7}
def test_hook_add_field(self):
"""
It is possible to add fields via the hook.
"""
def hook(cls, attribs):
a1 = attribs[0]
a2 = a1.assoc(name="new")
return [a1, a2]
@attr.s(auto_attribs=True, field_transformer=hook)
class C:
x: int
assert attr.asdict(C(1, 2)) == {"x": 1, "new": 2}
def test_hook_with_inheritance(self):
"""
The hook receives all fields from base classes.
"""
def hook(cls, attribs):
assert [a.name for a in attribs] == ["x", "y"]
# Remove Base' "x"
return attribs[1:]
@attr.s(auto_attribs=True)
class Base:
x: int
@attr.s(auto_attribs=True, field_transformer=hook)
class Sub(Base):
y: int
assert attr.asdict(Sub(2)) == {"y": 2}
class TestAsDictHook:
def test_asdict(self):
"""
asdict() calls the hooks in attrs classes and in other datastructures
like lists or dicts.
"""
def hook(inst, a, v):
if isinstance(v, datetime):
return v.isoformat()
return v
@attr.dataclass
class Child:
x: datetime
y: List[datetime]
@attr.dataclass
class Parent:
a: Child
b: List[Child]
c: Dict[str, Child]
d: Dict[str, datetime]
inst = Parent(
a=Child(1, [datetime(2020, 7, 1)]),
b=[Child(2, [datetime(2020, 7, 2)])],
c={"spam": Child(3, [datetime(2020, 7, 3)])},
d={"eggs": datetime(2020, 7, 4)},
)
result = attr.asdict(inst, value_serializer=hook)
assert result == {
"a": {"x": 1, "y": ["2020-07-01T00:00:00"]},
"b": [{"x": 2, "y": ["2020-07-02T00:00:00"]}],
"c": {"spam": {"x": 3, "y": ["2020-07-03T00:00:00"]}},
"d": {"eggs": "2020-07-04T00:00:00"},
}
def test_asdict_calls(self):
"""
The correct instances and attribute names are passed to the hook.
"""
calls = []
def hook(inst, a, v):
calls.append((inst, a.name if a else a, v))
return v
@attr.dataclass
class Child:
x: int
@attr.dataclass
class Parent:
a: Child
b: List[Child]
c: Dict[str, Child]
inst = Parent(a=Child(1), b=[Child(2)], c={"spam": Child(3)})
attr.asdict(inst, value_serializer=hook)
assert calls == [
(inst, "a", inst.a),
(inst.a, "x", inst.a.x),
(inst, "b", inst.b),
(inst.b[0], "x", inst.b[0].x),
(inst, "c", inst.c),
(None, None, "spam"),
(inst.c["spam"], "x", inst.c["spam"].x),
]

21
tests/test_make.py
View File

@ -167,7 +167,7 @@ class TestTransformAttrs(object):
Does not attach __attrs_attrs__ to the class.
"""
C = make_tc()
_transform_attrs(C, None, False, False, True)
_transform_attrs(C, None, False, False, True, None)
assert None is getattr(C, "__attrs_attrs__", None)
@ -176,7 +176,7 @@ class TestTransformAttrs(object):
Transforms every `_CountingAttr` and leaves others (a) be.
"""
C = make_tc()
attrs, _, _ = _transform_attrs(C, None, False, False, True)
attrs, _, _ = _transform_attrs(C, None, False, False, True, None)
assert ["z", "y", "x"] == [a.name for a in attrs]
@ -190,7 +190,7 @@ class TestTransformAttrs(object):
pass
assert _Attributes(((), [], {})) == _transform_attrs(
C, None, False, False, True
C, None, False, False, True, None
)
def test_transforms_to_attribute(self):
@ -198,7 +198,9 @@ class TestTransformAttrs(object):
All `_CountingAttr`s are transformed into `Attribute`s.
"""
C = make_tc()
attrs, base_attrs, _ = _transform_attrs(C, None, False, False, True)
attrs, base_attrs, _ = _transform_attrs(
C, None, False, False, True, None
)
assert [] == base_attrs
assert 3 == len(attrs)
@ -215,7 +217,7 @@ class TestTransformAttrs(object):
y = attr.ib()
with pytest.raises(ValueError) as e:
_transform_attrs(C, None, False, False, True)
_transform_attrs(C, None, False, False, True, None)
assert (
"No mandatory attributes allowed after an attribute with a "
"default value or factory. Attribute in question: Attribute"
@ -245,7 +247,9 @@ class TestTransformAttrs(object):
x = attr.ib(default=None)
y = attr.ib()
attrs, base_attrs, _ = _transform_attrs(C, None, False, True, True)
attrs, base_attrs, _ = _transform_attrs(
C, None, False, True, True, None
)
assert len(attrs) == 3
assert len(base_attrs) == 1
@ -268,7 +272,7 @@ class TestTransformAttrs(object):
y = attr.ib()
attrs, base_attrs, _ = _transform_attrs(
C, {"x": attr.ib()}, False, False, True
C, {"x": attr.ib()}, False, False, True, None
)
assert [] == base_attrs
@ -1487,6 +1491,7 @@ class TestClassBuilder(object):
True,
None,
False,
None,
)
assert "<_ClassBuilder(cls=C)>" == repr(b)
@ -1513,6 +1518,7 @@ class TestClassBuilder(object):
True,
None,
False,
None,
)
cls = (
@ -1591,6 +1597,7 @@ class TestClassBuilder(object):
collect_by_mro=True,
on_setattr=None,
has_custom_setattr=False,
field_transformer=None,
)
b._cls = {} # no __module__; no __qualname__

10
tests/typing_example.py
View File

@ -199,6 +199,16 @@ class ValidatedSetter:
)
# field_transformer
def ft_hook(cls: type, attribs: List[attr.Attribute]) -> List[attr.Attribute]:
return attribs
@attr.s(field_transformer=ft_hook)
class TransformedAttrs:
x: int
# Auto-detect
# XXX: needs support in mypy
# @attr.s(auto_detect=True)