diff --git a/spacy/errors.py b/spacy/errors.py index 2b63b6cda..567e29cd0 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -303,7 +303,11 @@ class Errors(object): E109 = ("Model for component '{name}' not initialized. Did you forget to load " "a model, or forget to call begin_training()?") E110 = ("Invalid displaCy render wrapper. Expected callable, got: {obj}") - + E111 = ("Pickling a token is not supported, because tokens are only views " + "of the parent Doc and can't exist on their own. A pickled token " + "would always have to include its Doc and Vocab, which has " + "practically no advantage over pickling the parent Doc directly. " + "So instead of pickling the token, pickle the Doc it belongs to.") @add_codes class TempErrors(object): diff --git a/spacy/tests/regression/test_issue2833.py b/spacy/tests/regression/test_issue2833.py new file mode 100644 index 000000000..81aa40eb3 --- /dev/null +++ b/spacy/tests/regression/test_issue2833.py @@ -0,0 +1,13 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import pytest +from spacy.tokens import Doc +from spacy.compat import pickle + + +def test_issue2833(en_vocab): + """Test that a custom error is raised if a token is pickled.""" + doc = Doc(en_vocab, words=["Hello", "world"]) + with pytest.raises(NotImplementedError): + pickle.dumps(doc[0]) diff --git a/spacy/tokens/token.pyx b/spacy/tokens/token.pyx index 7f416b7ed..a69a0def8 100644 --- a/spacy/tokens/token.pyx +++ b/spacy/tokens/token.pyx @@ -115,6 +115,9 @@ cdef class Token: else: raise ValueError(Errors.E041.format(op=op)) + def __reduce__(self): + raise NotImplementedError(Errors.E111) + @property def _(self): return Underscore(Underscore.token_extensions, self,