Raise better error if token is pickled (resolves #2833) (#3267)

spacy/errors.py

@@ -303,7 +303,11 @@ class Errors(object):
     E109 = ("Model for component '{name}' not initialized. Did you forget to load "
             "a model, or forget to call begin_training()?")
     E110 = ("Invalid displaCy render wrapper. Expected callable, got: {obj}")
-
+    E111 = ("Pickling a token is not supported, because tokens are only views "
+            "of the parent Doc and can't exist on their own. A pickled token "
+            "would always have to include its Doc and Vocab, which has "
+            "practically no advantage over pickling the parent Doc directly. "
+            "So instead of pickling the token, pickle the Doc it belongs to.")
 
 @add_codes
 class TempErrors(object):

spacy/tests/regression/test_issue2833.py

@@ -0,0 +1,13 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import pytest
+from spacy.tokens import Doc
+from spacy.compat import pickle
+
+
+def test_issue2833(en_vocab):
+    """Test that a custom error is raised if a token is pickled."""
+    doc = Doc(en_vocab, words=["Hello", "world"])
+    with pytest.raises(NotImplementedError):
+        pickle.dumps(doc[0])

spacy/tokens/token.pyx

@@ -115,6 +115,9 @@ cdef class Token:
         else:
             raise ValueError(Errors.E041.format(op=op))
 
+    def __reduce__(self):
+        raise NotImplementedError(Errors.E111)
+
     @property
     def _(self):
         return Underscore(Underscore.token_extensions, self,