From 05fe6758a71c0e524405d59b005eab0656f41098 Mon Sep 17 00:00:00 2001
From: ines <ines@ines.io>
Date: Sat, 3 Jun 2017 19:44:39 +0200
Subject: [PATCH] Set lexeme attributes for tokenizer special cases

---
 spacy/vocab.pyx | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx
index d3aa426cd..6655925e4 100644
--- a/spacy/vocab.pyx
+++ b/spacy/vocab.pyx
@@ -231,11 +231,13 @@ cdef class Vocab:
             props = intify_attrs(props, strings_map=self.strings, _do_deprecated=True)
             token = &tokens[i]
             # Set the special tokens up to have arbitrary attributes
-            token.lex = <LexemeC*>self.get_by_orth(self.mem, props[attrs.ORTH])
+            lex = <LexemeC*>self.get_by_orth(self.mem, props[attrs.ORTH])
+            token.lex = lex
             if attrs.TAG in props:
                 self.morphology.assign_tag(token, props[attrs.TAG])
             for attr_id, value in props.items():
                 Token.set_struct_attr(token, attr_id, value)
+                Lexeme.set_struct_attr(lex, attr_id, value)
         return tokens
 
     @property