Port over changes from #1294

spacy/tests/lang/en/test_customized_tokenizer.py

@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import pytest
+
+from ....lang.en import English
+from ....tokenizer import Tokenizer
+from .... import util
+
+
+@pytest.fixture
+def custom_en_tokenizer(en_vocab):
+    prefix_re = util.compile_prefix_regex(English.Defaults.prefixes)
+    suffix_re = util.compile_suffix_regex(English.Defaults.suffixes)
+    custom_infixes = ['\.\.\.+',
+                      '(?<=[0-9])-(?=[0-9])',
+                      # '(?<=[0-9]+),(?=[0-9]+)',
+                      '[0-9]+(,[0-9]+)+',
+                      '[\[\]!&:,()\*—–\/-]']
+
+    infix_re = util.compile_infix_regex(custom_infixes)
+    return Tokenizer(en_vocab,
+                     English.Defaults.tokenizer_exceptions,
+                     prefix_re.search,
+                     suffix_re.search,
+                     infix_re.finditer,
+                     token_match=None)
+
+
+def test_customized_tokenizer_handles_infixes(custom_en_tokenizer):
+    sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion."
+    context = [word.text for word in custom_en_tokenizer(sentence)]
+    assert context == ['The', '8', 'and', '10', '-', 'county', 'definitions',
+                       'are', 'not', 'used', 'for', 'the', 'greater',
+                       'Southern', 'California', 'Megaregion', '.']
+
+    # the trailing '-' may cause Assertion Error
+    sentence = "The 8- and 10-county definitions are not used for the greater Southern California Megaregion."
+    context = [word.text for word in custom_en_tokenizer(sentence)]
+    assert context == ['The', '8', '-', 'and', '10', '-', 'county',
+                       'definitions', 'are', 'not', 'used', 'for', 'the',
+                       'greater', 'Southern', 'California', 'Megaregion', '.']

spacy/tokenizer.pyx

@@ -248,7 +248,8 @@ cdef class Tokenizer:
 
                         start = infix_end
                     span = string[start:]
-                    tokens.push_back(self.vocab.get(tokens.mem, span), False)
+                    if span:
+                        tokens.push_back(self.vocab.get(tokens.mem, span), False)
         cdef vector[const LexemeC*].reverse_iterator it = suffixes.rbegin()
         while it != suffixes.rend():
             lexeme = deref(it)