Revert "Merge remote-tracking branch 'origin/master'"

This reverts commit d3b181cdf1, reversing
changes made to b19cfcc144.

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,6 +12,8 @@
 <!--- Include details of your testing environment, and the tests you ran too -->
 <!--- How were other areas of the code affected? -->
 
+## Screenshots (if appropriate):
+
 ## Types of changes
 <!--- What types of changes does your code introduce? Put an `x` in all applicable boxes.: -->
 - [ ] Bug fix (non-breaking change fixing an issue)
@@ -25,4 +27,4 @@
 - [ ] My change requires a change to spaCy's documentation.
 - [ ] I have updated the documentation accordingly.
 - [ ] I have added tests to cover my changes.
-- [ ] All new and existing tests passed.
+- [ ] All new and existing tests passed.

spacy/language.py

@@ -67,8 +67,6 @@ class BaseDefaults(object):
     @classmethod
     def create_tokenizer(cls, nlp=None):
         rules = cls.tokenizer_exceptions
-        if cls.token_match:
-            token_match = cls.token_match
         if cls.prefixes:
             prefix_search  = util.compile_prefix_regex(cls.prefixes).search
         else:
@@ -84,7 +82,7 @@ class BaseDefaults(object):
         vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)
         return Tokenizer(vocab, rules=rules,
                          prefix_search=prefix_search, suffix_search=suffix_search,
-                         infix_finditer=infix_finditer, token_match=token_match)
+                         infix_finditer=infix_finditer)
 
     @classmethod
     def create_tagger(cls, nlp=None):
@@ -144,8 +142,6 @@ class BaseDefaults(object):
             pipeline.append(nlp.entity)
         return pipeline
 
-    token_match = language_data.TOKEN_MATCH
-
     prefixes = tuple(language_data.TOKENIZER_PREFIXES)
 
     suffixes = tuple(language_data.TOKENIZER_SUFFIXES)

spacy/language_data/__init__.py

@@ -3,4 +3,3 @@ from .punctuation import *
 from .tag_map import *
 from .entity_rules import *
 from .util import *
-from .tokenizer_exceptions import *

spacy/language_data/tokenizer_exceptions.py

@@ -1,11 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-_URL_PATTERN = r'''
-^((([A-Za-z]{3,9}:(?:\/\/)?)(?:[-;:&=\+\$,\w]+@)?[A-Za-z0-9.-]+|(?:www.|[-;:&=\+\$,\w]+@)[A-Za-z0-9.-]+)((?:\/[\+~%\/.\w\-_]*)?\??(?:[-\+=&;%@.\w_]*)#?(?:[\w]*))?)$
-'''.strip()
-
-TOKEN_MATCH = re.compile(_URL_PATTERN).match
-
-__all__ = ['TOKEN_MATCH']

spacy/tests/tokenizer/test_urls.py

@@ -1,77 +0,0 @@
-from __future__ import unicode_literals
-
-import pytest
-
-URLS = [
-    u"http://www.nytimes.com/2016/04/20/us/politics/new-york-primary-preview.html?hp&action=click&pgtype=Homepage&clickSource=story-heading&module=a-lede-package-region&region=top-news&WT.nav=top-news&_r=0",
-    u"www.google.com?q=google",
-    u"google.com",
-    u"www.red-stars.com",
-    pytest.mark.xfail(u"red-stars.com"),
-    u"http://foo.com/blah_(wikipedia)#cite-1",
-    u"http://www.example.com/wpstyle/?bar=baz&inga=42&quux",
-    u"mailto:foo.bar@baz.com",
-    u"mailto:foo-bar@baz-co.com"
-]
-
-# Punctuation we want to check is split away before the URL
-PREFIXES = [
-    "(", '"', "...",  ">"
-]
-
-# Punctuation we want to check is split away after the URL
-SUFFIXES = [
-    '"', ":", ">"]
-
-@pytest.mark.parametrize("text", URLS)
-def test_simple_url(en_tokenizer, text):
-    tokens = en_tokenizer(text)
-    assert tokens[0].orth_ == text
-    assert len(tokens) == 1
-
-
-@pytest.mark.parametrize("prefix", PREFIXES)
-@pytest.mark.parametrize("url", URLS)
-def test_prefixed_url(en_tokenizer, prefix, url):
-    tokens = en_tokenizer(prefix + url)
-    assert tokens[0].text == prefix
-    assert tokens[1].text == url
-    assert len(tokens) == 2
-    
-@pytest.mark.parametrize("suffix", SUFFIXES)
-@pytest.mark.parametrize("url", URLS)
-def test_suffixed_url(en_tokenizer, url, suffix):
-    tokens = en_tokenizer(url + suffix)
-    assert tokens[0].text == url
-    assert tokens[1].text == suffix
-    assert len(tokens) == 2
-    
-@pytest.mark.parametrize("prefix", PREFIXES)
-@pytest.mark.parametrize("suffix", SUFFIXES)
-@pytest.mark.parametrize("url", URLS)
-def test_surround_url(en_tokenizer, prefix, suffix, url):
-    tokens = en_tokenizer(prefix + url + suffix)
-    assert tokens[0].text == prefix
-    assert tokens[1].text == url
-    assert tokens[2].text == suffix
-    assert len(tokens) == 3
-    
-@pytest.mark.parametrize("prefix1", PREFIXES)
-@pytest.mark.parametrize("prefix2", PREFIXES)
-@pytest.mark.parametrize("url", URLS)
-def test_two_prefix_url(en_tokenizer, prefix1, prefix2, url):
-    tokens = en_tokenizer(prefix1 + prefix2 + url)
-    assert tokens[0].text == prefix1
-    assert tokens[1].text == prefix2
-    assert tokens[2].text == url
-    assert len(tokens) == 3
-    
-@pytest.mark.parametrize("suffix1", SUFFIXES)
-@pytest.mark.parametrize("suffix2", SUFFIXES)
-@pytest.mark.parametrize("url", URLS)
-def test_two_prefix_url(en_tokenizer, suffix1, suffix2, url):
-    tokens = en_tokenizer(url + suffix1 + suffix2)
-    assert tokens[0].text == url
-    assert tokens[1].text == suffix1
-    assert tokens[2].text == suffix2
-    assert len(tokens) == 3

spacy/tokenizer.pxd

@@ -16,7 +16,6 @@ cdef class Tokenizer:
     cdef PreshMap _specials
     cpdef readonly Vocab vocab
 
-    cdef public object token_match
     cdef public object prefix_search
     cdef public object suffix_search
     cdef public object infix_finditer

spacy/tokenizer.pyx

@@ -29,7 +29,7 @@ cdef class Tokenizer:
     """Segment text, and create Doc objects with the discovered segment boundaries."""
     @classmethod
     def load(cls, path, Vocab vocab, rules=None, prefix_search=None, suffix_search=None,
-             infix_finditer=None, token_match=None):
+             infix_finditer=None):
         '''Load a Tokenizer, reading unsupplied components from the path.
         
         Arguments:
@@ -39,8 +39,6 @@ cdef class Tokenizer:
                 A storage container for lexical types.
             rules (dict):
                 Exceptions and special-cases for the tokenizer.
-            token_match:
-                A boolean function matching strings that becomes tokens.
             prefix_search:
                 Signature of re.compile(string).search
             suffix_search:
@@ -67,9 +65,10 @@ cdef class Tokenizer:
             with (path / 'tokenizer' / 'infix.txt').open() as file_:
                 entries = file_.read().split('\n')
             infix_finditer = util.compile_infix_regex(entries).finditer
-        return cls(vocab, rules, prefix_search, suffix_search, infix_finditer, token_match)
+        return cls(vocab, rules, prefix_search, suffix_search, infix_finditer)
 
-    def __init__(self, Vocab vocab, rules, prefix_search, suffix_search, infix_finditer, token_match=None):
+
+    def __init__(self, Vocab vocab, rules, prefix_search, suffix_search, infix_finditer):
         '''Create a Tokenizer, to create Doc objects given unicode text.
         
         Arguments:
@@ -86,13 +85,10 @@ cdef class Tokenizer:
             infix_finditer:
                 A function matching the signature of re.compile(string).finditer
                 to find infixes.
-            token_match:
-                A boolean function matching strings that becomes tokens.
         '''
         self.mem = Pool()
         self._cache = PreshMap()
         self._specials = PreshMap()
-        self.token_match = token_match
         self.prefix_search = prefix_search
         self.suffix_search = suffix_search
         self.infix_finditer = infix_finditer
@@ -104,10 +100,9 @@ cdef class Tokenizer:
     def __reduce__(self):
         args = (self.vocab,
                 self._rules,
-                self._prefix_re,
+                self._prefix_re, 
-                self._suffix_re,
+                self._suffix_re, 
-                self._infix_re,
+                self._infix_re)
-                self.token_match)
 
         return (self.__class__, args, None, None)
     
@@ -221,8 +216,6 @@ cdef class Tokenizer:
         cdef unicode minus_suf
         cdef size_t last_size = 0
         while string and len(string) != last_size:
-            if self.token_match and self.token_match(string):
-                break
             last_size = len(string)
             pre_len = self.find_prefix(string)
             if pre_len != 0:
@@ -233,8 +226,6 @@ cdef class Tokenizer:
                     string = minus_pre
                     prefixes.push_back(self.vocab.get(mem, prefix))
                     break
-                if self.token_match and self.token_match(string):
-                    break
             suf_len = self.find_suffix(string)
             if suf_len != 0:
                 suffix = string[-suf_len:]
@@ -272,11 +263,7 @@ cdef class Tokenizer:
                 tokens.push_back(prefixes[0][i], False)
         if string:
             cache_hit = self._try_cache(hash_string(string), tokens)
-            if cache_hit:
+            if not cache_hit:
-                pass
-            elif self.token_match and self.token_match(string): 
-                tokens.push_back(self.vocab.get(tokens.mem, string), not suffixes.size())
-            else:
                 matches = self.find_infix(string)
                 if not matches:
                     tokens.push_back(self.vocab.get(tokens.mem, string), False)