Merge branch 'develop' of https://github.com/explosion/spaCy into develop

requirements.txt

@@ -13,7 +13,6 @@ requests>=2.13.0,<3.0.0
 regex==2017.4.5
 ftfy>=4.4.2,<5.0.0
 pytest>=3.0.6,<4.0.0
-pip>=9.0.0,<10.0.0
 mock>=2.0.0,<3.0.0
 msgpack-python
 msgpack-numpy

setup.py

@@ -197,7 +197,6 @@ def setup_package():
                 'preshed>=1.0.0,<2.0.0',
                 'thinc>=6.8.1,<6.9.0',
                 'plac<1.0.0,>=0.9.6',
-                'pip>=9.0.0,<10.0.0',
                 'six',
                 'pathlib',
                 'ujson>=1.35',

spacy/lang/de/__init__.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
 from .norm_exceptions import NORM_EXCEPTIONS
+from .punctuation import TOKENIZER_INFIXES
 from .tag_map import TAG_MAP
 from .stop_words import STOP_WORDS
 from .lemmatizer import LOOKUP
@@ -23,6 +24,7 @@ class GermanDefaults(Language.Defaults):
                                          NORM_EXCEPTIONS, BASE_NORMS)
 
     tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS)
+    infixes = tuple(TOKENIZER_INFIXES)
     tag_map = dict(TAG_MAP)
     stop_words = set(STOP_WORDS)
     syntax_iterators = dict(SYNTAX_ITERATORS)

spacy/lang/de/punctuation.py

@@ -0,0 +1,20 @@
+# coding: utf8
+from __future__ import unicode_literals
+
+from ..char_classes import LIST_ELLIPSES, LIST_ICONS
+from ..char_classes import QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
+
+
+_quotes = QUOTES.replace("'", '')
+
+_infixes = (LIST_ELLIPSES + LIST_ICONS +
+            [r'(?<=[{}])\.(?=[{}])'.format(ALPHA_LOWER, ALPHA_UPPER),
+             r'(?<=[{a}])[,!?](?=[{a}])'.format(a=ALPHA),
+             r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
+             r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),
+             r'(?<=[{a}])([{q}\)\]\(\[])(?=[\{a}])'.format(a=ALPHA, q=_quotes),
+             r'(?<=[{a}])--(?=[{a}])'.format(a=ALPHA),
+             r'(?<=[0-9])-(?=[0-9])'])
+
+
+TOKENIZER_INFIXES = _infixes

spacy/tests/lang/de/test_prefix_suffix_infix.py

@@ -67,12 +67,6 @@ def test_tokenizer_splits_uneven_wrap_interact(de_tokenizer, text):
     assert len(tokens) == 4
 
 
-@pytest.mark.parametrize('text', ["blau-rot"])
-def test_tokenizer_splits_hyphens(de_tokenizer, text):
-    tokens = de_tokenizer(text)
-    assert len(tokens) == 3
-
-
 @pytest.mark.parametrize('text', ["0.1-13.5", "0.0-0.1", "103.27-300"])
 def test_tokenizer_splits_numeric_range(de_tokenizer, text):
     tokens = de_tokenizer(text)
@@ -100,17 +94,21 @@ def test_tokenizer_splits_ellipsis_infix(de_tokenizer, text):
     assert len(tokens) == 3
 
 
+@pytest.mark.parametrize('text', ['Islam-Konferenz', 'Ost-West-Konflikt'])
+def test_tokenizer_keeps_hyphens(de_tokenizer, text):
+    tokens = de_tokenizer(text)
+    assert len(tokens) == 1
+
+
 def test_tokenizer_splits_double_hyphen_infix(de_tokenizer):
     tokens = de_tokenizer("Viele Regeln--wie die Bindestrich-Regeln--sind kompliziert.")
-    assert len(tokens) == 12
+    assert len(tokens) == 10
     assert tokens[0].text == "Viele"
     assert tokens[1].text == "Regeln"
     assert tokens[2].text == "--"
     assert tokens[3].text == "wie"
     assert tokens[4].text == "die"
-    assert tokens[5].text == "Bindestrich"
+    assert tokens[5].text == "Bindestrich-Regeln"
-    assert tokens[6].text == "-"
+    assert tokens[6].text == "--"
-    assert tokens[7].text == "Regeln"
+    assert tokens[7].text == "sind"
-    assert tokens[8].text == "--"
+    assert tokens[8].text == "kompliziert"
-    assert tokens[9].text == "sind"
-    assert tokens[10].text == "kompliziert"

spacy/tests/lang/de/test_text.py

@@ -25,15 +25,15 @@ Umfang kläglich dünnen Beine flimmerten ihm hilflos vor den Augen.
     assert len(tokens) == 109
 
 
-@pytest.mark.parametrize('text,length', [
+@pytest.mark.parametrize('text', [
-    ("Donaudampfschifffahrtsgesellschaftskapitänsanwärterposten", 1),
+    "Donaudampfschifffahrtsgesellschaftskapitänsanwärterposten",
-    ("Rindfleischetikettierungsüberwachungsaufgabenübertragungsgesetz", 1),
+    "Rindfleischetikettierungsüberwachungsaufgabenübertragungsgesetz",
-    ("Kraftfahrzeug-Haftpflichtversicherung", 3),
+    "Kraftfahrzeug-Haftpflichtversicherung",
-    ("Vakuum-Mittelfrequenz-Induktionsofen", 5)
+    "Vakuum-Mittelfrequenz-Induktionsofen"
     ])
-def test_tokenizer_handles_long_words(de_tokenizer, text, length):
+def test_tokenizer_handles_long_words(de_tokenizer, text):
     tokens = de_tokenizer(text)
-    assert len(tokens) == length
+    assert len(tokens) == 1
 
 
 @pytest.mark.parametrize('text,length', [

spacy/util.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals, print_function
 
 import os
 import ujson
-import pip
+import pkg_resources
 import importlib
 import regex as re
 from pathlib import Path
@@ -180,9 +180,9 @@ def is_package(name):
     name (unicode): Name of package.
     RETURNS (bool): True if installed package, False if not.
     """
-    packages = pip.get_installed_distributions()
+    packages = pkg_resources.working_set.by_key.keys()
     for package in packages:
-        if package.project_name.replace('-', '_') == name:
+        if package.replace('-', '_') == name:
             return True
     return False