Improve tokenization for UD Dutch corpora (#5259)

* Improve tokenization for UD Dutch corpora

Improve tokenization for UD Dutch Alpino and LassySmall.

* Format Dutch tokenizer exceptions

spacy/lang/nl/__init__.py

@@ -5,7 +5,8 @@ from .stop_words import STOP_WORDS
 from .lex_attrs import LEX_ATTRS
 from .tag_map import TAG_MAP
 from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES
+from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES
+from .punctuation import TOKENIZER_SUFFIXES
 from .lemmatizer import DutchLemmatizer
 from ..tokenizer_exceptions import BASE_EXCEPTIONS
 from ..norm_exceptions import BASE_NORMS
@@ -25,6 +26,7 @@ class DutchDefaults(Language.Defaults):
     tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS)
     stop_words = STOP_WORDS
     tag_map = TAG_MAP
+    prefixes = TOKENIZER_PREFIXES
     infixes = TOKENIZER_INFIXES
     suffixes = TOKENIZER_SUFFIXES
 

spacy/lang/nl/punctuation.py

@@ -1,10 +1,14 @@
 # coding: utf8
 from __future__ import unicode_literals
 
-from ..char_classes import LIST_ELLIPSES, LIST_ICONS
+from ..char_classes import LIST_ELLIPSES, LIST_ICONS, LIST_UNITS, merge_chars
+from ..char_classes import LIST_PUNCT, LIST_QUOTES, CURRENCY, PUNCT
 from ..char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
 
-from ..punctuation import TOKENIZER_SUFFIXES as DEFAULT_TOKENIZER_SUFFIXES
+from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES
+
+
+_prefixes = [",,"] + BASE_TOKENIZER_PREFIXES
 
 
 # Copied from `de` package. Main purpose is to ensure that hyphens are not
@@ -22,20 +26,33 @@ _infixes = (
         r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
         r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes),
         r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA),
-        r"(?<=[0-9])-(?=[0-9])",
     ]
 )
 
 
-# Remove "'s" suffix from suffix list. In Dutch, "'s" is a plural ending when
+_list_units = [u for u in LIST_UNITS if u != "%"]
-# it occurs as a suffix and a clitic for "eens" in standalone use. To avoid
+_units = merge_chars(" ".join(_list_units))
-# ambiguity it's better to just leave it attached when it occurs as a suffix.
-default_suffix_blacklist = ("'s", "'S", "’s", "’S")
-_suffixes = [
-    suffix
-    for suffix in DEFAULT_TOKENIZER_SUFFIXES
-    if suffix not in default_suffix_blacklist
-]
 
+_suffixes = (
+    ["''"]
+    + LIST_PUNCT
+    + LIST_ELLIPSES
+    + LIST_QUOTES
+    + LIST_ICONS
+    + ["—", "–"]
+    + [
+        r"(?<=[0-9])\+",
+        r"(?<=°[FfCcKk])\.",
+        r"(?<=[0-9])(?:{c})".format(c=CURRENCY),
+        r"(?<=[0-9])(?:{u})".format(u=_units),
+        r"(?<=[0-9{al}{e}{p}(?:{q})])\.".format(
+            al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT
+        ),
+        r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER),
+    ]
+)
+
+
+TOKENIZER_PREFIXES = _prefixes
 TOKENIZER_INFIXES = _infixes
 TOKENIZER_SUFFIXES = _suffixes