Merge pull request #1389 from mdcclv/lemmatizer_obey_exceptions

Lemmatizer obey exceptions

spacy/lemmatizer.py

@@ -78,6 +78,7 @@ def lemmatize(string, index, exceptions, rules):
     #    forms.append(string)
     forms.extend(exceptions.get(string, []))
     oov_forms = []
+    if not forms:
         for old, new in rules:
             if string.endswith(old):
                 form = string[:len(string) - len(old)] + new

spacy/tests/regression/test_issue1387.py

@@ -0,0 +1,22 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from ...symbols import POS, VERB, VerbForm_part
+from ...vocab import Vocab
+from ...lemmatizer import Lemmatizer
+from ..util import get_doc
+
+import pytest
+
+def test_issue1387():
+    tag_map = {'VBG': {POS: VERB, VerbForm_part: True}}
+    index = {"verb": ("cope","cop")}
+    exc = {"verb": {"coping": ("cope",)}}
+    rules = {"verb": [["ing", ""]]}
+    lemmatizer = Lemmatizer(index, exc, rules)
+    vocab = Vocab(lemmatizer=lemmatizer, tag_map=tag_map)
+    doc = get_doc(vocab, ["coping"])
+    doc[0].tag_ = 'VBG'
+    assert doc[0].text == "coping"
+    assert doc[0].lemma_ == "cope"
+

spacy/tests/tagger/test_lemmatizer.py

@@ -47,3 +47,20 @@ def test_tagger_lemmatizer_lemma_assignment(EN):
     assert all(t.lemma_ == '' for t in doc)
     EN.tagger(doc)
     assert all(t.lemma_ != '' for t in doc)
+
+
+from ...symbols import POS, VERB, VerbForm_part
+from ...vocab import Vocab
+from ...lemmatizer import Lemmatizer
+from ..util import get_doc
+def test_tagger_lemmatizer_exceptions():
+    index = {"verb": ("cope","cop")}
+    exc = {"verb": {"coping": ("cope",)}}
+    rules = {"verb": [["ing", ""]]}
+    tag_map = {'VBG': {POS: VERB, VerbForm_part: True}}
+    lemmatizer = Lemmatizer(index, exc, rules)
+    vocab = Vocab(lemmatizer=lemmatizer, tag_map=tag_map)
+    doc = get_doc(vocab, ["coping"])
+    doc[0].tag_ = 'VBG'
+    assert doc[0].text == "coping"
+    assert doc[0].lemma_ == "cope"