mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-26 18:06:29 +03:00
2eb925bd05
* Perserve flags in EntityRuler The EntityRuler (explosion/spaCy#3526) does not preserve overwrite flags (or `ent_id_sep`) when serialized. This commit adds support for serialization/deserialization preserving overwrite and ent_id_sep flags. * add signed contributor agreement * flake8 cleanup mostly blank line issues. * mark test from the issue as needing a model The test from the issue needs some language model for serialization but the test wasn't originally marked correctly. * Adds `phrase_matcher_attr` to allow args to PhraseMatcher This is an added arg to pass to the `PhraseMatcher`. For example, this allows creation of a case insensitive phrase matcher when the `EntityRuler` is created. References explosion/spaCy#3822 * remove unneeded model loading The model didn't need to be loaded, and I replaced it with a change that doesn't require it (using existings fixtures) * updated docstring for new argument * updated docs to reflect new argument to the EntityRuler constructor * change tempdir handling to be compatible with python 2.7 * return conflicted code to entityruler Some stuff got cut out because of merge conflicts, this returns that code for the phrase_matcher_attr. * fixed typo in the code added back after conflicts * flake8 compliance When I deconflicted the branch there were some flake8 issues introduced. This resolves the spacing problems. * test changes: attempts to fix flaky test in python3.5 These tests seem to be alittle flaky in 3.5 so I changed the check to avoid the comparisons that seem to be fail sometimes.
130 lines
4.1 KiB
Python
130 lines
4.1 KiB
Python
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
import pytest
|
|
from spacy.tokens import Span
|
|
from spacy.language import Language
|
|
from spacy.pipeline import EntityRuler
|
|
|
|
|
|
@pytest.fixture
|
|
def nlp():
|
|
return Language()
|
|
|
|
|
|
@pytest.fixture
|
|
def patterns():
|
|
return [
|
|
{"label": "HELLO", "pattern": "hello world"},
|
|
{"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]},
|
|
{"label": "HELLO", "pattern": [{"ORTH": "HELLO"}]},
|
|
{"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]},
|
|
{"label": "TECH_ORG", "pattern": "Apple", "id": "a1"},
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
def add_ent():
|
|
def add_ent_component(doc):
|
|
doc.ents = [Span(doc, 0, 3, label=doc.vocab.strings["ORG"])]
|
|
return doc
|
|
|
|
return add_ent_component
|
|
|
|
|
|
def test_entity_ruler_init(nlp, patterns):
|
|
ruler = EntityRuler(nlp, patterns=patterns)
|
|
assert len(ruler) == len(patterns)
|
|
assert len(ruler.labels) == 4
|
|
assert "HELLO" in ruler
|
|
assert "BYE" in ruler
|
|
nlp.add_pipe(ruler)
|
|
doc = nlp("hello world bye bye")
|
|
assert len(doc.ents) == 2
|
|
assert doc.ents[0].label_ == "HELLO"
|
|
assert doc.ents[1].label_ == "BYE"
|
|
|
|
|
|
def test_entity_ruler_existing(nlp, patterns, add_ent):
|
|
ruler = EntityRuler(nlp, patterns=patterns)
|
|
nlp.add_pipe(add_ent)
|
|
nlp.add_pipe(ruler)
|
|
doc = nlp("OH HELLO WORLD bye bye")
|
|
assert len(doc.ents) == 2
|
|
assert doc.ents[0].label_ == "ORG"
|
|
assert doc.ents[1].label_ == "BYE"
|
|
|
|
|
|
def test_entity_ruler_existing_overwrite(nlp, patterns, add_ent):
|
|
ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True)
|
|
nlp.add_pipe(add_ent)
|
|
nlp.add_pipe(ruler)
|
|
doc = nlp("OH HELLO WORLD bye bye")
|
|
assert len(doc.ents) == 2
|
|
assert doc.ents[0].label_ == "HELLO"
|
|
assert doc.ents[0].text == "HELLO"
|
|
assert doc.ents[1].label_ == "BYE"
|
|
|
|
|
|
def test_entity_ruler_existing_complex(nlp, patterns, add_ent):
|
|
ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True)
|
|
nlp.add_pipe(add_ent)
|
|
nlp.add_pipe(ruler)
|
|
doc = nlp("foo foo bye bye")
|
|
assert len(doc.ents) == 2
|
|
assert doc.ents[0].label_ == "COMPLEX"
|
|
assert doc.ents[1].label_ == "BYE"
|
|
assert len(doc.ents[0]) == 2
|
|
assert len(doc.ents[1]) == 2
|
|
|
|
|
|
def test_entity_ruler_entity_id(nlp, patterns):
|
|
ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True)
|
|
nlp.add_pipe(ruler)
|
|
doc = nlp("Apple is a technology company")
|
|
assert len(doc.ents) == 1
|
|
assert doc.ents[0].label_ == "TECH_ORG"
|
|
assert doc.ents[0].ent_id_ == "a1"
|
|
|
|
|
|
def test_entity_ruler_cfg_ent_id_sep(nlp, patterns):
|
|
ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True, ent_id_sep="**")
|
|
assert "TECH_ORG**a1" in ruler.phrase_patterns
|
|
nlp.add_pipe(ruler)
|
|
doc = nlp("Apple is a technology company")
|
|
assert len(doc.ents) == 1
|
|
assert doc.ents[0].label_ == "TECH_ORG"
|
|
assert doc.ents[0].ent_id_ == "a1"
|
|
|
|
|
|
def test_entity_ruler_serialize_bytes(nlp, patterns):
|
|
ruler = EntityRuler(nlp, patterns=patterns)
|
|
assert len(ruler) == len(patterns)
|
|
assert len(ruler.labels) == 4
|
|
ruler_bytes = ruler.to_bytes()
|
|
new_ruler = EntityRuler(nlp)
|
|
assert len(new_ruler) == 0
|
|
assert len(new_ruler.labels) == 0
|
|
new_ruler = new_ruler.from_bytes(ruler_bytes)
|
|
assert len(new_ruler) == len(patterns)
|
|
assert len(new_ruler.labels) == 4
|
|
assert len(new_ruler.patterns) == len(ruler.patterns)
|
|
for pattern in ruler.patterns:
|
|
assert pattern in new_ruler.patterns
|
|
assert new_ruler.labels == ruler.labels
|
|
|
|
|
|
def test_entity_ruler_serialize_phrase_matcher_attr_bytes(nlp, patterns):
|
|
ruler = EntityRuler(nlp, phrase_matcher_attr="LOWER", patterns=patterns)
|
|
assert len(ruler) == len(patterns)
|
|
assert len(ruler.labels) == 4
|
|
ruler_bytes = ruler.to_bytes()
|
|
new_ruler = EntityRuler(nlp)
|
|
assert len(new_ruler) == 0
|
|
assert len(new_ruler.labels) == 0
|
|
assert new_ruler.phrase_matcher_attr is None
|
|
new_ruler = new_ruler.from_bytes(ruler_bytes)
|
|
assert len(new_ruler) == len(patterns)
|
|
assert len(new_ruler.labels) == 4
|
|
assert new_ruler.phrase_matcher_attr == "LOWER"
|