mirror of
https://github.com/explosion/spaCy.git
synced 2025-01-31 19:54:10 +03:00
c0f4a1e43b
* verbose and tag_map options * adding init_tok2vec option and only changing the tok2vec that is specified * adding omit_extra_lookups and verifying textcat config * wip * pretrain bugfix * add replace and resume options * train_textcat fix * raw text functionality * improve UX when KeyError or when input data can't be parsed * avoid unnecessary access to goldparse in TextCat pipe * save performance information in nlp.meta * add noise_level to config * move nn_parser's defaults to config file * multitask in config - doesn't work yet * scorer offering both F and AUC options, need to be specified in config * add textcat verification code from old train script * small fixes to config files * clean up * set default config for ner/parser to allow create_pipe to work as before * two more test fixes * small fixes * cleanup * fix NER pickling + additional unit test * create_pipe as before
47 lines
1.6 KiB
Python
47 lines
1.6 KiB
Python
from spacy.pipeline import EntityRecognizer
|
|
from spacy.tokens import Span
|
|
import pytest
|
|
|
|
from ..util import get_doc
|
|
from spacy.pipeline.defaults import default_ner
|
|
|
|
|
|
def test_doc_add_entities_set_ents_iob(en_vocab):
|
|
text = ["This", "is", "a", "lion"]
|
|
doc = get_doc(en_vocab, text)
|
|
config = {"learn_tokens": False, "min_action_freq": 30, "beam_width": 1, "beam_update_prob": 1.0}
|
|
ner = EntityRecognizer(en_vocab, default_ner(), **config)
|
|
ner.begin_training([])
|
|
ner(doc)
|
|
assert len(list(doc.ents)) == 0
|
|
assert [w.ent_iob_ for w in doc] == (["O"] * len(doc))
|
|
|
|
doc.ents = [(doc.vocab.strings["ANIMAL"], 3, 4)]
|
|
assert [w.ent_iob_ for w in doc] == ["O", "O", "O", "B"]
|
|
|
|
doc.ents = [(doc.vocab.strings["WORD"], 0, 2)]
|
|
assert [w.ent_iob_ for w in doc] == ["B", "I", "O", "O"]
|
|
|
|
|
|
def test_ents_reset(en_vocab):
|
|
text = ["This", "is", "a", "lion"]
|
|
doc = get_doc(en_vocab, text)
|
|
config = {"learn_tokens": False, "min_action_freq": 30, "beam_width": 1, "beam_update_prob": 1.0}
|
|
ner = EntityRecognizer(en_vocab, default_ner(), **config)
|
|
ner.begin_training([])
|
|
ner(doc)
|
|
assert [t.ent_iob_ for t in doc] == (["O"] * len(doc))
|
|
doc.ents = list(doc.ents)
|
|
assert [t.ent_iob_ for t in doc] == (["O"] * len(doc))
|
|
|
|
|
|
def test_add_overlapping_entities(en_vocab):
|
|
text = ["Louisiana", "Office", "of", "Conservation"]
|
|
doc = get_doc(en_vocab, text)
|
|
entity = Span(doc, 0, 4, label=391)
|
|
doc.ents = [entity]
|
|
|
|
new_entity = Span(doc, 0, 1, label=392)
|
|
with pytest.raises(ValueError):
|
|
doc.ents = list(doc.ents) + [new_entity]
|