mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 11:29:13 +03:00
b71a11ff6d
* Add pos and morph scoring to Scorer Add pos, morph, and morph_per_type to `Scorer`. Report pos and morph accuracy in `spacy evaluate`. * Update morphologizer for v3 * switch to tagger-based morphologizer * use `spacy.HashCharEmbedCNN` for morphologizer defaults * add `Doc.is_morphed` flag * Add morphologizer to train CLI * Add basic morphologizer pipeline tests * Add simple morphologizer training example * Remove subword_features from CharEmbed models Remove `subword_features` argument from `spacy.HashCharEmbedCNN.v1` and `spacy.HashCharEmbedBiLSTM.v1` since in these cases `subword_features` is always `False`. * Rename setting in morphologizer example Use `with_pos_tags` instead of `without_pos_tags`. * Fix kwargs for spacy.HashCharEmbedBiLSTM.v1 * Remove defaults for spacy.HashCharEmbedBiLSTM.v1 Remove default `nM/nC` for `spacy.HashCharEmbedBiLSTM.v1`. * Set random seed for textcat overfitting test
50 lines
1.7 KiB
Python
50 lines
1.7 KiB
Python
import pytest
|
|
|
|
from spacy import util
|
|
from spacy.lang.en import English
|
|
from spacy.language import Language
|
|
from spacy.tests.util import make_tempdir
|
|
|
|
|
|
def test_label_types():
|
|
nlp = Language()
|
|
nlp.add_pipe(nlp.create_pipe("morphologizer"))
|
|
nlp.get_pipe("morphologizer").add_label("Feat=A")
|
|
with pytest.raises(ValueError):
|
|
nlp.get_pipe("morphologizer").add_label(9)
|
|
|
|
|
|
TRAIN_DATA = [
|
|
("I like green eggs", {"morphs": ["Feat=N", "Feat=V", "Feat=J", "Feat=N"], "pos": ["NOUN", "VERB", "ADJ", "NOUN"]}),
|
|
("Eat blue ham", {"morphs": ["Feat=V", "Feat=J", "Feat=N"], "pos": ["VERB", "ADJ", "NOUN"]}),
|
|
]
|
|
|
|
|
|
def test_overfitting_IO():
|
|
# Simple test to try and quickly overfit the morphologizer - ensuring the ML models work correctly
|
|
nlp = English()
|
|
morphologizer = nlp.create_pipe("morphologizer")
|
|
for inst in TRAIN_DATA:
|
|
for morph, pos in zip(inst[1]["morphs"], inst[1]["pos"]):
|
|
morphologizer.add_label(morph + "|POS=" + pos)
|
|
nlp.add_pipe(morphologizer)
|
|
optimizer = nlp.begin_training()
|
|
|
|
for i in range(50):
|
|
losses = {}
|
|
nlp.update(TRAIN_DATA, sgd=optimizer, losses=losses)
|
|
assert losses["morphologizer"] < 0.00001
|
|
|
|
# test the trained model
|
|
test_text = "I like blue eggs"
|
|
doc = nlp(test_text)
|
|
gold_morphs = ["Feat=N|POS=NOUN", "Feat=V|POS=VERB", "Feat=J|POS=ADJ", "Feat=N|POS=NOUN"]
|
|
assert gold_morphs == [t.morph_ for t in doc]
|
|
|
|
# Also test the results are still the same after IO
|
|
with make_tempdir() as tmp_dir:
|
|
nlp.to_disk(tmp_dir)
|
|
nlp2 = util.load_model_from_path(tmp_dir)
|
|
doc2 = nlp2(test_text)
|
|
assert gold_morphs == [t.morph_ for t in doc2]
|