mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-28 19:06:33 +03:00
06f0a8daa0
* fix grad_clip naming * cleaning up pretrained_vectors out of cfg * further refactoring Model init's * move Model building out of pipes * further refactor to require a model config when creating a pipe * small fixes * making cfg in nn_parser more consistent * fixing nr_class for parser * fixing nn_parser's nO * fix printing of loss * architectures in own file per type, consistent naming * convenience methods default_tagger_config and default_tok2vec_config * let create_pipe access default config if available for that component * default_parser_config * move defaults to separate folder * allow reading nlp from package or dir with argument 'name' * architecture spacy.VocabVectors.v1 to read static vectors from file * cleanup * default configs for nel, textcat, morphologizer, tensorizer * fix imports * fixing unit tests * fixes and clean up * fixing defaults, nO, fix unit tests * restore parser IO * fix IO * 'fix' serialization test * add *.cfg to manifest * fix example configs with additional arguments * replace Morpohologizer with Tagger * add IO bit when testing overfitting of tagger (currently failing) * fix IO - don't initialize when reading from disk * expand overfitting tests to also check IO goes OK * remove dropout from HashEmbed to fix Tagger performance * add defaults for sentrec * update thinc * always pass a Model instance to a Pipe * fix piped_added statement * remove obsolete W029 * remove obsolete errors * restore byte checking tests (work again) * clean up test * further test cleanup * convert from config to Model in create_pipe * bring back error when component is not initialized * cleanup * remove calls for nlp2.begin_training * use thinc.api in imports * allow setting charembed's nM and nC * fix for hardcoded nM/nC + unit test * formatting fixes * trigger build
74 lines
2.0 KiB
Python
74 lines
2.0 KiB
Python
import pytest
|
|
from thinc.api import Adam
|
|
from spacy.attrs import NORM
|
|
from spacy.gold import GoldParse
|
|
from spacy.vocab import Vocab
|
|
|
|
from spacy.ml.models.defaults import default_parser
|
|
from spacy.tokens import Doc
|
|
from spacy.pipeline import DependencyParser
|
|
|
|
|
|
@pytest.fixture
|
|
def vocab():
|
|
return Vocab(lex_attr_getters={NORM: lambda s: s})
|
|
|
|
|
|
@pytest.fixture
|
|
def parser(vocab):
|
|
parser = DependencyParser(vocab, default_parser())
|
|
parser.cfg["token_vector_width"] = 4
|
|
parser.cfg["hidden_width"] = 32
|
|
# parser.add_label('right')
|
|
parser.add_label("left")
|
|
parser.begin_training([], **parser.cfg)
|
|
sgd = Adam(0.001)
|
|
|
|
for i in range(10):
|
|
losses = {}
|
|
doc = Doc(vocab, words=["a", "b", "c", "d"])
|
|
gold = GoldParse(doc, heads=[1, 1, 3, 3], deps=["left", "ROOT", "left", "ROOT"])
|
|
parser.update((doc, gold), sgd=sgd, losses=losses)
|
|
return parser
|
|
|
|
|
|
def test_no_sentences(parser):
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) >= 1
|
|
|
|
|
|
def test_sents_1(parser):
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[2].sent_start = True
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) >= 2
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[1].sent_start = False
|
|
doc[2].sent_start = True
|
|
doc[3].sent_start = False
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) == 2
|
|
|
|
|
|
def test_sents_1_2(parser):
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[1].sent_start = True
|
|
doc[2].sent_start = True
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) >= 3
|
|
|
|
|
|
def test_sents_1_3(parser):
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[1].sent_start = True
|
|
doc[3].sent_start = True
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) >= 3
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[1].sent_start = True
|
|
doc[2].sent_start = False
|
|
doc[3].sent_start = True
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) == 3
|