mirror of
https://github.com/explosion/spaCy.git
synced 2025-02-06 14:40:34 +03:00
084271c9e9
* Move get_parses_from_example to spacy.syntax * Get GoldParse out of Example * Avoid expecting GoldParse input in parser * Add Alignment to spacy.gold.align * Update Example object * Add comment * Update pipeline * Fix imports * Simplify gold_io * WIP on GoldCorpus * Update test * Xfail some gold tests * Remove ignore_misaligned option from GoldCorpus * Fix Example constructor * Update test * Fix usage of Example * Add deprecated_get_gold method on Example * Patch scorer * Fix test * Fix test * Update tests * Xfail a test * Fix passing of make_projective * Pass make_projective by default * Hack data format in Example.from_dict * Update tests * Fix example.from_dict * Update morphologizer * Fix entity linker * Add get_field to TokenAnnotation * Fix Example.get_aligned * Update test * Fix alignment * Fix corpus * Fix GoldCorpus * Handle misaligned * Format * Fix missing import
73 lines
1.9 KiB
Python
73 lines
1.9 KiB
Python
import pytest
|
|
from thinc.api import Adam
|
|
from spacy.attrs import NORM
|
|
from spacy.vocab import Vocab
|
|
|
|
from spacy.pipeline.defaults import default_parser
|
|
from spacy.tokens import Doc
|
|
from spacy.pipeline import DependencyParser
|
|
|
|
|
|
@pytest.fixture
|
|
def vocab():
|
|
return Vocab(lex_attr_getters={NORM: lambda s: s})
|
|
|
|
|
|
@pytest.fixture
|
|
def parser(vocab):
|
|
parser = DependencyParser(vocab, default_parser())
|
|
parser.cfg["token_vector_width"] = 4
|
|
parser.cfg["hidden_width"] = 32
|
|
# parser.add_label('right')
|
|
parser.add_label("left")
|
|
parser.begin_training([], **parser.cfg)
|
|
sgd = Adam(0.001)
|
|
|
|
for i in range(10):
|
|
losses = {}
|
|
doc = Doc(vocab, words=["a", "b", "c", "d"])
|
|
gold = dict(heads=[1, 1, 3, 3], deps=["left", "ROOT", "left", "ROOT"])
|
|
parser.update((doc, gold), sgd=sgd, losses=losses)
|
|
return parser
|
|
|
|
|
|
def test_no_sentences(parser):
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) >= 1
|
|
|
|
|
|
def test_sents_1(parser):
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[2].sent_start = True
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) >= 2
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[1].sent_start = False
|
|
doc[2].sent_start = True
|
|
doc[3].sent_start = False
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) == 2
|
|
|
|
|
|
def test_sents_1_2(parser):
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[1].sent_start = True
|
|
doc[2].sent_start = True
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) >= 3
|
|
|
|
|
|
def test_sents_1_3(parser):
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[1].sent_start = True
|
|
doc[3].sent_start = True
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) >= 3
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
doc[1].sent_start = True
|
|
doc[2].sent_start = False
|
|
doc[3].sent_start = True
|
|
doc = parser(doc)
|
|
assert len(list(doc.sents)) == 3
|