explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-12 02:06:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix Sentencizer.pipe() for empty doc (#4940)

Browse Source
This commit is contained in:
adrianeboyd 2020-01-28 11:36:49 +01:00 committed by Matthew Honnibal
parent 7ad000fce7
commit a938566b62
2 changed files with 29 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spacy/pipeline/pipes.pyx
View File

@ -1492,9 +1492,10 @@ class Sentencizer(object):
return guesses return guesses
guesses = [] guesses = []
for doc in docs: for doc in docs:
doc_guesses = [False] * len(doc)
if len(doc) > 0:
start = 0 start = 0
seen_period = False seen_period = False
doc_guesses = [False] * len(doc)
doc_guesses[0] = True doc_guesses[0] = True
for i, token in enumerate(doc): for i, token in enumerate(doc):
is_in_punct_chars = token.text in self.punct_chars is_in_punct_chars = token.text in self.punct_chars

16
spacy/tests/pipeline/test_sentencizer.py
View File

@ -29,6 +29,22 @@ def test_sentencizer_pipe():
assert len(list(doc.sents)) == 2 assert len(list(doc.sents)) == 2
def test_sentencizer_empty_docs():
one_empty_text = [""]
many_empty_texts = ["", "", ""]
some_empty_texts = ["hi", "", "This is a test. Here are two sentences.", ""]
nlp = English()
nlp.add_pipe(nlp.create_pipe("sentencizer"))
for texts in [one_empty_text, many_empty_texts, some_empty_texts]:
for doc in nlp.pipe(texts):
assert doc.is_sentenced
sent_starts = [t.is_sent_start for t in doc]
if len(doc) == 0:
assert sent_starts == []
else:
assert len(sent_starts) > 0
@pytest.mark.parametrize( @pytest.mark.parametrize(
"words,sent_starts,n_sents", "words,sent_starts,n_sents",
[ [