explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 09:26:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Improve piping in language.pipe

Browse Source
This commit is contained in:
Matthew Honnibal 2017-10-18 21:46:12 +02:00
parent 633a75c7e0
commit 65bf5e85bd
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
spacy/language.py
View File

@ -10,6 +10,7 @@ from collections import OrderedDict
import itertools
import weakref
import functools
import tqdm
from .tokenizer import Tokenizer
from .vocab import Vocab
@ -447,11 +448,9 @@ class Language(object):
golds = list(golds)
for name, pipe in self.pipeline:
if not hasattr(pipe, 'pipe'):
for doc in docs:
pipe(doc)
docs = (pipe(doc) for doc in docs)
else:
docs = list(pipe.pipe(docs))
assert len(docs) == len(golds)
docs = pipe.pipe(docs, batch_size=256)
for doc, gold in zip(docs, golds):
if verbose:
print(doc)