explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-13 18:56:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix morphologizer

Browse Source
This commit is contained in:
Matthew Honnibal 2018-09-26 21:02:13 +02:00
parent 3b6b018904
commit 1f9f834dc0
1 changed files with 42 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
spacy/_morphologizer.pyx
View File

@ -20,7 +20,7 @@ from .compat import json_dumps, basestring_
from .tokens.doc cimport Doc from .tokens.doc cimport Doc
from .vocab cimport Vocab from .vocab cimport Vocab
from .morphology cimport Morphology from .morphology cimport Morphology
from .morphology import parse_feature from .morphology import parse_feature, IDS, FIELDS, FIELD_SIZES, NAMES
from .pipeline import Pipe from .pipeline import Pipe
@ -28,9 +28,11 @@ class Morphologizer(Pipe):
name = 'morphologizer' name = 'morphologizer'
@classmethod @classmethod
def Model(cls, attr_nums, **cfg): def Model(cls, attr_nums=None, **cfg):
if cfg.get('pretrained_dims') and not cfg.get('pretrained_vectors'): if cfg.get('pretrained_dims') and not cfg.get('pretrained_vectors'):
raise ValueError(TempErrors.T008) raise ValueError(TempErrors.T008)
if attr_nums is None:
attr_nums = list(FIELD_SIZES)
return build_morphologizer_model(attr_nums, **cfg) return build_morphologizer_model(attr_nums, **cfg)
def __init__(self, vocab, model=True, **cfg): def __init__(self, vocab, model=True, **cfg):
@ -71,29 +73,34 @@ class Morphologizer(Pipe):
return guesses, tokvecs return guesses, tokvecs
tokvecs = self.model.tok2vec(docs) tokvecs = self.model.tok2vec(docs)
scores = self.model.softmax(tokvecs) scores = self.model.softmax(tokvecs)
guesses = [] return scores, tokvecs
# Resolve multisoftmax into guesses
for doc_scores in scores:
guesses.append(scores_to_guesses(doc_scores, self.model.softmax.out_sizes))
return guesses, tokvecs
def set_annotations(self, docs, batch_feature_ids, tensors=None): def set_annotations(self, docs, batch_scores, tensors=None):
if isinstance(docs, Doc): if isinstance(docs, Doc):
docs = [docs] docs = [docs]
cdef Doc doc cdef Doc doc
cdef Vocab vocab = self.vocab cdef Vocab vocab = self.vocab
field_names = list(FIELDS)
offsets = [IDS['begin_%s' % field] for field in field_names]
for i, doc in enumerate(docs): for i, doc in enumerate(docs):
doc_feat_ids = batch_feature_ids[i] doc_scores = batch_scores[i]
if hasattr(doc_feat_ids, 'get'): doc_guesses = scores_to_guesses(doc_scores, self.model.softmax.out_sizes)
doc_feat_ids = doc_feat_ids.get()
# Convert the neuron indices into feature IDs. # Convert the neuron indices into feature IDs.
offset = self.vocab.morphology.first_feature doc_feat_ids = self.model.ops.allocate((len(doc), len(field_names)), dtype='i')
for j, nr_feat in enumerate(self.model.softmax.out_sizes): for j in range(len(doc)):
doc_feat_ids[:, j] += offset for k, offset in enumerate(offsets):
offset += nr_feat if doc_guesses[j, k] == 0:
# Now add the analysis, and set the hash. doc_feat_ids[j, k] = 0
for j in range(doc_feat_ids.shape[0]): else:
doc.c[j].morph = self.vocab.morphology.add(doc_feat_ids[j]) doc_feat_ids[j, k] = offset + doc_guesses[j, k]
# Now add the analysis, and set the hash.
try:
doc.c[j].morph = self.vocab.morphology.add(doc_feat_ids[j])
except:
print(offsets)
print(doc_guesses[j])
print(doc_feat_ids[j])
raise
def update(self, docs, golds, drop=0., sgd=None, losses=None): def update(self, docs, golds, drop=0., sgd=None, losses=None):
if losses is not None and self.name not in losses: if losses is not None and self.name not in losses:
@ -110,17 +117,27 @@ class Morphologizer(Pipe):
guesses = [] guesses = []
for doc_scores in scores: for doc_scores in scores:
guesses.append(scores_to_guesses(doc_scores, self.model.softmax.out_sizes)) guesses.append(scores_to_guesses(doc_scores, self.model.softmax.out_sizes))
guesses = self.model.ops.flatten(guesses) guesses = self.model.ops.xp.vstack(guesses)
scores = self.model.ops.xp.vstack(scores)
cdef int idx = 0 cdef int idx = 0
target = numpy.zeros(scores.shape, dtype='f') target = numpy.zeros(scores.shape, dtype='f')
field_sizes = self.model.softmax.out_sizes
for gold in golds: for gold in golds:
for features in gold.morphology: for features in gold.morphology:
if features is None: if features is None:
target[idx] = guesses[idx] target[idx] = scores[idx]
else: else:
by_field = {}
for feature in features: for feature in features:
_, column = parse_feature(feature) field, column = parse_feature(feature)
target[idx, column] = 1 by_field[field] = column
col_offset = 0
for field, field_size in enumerate(field_sizes):
if field in by_field:
target[idx, col_offset + by_field[field]] = 1.
else:
target[idx, col_offset] = 1.
col_offset += field_size
idx += 1 idx += 1
target = self.model.ops.xp.array(target, dtype='f') target = self.model.ops.xp.array(target, dtype='f')
d_scores = scores - target d_scores = scores - target
@ -137,6 +154,8 @@ def scores_to_guesses(scores, out_sizes):
guesses = xp.zeros((scores.shape[0], len(out_sizes)), dtype='i') guesses = xp.zeros((scores.shape[0], len(out_sizes)), dtype='i')
offset = 0 offset = 0
for i, size in enumerate(out_sizes): for i, size in enumerate(out_sizes):
guesses[:, i] = scores[:, offset : offset + size].argmax(axis=1) slice_ = scores[:, offset : offset + size]
col_guesses = slice_.argmax(axis=1)
guesses[:, i] = col_guesses
offset += size offset += size
return guesses return guesses