explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-07-10 00:02:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix for context encoder optimizer

Browse Source
This commit is contained in:
svlandeg 2019-07-03 13:35:36 +02:00
parent 3420cbe496
commit 8840d4b1b3
2 changed files with 23 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
examples/pipeline/wikidata_entity_linking.py
View File

@ -73,11 +73,11 @@ def run_pipeline():
measure_performance = True measure_performance = True
# test the EL pipe on a simple example # test the EL pipe on a simple example
to_test_pipeline = False to_test_pipeline = True
# write the NLP object, read back in and test again # write the NLP object, read back in and test again
to_write_nlp = False to_write_nlp = True
to_read_nlp = False to_read_nlp = True
test_from_file = False test_from_file = False
# STEP 1 : create prior probabilities from WP (run only once) # STEP 1 : create prior probabilities from WP (run only once)
@ -154,8 +154,8 @@ def run_pipeline():
optimizer.L2 = L2 optimizer.L2 = L2
# define the size (nr of entities) of training and dev set # define the size (nr of entities) of training and dev set
train_limit = 50000 train_limit = 5
dev_limit = 50000 dev_limit = 5
train_data = training_set_creator.read_training(nlp=nlp_2, train_data = training_set_creator.read_training(nlp=nlp_2,
training_dir=TRAINING_DIR, training_dir=TRAINING_DIR,
@ -250,7 +250,8 @@ def run_pipeline():
print("STEP 9: testing NLP IO", datetime.datetime.now()) print("STEP 9: testing NLP IO", datetime.datetime.now())
print() print()
print("writing to", NLP_2_DIR) print("writing to", NLP_2_DIR)
nlp_2.to_disk(NLP_2_DIR) with el_pipe.model.use_params(optimizer.averages) and el_pipe.model.tok2vec.use_params(el_pipe.sgd_context.averages):
nlp_2.to_disk(NLP_2_DIR)
print() print()
# verify that the IO has gone correctly # verify that the IO has gone correctly

31
spacy/pipeline/pipes.pyx
View File

@ -1082,12 +1082,8 @@ class EntityLinker(Pipe):
def __init__(self, **cfg): def __init__(self, **cfg):
self.model = True self.model = True
self.kb = None self.kb = None
self.sgd_context = None
self.cfg = dict(cfg) self.cfg = dict(cfg)
self.context_weight = cfg.get("context_weight", 1) self.sgd_context = None
self.prior_weight = cfg.get("prior_weight", 1)
self.context_width = cfg.get("context_width")
self.type_to_int = cfg.get("type_to_int", dict())
def set_kb(self, kb): def set_kb(self, kb):
self.kb = kb self.kb = kb
@ -1112,6 +1108,7 @@ class EntityLinker(Pipe):
if sgd is None: if sgd is None:
sgd = self.create_optimizer() sgd = self.create_optimizer()
return sgd return sgd
def update(self, docs, golds, state=None, drop=0.0, sgd=None, losses=None): def update(self, docs, golds, state=None, drop=0.0, sgd=None, losses=None):
@ -1138,6 +1135,8 @@ class EntityLinker(Pipe):
priors = [] priors = []
type_vectors = [] type_vectors = []
type_to_int = self.cfg.get("type_to_int", dict())
for doc, gold in zip(docs, golds): for doc, gold in zip(docs, golds):
ents_by_offset = dict() ents_by_offset = dict()
for ent in doc.ents: for ent in doc.ents:
@ -1148,9 +1147,9 @@ class EntityLinker(Pipe):
gold_ent = ents_by_offset[str(ent.start_char) + "_" + str(ent.end_char)] gold_ent = ents_by_offset[str(ent.start_char) + "_" + str(ent.end_char)]
assert gold_ent is not None assert gold_ent is not None
type_vector = [0 for i in range(len(self.type_to_int))] type_vector = [0 for i in range(len(type_to_int))]
if len(self.type_to_int) > 0: if len(type_to_int) > 0:
type_vector[self.type_to_int[gold_ent.label_]] = 1 type_vector[type_to_int[gold_ent.label_]] = 1
candidates = self.kb.get_candidates(mention) candidates = self.kb.get_candidates(mention)
random.shuffle(candidates) random.shuffle(candidates)
@ -1162,7 +1161,7 @@ class EntityLinker(Pipe):
context_docs.append(doc) context_docs.append(doc)
type_vectors.append(type_vector) type_vectors.append(type_vector)
if self.prior_weight > 0: if self.cfg.get("prior_weight", 1) > 0:
priors.append([c.prior_prob]) priors.append([c.prior_prob])
else: else:
priors.append([0]) priors.append([0])
@ -1187,7 +1186,7 @@ class EntityLinker(Pipe):
loss, d_scores = self.get_loss(prediction=pred, golds=cats, docs=None) loss, d_scores = self.get_loss(prediction=pred, golds=cats, docs=None)
mention_gradient = bp_mention(d_scores, sgd=sgd) mention_gradient = bp_mention(d_scores, sgd=sgd)
context_gradients = [list(x[0:self.context_width]) for x in mention_gradient] context_gradients = [list(x[0:self.cfg.get("context_width")]) for x in mention_gradient]
bp_context(self.model.ops.asarray(context_gradients, dtype="float32"), sgd=self.sgd_context) bp_context(self.model.ops.asarray(context_gradients, dtype="float32"), sgd=self.sgd_context)
if losses is not None: if losses is not None:
@ -1235,13 +1234,15 @@ class EntityLinker(Pipe):
context_encodings = self.model.tok2vec(docs) context_encodings = self.model.tok2vec(docs)
xp = get_array_module(context_encodings) xp = get_array_module(context_encodings)
type_to_int = self.cfg.get("type_to_int", dict())
for i, doc in enumerate(docs): for i, doc in enumerate(docs):
if len(doc) > 0: if len(doc) > 0:
context_encoding = context_encodings[i] context_encoding = context_encodings[i]
for ent in doc.ents: for ent in doc.ents:
type_vector = [0 for i in range(len(self.type_to_int))] type_vector = [0 for i in range(len(type_to_int))]
if len(self.type_to_int) > 0: if len(type_to_int) > 0:
type_vector[self.type_to_int[ent.label_]] = 1 type_vector[type_to_int[ent.label_]] = 1
candidates = self.kb.get_candidates(ent.text) candidates = self.kb.get_candidates(ent.text)
if candidates: if candidates:
@ -1249,10 +1250,10 @@ class EntityLinker(Pipe):
# this will set the prior probabilities to 0 (just like in training) if their weight is 0 # this will set the prior probabilities to 0 (just like in training) if their weight is 0
prior_probs = xp.asarray([[c.prior_prob] for c in candidates]) prior_probs = xp.asarray([[c.prior_prob] for c in candidates])
prior_probs *= self.prior_weight prior_probs *= self.cfg.get("prior_weight", 1)
scores = prior_probs scores = prior_probs
if self.context_weight > 0: if self.cfg.get("context_weight", 1) > 0:
entity_encodings = xp.asarray([c.entity_vector for c in candidates]) entity_encodings = xp.asarray([c.entity_vector for c in candidates])
assert len(entity_encodings) == len(prior_probs) assert len(entity_encodings) == len(prior_probs)
mention_encodings = [list(context_encoding) + list(entity_encodings[i]) mention_encodings = [list(context_encoding) + list(entity_encodings[i])