explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-07-14 18:22:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

learn rate en epochs

Browse Source
This commit is contained in:
svlandeg 2019-05-24 22:04:25 +02:00
parent 86ed771e0b
commit abf9af81c9
2 changed files with 48 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
examples/pipeline/wiki_entity_linking/train_el.py
View File

@ -34,6 +34,7 @@ class EL_Model:
CUTOFF = 0.5 CUTOFF = 0.5
BATCH_SIZE = 5 BATCH_SIZE = 5
UPSAMPLE = True
DOC_CUTOFF = 300 # number of characters from the doc context DOC_CUTOFF = 300 # number of characters from the doc context
INPUT_DIM = 300 # dimension of pre-trained vectors INPUT_DIM = 300 # dimension of pre-trained vectors
@ -45,6 +46,8 @@ class EL_Model:
SENT_WIDTH = 64 SENT_WIDTH = 64
DROP = 0.1 DROP = 0.1
LEARN_RATE = 0.01
EPOCHS = 10
name = "entity_linker" name = "entity_linker"
@ -67,6 +70,12 @@ class EL_Model:
train_ent, train_gold, train_desc, train_art, train_art_texts, train_sent, train_sent_texts = \ train_ent, train_gold, train_desc, train_art, train_art_texts, train_sent, train_sent_texts = \
self._get_training_data(training_dir, entity_descr_output, False, trainlimit, to_print=False) self._get_training_data(training_dir, entity_descr_output, False, trainlimit, to_print=False)
dev_ent, dev_gold, dev_desc, dev_art, dev_art_texts, dev_sent, dev_sent_texts = \
self._get_training_data(training_dir, entity_descr_output, True, devlimit, to_print=False)
dev_pos_count = len([g for g in dev_gold.values() if g])
dev_neg_count = len([g for g in dev_gold.values() if not g])
# inspect data # inspect data
if self.PRINT_INSPECT: if self.PRINT_INSPECT:
for entity in train_ent: for entity in train_ent:
@ -85,28 +94,20 @@ class EL_Model:
train_pos_count = len(train_pos_entities) train_pos_count = len(train_pos_entities)
train_neg_count = len(train_neg_entities) train_neg_count = len(train_neg_entities)
if to_print: if self.UPSAMPLE:
print() if to_print:
print("Upsampling, original training instances pos/neg:", train_pos_count, train_neg_count) print()
print("Upsampling, original training instances pos/neg:", train_pos_count, train_neg_count)
# upsample positives to 50-50 distribution # upsample positives to 50-50 distribution
while train_pos_count < train_neg_count: while train_pos_count < train_neg_count:
train_ent.append(random.choice(train_pos_entities)) train_ent.append(random.choice(train_pos_entities))
train_pos_count += 1 train_pos_count += 1
# upsample negatives to 50-50 distribution # upsample negatives to 50-50 distribution
while train_neg_count < train_pos_count: while train_neg_count < train_pos_count:
train_ent.append(random.choice(train_neg_entities)) train_ent.append(random.choice(train_neg_entities))
train_neg_count += 1 train_neg_count += 1
shuffle(train_ent)
dev_ent, dev_gold, dev_desc, dev_art, dev_art_texts, dev_sent, dev_sent_texts = \
self._get_training_data(training_dir, entity_descr_output, True, devlimit, to_print=False)
shuffle(dev_ent)
dev_pos_count = len([g for g in dev_gold.values() if g])
dev_neg_count = len([g for g in dev_gold.values() if not g])
self._begin_training() self._begin_training()
@ -135,30 +136,34 @@ class EL_Model:
print_string="dev_pre", avg=True) print_string="dev_pre", avg=True)
print() print()
start = 0 for i in range(self.EPOCHS):
stop = min(self.BATCH_SIZE, len(train_ent)) print("EPOCH", i)
processed = 0 shuffle(train_ent)
while start < len(train_ent): start = 0
next_batch = train_ent[start:stop] stop = min(self.BATCH_SIZE, len(train_ent))
processed = 0
golds = [train_gold[e] for e in next_batch] while start < len(train_ent):
descs = [train_desc[e] for e in next_batch] next_batch = train_ent[start:stop]
article_texts = [train_art_texts[train_art[e]] for e in next_batch]
sent_texts = [train_sent_texts[train_sent[e]] for e in next_batch]
self.update(entities=next_batch, golds=golds, descs=descs, art_texts=article_texts, sent_texts=sent_texts) golds = [train_gold[e] for e in next_batch]
self._test_dev(dev_ent, dev_gold, dev_desc, dev_art, dev_art_texts, dev_sent, dev_sent_texts, descs = [train_desc[e] for e in next_batch]
print_string="dev_inter", avg=True) article_texts = [train_art_texts[train_art[e]] for e in next_batch]
sent_texts = [train_sent_texts[train_sent[e]] for e in next_batch]
processed += len(next_batch) self.update(entities=next_batch, golds=golds, descs=descs, art_texts=article_texts, sent_texts=sent_texts)
self._test_dev(dev_ent, dev_gold, dev_desc, dev_art, dev_art_texts, dev_sent, dev_sent_texts,
print_string="dev_inter", avg=True)
start = start + self.BATCH_SIZE processed += len(next_batch)
stop = min(stop + self.BATCH_SIZE, len(train_ent))
if to_print: start = start + self.BATCH_SIZE
print() stop = min(stop + self.BATCH_SIZE, len(train_ent))
print("Trained on", processed, "entities in total")
if to_print:
print()
print("Trained on", processed, "entities in total")
def _test_dev(self, entities, gold_by_entity, desc_by_entity, art_by_entity, art_texts, sent_by_entity, sent_texts, def _test_dev(self, entities, gold_by_entity, desc_by_entity, art_by_entity, art_texts, sent_by_entity, sent_texts,
print_string, avg=True, calc_random=False): print_string, avg=True, calc_random=False):
@ -257,9 +262,13 @@ class EL_Model:
def _begin_training(self): def _begin_training(self):
self.sgd_article = create_default_optimizer(self.article_encoder.ops) self.sgd_article = create_default_optimizer(self.article_encoder.ops)
self.sgd_article.learn_rate = self.LEARN_RATE
self.sgd_sent = create_default_optimizer(self.sent_encoder.ops) self.sgd_sent = create_default_optimizer(self.sent_encoder.ops)
self.sgd_sent.learn_rate = self.LEARN_RATE
self.sgd_desc = create_default_optimizer(self.desc_encoder.ops) self.sgd_desc = create_default_optimizer(self.desc_encoder.ops)
self.sgd_desc.learn_rate = self.LEARN_RATE
self.sgd = create_default_optimizer(self.model.ops) self.sgd = create_default_optimizer(self.model.ops)
self.sgd.learn_rate = self.LEARN_RATE
@staticmethod @staticmethod
def get_loss(predictions, golds): def get_loss(predictions, golds):

2
examples/pipeline/wiki_entity_linking/wiki_nel_pipeline.py
View File

@ -111,7 +111,7 @@ if __name__ == "__main__":
print("STEP 6: training", datetime.datetime.now()) print("STEP 6: training", datetime.datetime.now())
my_nlp = spacy.load('en_core_web_md') my_nlp = spacy.load('en_core_web_md')
trainer = EL_Model(kb=my_kb, nlp=my_nlp) trainer = EL_Model(kb=my_kb, nlp=my_nlp)
trainer.train_model(training_dir=TRAINING_DIR, entity_descr_output=ENTITY_DESCR, trainlimit=1000, devlimit=50) trainer.train_model(training_dir=TRAINING_DIR, entity_descr_output=ENTITY_DESCR, trainlimit=20, devlimit=20)
print() print()
# STEP 7: apply the EL algorithm on the dev dataset # STEP 7: apply the EL algorithm on the dev dataset