diff --git a/examples/train_ner.py b/examples/train_ner.py deleted file mode 100644 index 3f748a488..000000000 --- a/examples/train_ner.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import unicode_literals, print_function -import json -import pathlib -import random - -import spacy -from spacy.pipeline import EntityRecognizer -from spacy.gold import GoldParse - - -def train_ner(nlp, train_data, entity_types): - ner = EntityRecognizer.blank(nlp.vocab, entity_types=entity_types, - features=nlp.Defaults.entity_features) - for itn in range(5): - random.shuffle(train_data) - for raw_text, entity_offsets in train_data: - doc = nlp.make_doc(raw_text) - gold = GoldParse(doc, entities=entity_offsets) - ner.update(doc, gold) - ner.model.end_training() - return ner - - -def main(model_dir=None): - if model_dir is not None: - model_dir = pathlb.Path(model_dir) - if not model_dir.exists(): - model_dir.mkdir() - assert model_dir.isdir() - - nlp = spacy.load('en', parser=False, entity=False, vectors=False) - - train_data = [ - ( - 'Who is Shaka Khan?', - [(len('Who is '), len('Who is Shaka Khan'), 'PERSON')] - ), - ( - 'I like London and Berlin.', - [(len('I like '), len('I like London'), 'LOC'), - (len('I like London and '), len('I like London and Berlin'), 'LOC')] - ) - ] - ner = train_ner(nlp, train_data, ['PERSON', 'LOC']) - - doc = nlp.make_doc('Who is Shaka Khan?') - nlp.tagger(doc) - ner(doc) - for word in doc: - print(word.text, word.tag_, word.ent_type_, word.ent_iob) - - if model_dir is not None: - with (model_dir / 'config.json').open('wb') as file_: - json.dump(ner.cfg, file_) - ner.model.dump(str(model_dir / 'model')) - - -if __name__ == '__main__': - main() - # Who "" 2 - # is "" 2 - # Shaka "" PERSON 3 - # Khan "" PERSON 1 - # ? "" 2 diff --git a/examples/train_pos_tagger.py b/examples/train_pos_tagger.py deleted file mode 100644 index 43bd607c7..000000000 --- a/examples/train_pos_tagger.py +++ /dev/null @@ -1,72 +0,0 @@ -"""A quick example for training a part-of-speech tagger, without worrying -about the tokenization, or other language-specific customizations.""" - -from __future__ import unicode_literals -from __future__ import print_function - -import plac -from os import path -import os - -from spacy.vocab import Vocab -from spacy.tokenizer import Tokenizer -from spacy.tagger import Tagger -import random - - -# You need to define a mapping from your data's part-of-speech tag names to the -# Universal Part-of-Speech tag set, as spaCy includes an enum of these tags. -# See here for the Universal Tag Set: -# http://universaldependencies.github.io/docs/u/pos/index.html -# You may also specify morphological features for your tags, from the universal -# scheme. -TAG_MAP = { - 'N': {"pos": "NOUN"}, - 'V': {"pos": "VERB"}, - 'J': {"pos": "ADJ"} - } - -# Usually you'll read this in, of course. Data formats vary. -# Ensure your strings are unicode. -DATA = [ - ( - ["I", "like", "green", "eggs"], - ["N", "V", "J", "N"] - ), - ( - ["Eat", "blue", "ham"], - ["V", "J", "N"] - ) -] - -def ensure_dir(*parts): - path_ = path.join(*parts) - if not path.exists(path_): - os.mkdir(path_) - return path_ - - -def main(output_dir): - ensure_dir(output_dir) - ensure_dir(output_dir, "pos") - ensure_dir(output_dir, "vocab") - - vocab = Vocab(tag_map=TAG_MAP) - tokenizer = Tokenizer(vocab, {}, None, None, None) - # The default_templates argument is where features are specified. See - # spacy/tagger.pyx for the defaults. - tagger = Tagger.blank(vocab, Tagger.default_templates()) - - for i in range(5): - for words, tags in DATA: - tokens = tokenizer.tokens_from_list(words) - tagger.train(tokens, tags) - random.shuffle(DATA) - tagger.model.end_training() - tagger.model.dump(path.join(output_dir, 'pos', 'model')) - with io.open(output_dir, 'vocab', 'strings.json') as file_: - tagger.vocab.strings.dump(file_) - - -if __name__ == '__main__': - plac.call(main)