spaCy/spacy/language.py

396 lines
15 KiB
Python
Raw Normal View History

# coding: utf8
from __future__ import absolute_import, unicode_literals
from contextlib import contextmanager
import shutil
from .tokenizer import Tokenizer
from .vocab import Vocab
from .tagger import Tagger
from .matcher import Matcher
from .lemmatizer import Lemmatizer
from .train import Trainer
from .syntax.parser import get_templates
from .syntax.nonproj import PseudoProjectivity
from .pipeline import DependencyParser, EntityRecognizer
from .syntax.arc_eager import ArcEager
from .syntax.ner import BiluoPushDown
2017-04-17 02:46:14 +03:00
from .compat import json_dumps
from .attrs import IS_STOP
from . import attrs
from . import orth
from . import util
from . import language_data
2016-09-24 21:26:17 +03:00
class BaseDefaults(object):
2016-10-18 17:18:25 +03:00
@classmethod
def create_lemmatizer(cls, nlp=None):
return Lemmatizer(cls.lemma_index, cls.lemma_exc, cls.lemma_rules)
2016-10-18 17:18:25 +03:00
@classmethod
def create_vocab(cls, nlp=None):
lemmatizer = cls.create_lemmatizer(nlp)
if nlp is None or nlp.path is None:
lex_attr_getters = dict(cls.lex_attr_getters)
# This is very messy, but it's the minimal working fix to Issue #639.
# This defaults stuff needs to be refactored (again)
lex_attr_getters[IS_STOP] = lambda string: string.lower() in cls.stop_words
2017-03-15 17:24:40 +03:00
vocab = Vocab(lex_attr_getters=lex_attr_getters, tag_map=cls.tag_map,
2016-10-18 17:18:25 +03:00
lemmatizer=lemmatizer)
else:
2017-03-15 17:24:40 +03:00
vocab = Vocab.load(nlp.path, lex_attr_getters=cls.lex_attr_getters,
2016-10-18 17:18:25 +03:00
tag_map=cls.tag_map, lemmatizer=lemmatizer)
2017-03-15 17:24:40 +03:00
for tag_str, exc in cls.morph_rules.items():
for orth_str, attrs in exc.items():
vocab.morphology.add_special_case(tag_str, orth_str, attrs)
return vocab
2016-12-18 18:54:52 +03:00
2016-10-18 17:18:25 +03:00
@classmethod
def add_vectors(cls, nlp=None):
if nlp is None or nlp.path is None:
return False
else:
vec_path = nlp.path / 'vocab' / 'vec.bin'
if vec_path.exists():
return lambda vocab: vocab.load_vectors_from_bin_loc(vec_path)
2016-10-18 17:18:25 +03:00
@classmethod
def create_tokenizer(cls, nlp=None):
rules = cls.tokenizer_exceptions
if cls.token_match:
token_match = cls.token_match
if cls.prefixes:
prefix_search = util.compile_prefix_regex(cls.prefixes).search
else:
prefix_search = None
if cls.suffixes:
suffix_search = util.compile_suffix_regex(cls.suffixes).search
else:
suffix_search = None
if cls.infixes:
infix_finditer = util.compile_infix_regex(cls.infixes).finditer
else:
infix_finditer = None
2016-10-18 17:18:25 +03:00
vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)
2016-11-26 14:36:04 +03:00
return Tokenizer(vocab, rules=rules,
2016-10-18 17:18:25 +03:00
prefix_search=prefix_search, suffix_search=suffix_search,
infix_finditer=infix_finditer, token_match=token_match)
2016-10-18 17:18:25 +03:00
@classmethod
def create_tagger(cls, nlp=None):
if nlp is None:
return Tagger(cls.create_vocab(), features=cls.tagger_features)
2016-10-23 21:19:01 +03:00
elif nlp.path is False:
2016-10-18 17:18:25 +03:00
return Tagger(nlp.vocab, features=cls.tagger_features)
2016-10-23 21:19:01 +03:00
elif nlp.path is None or not (nlp.path / 'pos').exists():
return None
else:
2016-10-18 20:33:04 +03:00
return Tagger.load(nlp.path / 'pos', nlp.vocab)
2016-10-18 17:18:25 +03:00
@classmethod
def create_parser(cls, nlp=None, **cfg):
2016-10-18 17:18:25 +03:00
if nlp is None:
return DependencyParser(cls.create_vocab(), features=cls.parser_features,
**cfg)
2016-10-23 21:19:01 +03:00
elif nlp.path is False:
return DependencyParser(nlp.vocab, features=cls.parser_features, **cfg)
2016-10-23 21:19:01 +03:00
elif nlp.path is None or not (nlp.path / 'deps').exists():
return None
else:
return DependencyParser.load(nlp.path / 'deps', nlp.vocab, **cfg)
2016-10-18 17:18:25 +03:00
@classmethod
def create_entity(cls, nlp=None, **cfg):
2016-10-18 17:18:25 +03:00
if nlp is None:
return EntityRecognizer(cls.create_vocab(), features=cls.entity_features, **cfg)
2016-10-23 21:19:01 +03:00
elif nlp.path is False:
return EntityRecognizer(nlp.vocab, features=cls.entity_features, **cfg)
2016-10-23 21:19:01 +03:00
elif nlp.path is None or not (nlp.path / 'ner').exists():
return None
else:
return EntityRecognizer.load(nlp.path / 'ner', nlp.vocab, **cfg)
2016-10-18 17:18:25 +03:00
@classmethod
def create_matcher(cls, nlp=None):
if nlp is None:
return Matcher(cls.create_vocab())
2016-10-23 21:19:01 +03:00
elif nlp.path is False:
2016-10-18 17:18:25 +03:00
return Matcher(nlp.vocab)
2016-10-23 21:19:01 +03:00
elif nlp.path is None or not (nlp.path / 'vocab').exists():
return None
2016-10-18 17:18:25 +03:00
else:
return Matcher.load(nlp.path / 'vocab', nlp.vocab)
2016-10-18 17:18:25 +03:00
@classmethod
def create_pipeline(self, nlp=None):
pipeline = []
2016-10-18 17:18:25 +03:00
if nlp is None:
return []
if nlp.tagger:
pipeline.append(nlp.tagger)
if nlp.parser:
pipeline.append(nlp.parser)
pipeline.append(PseudoProjectivity.deprojectivize)
if nlp.entity:
pipeline.append(nlp.entity)
return pipeline
token_match = language_data.TOKEN_MATCH
2016-12-18 18:55:25 +03:00
prefixes = tuple(language_data.TOKENIZER_PREFIXES)
suffixes = tuple(language_data.TOKENIZER_SUFFIXES)
infixes = tuple(language_data.TOKENIZER_INFIXES)
tag_map = dict(language_data.TAG_MAP)
tokenizer_exceptions = {}
parser_features = get_templates('parser')
entity_features = get_templates('ner')
2016-10-18 17:18:25 +03:00
tagger_features = Tagger.feature_templates # TODO -- fix this
2016-09-24 21:26:17 +03:00
stop_words = set()
2016-12-18 17:50:09 +03:00
lemma_rules = {}
lemma_exc = {}
lemma_index = {}
2017-03-15 17:24:40 +03:00
morph_rules = {}
2016-12-18 17:50:09 +03:00
lex_attr_getters = {
attrs.LOWER: lambda string: string.lower(),
attrs.NORM: lambda string: string,
attrs.SHAPE: orth.word_shape,
attrs.PREFIX: lambda string: string[0],
attrs.SUFFIX: lambda string: string[-3:],
attrs.CLUSTER: lambda string: 0,
attrs.IS_ALPHA: orth.is_alpha,
attrs.IS_ASCII: orth.is_ascii,
attrs.IS_DIGIT: lambda string: string.isdigit(),
attrs.IS_LOWER: orth.is_lower,
attrs.IS_PUNCT: orth.is_punct,
attrs.IS_SPACE: lambda string: string.isspace(),
attrs.IS_TITLE: orth.is_title,
attrs.IS_UPPER: orth.is_upper,
attrs.IS_BRACKET: orth.is_bracket,
attrs.IS_QUOTE: orth.is_quote,
attrs.IS_LEFT_PUNCT: orth.is_left_punct,
attrs.IS_RIGHT_PUNCT: orth.is_right_punct,
attrs.LIKE_URL: orth.like_url,
attrs.LIKE_NUM: orth.like_number,
attrs.LIKE_EMAIL: orth.like_email,
attrs.IS_STOP: lambda string: False,
attrs.IS_OOV: lambda string: True
}
2015-09-14 10:48:51 +03:00
class Language(object):
"""
A text-processing pipeline. Usually you'll load this once per process, and
pass the instance around your program.
"""
2016-09-24 21:26:17 +03:00
Defaults = BaseDefaults
lang = None
2015-08-25 16:37:17 +03:00
@classmethod
def setup_directory(cls, path, **configs):
2017-04-17 02:40:26 +03:00
"""
Initialise a model directory.
"""
for name, config in configs.items():
directory = path / name
if directory.exists():
shutil.rmtree(str(directory))
directory.mkdir()
2017-04-23 18:05:53 +03:00
with (directory / 'config.json').open('w') as file_:
2017-04-17 02:46:14 +03:00
data = json_dumps(config)
file_.write(data)
if not (path / 'vocab').exists():
(path / 'vocab').mkdir()
@classmethod
@contextmanager
def train(cls, path, gold_tuples, **configs):
2017-04-16 21:00:37 +03:00
parser_cfg = configs.get('deps', {})
if parser_cfg.get('pseudoprojective'):
# preprocess training data here before ArcEager.get_labels() is called
gold_tuples = PseudoProjectivity.preprocess_training_data(gold_tuples)
for subdir in ('deps', 'ner', 'pos'):
if subdir not in configs:
configs[subdir] = {}
2017-04-16 21:00:37 +03:00
if parser_cfg:
configs['deps']['actions'] = ArcEager.get_actions(gold_parses=gold_tuples)
if 'ner' in configs:
configs['ner']['actions'] = BiluoPushDown.get_actions(gold_parses=gold_tuples)
cls.setup_directory(path, **configs)
self = cls(
path=path,
vocab=False,
tokenizer=False,
tagger=False,
parser=False,
entity=False,
matcher=False,
serializer=False,
vectors=False,
pipeline=False)
self.vocab = self.Defaults.create_vocab(self)
self.tokenizer = self.Defaults.create_tokenizer(self)
self.tagger = self.Defaults.create_tagger(self)
self.parser = self.Defaults.create_parser(self)
self.entity = self.Defaults.create_entity(self)
self.pipeline = self.Defaults.create_pipeline(self)
yield Trainer(self, gold_tuples)
self.end_training()
2017-04-16 23:28:27 +03:00
self.save_to_directory(path)
def __init__(self, **overrides):
2017-04-17 02:52:13 +03:00
"""
Create or load the pipeline.
Arguments:
**overrides: Keyword arguments indicating which defaults to override.
Returns:
Language: The newly constructed object.
"""
2016-11-23 18:12:45 +03:00
if 'data_dir' in overrides and 'path' not in overrides:
2016-10-18 17:18:25 +03:00
raise ValueError("The argument 'data_dir' has been renamed to 'path'")
path = util.ensure_path(overrides.get('path', True))
2016-10-15 15:12:54 +03:00
if path is True:
path = util.get_data_path() / self.lang
2017-03-26 17:46:00 +03:00
if not path.exists() and 'path' not in overrides:
path = None
self.meta = overrides.get('meta', {})
self.path = path
2016-12-18 18:54:52 +03:00
2016-10-18 17:18:25 +03:00
self.vocab = self.Defaults.create_vocab(self) \
if 'vocab' not in overrides \
else overrides['vocab']
add_vectors = self.Defaults.add_vectors(self) \
if 'add_vectors' not in overrides \
else overrides['add_vectors']
if self.vocab and add_vectors:
add_vectors(self.vocab)
2016-10-18 17:18:25 +03:00
self.tokenizer = self.Defaults.create_tokenizer(self) \
if 'tokenizer' not in overrides \
else overrides['tokenizer']
self.tagger = self.Defaults.create_tagger(self) \
if 'tagger' not in overrides \
else overrides['tagger']
2016-10-18 20:36:44 +03:00
self.parser = self.Defaults.create_parser(self) \
2016-10-18 17:18:25 +03:00
if 'parser' not in overrides \
else overrides['parser']
self.entity = self.Defaults.create_entity(self) \
if 'entity' not in overrides \
else overrides['entity']
self.matcher = self.Defaults.create_matcher(self) \
if 'matcher' not in overrides \
else overrides['matcher']
if 'make_doc' in overrides:
self.make_doc = overrides['make_doc']
elif 'create_make_doc' in overrides:
2016-10-23 15:24:16 +03:00
self.make_doc = overrides['create_make_doc'](self)
elif not hasattr(self, 'make_doc'):
2016-10-18 17:18:25 +03:00
self.make_doc = lambda text: self.tokenizer(text)
if 'pipeline' in overrides:
self.pipeline = overrides['pipeline']
elif 'create_pipeline' in overrides:
2016-10-23 15:24:16 +03:00
self.pipeline = overrides['create_pipeline'](self)
else:
2016-10-18 17:18:25 +03:00
self.pipeline = [self.tagger, self.parser, self.matcher, self.entity]
def __call__(self, text, tag=True, parse=True, entity=True):
"""
Apply the pipeline to some text. The text can span multiple sentences,
2015-08-25 16:37:17 +03:00
and can contain arbtrary whitespace. Alignment into the original string
is preserved.
2016-12-18 18:54:52 +03:00
2017-04-17 02:40:26 +03:00
Argsuments:
2015-08-25 16:37:17 +03:00
text (unicode): The text to be processed.
Returns:
2016-11-01 14:25:36 +03:00
doc (Doc): A container for accessing the annotations.
Example:
>>> from spacy.en import English
>>> nlp = English()
>>> tokens = nlp('An example sentence. Another example sentence.')
>>> tokens[0].orth_, tokens[0].head.tag_
('An', 'NN')
2015-08-25 16:37:17 +03:00
"""
doc = self.make_doc(text)
2015-08-25 16:37:17 +03:00
if self.entity and entity:
# Add any of the entity labels already set, in case we don't have them.
for token in doc:
if token.ent_type != 0:
self.entity.add_label(token.ent_type)
skip = {self.tagger: not tag, self.parser: not parse, self.entity: not entity}
for proc in self.pipeline:
if proc and not skip.get(proc):
proc(doc)
return doc
2015-08-25 16:37:17 +03:00
def pipe(self, texts, tag=True, parse=True, entity=True, n_threads=2, batch_size=1000):
"""
Process texts as a stream, and yield Doc objects in order.
2016-12-18 18:54:52 +03:00
2016-11-01 14:25:36 +03:00
Supports GIL-free multi-threading.
2016-12-18 18:54:52 +03:00
2016-11-01 14:25:36 +03:00
Arguments:
texts (iterator)
tag (bool)
parse (bool)
entity (bool)
"""
skip = {self.tagger: not tag, self.parser: not parse, self.entity: not entity}
stream = (self.make_doc(text) for text in texts)
for proc in self.pipeline:
if proc and not skip.get(proc):
if hasattr(proc, 'pipe'):
stream = proc.pipe(stream, n_threads=n_threads, batch_size=batch_size)
else:
stream = (proc(item) for item in stream)
for doc in stream:
yield doc
def save_to_directory(self, path):
2017-04-17 02:40:26 +03:00
"""
Save the Vocab, StringStore and pipeline to a directory.
Arguments:
path (string or pathlib path): Path to save the model.
"""
configs = {
'pos': self.tagger.cfg if self.tagger else {},
'deps': self.parser.cfg if self.parser else {},
'ner': self.entity.cfg if self.entity else {},
}
2016-12-18 18:54:52 +03:00
2017-04-17 02:40:14 +03:00
path = util.ensure_path(path)
if not path.exists():
path.mkdir()
self.setup_directory(path, **configs)
strings_loc = path / 'vocab' / 'strings.json'
with strings_loc.open('w', encoding='utf8') as file_:
self.vocab.strings.dump(file_)
self.vocab.dump(path / 'vocab' / 'lexemes.bin')
# TODO: Word vectors?
if self.tagger:
self.tagger.model.dump(str(path / 'pos' / 'model'))
if self.parser:
2016-10-12 21:26:38 +03:00
self.parser.model.dump(str(path / 'deps' / 'model'))
if self.entity:
self.entity.model.dump(str(path / 'ner' / 'model'))
2016-12-18 18:54:52 +03:00
def end_training(self, path=None):
if self.tagger:
self.tagger.model.end_training()
if self.parser:
self.parser.model.end_training()
if self.entity:
self.entity.model.end_training()
# NB: This is slightly different from before --- we no longer default
# to taking nlp.path
if path is not None:
self.save_to_directory(path)