From 052c45dc2ff1c73ad208ae17c6a3cd6faefbd1f5 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Mon, 24 Sep 2018 15:25:20 +0200 Subject: [PATCH 001/287] Add as_int and as_string methods to StringStore --- spacy/strings.pyx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/spacy/strings.pyx b/spacy/strings.pyx index b54e3f155..4c6b5e1bb 100644 --- a/spacy/strings.pyx +++ b/spacy/strings.pyx @@ -98,9 +98,7 @@ cdef class StringStore: return u'' elif string_or_id in SYMBOLS_BY_STR: return SYMBOLS_BY_STR[string_or_id] - cdef hash_t key - if isinstance(string_or_id, unicode): key = hash_string(string_or_id) return key @@ -118,6 +116,20 @@ cdef class StringStore: else: return decode_Utf8Str(utf8str) + def as_int(self, key): + """If key is an int, return it; otherwise, get the int value.""" + if not isinstance(key, basestring): + return key + else: + return self[key] + + def as_string(self, key): + """If key is a string, return it; otherwise, get the string value.""" + if isinstance(key, basestring): + return key + else: + return self[key] + def add(self, string): """Add a string to the StringStore. From b10d0cce05ee6ff90362f0571ae386ab03da01ad Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Mon, 24 Sep 2018 17:35:28 +0200 Subject: [PATCH 002/287] Add MultiSoftmax class Add a new class for the Tagger model, MultiSoftmax. This allows softmax prediction of multiple classes on the same output layer, e.g. one variable with 3 classes, another with 4 classes. This makes a layer with 7 output neurons, which we softmax into two distributions. --- spacy/_ml.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/spacy/_ml.py b/spacy/_ml.py index 964b1fa7a..231f6a7a4 100644 --- a/spacy/_ml.py +++ b/spacy/_ml.py @@ -444,7 +444,46 @@ def getitem(i): return layerize(getitem_fwd) -def build_tagger_model(nr_class, **cfg): +@describe.attributes( + W=Synapses("Weights matrix", + lambda obj: (obj.nO, obj.nI), + lambda W, ops: None) +) +class MultiSoftmax(Affine): + '''Neural network layer that predicts several multi-class attributes at once. + For instance, we might predict one class with 6 variables, and another with 5. + We predict the 11 neurons required for this, and then softmax them such + that columns 0-6 make a probability distribution and coumns 6-11 make another. + ''' + name = 'multisoftmax' + + def __init__(self, out_sizes, nI=None, **kwargs): + Model.__init__(self, **kwargs) + self.out_sizes = out_sizes + self.nO = sum(out_sizes) + self.nI = nI + + def predict(self, input__BI): + output__BO = self.ops.affine(self.W, self.b, input__BI) + i = 0 + for out_size in self.out_sizes: + self.ops.softmax(output__BO[:, i : i+out_size], inplace=True) + i += out_size + return output__BO + + def begin_update(self, input__BI, drop=0.): + output__BO = self.predict(input__BI) + def finish_update(grad__BO, sgd=None): + self.d_W += self.ops.gemm(grad__BO, input__BI, trans1=True) + self.d_b += grad__BO.sum(axis=0) + grad__BI = self.ops.gemm(grad__BO, self.W) + if sgd is not None: + sgd(self._mem.weights, self._mem.gradient, key=self.id) + return grad__BI + return output__BO, finish_update + + +def build_tagger_model(class_nums, **cfg): embed_size = util.env_opt('embed_size', 7000) if 'token_vector_width' in cfg: token_vector_width = cfg['token_vector_width'] @@ -459,7 +498,8 @@ def build_tagger_model(nr_class, **cfg): tok2vec = Tok2Vec(token_vector_width, embed_size, subword_features=subword_features, pretrained_vectors=pretrained_vectors) - softmax = with_flatten(Softmax(nr_class, token_vector_width)) + softmax = with_flatten( + MultiSoftmax(class_nums, token_vector_width)) model = ( tok2vec >> softmax From ac5742223ae4c1d094dde4bd98dbbf51e7b19e5d Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Mon, 24 Sep 2018 23:14:06 +0200 Subject: [PATCH 003/287] Draft class to predict morphological tags --- spacy/_morphologizer.pyx | 131 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 spacy/_morphologizer.pyx diff --git a/spacy/_morphologizer.pyx b/spacy/_morphologizer.pyx new file mode 100644 index 000000000..ca857296e --- /dev/null +++ b/spacy/_morphologizer.pyx @@ -0,0 +1,131 @@ +from __future__ import unicode_literals +from collections import OrderedDict, defaultdict +import cytoolz +import ujson + +import numpy +cimport numpy as np +from .util import msgpack +from .util import msgpack_numpy + +from thinc.api import chain +from thinc.neural.util import to_categorical, copy_array +from . import util +from .pipe import Pipe +from ._ml import Tok2Vec, build_tagger_model +from ._ml import link_vectors_to_models, zero_init, flatten +from ._ml import create_default_optimizer +from .errors import Errors, TempErrors +from .compat import json_dumps, basestring_ +from .tokens.doc cimport Doc +from .vocab cimport Vocab +from .morphology cimport Morphology + + +class Morphologizer(Pipe): + name = 'morphologizer' + + @classmethod + def Model(cls, attr_nums, **cfg): + if cfg.get('pretrained_dims') and not cfg.get('pretrained_vectors'): + raise ValueError(TempErrors.T008) + return build_morphologizer_model(attr_nums, **cfg) + + def __init__(self, vocab, model=True, **cfg): + self.vocab = vocab + self.model = model + self.cfg = OrderedDict(sorted(cfg.items())) + self.cfg.setdefault('cnn_maxout_pieces', 2) + + @property + def labels(self): + return self.vocab.morphology.tag_names + + @property + def tok2vec(self): + if self.model in (None, True, False): + return None + else: + return chain(self.model.tok2vec, flatten) + + def __call__(self, doc): + features, tokvecs = self.predict([doc]) + self.set_annotations([doc], tags, tensors=tokvecs) + return doc + + def pipe(self, stream, batch_size=128, n_threads=-1): + for docs in cytoolz.partition_all(batch_size, stream): + docs = list(docs) + features, tokvecs = self.predict(docs) + self.set_annotations(docs, features, tensors=tokvecs) + yield from docs + + def predict(self, docs): + if not any(len(doc) for doc in docs): + # Handle case where there are no tokens in any docs. + n_labels = self.model.nO + guesses = [self.model.ops.allocate((0, n_labels)) for doc in docs] + tokvecs = self.model.ops.allocate((0, self.model.tok2vec.nO)) + return guesses, tokvecs + tokvecs = self.model.tok2vec(docs) + scores = self.model.softmax(tokvecs) + guesses = [] + # Resolve multisoftmax into guesses + for doc_scores in scores: + guesses.append(scores_to_guesses(doc_scores, self.model.softmax.out_sizes)) + return guesses, tokvecs + + def set_annotations(self, docs, batch_feature_ids, tensors=None): + if isinstance(docs, Doc): + docs = [docs] + cdef Doc doc + cdef Vocab vocab = self.vocab + for i, doc in enumerate(docs): + doc_feat_ids = batch_feat_ids[i] + if hasattr(doc_feat_ids, 'get'): + doc_feat_ids = doc_feat_ids.get() + # Convert the neuron indices into feature IDs. + offset = self.vocab.morphology.first_feature + for j, nr_feat in enumerate(self.model.softmax.out_sizes): + doc_feat_ids[:, j] += offset + offset += nr_feat + # Now add the analysis, and set the hash. + for j in range(doc_feat_ids.shape[0]): + doc.c[j].morph = self.vocab.morphology.add(doc_feat_ids[j]) + + def update(self, docs, golds, drop=0., sgd=None, losses=None): + if losses is not None and self.name not in losses: + losses[self.name] = 0. + + tag_scores, bp_tag_scores = self.model.begin_update(docs, drop=drop) + loss, d_tag_scores = self.get_loss(docs, golds, tag_scores) + bp_tag_scores(d_tag_scores, sgd=sgd) + + if losses is not None: + losses[self.name] += loss + + def get_loss(self, docs, golds, scores): + guesses = [] + for doc_scores in scores: + guesses.append(scores_to_guesses(doc_scores, self.model.softmax.out_sizes)) + guesses = self.model.ops.flatten(guesses) + cdef int idx = 0 + target = numpy.zeros(scores.shape, dtype='f') + for gold in golds: + for features in gold.morphology: + if features is None: + target[idx] = guesses[idx] + else: + for feature in features: + column = feature_to_column(feature) # TODO + target[idx, column] = 1 + idx += 1 + target = self.model.ops.xp.array(target, dtype='f') + d_scores = scores - target + loss = (d_scores**2).sum() + d_scores = self.model.ops.unflatten(d_scores, [len(d) for d in docs]) + return float(loss), d_scores + + def use_params(self, params): + with self.model.use_params(params): + yield From 6ae645c4ef67449992463979c5539118a3699a5e Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Mon, 24 Sep 2018 23:57:41 +0200 Subject: [PATCH 004/287] WIP on supporting morphology features --- spacy/morphology.pxd | 81 ++++++--- spacy/morphology.pyx | 402 +++++++++++++++++++++++++++++++++---------- 2 files changed, 366 insertions(+), 117 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index d0110b300..2220cfcfc 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -1,48 +1,30 @@ from cymem.cymem cimport Pool -from preshed.maps cimport PreshMapArray +from preshed.maps cimport PreshMap from libc.stdint cimport uint64_t +from murmurhash cimport mrmr from .structs cimport TokenC from .strings cimport StringStore -from .typedefs cimport attr_t, flags_t +from .typedefs cimport hash_t, attr_t, flags_t from .parts_of_speech cimport univ_pos_t from . cimport symbols - -cdef struct RichTagC: - uint64_t morph - int id - univ_pos_t pos - attr_t name - - -cdef struct MorphAnalysisC: - RichTagC tag - attr_t lemma - - cdef class Morphology: cdef readonly Pool mem cdef readonly StringStore strings + cdef PreshMap tags # Keyed by hash, value is pointer to tag + cdef public object lemmatizer cdef readonly object tag_map - cdef public object n_tags - cdef public object reverse_index - cdef public object tag_names - cdef public object exc - - cdef RichTagC* rich_tags - cdef PreshMapArray _cache + cdef hash_t insert(self, RichTagC tag) except 0 + cdef int assign_untagged(self, TokenC* token) except -1 - cdef int assign_tag(self, TokenC* token, tag) except -1 - cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1 - - cdef int assign_feature(self, uint64_t* morph, univ_morph_t feat_id, bint value) except -1 - + cdef update_token_morph(self, TokenC* token, features) + cdef set_token_morph(self, TokenC* token, pos, features) cdef enum univ_morph_t: NIL = 0 @@ -298,4 +280,47 @@ cdef enum univ_morph_t: VerbType_mod # U VerbType_light # U - +cdef struct RichTagC: + univ_pos_t pos + + univ_morph_t abbr + univ_morph_t adp_type + univ_morph_t adv_type + univ_morph_t animacy + univ_morph_t aspect + univ_morph_t case + univ_morph_t conj_type + univ_morph_t connegative + univ_morph_t definite + univ_morph_t degree + univ_morph_t derivation + univ_morph_t echo + univ_morph_t foreign + univ_morph_t gender + univ_morph_t hyph + univ_morph_t inf_form + univ_morph_t mood + univ_morph_t negative + univ_morph_t number + univ_morph_t name_type + univ_morph_t num_form + univ_morph_t num_type + univ_morph_t num_value + univ_morph_t part_form + univ_morph_t part_type + univ_morph_t person + univ_morph_t polite + univ_morph_t polarity + univ_morph_t poss + univ_morph_t prefix + univ_morph_t prep_case + univ_morph_t pron_type + univ_morph_t punct_side + univ_morph_t punct_type + univ_morph_t reflex + univ_morph_t style + univ_morph_t style_variant + univ_morph_t tense + univ_morph_t verb_form + univ_morph_t voice + univ_morph_t verb_type diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index bd821d76f..3b74ecaae 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -3,6 +3,7 @@ from __future__ import unicode_literals from libc.string cimport memset +import ujson as json from .attrs cimport POS, IS_SPACE from .attrs import LEMMA, intify_attrs @@ -12,6 +13,7 @@ from .lexeme cimport Lexeme from .errors import Errors + def _normalize_props(props): """Transform deprecated string keys to correct names.""" out = {} @@ -32,9 +34,17 @@ def _normalize_props(props): cdef class Morphology: + '''Store the possible morphological analyses for a language, and index them + by hash. + + To save space on each token, tokens only know the hash of their morphological + analysis, so queries of morphological attributes are delegated + to this class. + ''' def __init__(self, StringStore string_store, tag_map, lemmatizer, exc=None): self.mem = Pool() self.strings = string_store + self.tags = PreshMap() # Add special space symbol. We prefix with underscore, to make sure it # always sorts to the end. space_attrs = tag_map.get('SP', {POS: SPACE}) @@ -47,32 +57,46 @@ cdef class Morphology: self.lemmatizer = lemmatizer self.n_tags = len(tag_map) self.reverse_index = {} - - self.rich_tags = self.mem.alloc(self.n_tags+1, sizeof(RichTagC)) for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())): - self.strings.add(tag_str) self.tag_map[tag_str] = dict(attrs) - attrs = _normalize_props(attrs) - attrs = intify_attrs(attrs, self.strings, _do_deprecated=True) - self.rich_tags[i].id = i - self.rich_tags[i].name = self.strings.add(tag_str) - self.rich_tags[i].morph = 0 - self.rich_tags[i].pos = attrs[POS] - self.reverse_index[self.rich_tags[i].name] = i - # Add a 'null' tag, which we can reference when assign morphology to - # untagged tokens. - self.rich_tags[self.n_tags].id = self.n_tags + self.reverse_index[i] = self.strings.add(tag_str) self._cache = PreshMapArray(self.n_tags) self.exc = {} if exc is not None: for (tag_str, orth_str), attrs in exc.items(): self.add_special_case(tag_str, orth_str, attrs) + + def add(self, features): + """Insert a morphological analysis in the morphology table, if not already + present. Returns the hash of the new analysis. + """ + features = intify_features(self.strings, features) + cdef RichTagC tag = create_rich_tag(features) + cdef hash_t key = self.insert(tag) + return key - def __reduce__(self): - return (Morphology, (self.strings, self.tag_map, self.lemmatizer, - self.exc), None, None) - + def lemmatize(self, const univ_pos_t univ_pos, attr_t orth, morphology): + if orth not in self.strings: + return orth + cdef unicode py_string = self.strings[orth] + if self.lemmatizer is None: + return self.strings.add(py_string.lower()) + cdef list lemma_strings + cdef unicode lemma_string + lemma_strings = self.lemmatizer(py_string, univ_pos, morphology) + lemma_string = lemma_strings[0] + lemma = self.strings.add(lemma_string) + return lemma + + cdef hash_t insert(self, RichTagC tag) except 0: + cdef hash_t key = hash_tag(tag) + if self.tags.get(key) == NULL: + tag_ptr = self.mem.alloc(1, sizeof(RichTagC)) + tag_ptr[0] = tag + self.tags.set(key, tag_ptr) + return key + cdef int assign_untagged(self, TokenC* token) except -1: """Set morphological attributes on a token without a POS tag. Uses the lemmatizer's lookup() method, which looks up the string in the @@ -101,84 +125,284 @@ cdef class Morphology: # figure out why the statistical model fails. Related to Issue #220 if Lexeme.c_check_flag(token.lex, IS_SPACE): tag_id = self.reverse_index[self.strings.add('_SP')] - rich_tag = self.rich_tags[tag_id] - analysis = self._cache.get(tag_id, token.lex.orth) - if analysis is NULL: - analysis = self.mem.alloc(1, sizeof(MorphAnalysisC)) - tag_str = self.strings[self.rich_tags[tag_id].name] - analysis.tag = rich_tag - analysis.lemma = self.lemmatize(analysis.tag.pos, token.lex.orth, - self.tag_map.get(tag_str, {})) - self._cache.set(tag_id, token.lex.orth, analysis) - token.lemma = analysis.lemma - token.pos = analysis.tag.pos - token.tag = analysis.tag.name - token.morph = analysis.tag.morph + lemma = self._cache.get(tag_id, token.lex.orth) + if lemma == 0: + tag_str = self.tag_names[tag_id] + features = dict(self.tag_map.get(tag_str, {})) + pos = self.strings.as_int(features.pop('POS')) + lemma = self.lemmatize(pos, token.lex.orth, features) + self._cache.set(tag_id, token.lex.orth, lemma) + token.lemma = lemma + token.pos = pos + token.tag = self.strings[tag_str] + token.morph = self.add(attrs) - cdef int assign_feature(self, uint64_t* flags, univ_morph_t flag_id, bint value) except -1: - cdef flags_t one = 1 - if value: - flags[0] |= one << flag_id - else: - flags[0] &= ~(one << flag_id) + cdef update_morph(self, hash_t morph, features): + """Update a morphological analysis with new feature values.""" + tag = (self.tags.get(morph))[0] + cdef univ_morph_t feature + cdef int value + for feature_, value in features.items(): + feature = self.strings.as_int(feature_) + set_feature(&tag, feature, 1) + morph = self.insert_tag(tag) + return morph - def add_special_case(self, unicode tag_str, unicode orth_str, attrs, - force=False): - """Add a special-case rule to the morphological analyser. Tokens whose - tag and orth match the rule will receive the specified properties. + def to_bytes(self): + json_tags = [] + for key in self.tags: + tag_ptr = self.tags.get(key) + if tag_ptr != NULL: + json_tags.append(tag_to_json(tag_ptr[0])) + raise json.dumps(json_tags) - tag (unicode): The part-of-speech tag to key the exception. - orth (unicode): The word-form to key the exception. - """ - # TODO: Currently we've assumed that we know the number of tags -- - # RichTagC is an array, and _cache is a PreshMapArray - # This is really bad: it makes the morphology typed to the tagger - # classes, which is all wrong. - self.exc[(tag_str, orth_str)] = dict(attrs) - tag = self.strings.add(tag_str) - if tag not in self.reverse_index: - return - tag_id = self.reverse_index[tag] - orth = self.strings[orth_str] - cdef RichTagC rich_tag = self.rich_tags[tag_id] - attrs = intify_attrs(attrs, self.strings, _do_deprecated=True) - cached = self._cache.get(tag_id, orth) - if cached is NULL: - cached = self.mem.alloc(1, sizeof(MorphAnalysisC)) - elif force: - memset(cached, 0, sizeof(cached[0])) - else: - raise ValueError(Errors.E015.format(tag=tag_str, orth=orth_str)) + def from_bytes(self, byte_string): + raise NotImplementedError - cached.tag = rich_tag - # TODO: Refactor this to take arbitrary attributes. - for name_id, value_id in attrs.items(): - if name_id == LEMMA: - cached.lemma = value_id - else: - self.assign_feature(&cached.tag.morph, name_id, value_id) - if cached.lemma == 0: - cached.lemma = self.lemmatize(rich_tag.pos, orth, attrs) - self._cache.set(tag_id, orth, cached) + def to_disk(self, path): + raise NotImplementedError + + def from_disk(self, path): + raise NotImplementedError + + +cpdef univ_pos_t get_int_tag(pos_): + return 0 + +cpdef intify_features(StringStore strings, features): + return {strings.as_int(feature) for feature in features} + +cdef hash_t hash_tag(RichTagC tag) nogil: + return mrmr.hash64(&tag, sizeof(tag), 0) + +cdef RichTagC create_rich_tag(pos_, features): + cdef RichTagC tag + cdef univ_morph_t feature + tag.pos = get_int_tag(pos_) + for feature in features: + set_feature(&tag, feature, 1) + return tag + +cdef tag_to_json(RichTagC tag): + return {} + +cdef RichTagC tag_from_json(json_tag): + cdef RichTagC tag + return tag + +cdef int set_feature(RichTagC* tag, univ_morph_t feature, int value) nogil: + if value == True: + value_ = feature + else: + value_ = NIL + if feature == NIL: + pass + if is_abbr_feature(feature): + tag.abbr = value_ + elif is_adp_type_feature(feature): + tag.adp_type = value_ + elif is_adv_type_feature(feature): + tag.adv_type = value_ + elif is_animacy_feature(feature): + tag.animacy = value_ + elif is_aspect_feature(feature): + tag.aspect = value_ + elif is_case_feature(feature): + tag.case = value_ + elif is_conj_type_feature(feature): + tag.conj_type = value_ + elif is_connegative_feature(feature): + tag.connegative = value_ + elif is_definite_feature(feature): + tag.definite = value_ + elif is_degree_feature(feature): + tag.degree = value_ + elif is_derivation_feature(feature): + tag.derivation = value_ + elif is_echo_feature(feature): + tag.echo = value_ + elif is_foreign_feature(feature): + tag.foreign = value_ + elif is_gender_feature(feature): + tag.gender = value_ + elif is_hyph_feature(feature): + tag.hyph = value_ + elif is_inf_form_feature(feature): + tag.inf_form = value_ + elif is_mood_feature(feature): + tag.mood = value_ + elif is_negative_feature(feature): + tag.negative = value_ + elif is_number_feature(feature): + tag.number = value_ + elif is_name_type_feature(feature): + tag.name_type = value_ + elif is_num_form_feature(feature): + tag.num_form = value_ + elif is_num_value_feature(feature): + tag.num_value = value_ + elif is_part_form_feature(feature): + tag.part_form = value_ + elif is_part_type_feature(feature): + tag.part_type = value_ + elif is_person_feature(feature): + tag.person = value_ + elif is_polite_feature(feature): + tag.polite = value_ + elif is_polarity_feature(feature): + tag.polarity = value_ + elif is_poss_feature(feature): + tag.poss = value_ + elif is_prefix_feature(feature): + tag.prefix = value_ + elif is_prep_case_feature(feature): + tag.prep_case = value_ + elif is_pron_type_feature(feature): + tag.pron_type = value_ + elif is_punct_side_feature(feature): + tag.punct_type = value_ + elif is_reflex_feature(feature): + tag.reflex = value_ + elif is_style_feature(feature): + tag.style = value_ + elif is_style_variant_feature(feature): + tag.style_variant = value_ + elif is_tense_feature(feature): + tag.tense = value_ + elif is_verb_form_feature(feature): + tag.verb_form = value_ + elif is_voice_feature(feature): + tag.voice = value_ + elif is_verb_type_feature(feature): + tag.verb_type = value_ + else: + with gil: + raise ValueError("Unknown feature: %d" % feature) + +cdef int is_abbr_feature(univ_morph_t abbr) nogil: + return 0 + +cdef int is_adp_type_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_adv_type_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_animacy_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_aspect_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_case_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_conj_type_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_connegative_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_definite_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_degree_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_derivation_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_echo_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_foreign_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_gender_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_hyph_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_inf_form_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_mood_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_negative_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_number_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_name_type_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_num_form_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_num_type_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_num_value_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_part_form_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_part_type_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_person_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_polite_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_polarity_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_poss_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_prefix_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_prep_case_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_pron_type_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_punct_side_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_punct_type_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_reflex_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_style_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_style_variant_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_tense_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_verb_form_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_voice_feature(univ_morph_t feature) nogil: + return 0 + +cdef int is_verb_type_feature(univ_morph_t feature) nogil: + return 0 - def load_morph_exceptions(self, dict exc): - # Map (form, pos) to (lemma, rich tag) - for tag_str, entries in exc.items(): - for form_str, attrs in entries.items(): - self.add_special_case(tag_str, form_str, attrs) - def lemmatize(self, const univ_pos_t univ_pos, attr_t orth, morphology): - if orth not in self.strings: - return orth - cdef unicode py_string = self.strings[orth] - if self.lemmatizer is None: - return self.strings.add(py_string.lower()) - cdef list lemma_strings - cdef unicode lemma_string - lemma_strings = self.lemmatizer(py_string, univ_pos, morphology) - lemma_string = lemma_strings[0] - lemma = self.strings.add(lemma_string) - return lemma IDS = { From 3bba8e9245bc89494e5d0bf460397844e000d424 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Mon, 24 Sep 2018 23:58:08 +0200 Subject: [PATCH 005/287] Update structs --- spacy/structs.pxd | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spacy/structs.pxd b/spacy/structs.pxd index cfcadc3d0..954ea19fe 100644 --- a/spacy/structs.pxd +++ b/spacy/structs.pxd @@ -2,6 +2,7 @@ from libc.stdint cimport uint8_t, uint32_t, int32_t, uint64_t from .typedefs cimport flags_t, attr_t, hash_t from .parts_of_speech cimport univ_pos_t +from .morphology cimport univ_morph_t cdef struct LexemeC: @@ -71,3 +72,6 @@ cdef struct TokenC: int ent_iob attr_t ent_type # TODO: Is there a better way to do this? Multiple sources of truth.. hash_t ent_id + + + From a3d2e616d53b93eacf9124a50c618b1888eb3718 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 00:35:59 +0200 Subject: [PATCH 006/287] Restore previous morphology stuff --- spacy/morphology.pxd | 9 ++++++--- spacy/morphology.pyx | 11 +++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index 2220cfcfc..aa2a4cb3c 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -1,5 +1,5 @@ from cymem.cymem cimport Pool -from preshed.maps cimport PreshMap +from preshed.maps cimport PreshMap, PreshMapArray from libc.stdint cimport uint64_t from murmurhash cimport mrmr @@ -17,14 +17,17 @@ cdef class Morphology: cdef public object lemmatizer cdef readonly object tag_map + cdef readonly object tag_names + cdef readonly object reverse_index + cdef readonly object exc + cdef readonly int n_tags cdef hash_t insert(self, RichTagC tag) except 0 cdef int assign_untagged(self, TokenC* token) except -1 cdef int assign_tag(self, TokenC* token, tag) except -1 cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1 - cdef update_token_morph(self, TokenC* token, features) - cdef set_token_morph(self, TokenC* token, pos, features) + cdef update_morph(self, hash_t morph, features) cdef enum univ_morph_t: NIL = 0 diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 3b74ecaae..2eb20776f 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -125,17 +125,17 @@ cdef class Morphology: # figure out why the statistical model fails. Related to Issue #220 if Lexeme.c_check_flag(token.lex, IS_SPACE): tag_id = self.reverse_index[self.strings.add('_SP')] + tag_str = self.tag_names[tag_id] + features = dict(self.tag_map.get(tag_str, {})) lemma = self._cache.get(tag_id, token.lex.orth) - if lemma == 0: - tag_str = self.tag_names[tag_id] - features = dict(self.tag_map.get(tag_str, {})) + if lemma == 0 and features: pos = self.strings.as_int(features.pop('POS')) lemma = self.lemmatize(pos, token.lex.orth, features) self._cache.set(tag_id, token.lex.orth, lemma) token.lemma = lemma token.pos = pos token.tag = self.strings[tag_str] - token.morph = self.add(attrs) + token.morph = self.add(features) cdef update_morph(self, hash_t morph, features): """Update a morphological analysis with new feature values.""" @@ -175,10 +175,9 @@ cpdef intify_features(StringStore strings, features): cdef hash_t hash_tag(RichTagC tag) nogil: return mrmr.hash64(&tag, sizeof(tag), 0) -cdef RichTagC create_rich_tag(pos_, features): +cdef RichTagC create_rich_tag(features): cdef RichTagC tag cdef univ_morph_t feature - tag.pos = get_int_tag(pos_) for feature in features: set_feature(&tag, feature, 1) return tag From be8cf39e16c0a518f568600171c8a7f1c314fab4 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 10:57:33 +0200 Subject: [PATCH 007/287] Fix morphology --- spacy/morphology.pxd | 1 + spacy/morphology.pyx | 68 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index aa2a4cb3c..05bc8ccc0 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -20,6 +20,7 @@ cdef class Morphology: cdef readonly object tag_names cdef readonly object reverse_index cdef readonly object exc + cdef readonly PreshMapArray _cache cdef readonly int n_tags cdef hash_t insert(self, RichTagC tag) except 0 diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 2eb20776f..35571af49 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -58,15 +58,20 @@ cdef class Morphology: self.n_tags = len(tag_map) self.reverse_index = {} for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())): + print(tag_str, attrs) self.tag_map[tag_str] = dict(attrs) - self.reverse_index[i] = self.strings.add(tag_str) + self.reverse_index[self.strings.add(tag_str)] = i self._cache = PreshMapArray(self.n_tags) self.exc = {} if exc is not None: for (tag_str, orth_str), attrs in exc.items(): self.add_special_case(tag_str, orth_str, attrs) - + + def __reduce__(self): + return (Morphology, (self.strings, self.tag_map, self.lemmatizer, + self.exc), None, None) + def add(self, features): """Insert a morphological analysis in the morphology table, if not already present. Returns the hash of the new analysis. @@ -88,6 +93,46 @@ cdef class Morphology: lemma_string = lemma_strings[0] lemma = self.strings.add(lemma_string) return lemma + + def add_special_case(self, unicode tag_str, unicode orth_str, attrs, + force=False): + """Add a special-case rule to the morphological analyser. Tokens whose + tag and orth match the rule will receive the specified properties. + + tag (unicode): The part-of-speech tag to key the exception. + orth (unicode): The word-form to key the exception. + """ + pass + ## TODO: Currently we've assumed that we know the number of tags -- + ## RichTagC is an array, and _cache is a PreshMapArray + ## This is really bad: it makes the morphology typed to the tagger + ## classes, which is all wrong. + #self.exc[(tag_str, orth_str)] = dict(attrs) + #tag = self.strings.add(tag_str) + #if tag not in self.reverse_index: + # return + #tag_id = self.reverse_index[tag] + #orth = self.strings[orth_str] + #cdef RichTagC rich_tag = self.rich_tags[tag_id] + #attrs = intify_attrs(attrs, self.strings, _do_deprecated=True) + #cached = self._cache.get(tag_id, orth) + #if cached is NULL: + # cached = self.mem.alloc(1, sizeof(MorphAnalysisC)) + #elif force: + # memset(cached, 0, sizeof(cached[0])) + #else: + # raise ValueError(Errors.E015.format(tag=tag_str, orth=orth_str)) + + #cached.tag = rich_tag + ## TODO: Refactor this to take arbitrary attributes. + #for name_id, value_id in attrs.items(): + # if name_id == LEMMA: + # cached.lemma = value_id + # else: + # self.assign_feature(&cached.tag.morph, name_id, value_id) + #if cached.lemma == 0: + # cached.lemma = self.lemmatize(rich_tag.pos, orth, attrs) + #self._cache.set(tag_id, orth, cached) cdef hash_t insert(self, RichTagC tag) except 0: cdef hash_t key = hash_tag(tag) @@ -107,9 +152,8 @@ cdef class Morphology: lemma = self.lemmatizer.lookup(orth_str) token.lemma = self.strings.add(lemma) - cdef int assign_tag(self, TokenC* token, tag) except -1: - if isinstance(tag, basestring): - tag = self.strings.add(tag) + cdef int assign_tag(self, TokenC* token, tag_str) except -1: + cdef attr_t tag = self.strings.as_int(tag_str) if tag in self.reverse_index: tag_id = self.reverse_index[tag] self.assign_tag_id(token, tag_id) @@ -127,13 +171,15 @@ cdef class Morphology: tag_id = self.reverse_index[self.strings.add('_SP')] tag_str = self.tag_names[tag_id] features = dict(self.tag_map.get(tag_str, {})) - lemma = self._cache.get(tag_id, token.lex.orth) + cdef attr_t lemma = self._cache.get(tag_id, token.lex.orth) if lemma == 0 and features: - pos = self.strings.as_int(features.pop('POS')) + pos = self.strings.as_int(features.pop(POS)) lemma = self.lemmatize(pos, token.lex.orth, features) - self._cache.set(tag_id, token.lex.orth, lemma) + self._cache.set(tag_id, token.lex.orth, lemma) + else: + pos = 0 token.lemma = lemma - token.pos = pos + token.pos = pos token.tag = self.strings[tag_str] token.morph = self.add(features) @@ -178,8 +224,8 @@ cdef hash_t hash_tag(RichTagC tag) nogil: cdef RichTagC create_rich_tag(features): cdef RichTagC tag cdef univ_morph_t feature - for feature in features: - set_feature(&tag, feature, 1) + #for feature in features: + # set_feature(&tag, feature, 1) return tag cdef tag_to_json(RichTagC tag): From e6dde97295022efe299bfa65a73c8d9b96eba8c4 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 10:57:59 +0200 Subject: [PATCH 008/287] Add function to make morphologizer model --- spacy/_ml.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/spacy/_ml.py b/spacy/_ml.py index 231f6a7a4..f37938671 100644 --- a/spacy/_ml.py +++ b/spacy/_ml.py @@ -483,7 +483,33 @@ class MultiSoftmax(Affine): return output__BO, finish_update -def build_tagger_model(class_nums, **cfg): +def build_tagger_model(nr_class, **cfg): + embed_size = util.env_opt('embed_size', 7000) + if 'token_vector_width' in cfg: + token_vector_width = cfg['token_vector_width'] + else: + token_vector_width = util.env_opt('token_vector_width', 128) + pretrained_vectors = cfg.get('pretrained_vectors') + subword_features = cfg.get('subword_features', True) + with Model.define_operators({'>>': chain, '+': add}): + if 'tok2vec' in cfg: + tok2vec = cfg['tok2vec'] + else: + tok2vec = Tok2Vec(token_vector_width, embed_size, + subword_features=subword_features, + pretrained_vectors=pretrained_vectors) + softmax = with_flatten( + Softmax(nr_class, token_vector_width)) + model = ( + tok2vec + >> softmax + ) + model.nI = None + model.tok2vec = tok2vec + model.softmax = softmax + return model + +def build_morphologizer_model(class_nums, **cfg): embed_size = util.env_opt('embed_size', 7000) if 'token_vector_width' in cfg: token_vector_width = cfg['token_vector_width'] From c2357d3ba075f0733505ca69ac114705872bb07b Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 10:58:13 +0200 Subject: [PATCH 009/287] Fix morphologizer class --- spacy/_morphologizer.pyx | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/spacy/_morphologizer.pyx b/spacy/_morphologizer.pyx index ca857296e..2fa092faa 100644 --- a/spacy/_morphologizer.pyx +++ b/spacy/_morphologizer.pyx @@ -12,7 +12,7 @@ from thinc.api import chain from thinc.neural.util import to_categorical, copy_array from . import util from .pipe import Pipe -from ._ml import Tok2Vec, build_tagger_model +from ._ml import Tok2Vec, build_morphologizer_model from ._ml import link_vectors_to_models, zero_init, flatten from ._ml import create_default_optimizer from .errors import Errors, TempErrors @@ -20,6 +20,7 @@ from .compat import json_dumps, basestring_ from .tokens.doc cimport Doc from .vocab cimport Vocab from .morphology cimport Morphology +from .pipeline import Pipe class Morphologizer(Pipe): @@ -50,7 +51,7 @@ class Morphologizer(Pipe): def __call__(self, doc): features, tokvecs = self.predict([doc]) - self.set_annotations([doc], tags, tensors=tokvecs) + self.set_annotations([doc], features, tensors=tokvecs) return doc def pipe(self, stream, batch_size=128, n_threads=-1): @@ -81,7 +82,7 @@ class Morphologizer(Pipe): cdef Doc doc cdef Vocab vocab = self.vocab for i, doc in enumerate(docs): - doc_feat_ids = batch_feat_ids[i] + doc_feat_ids = batch_feature_ids[i] if hasattr(doc_feat_ids, 'get'): doc_feat_ids = doc_feat_ids.get() # Convert the neuron indices into feature IDs. @@ -129,3 +130,9 @@ class Morphologizer(Pipe): def use_params(self, params): with self.model.use_params(params): yield + +def scores_to_guesses(scores, out_sizes): + raise NotImplementedError + +def feature_to_column(feature): + raise NotImplementedError From 8308c1525e921c7655470fb1b2e255c37dca68b3 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 15:18:21 +0200 Subject: [PATCH 010/287] Fix exception loading --- spacy/lemmatizer.py | 8 ++--- spacy/morphology.pxd | 2 ++ spacy/morphology.pyx | 85 ++++++++++++++++++++++++-------------------- 3 files changed, 53 insertions(+), 42 deletions(-) diff --git a/spacy/lemmatizer.py b/spacy/lemmatizer.py index 93121a0c5..483debb67 100644 --- a/spacy/lemmatizer.py +++ b/spacy/lemmatizer.py @@ -60,13 +60,13 @@ class Lemmatizer(object): return True elif univ_pos == 'adj' and morphology.get('Degree') == 'pos': return True - elif VerbForm_inf in morphology: + elif VerbForm_inf in morphology or 'VerbForm_inf' in morphology: return True - elif VerbForm_none in morphology: + elif VerbForm_none in morphology or 'VerbForm_none' in morphology: return True - elif Number_sing in morphology: + elif Number_sing in morphology or 'Number_sing' in morphology: return True - elif Degree_pos in morphology: + elif Degree_pos in morphology or 'Degree_pos' in morphology: return True else: return False diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index 05bc8ccc0..7ba84d40c 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -30,6 +30,8 @@ cdef class Morphology: cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1 cdef update_morph(self, hash_t morph, features) + cdef int _assign_tag_from_exceptions(self, TokenC* token, int tag_id) except -1 + cdef enum univ_morph_t: NIL = 0 Animacy_anim = symbols.Animacy_anim diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 35571af49..f314a91a3 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -5,6 +5,7 @@ from __future__ import unicode_literals from libc.string cimport memset import ujson as json +from . import symbols from .attrs cimport POS, IS_SPACE from .attrs import LEMMA, intify_attrs from .parts_of_speech cimport SPACE @@ -17,6 +18,24 @@ from .errors import Errors def _normalize_props(props): """Transform deprecated string keys to correct names.""" out = {} + morph_keys = [ + 'PunctType', 'PunctSide', 'Other', 'Degree', 'AdvType', 'Number', + 'VerbForm', 'PronType', 'Aspect', 'Tense', 'PartType', 'Poss', + 'Hyph', 'ConjType', 'NumType', 'Foreign', 'VerbType', 'NounType', + 'Gender', 'Mood', 'Negative', 'Tense', 'Voice', 'Abbr', + 'Derivation', 'Echo', 'Foreign', 'NameType', 'NounType', 'NumForm', + 'NumValue', 'PartType', 'Polite', 'StyleVariant', + 'PronType', 'AdjType', 'Person', 'Variant', 'AdpType', + 'Reflex', 'Negative', 'Mood', 'Aspect', 'Case', + 'Polarity', 'PrepCase', 'Animacy' # U20 + ] + props = dict(props) + for key in morph_keys: + if key in props: + attr = '%s_%s' % (key, props[key]) + if attr in IDS: + props.pop(key) + props[attr] = True for key, value in props.items(): if key == POS: if hasattr(value, 'upper'): @@ -58,15 +77,16 @@ cdef class Morphology: self.n_tags = len(tag_map) self.reverse_index = {} for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())): - print(tag_str, attrs) + attrs = _normalize_props(attrs) self.tag_map[tag_str] = dict(attrs) self.reverse_index[self.strings.add(tag_str)] = i self._cache = PreshMapArray(self.n_tags) self.exc = {} if exc is not None: - for (tag_str, orth_str), attrs in exc.items(): - self.add_special_case(tag_str, orth_str, attrs) + for (tag, orth), attrs in exc.items(): + self.add_special_case( + self.strings.as_string(tag), self.strings.as_string(orth), attrs) def __reduce__(self): return (Morphology, (self.strings, self.tag_map, self.lemmatizer, @@ -102,37 +122,10 @@ cdef class Morphology: tag (unicode): The part-of-speech tag to key the exception. orth (unicode): The word-form to key the exception. """ - pass - ## TODO: Currently we've assumed that we know the number of tags -- - ## RichTagC is an array, and _cache is a PreshMapArray - ## This is really bad: it makes the morphology typed to the tagger - ## classes, which is all wrong. - #self.exc[(tag_str, orth_str)] = dict(attrs) - #tag = self.strings.add(tag_str) - #if tag not in self.reverse_index: - # return - #tag_id = self.reverse_index[tag] - #orth = self.strings[orth_str] - #cdef RichTagC rich_tag = self.rich_tags[tag_id] - #attrs = intify_attrs(attrs, self.strings, _do_deprecated=True) - #cached = self._cache.get(tag_id, orth) - #if cached is NULL: - # cached = self.mem.alloc(1, sizeof(MorphAnalysisC)) - #elif force: - # memset(cached, 0, sizeof(cached[0])) - #else: - # raise ValueError(Errors.E015.format(tag=tag_str, orth=orth_str)) - - #cached.tag = rich_tag - ## TODO: Refactor this to take arbitrary attributes. - #for name_id, value_id in attrs.items(): - # if name_id == LEMMA: - # cached.lemma = value_id - # else: - # self.assign_feature(&cached.tag.morph, name_id, value_id) - #if cached.lemma == 0: - # cached.lemma = self.lemmatize(rich_tag.pos, orth, attrs) - #self._cache.set(tag_id, orth, cached) + attrs = dict(attrs) + attrs = _normalize_props(attrs) + attrs = intify_attrs(attrs, self.strings, _do_deprecated=True) + self.exc[(tag_str, self.strings.add(orth_str))] = attrs cdef hash_t insert(self, RichTagC tag) except 0: cdef hash_t key = hash_tag(tag) @@ -171,17 +164,27 @@ cdef class Morphology: tag_id = self.reverse_index[self.strings.add('_SP')] tag_str = self.tag_names[tag_id] features = dict(self.tag_map.get(tag_str, {})) - cdef attr_t lemma = self._cache.get(tag_id, token.lex.orth) - if lemma == 0 and features: + if features: pos = self.strings.as_int(features.pop(POS)) - lemma = self.lemmatize(pos, token.lex.orth, features) - self._cache.set(tag_id, token.lex.orth, lemma) else: pos = 0 + cdef attr_t lemma = self._cache.get(tag_id, token.lex.orth) + if lemma == 0: + lemma = self.lemmatize(pos, token.lex.orth, features) + self._cache.set(tag_id, token.lex.orth, lemma) token.lemma = lemma token.pos = pos token.tag = self.strings[tag_str] token.morph = self.add(features) + if (self.tag_names[tag_id], token.lex.orth) in self.exc: + self._assign_tag_from_exceptions(token, tag_id) + + cdef int _assign_tag_from_exceptions(self, TokenC* token, int tag_id) except -1: + key = (self.tag_names[tag_id], token.lex.orth) + cdef dict attrs + attrs = self.exc[key] + token.pos = attrs.get(POS, token.pos) + token.lemma = attrs.get(LEMMA, token.lemma) cdef update_morph(self, hash_t morph, features): """Update a morphological analysis with new feature values.""" @@ -194,6 +197,12 @@ cdef class Morphology: morph = self.insert_tag(tag) return morph + def load_morph_exceptions(self, dict exc): + # Map (form, pos) to (lemma, rich tag) + for tag_str, entries in exc.items(): + for form_str, attrs in entries.items(): + self.add_special_case(tag_str, form_str, attrs) + def to_bytes(self): json_tags = [] for key in self.tags: From 6fe7c7256053aea202bfef155d4034149051fc1d Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 17:28:13 +0200 Subject: [PATCH 011/287] Reorder morphology enum, and add begin and end markers --- spacy/morphology.pxd | 190 +++++++++++++++++++++++++++++++------------ 1 file changed, 138 insertions(+), 52 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index 7ba84d40c..96bba5260 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -34,15 +34,41 @@ cdef class Morphology: cdef enum univ_morph_t: NIL = 0 + begin_Abbr + Abbr_yes # cz, fi, sl, U + end_Abbr + begin_AdpType + AdpType_circ # U + AdpType_comprep # cz + AdpType_prep # cz, U + AdpType_post # U + AdpType_voc # cz + end_AdpType + begin_AdvType + AdvType_adadj + AdvType_cau + AdvType_deg + AdvType_ex + AdvType_loc + AdvType_man + AdvType_mod + AdvType_sta + AdvType_tim + end_AdvType + begin_Animacy Animacy_anim = symbols.Animacy_anim - Animacy_inan Animacy_hum + Animacy_inan Animacy_nhum + end_Animacy + begin_Aspect Aspect_freq Aspect_imp Aspect_mod Aspect_none Aspect_perf + end_Aspect + begin_Case Case_abe Case_abl Case_abs @@ -70,23 +96,46 @@ cdef enum univ_morph_t: Case_ter Case_tra Case_voc - Definite_two - Definite_def - Definite_red + end_Case + begin_ConjType + ConjType_comp # cz, U + ConjType_oper # cz, U + end_ConjType + begin_Connegative + Connegative_yes # fi + end_Connegative + begin_Definite Definite_cons # U20 + Definite_def Definite_ind + Definite_red + Definite_two + end_Definite + begin_Degree + Degree_abs Degree_cmp Degree_comp Degree_none Degree_pos Degree_sup - Degree_abs Degree_com Degree_dim # du + end_Degree + begin_Gender Gender_com Gender_fem Gender_masc Gender_neut + Gender_dat_masc # bq, U + Gender_dat_fem # bq, U + Gender_erg_masc # bq + Gender_erg_fem # bq + Gender_psor_masc # cz, sl, U + Gender_psor_fem # cz, sl, U + Gender_psor_neut # sl + + end_Gender + begin_Mood Mood_cnd Mood_imp Mood_ind @@ -94,11 +143,17 @@ cdef enum univ_morph_t: Mood_pot Mood_sub Mood_opt + end_Mood + begin_Negative Negative_neg Negative_pos Negative_yes + end_Negative + begin_Polarity Polarity_neg # U20 Polarity_pos # U20 + end_Polarity + begin_Number Number_com Number_dual Number_none @@ -106,6 +161,19 @@ cdef enum univ_morph_t: Number_sing Number_ptan # bg Number_count # bg + Number_abs_sing # bq, U + Number_abs_plur # bq, U + Number_dat_sing # bq, U + Number_dat_plur # bq, U + Number_erg_sing # bq, U + Number_erg_plur # bq, U + Number_psee_sing # U + Number_psee_plur # U + Number_psor_sing # cz, fi, sl, U + Number_psor_plur # cz, fi, sl, U + + end_Number + begin_NumType NumType_card NumType_dist NumType_frac @@ -114,11 +182,29 @@ cdef enum univ_morph_t: NumType_none NumType_ord NumType_sets + end_NumType + begin_Person Person_one Person_two Person_three Person_none + Person_abs_one # bq, U + Person_abs_two # bq, U + Person_abs_three # bq, U + Person_dat_one # bq, U + Person_dat_two # bq, U + Person_dat_three # bq, U + Person_erg_one # bq, U + Person_erg_two # bq, U + Person_erg_three # bq, U + Person_psor_one # fi, U + Person_psor_two # fi, U + Person_psor_three # fi, U + end_Person + begin_Poss Poss_yes + end_Poss + begin_PronType PronType_advPart PronType_art PronType_default @@ -132,11 +218,17 @@ cdef enum univ_morph_t: PronType_tot PronType_clit PronType_exc # es, ca, it, fa + end_PronType + begin_Reflex Reflex_yes + end_Reflex + begin_Tense Tense_fut Tense_imp Tense_past Tense_pres + end_Tense + begin_VerbForm VerbForm_fin VerbForm_ger VerbForm_inf @@ -149,29 +241,15 @@ cdef enum univ_morph_t: VerbForm_trans VerbForm_conv # U20 VerbForm_gdv # la + end_VerbForm + begin_Voice Voice_act Voice_cau Voice_pass Voice_mid # gkc Voice_int # hb - Abbr_yes # cz, fi, sl, U - AdpType_prep # cz, U - AdpType_post # U - AdpType_voc # cz - AdpType_comprep # cz - AdpType_circ # U - AdvType_man - AdvType_loc - AdvType_tim - AdvType_deg - AdvType_cau - AdvType_mod - AdvType_sta - AdvType_ex - AdvType_adadj - ConjType_oper # cz, U - ConjType_comp # cz, U - Connegative_yes # fi + end_Voice + begin_Derivation Derivation_minen # fi Derivation_sti # fi Derivation_inen # fi @@ -181,23 +259,26 @@ cdef enum univ_morph_t: Derivation_vs # fi Derivation_ttain # fi Derivation_ttaa # fi + end_Derivation + begin_Echo Echo_rdp # U Echo_ech # U + end_Echo + begin_Foreign Foreign_foreign # cz, fi, U Foreign_fscript # cz, fi, U Foreign_tscript # cz, U Foreign_yes # sl - Gender_dat_masc # bq, U - Gender_dat_fem # bq, U - Gender_erg_masc # bq - Gender_erg_fem # bq - Gender_psor_masc # cz, sl, U - Gender_psor_fem # cz, sl, U - Gender_psor_neut # sl + end_Foreign + begin_Hyph Hyph_yes # cz, U + end_Hyph + begin_InfForm InfForm_one # fi InfForm_two # fi InfForm_three # fi + end_InfForm + begin_NameType NameType_geo # U, cz NameType_prs # U, cz NameType_giv # U, cz @@ -206,46 +287,36 @@ cdef enum univ_morph_t: NameType_com # U, cz NameType_pro # U, cz NameType_oth # U, cz + end_NameType + begin_NounType NounType_com # U NounType_prop # U NounType_class # U - Number_abs_sing # bq, U - Number_abs_plur # bq, U - Number_dat_sing # bq, U - Number_dat_plur # bq, U - Number_erg_sing # bq, U - Number_erg_plur # bq, U - Number_psee_sing # U - Number_psee_plur # U - Number_psor_sing # cz, fi, sl, U - Number_psor_plur # cz, fi, sl, U + end_NounType + begin_NumForm NumForm_digit # cz, sl, U NumForm_roman # cz, sl, U NumForm_word # cz, sl, U + end_NumForm + begin_NumValue NumValue_one # cz, U NumValue_two # cz, U NumValue_three # cz, U + end_NumValue + begin_PartForm PartForm_pres # fi PartForm_past # fi PartForm_agt # fi PartForm_neg # fi + end_PartForm + begin_PartType PartType_mod # U PartType_emp # U PartType_res # U PartType_inf # U PartType_vbp # U - Person_abs_one # bq, U - Person_abs_two # bq, U - Person_abs_three # bq, U - Person_dat_one # bq, U - Person_dat_two # bq, U - Person_dat_three # bq, U - Person_erg_one # bq, U - Person_erg_two # bq, U - Person_erg_three # bq, U - Person_psor_one # fi, U - Person_psor_two # fi, U - Person_psor_three # fi, U + end_PartType + begin_Polite Polite_inf # bq, U Polite_pol # bq, U Polite_abs_inf # bq, U @@ -254,11 +325,19 @@ cdef enum univ_morph_t: Polite_erg_pol # bq, U Polite_dat_inf # bq, U Polite_dat_pol # bq, U + end_Polite + begin_Prefix Prefix_yes # U + end_Prefix + begin_PrepCase PrepCase_npr # cz PrepCase_pre # U + end_PrepCase + begin_PunctSide PunctSide_ini # U PunctSide_fin # U + end_PunctSide + begin_PunctType PunctType_peri # U PunctType_qest # U PunctType_excl # U @@ -268,6 +347,8 @@ cdef enum univ_morph_t: PunctType_colo # U PunctType_semi # U PunctType_dash # U + end_PunctType + begin_Style Style_arch # cz, fi, U Style_rare # cz, fi, U Style_poet # cz, U @@ -279,12 +360,17 @@ cdef enum univ_morph_t: Style_derg # cz, U Style_vulg # cz, U Style_yes # fi, U + end_Style + begin_StyleVariant StyleVariant_styleShort # cz StyleVariant_styleBound # cz, sl + end_StyleVariant + begin_VerbType VerbType_aux # U VerbType_cop # U VerbType_mod # U VerbType_light # U + end_VerbType cdef struct RichTagC: univ_pos_t pos From 4b7e772f5dbf75973320662976b997298c5d243d Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 17:28:34 +0200 Subject: [PATCH 012/287] Implement the is_animacy_feature etc functions --- spacy/morphology.pyx | 98 ++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index f314a91a3..b37107f09 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -156,10 +156,12 @@ cdef class Morphology: cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1: if tag_id > self.n_tags: raise ValueError(Errors.E014.format(tag=tag_id)) - # TODO: It's pretty arbitrary to put this logic here. I guess the - # justification is that this is where the specific word and the tag - # interact. Still, we should have a better way to enforce this rule, or - # figure out why the statistical model fails. Related to Issue #220 + # Ensure spaces get tagged as space. + # It seems pretty arbitrary to put this logic here, but there's really + # nowhere better. I guess the justification is that this is where the + # specific word and the tag interact. Still, we should have a better + # way to enforce this rule, or figure out why the statistical model fails. + # Related to Issue #220 if Lexeme.c_check_flag(token.lex, IS_SPACE): tag_id = self.reverse_index[self.strings.add('_SP')] tag_str = self.tag_names[tag_id] @@ -198,7 +200,7 @@ cdef class Morphology: return morph def load_morph_exceptions(self, dict exc): - # Map (form, pos) to (lemma, rich tag) + # Map (form, pos) to attributes for tag_str, entries in exc.items(): for form_str, attrs in entries.items(): self.add_special_case(tag_str, form_str, attrs) @@ -333,130 +335,128 @@ cdef int set_feature(RichTagC* tag, univ_morph_t feature, int value) nogil: with gil: raise ValueError("Unknown feature: %d" % feature) -cdef int is_abbr_feature(univ_morph_t abbr) nogil: - return 0 +cdef int is_abbr_feature(univ_morph_t feature) nogil: + return feature > begin_Abbr and feature < end_Abbr cdef int is_adp_type_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_AdpType and feature < end_AdpType cdef int is_adv_type_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_AdvType and feature < end_AdvType cdef int is_animacy_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Animacy and feature < end_Animacy cdef int is_aspect_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Aspect and feature < end_Aspect cdef int is_case_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Case and feature < end_Case cdef int is_conj_type_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_ConjType and feature < end_ConjType cdef int is_connegative_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Connegative and feature < end_Connegative cdef int is_definite_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Definite and feature < end_Definite cdef int is_degree_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Degree and feature < end_Degree cdef int is_derivation_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Derivation and feature < end_Derivation cdef int is_echo_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Echo and feature < end_Echo cdef int is_foreign_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Foreign and feature < end_Foreign cdef int is_gender_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Gender and feature < end_Gender cdef int is_hyph_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Hyph and feature < begin_Hyph cdef int is_inf_form_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_InfForm and feature < end_InfForm cdef int is_mood_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Mood and feature < end_Mood cdef int is_negative_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Negative and feature < end_Negative cdef int is_number_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Number and feature < end_Number cdef int is_name_type_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_NameType and feature < end_NameType cdef int is_num_form_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_NumForm and feature < end_NumForm cdef int is_num_type_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_NumType and feature < end_NumType cdef int is_num_value_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_NumValue and feature < end_NumValue cdef int is_part_form_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_PartForm and feature < end_PartForm cdef int is_part_type_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_PartType and feature < end_PartType cdef int is_person_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Person and feature < end_Person cdef int is_polite_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Polite and feature < end_Polite cdef int is_polarity_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Polarity and feature < end_Polarity cdef int is_poss_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Poss and feature < end_Poss cdef int is_prefix_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Prefix and feature < end_Prefix cdef int is_prep_case_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_PrepCase and feature < end_PrepCase cdef int is_pron_type_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_PronType and feature < end_PronType cdef int is_punct_side_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_PunctSide and feature < end_PunctSide cdef int is_punct_type_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_PunctType and feature < end_PunctType cdef int is_reflex_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Reflex and feature < end_Reflex cdef int is_style_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Style and feature < end_Style cdef int is_style_variant_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_StyleVariant and feature < end_StyleVariant cdef int is_tense_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Tense and feature < end_Tense cdef int is_verb_form_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_VerbForm and feature < end_VerbForm cdef int is_voice_feature(univ_morph_t feature) nogil: - return 0 + return feature > begin_Voice and feature < end_Voice cdef int is_verb_type_feature(univ_morph_t feature) nogil: - return 0 - - + return feature > begin_VerbType and feature < end_VerbType IDS = { From 9998d9b9ffd18fd4752dad7f77508502b7f2958d Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 20:38:08 +0200 Subject: [PATCH 013/287] Start testing morphology class --- spacy/tests/morphology/__init__.py | 0 spacy/tests/morphology/test_morph_features.py | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 spacy/tests/morphology/__init__.py create mode 100644 spacy/tests/morphology/test_morph_features.py diff --git a/spacy/tests/morphology/__init__.py b/spacy/tests/morphology/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/spacy/tests/morphology/test_morph_features.py b/spacy/tests/morphology/test_morph_features.py new file mode 100644 index 000000000..f0610b745 --- /dev/null +++ b/spacy/tests/morphology/test_morph_features.py @@ -0,0 +1,25 @@ +from __future__ import unicode_literals +import pytest + +from ...morphology import Morphology +from ...strings import StringStore +from ...lemmatizer import Lemmatizer +from ...symbols import * + +@pytest.fixture +def morphology(): + return Morphology(StringStore(), {}, Lemmatizer()) + +def test_init(morphology): + pass + +def test_add_tag_with_string_names(morphology): + morphology.add({"Case_gen", "Number_Sing"}) + +def test_add_tag_with_int_ids(morphology): + morphology.add({Case_gen, Number_sing}) + +def test_add_tag_with_mix_strings_and_ints(morphology): + morphology.add({PunctSide_ini, 'VerbType_aux'}) + + From 34cab8cc4956305fad928c681b7911c004e51e8f Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 20:53:24 +0200 Subject: [PATCH 014/287] Update morphology API --- spacy/morphology.pxd | 2 +- spacy/morphology.pyx | 30 ++++++++++++++++-------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index 96bba5260..589f500c2 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -28,7 +28,7 @@ cdef class Morphology: cdef int assign_untagged(self, TokenC* token) except -1 cdef int assign_tag(self, TokenC* token, tag) except -1 cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1 - cdef update_morph(self, hash_t morph, features) + cpdef update_morph_key(self, hash_t morph, features) cdef int _assign_tag_from_exceptions(self, TokenC* token, int tag_id) except -1 diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index b37107f09..cc8cb1b19 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -96,10 +96,23 @@ cdef class Morphology: """Insert a morphological analysis in the morphology table, if not already present. Returns the hash of the new analysis. """ - features = intify_features(self.strings, features) + features = intify_features(features) cdef RichTagC tag = create_rich_tag(features) cdef hash_t key = self.insert(tag) return key + + cpdef update_morph_key(self, hash_t morph, features): + """Update a morphological analysis with new feature values.""" + tag = (self.tags.get(morph))[0] + cdef univ_morph_t feature + cdef int value + for feature_, value in features.items(): + feature = self.strings.as_int(feature_) + set_feature(&tag, feature, 1) + morph = self.insert_tag(tag) + return morph + + def lemmatize(self, const univ_pos_t univ_pos, attr_t orth, morphology): if orth not in self.strings: @@ -188,17 +201,6 @@ cdef class Morphology: token.pos = attrs.get(POS, token.pos) token.lemma = attrs.get(LEMMA, token.lemma) - cdef update_morph(self, hash_t morph, features): - """Update a morphological analysis with new feature values.""" - tag = (self.tags.get(morph))[0] - cdef univ_morph_t feature - cdef int value - for feature_, value in features.items(): - feature = self.strings.as_int(feature_) - set_feature(&tag, feature, 1) - morph = self.insert_tag(tag) - return morph - def load_morph_exceptions(self, dict exc): # Map (form, pos) to attributes for tag_str, entries in exc.items(): @@ -226,8 +228,8 @@ cdef class Morphology: cpdef univ_pos_t get_int_tag(pos_): return 0 -cpdef intify_features(StringStore strings, features): - return {strings.as_int(feature) for feature in features} +cpdef intify_features(features): + return {IDS.get(feature, feature) for feature in features} cdef hash_t hash_tag(RichTagC tag) nogil: return mrmr.hash64(&tag, sizeof(tag), 0) From 51a297f93448df314612cf46d5a1946a6afb880b Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 21:07:08 +0200 Subject: [PATCH 015/287] Fix morphology add and update --- spacy/morphology.pxd | 2 +- spacy/morphology.pyx | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index 589f500c2..bc8c44417 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -23,12 +23,12 @@ cdef class Morphology: cdef readonly PreshMapArray _cache cdef readonly int n_tags + cpdef update(self, hash_t morph, features) cdef hash_t insert(self, RichTagC tag) except 0 cdef int assign_untagged(self, TokenC* token) except -1 cdef int assign_tag(self, TokenC* token, tag) except -1 cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1 - cpdef update_morph_key(self, hash_t morph, features) cdef int _assign_tag_from_exceptions(self, TokenC* token, int tag_id) except -1 diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index cc8cb1b19..6e45cab81 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -101,15 +101,14 @@ cdef class Morphology: cdef hash_t key = self.insert(tag) return key - cpdef update_morph_key(self, hash_t morph, features): + cpdef update(self, hash_t morph, features): """Update a morphological analysis with new feature values.""" tag = (self.tags.get(morph))[0] + features = intify_features(features) cdef univ_morph_t feature - cdef int value - for feature_, value in features.items(): - feature = self.strings.as_int(feature_) + for feature in features: set_feature(&tag, feature, 1) - morph = self.insert_tag(tag) + morph = self.insert(tag) return morph @@ -237,8 +236,9 @@ cdef hash_t hash_tag(RichTagC tag) nogil: cdef RichTagC create_rich_tag(features): cdef RichTagC tag cdef univ_morph_t feature - #for feature in features: - # set_feature(&tag, feature, 1) + memset(&tag, 0, sizeof(tag)) + for feature in features: + set_feature(&tag, feature, 1) return tag cdef tag_to_json(RichTagC tag): From d89a1a91ac9dfe8f9a4941ece7a76aab8453c591 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 21:07:48 +0200 Subject: [PATCH 016/287] Update morphology tests --- spacy/tests/morphology/test_morph_features.py | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/spacy/tests/morphology/test_morph_features.py b/spacy/tests/morphology/test_morph_features.py index f0610b745..391fd1337 100644 --- a/spacy/tests/morphology/test_morph_features.py +++ b/spacy/tests/morphology/test_morph_features.py @@ -13,13 +13,29 @@ def morphology(): def test_init(morphology): pass -def test_add_tag_with_string_names(morphology): - morphology.add({"Case_gen", "Number_Sing"}) +def test_add_morphology_with_string_names(morphology): + morphology.add({"Case_gen", "Number_sing"}) -def test_add_tag_with_int_ids(morphology): +def test_add_morphology_with_int_ids(morphology): morphology.add({Case_gen, Number_sing}) -def test_add_tag_with_mix_strings_and_ints(morphology): +def test_add_morphology_with_mix_strings_and_ints(morphology): morphology.add({PunctSide_ini, 'VerbType_aux'}) +def test_morphology_tags_hash_distinctly(morphology): + tag1 = morphology.add({PunctSide_ini, 'VerbType_aux'}) + tag2 = morphology.add({"Case_gen", 'Number_sing'}) + assert tag1 != tag2 + +def test_morphology_tags_hash_independent_of_order(morphology): + tag1 = morphology.add({"Case_gen", 'Number_sing'}) + tag2 = morphology.add({"Number_sing", "Case_gen"}) + assert tag1 == tag2 + +def test_update_morphology_tag(morphology): + tag1 = morphology.add({"Case_gen"}) + tag2 = morphology.update(tag1, {"Number_sing"}) + assert tag1 != tag2 + tag3 = morphology.add({"Number_sing", "Case_gen"}) + assert tag2 == tag3 From 834dfb0e9da3d2aef4b0aa4af9464983ee625ca6 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 21:32:05 +0200 Subject: [PATCH 017/287] Add morph attribute to GoldParse --- spacy/gold.pxd | 1 + spacy/gold.pyx | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/spacy/gold.pxd b/spacy/gold.pxd index a1550b1ef..fdf6f5440 100644 --- a/spacy/gold.pxd +++ b/spacy/gold.pxd @@ -24,6 +24,7 @@ cdef class GoldParse: cdef public int loss cdef public list words cdef public list tags + cdef public list morph cdef public list heads cdef public list labels cdef public dict orths diff --git a/spacy/gold.pyx b/spacy/gold.pyx index 20a319f5d..c9be6d6f1 100644 --- a/spacy/gold.pyx +++ b/spacy/gold.pyx @@ -399,7 +399,7 @@ cdef class GoldParse: return cls(doc, words=words, tags=tags, heads=heads, deps=deps, entities=entities, make_projective=make_projective) - def __init__(self, doc, annot_tuples=None, words=None, tags=None, + def __init__(self, doc, annot_tuples=None, words=None, tags=None, morph=None, heads=None, deps=None, entities=None, make_projective=False, cats=None, **_): """Create a GoldParse. @@ -436,6 +436,8 @@ cdef class GoldParse: deps = [None for _ in doc] if entities is None: entities = [None for _ in doc] + if morph is None: + morph = [None for _ in doc] elif len(entities) == 0: entities = ['O' for _ in doc] elif not isinstance(entities[0], basestring): @@ -460,6 +462,7 @@ cdef class GoldParse: self.heads = [None] * len(doc) self.labels = [None] * len(doc) self.ner = [None] * len(doc) + self.morph = [None] * len(doc) # This needs to be done before we align the words if make_projective and heads is not None and deps is not None: @@ -487,10 +490,12 @@ cdef class GoldParse: self.heads[i] = None self.labels[i] = None self.ner[i] = 'O' + self.morph[i] = set() if gold_i is None: if i in i2j_multi: self.words[i] = words[i2j_multi[i]] self.tags[i] = tags[i2j_multi[i]] + self.morph[i] = morph[i2j_multi[i]] is_last = i2j_multi[i] != i2j_multi.get(i+1) is_first = i2j_multi[i] != i2j_multi.get(i-1) # Set next word in multi-token span as head, until last From 2ba10493f719d442c7d56f07883b195bbc8217f8 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 21:32:24 +0200 Subject: [PATCH 018/287] Read morphology into gold standard in ud-train --- spacy/cli/ud_train.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/spacy/cli/ud_train.py b/spacy/cli/ud_train.py index 4c0b3c7eb..b7e283cfb 100644 --- a/spacy/cli/ud_train.py +++ b/spacy/cli/ud_train.py @@ -74,6 +74,7 @@ def read_data(nlp, conllu_file, text_file, raw_text=True, oracle_segments=False, head = int(head)-1 if head != '0' else id_ sent['words'].append(word) sent['tags'].append(tag) + sent['morph'].append(_parse_morph_string(morph)) sent['heads'].append(head) sent['deps'].append('ROOT' if dep == 'root' else dep) sent['spaces'].append(space_after == '_') @@ -101,6 +102,16 @@ def read_data(nlp, conllu_file, text_file, raw_text=True, oracle_segments=False, return docs, golds return docs, golds +def _parse_morph_string(morph_string): + if morph_string == '_': + return None + output = [] + replacements = {'1': 'one', '2': 'two', '3': 'three'} + for feature in morph_string.split('|'): + key, value = feature.split('=') + value = replacements.get(value, value) + output.append('%s_%s' % (key, value.lower())) + return set(output) def read_conllu(file_): docs = [] From fb0abddd9ebfed4017777e2483fd630c2f711752 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 21:34:53 +0200 Subject: [PATCH 019/287] Call morph morphology in GoldParse --- spacy/cli/ud_train.py | 2 +- spacy/gold.pxd | 2 +- spacy/gold.pyx | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/spacy/cli/ud_train.py b/spacy/cli/ud_train.py index b7e283cfb..9a0b5e10c 100644 --- a/spacy/cli/ud_train.py +++ b/spacy/cli/ud_train.py @@ -74,7 +74,7 @@ def read_data(nlp, conllu_file, text_file, raw_text=True, oracle_segments=False, head = int(head)-1 if head != '0' else id_ sent['words'].append(word) sent['tags'].append(tag) - sent['morph'].append(_parse_morph_string(morph)) + sent['morphology'].append(_parse_morph_string(morph)) sent['heads'].append(head) sent['deps'].append('ROOT' if dep == 'root' else dep) sent['spaces'].append(space_after == '_') diff --git a/spacy/gold.pxd b/spacy/gold.pxd index fdf6f5440..ce066f049 100644 --- a/spacy/gold.pxd +++ b/spacy/gold.pxd @@ -24,7 +24,7 @@ cdef class GoldParse: cdef public int loss cdef public list words cdef public list tags - cdef public list morph + cdef public list morphology cdef public list heads cdef public list labels cdef public dict orths diff --git a/spacy/gold.pyx b/spacy/gold.pyx index c9be6d6f1..65a3932be 100644 --- a/spacy/gold.pyx +++ b/spacy/gold.pyx @@ -399,7 +399,7 @@ cdef class GoldParse: return cls(doc, words=words, tags=tags, heads=heads, deps=deps, entities=entities, make_projective=make_projective) - def __init__(self, doc, annot_tuples=None, words=None, tags=None, morph=None, + def __init__(self, doc, annot_tuples=None, words=None, tags=None, morphology=None, heads=None, deps=None, entities=None, make_projective=False, cats=None, **_): """Create a GoldParse. @@ -436,8 +436,8 @@ cdef class GoldParse: deps = [None for _ in doc] if entities is None: entities = [None for _ in doc] - if morph is None: - morph = [None for _ in doc] + if morphology is None: + morphology = [None for _ in doc] elif len(entities) == 0: entities = ['O' for _ in doc] elif not isinstance(entities[0], basestring): @@ -462,7 +462,7 @@ cdef class GoldParse: self.heads = [None] * len(doc) self.labels = [None] * len(doc) self.ner = [None] * len(doc) - self.morph = [None] * len(doc) + self.morphology = [None] * len(doc) # This needs to be done before we align the words if make_projective and heads is not None and deps is not None: @@ -490,12 +490,12 @@ cdef class GoldParse: self.heads[i] = None self.labels[i] = None self.ner[i] = 'O' - self.morph[i] = set() + self.morphology[i] = set() if gold_i is None: if i in i2j_multi: self.words[i] = words[i2j_multi[i]] self.tags[i] = tags[i2j_multi[i]] - self.morph[i] = morph[i2j_multi[i]] + self.morphology[i] = morphology[i2j_multi[i]] is_last = i2j_multi[i] != i2j_multi.get(i+1) is_first = i2j_multi[i] != i2j_multi.get(i-1) # Set next word in multi-token span as head, until last From 53eb96db0908cb9884dec3cfcdc591ea2f134488 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 22:12:32 +0200 Subject: [PATCH 020/287] Fix definition of morphology model --- spacy/_ml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/_ml.py b/spacy/_ml.py index f37938671..813f5ab7f 100644 --- a/spacy/_ml.py +++ b/spacy/_ml.py @@ -524,8 +524,8 @@ def build_morphologizer_model(class_nums, **cfg): tok2vec = Tok2Vec(token_vector_width, embed_size, subword_features=subword_features, pretrained_vectors=pretrained_vectors) - softmax = with_flatten( - MultiSoftmax(class_nums, token_vector_width)) + softmax = with_flatten(MultiSoftmax(class_nums, token_vector_width)) + softmax.out_sizes = class_nums model = ( tok2vec >> softmax From d0dc032842ed2e3f7ace3a4267fa6ad2e6d4e85d Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 22:12:54 +0200 Subject: [PATCH 021/287] Fill in missing morphologizer methods --- spacy/_morphologizer.pyx | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/spacy/_morphologizer.pyx b/spacy/_morphologizer.pyx index 2fa092faa..a3d3a301a 100644 --- a/spacy/_morphologizer.pyx +++ b/spacy/_morphologizer.pyx @@ -9,9 +9,9 @@ from .util import msgpack from .util import msgpack_numpy from thinc.api import chain -from thinc.neural.util import to_categorical, copy_array +from thinc.neural.util import to_categorical, copy_array, get_array_module from . import util -from .pipe import Pipe +from .pipeline import Pipe from ._ml import Tok2Vec, build_morphologizer_model from ._ml import link_vectors_to_models, zero_init, flatten from ._ml import create_default_optimizer @@ -20,6 +20,7 @@ from .compat import json_dumps, basestring_ from .tokens.doc cimport Doc from .vocab cimport Vocab from .morphology cimport Morphology +from .morphology import parse_feature from .pipeline import Pipe @@ -118,7 +119,7 @@ class Morphologizer(Pipe): target[idx] = guesses[idx] else: for feature in features: - column = feature_to_column(feature) # TODO + _, column = parse_feature(feature) target[idx, column] = 1 idx += 1 target = self.model.ops.xp.array(target, dtype='f') @@ -132,7 +133,10 @@ class Morphologizer(Pipe): yield def scores_to_guesses(scores, out_sizes): - raise NotImplementedError - -def feature_to_column(feature): - raise NotImplementedError + xp = get_array_module(scores) + guesses = xp.zeros((scores.shape[0], len(out_sizes)), dtype='i') + offset = 0 + for i, size in enumerate(out_sizes): + guesses[:, i] = scores[:, offset : offset + size].argmax(axis=1) + offset += size + return guesses From a4fc39788014c7929bc579b699d56853947b4945 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 22:13:10 +0200 Subject: [PATCH 022/287] Add helper to parse features into field and column IDs --- spacy/morphology.pyx | 57 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 6e45cab81..3ba50123c 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -52,6 +52,16 @@ def _normalize_props(props): return out +def parse_feature(feature): + if not hasattr(feature, 'split'): + feature = NAMES[feature] + key, value = feature.split('_') + begin = 'begin_%s' % key + offset = IDS[feature] - IDS[begin] + field_id = FIELDS[key] + return (field_id, offset) + + cdef class Morphology: '''Store the possible morphological analyses for a language, and index them by hash. @@ -716,7 +726,52 @@ IDS = { } -NAMES = [key for key, value in sorted(IDS.items(), key=lambda item: item[1])] +FIELDS = { + 'Abbr': 0, + 'AdpType': 1, + 'AdvType': 2, + 'Animacy': 3, + 'Aspect': 4, + 'Case': 5, + 'ConjType': 6, + 'Connegative': 7, + 'Definite': 8, + 'Degree': 9, + 'Derivation': 10, + 'Echo': 11, + 'Foreign': 12, + 'Gender': 13, + 'Hyph': 14, + 'InfForm': 15, + 'Mood': 16, + 'Negative': 17, + 'Number': 18, + 'NameType': 19, + 'NumForm': 20, + 'NumType': 21, + 'NumValue': 22, + 'PartForm': 23, + 'PartType': 24, + 'Person': 25, + 'Polite': 26, + 'Polarity': 27, + 'Poss': 28, + 'Prefix': 29, + 'PrepCase': 30, + 'PronType': 31, + 'PunctSide': 32, + 'PunctType': 33, + 'Reflex': 34, + 'Style': 35, + 'StyleVariant': 36, + 'Tense': 37, + 'VerbForm': 38, + 'Voice': 39, + 'VerbType': 40 +} + + +NAMES = {value: key for key, value in IDS.items()} # Unfortunate hack here, to work around problem with long cpdef enum # (which is generating an enormous amount of C++ in Cython 0.24+) # We keep the enum cdef, and just make sure the names are available to Python From 031b0d2a3ad5bd8b9a432acb857b4babbc153923 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 22:13:22 +0200 Subject: [PATCH 023/287] Build morphologizer in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e22e28f47..4a587386b 100755 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ MOD_NAMES = [ 'spacy.attrs', 'spacy.morphology', 'spacy.pipeline', + 'spacy._morphologizer', 'spacy.syntax.stateclass', 'spacy.syntax._state', 'spacy.tokenizer', From 2be15fa7d25709599ea0e4af26933a423529ece1 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 25 Sep 2018 23:03:43 +0200 Subject: [PATCH 024/287] Fix Python feature enum in morphology --- spacy/morphology.pyx | 601 +++++++++++++++++++++++++------------------ 1 file changed, 346 insertions(+), 255 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 3ba50123c..870f05a87 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -62,6 +62,12 @@ def parse_feature(feature): return (field_id, offset) +def get_field_size(field): + begin = 'begin_%s' % field + end = 'end_%s' % field + return (IDS[end] - IDS[begin]) - 1 + + cdef class Morphology: '''Store the possible morphological analyses for a language, and index them by hash. @@ -471,261 +477,6 @@ cdef int is_verb_type_feature(univ_morph_t feature) nogil: return feature > begin_VerbType and feature < end_VerbType -IDS = { - "Animacy_anim": Animacy_anim, - "Animacy_inan": Animacy_inan, - "Animacy_hum": Animacy_hum, # U20 - "Animacy_nhum": Animacy_nhum, - "Aspect_freq": Aspect_freq, - "Aspect_imp": Aspect_imp, - "Aspect_mod": Aspect_mod, - "Aspect_none": Aspect_none, - "Aspect_perf": Aspect_perf, - "Case_abe": Case_abe, - "Case_abl": Case_abl, - "Case_abs": Case_abs, - "Case_acc": Case_acc, - "Case_ade": Case_ade, - "Case_all": Case_all, - "Case_cau": Case_cau, - "Case_com": Case_com, - "Case_dat": Case_dat, - "Case_del": Case_del, - "Case_dis": Case_dis, - "Case_ela": Case_ela, - "Case_ess": Case_ess, - "Case_gen": Case_gen, - "Case_ill": Case_ill, - "Case_ine": Case_ine, - "Case_ins": Case_ins, - "Case_loc": Case_loc, - "Case_lat": Case_lat, - "Case_nom": Case_nom, - "Case_par": Case_par, - "Case_sub": Case_sub, - "Case_sup": Case_sup, - "Case_tem": Case_tem, - "Case_ter": Case_ter, - "Case_tra": Case_tra, - "Case_voc": Case_voc, - "Definite_two": Definite_two, - "Definite_def": Definite_def, - "Definite_red": Definite_red, - "Definite_cons": Definite_cons, # U20 - "Definite_ind": Definite_ind, - "Degree_cmp": Degree_cmp, - "Degree_comp": Degree_comp, - "Degree_none": Degree_none, - "Degree_pos": Degree_pos, - "Degree_sup": Degree_sup, - "Degree_abs": Degree_abs, - "Degree_com": Degree_com, - "Degree_dim ": Degree_dim, # du - "Gender_com": Gender_com, - "Gender_fem": Gender_fem, - "Gender_masc": Gender_masc, - "Gender_neut": Gender_neut, - "Mood_cnd": Mood_cnd, - "Mood_imp": Mood_imp, - "Mood_ind": Mood_ind, - "Mood_n": Mood_n, - "Mood_pot": Mood_pot, - "Mood_sub": Mood_sub, - "Mood_opt": Mood_opt, - "Negative_neg": Negative_neg, - "Negative_pos": Negative_pos, - "Negative_yes": Negative_yes, - "Polarity_neg": Polarity_neg, # U20 - "Polarity_pos": Polarity_pos, # U20 - "Number_com": Number_com, - "Number_dual": Number_dual, - "Number_none": Number_none, - "Number_plur": Number_plur, - "Number_sing": Number_sing, - "Number_ptan ": Number_ptan, # bg - "Number_count ": Number_count, # bg - "NumType_card": NumType_card, - "NumType_dist": NumType_dist, - "NumType_frac": NumType_frac, - "NumType_gen": NumType_gen, - "NumType_mult": NumType_mult, - "NumType_none": NumType_none, - "NumType_ord": NumType_ord, - "NumType_sets": NumType_sets, - "Person_one": Person_one, - "Person_two": Person_two, - "Person_three": Person_three, - "Person_none": Person_none, - "Poss_yes": Poss_yes, - "PronType_advPart": PronType_advPart, - "PronType_art": PronType_art, - "PronType_default": PronType_default, - "PronType_dem": PronType_dem, - "PronType_ind": PronType_ind, - "PronType_int": PronType_int, - "PronType_neg": PronType_neg, - "PronType_prs": PronType_prs, - "PronType_rcp": PronType_rcp, - "PronType_rel": PronType_rel, - "PronType_tot": PronType_tot, - "PronType_clit": PronType_clit, - "PronType_exc ": PronType_exc, # es, ca, it, fa, - "Reflex_yes": Reflex_yes, - "Tense_fut": Tense_fut, - "Tense_imp": Tense_imp, - "Tense_past": Tense_past, - "Tense_pres": Tense_pres, - "VerbForm_fin": VerbForm_fin, - "VerbForm_ger": VerbForm_ger, - "VerbForm_inf": VerbForm_inf, - "VerbForm_none": VerbForm_none, - "VerbForm_part": VerbForm_part, - "VerbForm_partFut": VerbForm_partFut, - "VerbForm_partPast": VerbForm_partPast, - "VerbForm_partPres": VerbForm_partPres, - "VerbForm_sup": VerbForm_sup, - "VerbForm_trans": VerbForm_trans, - "VerbForm_conv": VerbForm_conv, # U20 - "VerbForm_gdv ": VerbForm_gdv, # la, - "Voice_act": Voice_act, - "Voice_cau": Voice_cau, - "Voice_pass": Voice_pass, - "Voice_mid ": Voice_mid, # gkc, - "Voice_int ": Voice_int, # hb, - "Abbr_yes ": Abbr_yes, # cz, fi, sl, U, - "AdpType_prep ": AdpType_prep, # cz, U, - "AdpType_post ": AdpType_post, # U, - "AdpType_voc ": AdpType_voc, # cz, - "AdpType_comprep ": AdpType_comprep, # cz, - "AdpType_circ ": AdpType_circ, # U, - "AdvType_man": AdvType_man, - "AdvType_loc": AdvType_loc, - "AdvType_tim": AdvType_tim, - "AdvType_deg": AdvType_deg, - "AdvType_cau": AdvType_cau, - "AdvType_mod": AdvType_mod, - "AdvType_sta": AdvType_sta, - "AdvType_ex": AdvType_ex, - "AdvType_adadj": AdvType_adadj, - "ConjType_oper ": ConjType_oper, # cz, U, - "ConjType_comp ": ConjType_comp, # cz, U, - "Connegative_yes ": Connegative_yes, # fi, - "Derivation_minen ": Derivation_minen, # fi, - "Derivation_sti ": Derivation_sti, # fi, - "Derivation_inen ": Derivation_inen, # fi, - "Derivation_lainen ": Derivation_lainen, # fi, - "Derivation_ja ": Derivation_ja, # fi, - "Derivation_ton ": Derivation_ton, # fi, - "Derivation_vs ": Derivation_vs, # fi, - "Derivation_ttain ": Derivation_ttain, # fi, - "Derivation_ttaa ": Derivation_ttaa, # fi, - "Echo_rdp ": Echo_rdp, # U, - "Echo_ech ": Echo_ech, # U, - "Foreign_foreign ": Foreign_foreign, # cz, fi, U, - "Foreign_fscript ": Foreign_fscript, # cz, fi, U, - "Foreign_tscript ": Foreign_tscript, # cz, U, - "Foreign_yes ": Foreign_yes, # sl, - "Gender_dat_masc ": Gender_dat_masc, # bq, U, - "Gender_dat_fem ": Gender_dat_fem, # bq, U, - "Gender_erg_masc ": Gender_erg_masc, # bq, - "Gender_erg_fem ": Gender_erg_fem, # bq, - "Gender_psor_masc ": Gender_psor_masc, # cz, sl, U, - "Gender_psor_fem ": Gender_psor_fem, # cz, sl, U, - "Gender_psor_neut ": Gender_psor_neut, # sl, - "Hyph_yes ": Hyph_yes, # cz, U, - "InfForm_one ": InfForm_one, # fi, - "InfForm_two ": InfForm_two, # fi, - "InfForm_three ": InfForm_three, # fi, - "NameType_geo ": NameType_geo, # U, cz, - "NameType_prs ": NameType_prs, # U, cz, - "NameType_giv ": NameType_giv, # U, cz, - "NameType_sur ": NameType_sur, # U, cz, - "NameType_nat ": NameType_nat, # U, cz, - "NameType_com ": NameType_com, # U, cz, - "NameType_pro ": NameType_pro, # U, cz, - "NameType_oth ": NameType_oth, # U, cz, - "NounType_com ": NounType_com, # U, - "NounType_prop ": NounType_prop, # U, - "NounType_class ": NounType_class, # U, - "Number_abs_sing ": Number_abs_sing, # bq, U, - "Number_abs_plur ": Number_abs_plur, # bq, U, - "Number_dat_sing ": Number_dat_sing, # bq, U, - "Number_dat_plur ": Number_dat_plur, # bq, U, - "Number_erg_sing ": Number_erg_sing, # bq, U, - "Number_erg_plur ": Number_erg_plur, # bq, U, - "Number_psee_sing ": Number_psee_sing, # U, - "Number_psee_plur ": Number_psee_plur, # U, - "Number_psor_sing ": Number_psor_sing, # cz, fi, sl, U, - "Number_psor_plur ": Number_psor_plur, # cz, fi, sl, U, - "NumForm_digit ": NumForm_digit, # cz, sl, U, - "NumForm_roman ": NumForm_roman, # cz, sl, U, - "NumForm_word ": NumForm_word, # cz, sl, U, - "NumValue_one ": NumValue_one, # cz, U, - "NumValue_two ": NumValue_two, # cz, U, - "NumValue_three ": NumValue_three, # cz, U, - "PartForm_pres ": PartForm_pres, # fi, - "PartForm_past ": PartForm_past, # fi, - "PartForm_agt ": PartForm_agt, # fi, - "PartForm_neg ": PartForm_neg, # fi, - "PartType_mod ": PartType_mod, # U, - "PartType_emp ": PartType_emp, # U, - "PartType_res ": PartType_res, # U, - "PartType_inf ": PartType_inf, # U, - "PartType_vbp ": PartType_vbp, # U, - "Person_abs_one ": Person_abs_one, # bq, U, - "Person_abs_two ": Person_abs_two, # bq, U, - "Person_abs_three ": Person_abs_three, # bq, U, - "Person_dat_one ": Person_dat_one, # bq, U, - "Person_dat_two ": Person_dat_two, # bq, U, - "Person_dat_three ": Person_dat_three, # bq, U, - "Person_erg_one ": Person_erg_one, # bq, U, - "Person_erg_two ": Person_erg_two, # bq, U, - "Person_erg_three ": Person_erg_three, # bq, U, - "Person_psor_one ": Person_psor_one, # fi, U, - "Person_psor_two ": Person_psor_two, # fi, U, - "Person_psor_three ": Person_psor_three, # fi, U, - "Polite_inf ": Polite_inf, # bq, U, - "Polite_pol ": Polite_pol, # bq, U, - "Polite_abs_inf ": Polite_abs_inf, # bq, U, - "Polite_abs_pol ": Polite_abs_pol, # bq, U, - "Polite_erg_inf ": Polite_erg_inf, # bq, U, - "Polite_erg_pol ": Polite_erg_pol, # bq, U, - "Polite_dat_inf ": Polite_dat_inf, # bq, U, - "Polite_dat_pol ": Polite_dat_pol, # bq, U, - "Prefix_yes ": Prefix_yes, # U, - "PrepCase_npr ": PrepCase_npr, # cz, - "PrepCase_pre ": PrepCase_pre, # U, - "PunctSide_ini ": PunctSide_ini, # U, - "PunctSide_fin ": PunctSide_fin, # U, - "PunctType_peri ": PunctType_peri, # U, - "PunctType_qest ": PunctType_qest, # U, - "PunctType_excl ": PunctType_excl, # U, - "PunctType_quot ": PunctType_quot, # U, - "PunctType_brck ": PunctType_brck, # U, - "PunctType_comm ": PunctType_comm, # U, - "PunctType_colo ": PunctType_colo, # U, - "PunctType_semi ": PunctType_semi, # U, - "PunctType_dash ": PunctType_dash, # U, - "Style_arch ": Style_arch, # cz, fi, U, - "Style_rare ": Style_rare, # cz, fi, U, - "Style_poet ": Style_poet, # cz, U, - "Style_norm ": Style_norm, # cz, U, - "Style_coll ": Style_coll, # cz, U, - "Style_vrnc ": Style_vrnc, # cz, U, - "Style_sing ": Style_sing, # cz, U, - "Style_expr ": Style_expr, # cz, U, - "Style_derg ": Style_derg, # cz, U, - "Style_vulg ": Style_vulg, # cz, U, - "Style_yes ": Style_yes, # fi, U, - "StyleVariant_styleShort ": StyleVariant_styleShort, # cz, - "StyleVariant_styleBound ": StyleVariant_styleBound, # cz, sl, - "VerbType_aux ": VerbType_aux, # U, - "VerbType_cop ": VerbType_cop, # U, - "VerbType_mod ": VerbType_mod, # U, - "VerbType_light ": VerbType_light, # U, -} - - FIELDS = { 'Abbr': 0, 'AdpType': 1, @@ -770,6 +521,346 @@ FIELDS = { 'VerbType': 40 } +IDS = { + "begin_Abbr": begin_Abbr, + "Abbr_yes ": Abbr_yes , + "end_Abbr": end_Abbr, + "begin_AdpType": begin_AdpType, + "AdpType_circ": AdpType_circ, + "AdpType_comprep": AdpType_comprep, + "AdpType_prep ": AdpType_prep , + "AdpType_post": AdpType_post, + "AdpType_voc": AdpType_voc, + "end_AdpType": end_AdpType, + "begin_AdvType": begin_AdvType, + "AdvType_adadj": AdvType_adadj, + "AdvType_cau": AdvType_cau, + "AdvType_deg": AdvType_deg, + "AdvType_ex": AdvType_ex, + "AdvType_loc": AdvType_loc, + "AdvType_man": AdvType_man, + "AdvType_mod": AdvType_mod, + "AdvType_sta": AdvType_sta, + "AdvType_tim": AdvType_tim, + "end_AdvType": end_AdvType, + "begin_Animacy": begin_Animacy, + "Animacy_anim": Animacy_anim, + "Animacy_hum": Animacy_hum, + "Animacy_inan": Animacy_inan, + "Animacy_nhum": Animacy_nhum, + "end_Animacy": end_Animacy, + "begin_Aspect": begin_Aspect, + "Aspect_freq": Aspect_freq, + "Aspect_imp": Aspect_imp, + "Aspect_mod": Aspect_mod, + "Aspect_none": Aspect_none, + "Aspect_perf": Aspect_perf, + "end_Aspect": end_Aspect, + "begin_Case": begin_Case, + "Case_abe": Case_abe, + "Case_abl": Case_abl, + "Case_abs": Case_abs, + "Case_acc": Case_acc, + "Case_ade": Case_ade, + "Case_all": Case_all, + "Case_cau": Case_cau, + "Case_com": Case_com, + "Case_dat": Case_dat, + "Case_del": Case_del, + "Case_dis": Case_dis, + "Case_ela": Case_ela, + "Case_ess": Case_ess, + "Case_gen": Case_gen, + "Case_ill": Case_ill, + "Case_ine": Case_ine, + "Case_ins": Case_ins, + "Case_loc": Case_loc, + "Case_lat": Case_lat, + "Case_nom": Case_nom, + "Case_par": Case_par, + "Case_sub": Case_sub, + "Case_sup": Case_sup, + "Case_tem": Case_tem, + "Case_ter": Case_ter, + "Case_tra": Case_tra, + "Case_voc": Case_voc, + "end_Case": end_Case, + "begin_ConjType": begin_ConjType, + "ConjType_comp ": ConjType_comp , + "ConjType_oper": ConjType_oper, + "end_ConjType": end_ConjType, + "begin_Connegative": begin_Connegative, + "Connegative_yes": Connegative_yes, + "end_Connegative": end_Connegative, + "begin_Definite": begin_Definite, + "Definite_cons": Definite_cons, + "Definite_def": Definite_def, + "Definite_ind": Definite_ind, + "Definite_red": Definite_red, + "Definite_two": Definite_two, + "end_Definite": end_Definite, + "begin_Degree": begin_Degree, + "Degree_abs": Degree_abs, + "Degree_cmp": Degree_cmp, + "Degree_comp": Degree_comp, + "Degree_none": Degree_none, + "Degree_pos": Degree_pos, + "Degree_sup": Degree_sup, + "Degree_com": Degree_com, + "Degree_dim": Degree_dim, + "end_Degree": end_Degree, + "begin_Gender": begin_Gender, + "Gender_com": Gender_com, + "Gender_fem": Gender_fem, + "Gender_masc": Gender_masc, + "Gender_neut": Gender_neut, + "Gender_dat_masc": Gender_dat_masc, + "Gender_dat_fem": Gender_dat_fem, + "Gender_erg_masc": Gender_erg_masc, + "Gender_erg_fem": Gender_erg_fem, + "Gender_psor_masc": Gender_psor_masc, + "Gender_psor_fem": Gender_psor_fem, + "Gender_psor_neut": Gender_psor_neut, + "end_Gender": end_Gender, + "begin_Mood": begin_Mood, + "Mood_cnd": Mood_cnd, + "Mood_imp": Mood_imp, + "Mood_ind": Mood_ind, + "Mood_n": Mood_n, + "Mood_pot": Mood_pot, + "Mood_sub": Mood_sub, + "Mood_opt": Mood_opt, + "end_Mood": end_Mood, + "begin_Negative": begin_Negative, + "Negative_neg": Negative_neg, + "Negative_pos": Negative_pos, + "Negative_yes": Negative_yes, + "end_Negative": end_Negative, + "begin_Polarity": begin_Polarity, + "Polarity_neg": Polarity_neg, + "Polarity_pos": Polarity_pos, + "end_Polarity": end_Polarity, + "begin_Number": begin_Number, + "Number_com": Number_com, + "Number_dual": Number_dual, + "Number_none": Number_none, + "Number_plur": Number_plur, + "Number_sing": Number_sing, + "Number_ptan": Number_ptan, + "Number_count": Number_count, + "Number_abs_sing": Number_abs_sing, + "Number_abs_plur": Number_abs_plur, + "Number_dat_sing": Number_dat_sing, + "Number_dat_plur": Number_dat_plur, + "Number_erg_sing": Number_erg_sing, + "Number_erg_plur": Number_erg_plur, + "Number_psee_sing": Number_psee_sing, + "Number_psee_plur": Number_psee_plur, + "Number_psor_sing": Number_psor_sing, + "Number_psor_plur": Number_psor_plur, + "end_Number": end_Number, + "begin_NumType": begin_NumType, + "NumType_card": NumType_card, + "NumType_dist": NumType_dist, + "NumType_frac": NumType_frac, + "NumType_gen": NumType_gen, + "NumType_mult": NumType_mult, + "NumType_none": NumType_none, + "NumType_ord": NumType_ord, + "NumType_sets": NumType_sets, + "end_NumType": end_NumType, + "begin_Person": begin_Person, + "Person_one": Person_one, + "Person_two": Person_two, + "Person_three": Person_three, + "Person_none": Person_none, + "Person_abs_one": Person_abs_one, + "Person_abs_two": Person_abs_two, + "Person_abs_three": Person_abs_three, + "Person_dat_one": Person_dat_one, + "Person_dat_two": Person_dat_two, + "Person_dat_three": Person_dat_three, + "Person_erg_one": Person_erg_one, + "Person_erg_two": Person_erg_two, + "Person_erg_three": Person_erg_three, + "Person_psor_one": Person_psor_one, + "Person_psor_two": Person_psor_two, + "Person_psor_three": Person_psor_three, + "end_Person": end_Person, + "begin_Poss": begin_Poss, + "Poss_yes": Poss_yes, + "end_Poss": end_Poss, + "begin_PronType": begin_PronType, + "PronType_advPart": PronType_advPart, + "PronType_art": PronType_art, + "PronType_default": PronType_default, + "PronType_dem": PronType_dem, + "PronType_ind": PronType_ind, + "PronType_int": PronType_int, + "PronType_neg": PronType_neg, + "PronType_prs": PronType_prs, + "PronType_rcp": PronType_rcp, + "PronType_rel": PronType_rel, + "PronType_tot": PronType_tot, + "PronType_clit": PronType_clit, + "PronType_exc": PronType_exc, + "end_PronType": end_PronType, + "begin_Reflex": begin_Reflex, + "Reflex_yes": Reflex_yes, + "end_Reflex": end_Reflex, + "begin_Tense": begin_Tense, + "Tense_fut": Tense_fut, + "Tense_imp": Tense_imp, + "Tense_past": Tense_past, + "Tense_pres": Tense_pres, + "end_Tense": end_Tense, + "begin_VerbForm": begin_VerbForm, + "VerbForm_fin": VerbForm_fin, + "VerbForm_ger": VerbForm_ger, + "VerbForm_inf": VerbForm_inf, + "VerbForm_none": VerbForm_none, + "VerbForm_part": VerbForm_part, + "VerbForm_partFut": VerbForm_partFut, + "VerbForm_partPast": VerbForm_partPast, + "VerbForm_partPres": VerbForm_partPres, + "VerbForm_sup": VerbForm_sup, + "VerbForm_trans": VerbForm_trans, + "VerbForm_conv": VerbForm_conv, + "VerbForm_gdv": VerbForm_gdv, + "end_VerbForm": end_VerbForm, + "begin_Voice": begin_Voice, + "Voice_act": Voice_act, + "Voice_cau": Voice_cau, + "Voice_pass": Voice_pass, + "Voice_mid": Voice_mid, + "Voice_int": Voice_int, + "end_Voice": end_Voice, + "begin_Derivation": begin_Derivation, + "Derivation_minen": Derivation_minen, + "Derivation_sti": Derivation_sti, + "Derivation_inen": Derivation_inen, + "Derivation_lainen": Derivation_lainen, + "Derivation_ja": Derivation_ja, + "Derivation_ton": Derivation_ton, + "Derivation_vs": Derivation_vs, + "Derivation_ttain": Derivation_ttain, + "Derivation_ttaa": Derivation_ttaa, + "end_Derivation": end_Derivation, + "begin_Echo": begin_Echo, + "Echo_rdp": Echo_rdp, + "Echo_ech": Echo_ech, + "end_Echo": end_Echo, + "begin_Foreign": begin_Foreign, + "Foreign_foreign": Foreign_foreign, + "Foreign_fscript": Foreign_fscript, + "Foreign_tscript": Foreign_tscript, + "Foreign_yes": Foreign_yes, + "end_Foreign": end_Foreign, + "begin_Hyph": begin_Hyph, + "Hyph_yes": Hyph_yes, + "end_Hyph": end_Hyph, + "begin_InfForm": begin_InfForm, + "InfForm_one": InfForm_one, + "InfForm_two": InfForm_two, + "InfForm_three": InfForm_three, + "end_InfForm": end_InfForm, + "begin_NameType": begin_NameType, + "NameType_geo": NameType_geo, + "NameType_prs": NameType_prs, + "NameType_giv": NameType_giv, + "NameType_sur": NameType_sur, + "NameType_nat": NameType_nat, + "NameType_com": NameType_com, + "NameType_pro": NameType_pro, + "NameType_oth": NameType_oth, + "end_NameType": end_NameType, + "begin_NounType": begin_NounType, + "NounType_com": NounType_com, + "NounType_prop": NounType_prop, + "NounType_class": NounType_class, + "end_NounType": end_NounType, + "begin_NumForm": begin_NumForm, + "NumForm_digit": NumForm_digit, + "NumForm_roman": NumForm_roman, + "NumForm_word": NumForm_word, + "end_NumForm": end_NumForm, + "begin_NumValue": begin_NumValue, + "NumValue_one": NumValue_one, + "NumValue_two": NumValue_two, + "NumValue_three": NumValue_three, + "end_NumValue": end_NumValue, + "begin_PartForm": begin_PartForm, + "PartForm_pres": PartForm_pres, + "PartForm_past": PartForm_past, + "PartForm_agt": PartForm_agt, + "PartForm_neg": PartForm_neg, + "end_PartForm": end_PartForm, + "begin_PartType": begin_PartType, + "PartType_mod": PartType_mod, + "PartType_emp": PartType_emp, + "PartType_res": PartType_res, + "PartType_inf": PartType_inf, + "PartType_vbp": PartType_vbp, + "end_PartType": end_PartType, + "begin_Polite": begin_Polite, + "Polite_inf": Polite_inf, + "Polite_pol": Polite_pol, + "Polite_abs_inf": Polite_abs_inf, + "Polite_abs_pol": Polite_abs_pol, + "Polite_erg_inf": Polite_erg_inf, + "Polite_erg_pol": Polite_erg_pol, + "Polite_dat_inf": Polite_dat_inf, + "Polite_dat_pol": Polite_dat_pol, + "end_Polite": end_Polite, + "begin_Prefix": begin_Prefix, + "Prefix_yes": Prefix_yes, + "end_Prefix": end_Prefix, + "begin_PrepCase": begin_PrepCase, + "PrepCase_npr": PrepCase_npr, + "PrepCase_pre": PrepCase_pre, + "end_PrepCase": end_PrepCase, + "begin_PunctSide": begin_PunctSide, + "PunctSide_ini": PunctSide_ini, + "PunctSide_fin": PunctSide_fin, + "end_PunctSide": end_PunctSide, + "begin_PunctType": begin_PunctType, + "PunctType_peri": PunctType_peri, + "PunctType_qest": PunctType_qest, + "PunctType_excl": PunctType_excl, + "PunctType_quot": PunctType_quot, + "PunctType_brck": PunctType_brck, + "PunctType_comm": PunctType_comm, + "PunctType_colo": PunctType_colo, + "PunctType_semi": PunctType_semi, + "PunctType_dash": PunctType_dash, + "end_PunctType": end_PunctType, + "begin_Style": begin_Style, + "Style_arch": Style_arch, + "Style_rare": Style_rare, + "Style_poet": Style_poet, + "Style_norm": Style_norm, + "Style_coll": Style_coll, + "Style_vrnc": Style_vrnc, + "Style_sing": Style_sing, + "Style_expr": Style_expr, + "Style_derg": Style_derg, + "Style_vulg": Style_vulg, + "Style_yes": Style_yes, + "end_Style": end_Style, + "begin_StyleVariant": begin_StyleVariant, + "StyleVariant_styleShort": StyleVariant_styleShort, + "StyleVariant_styleBound": StyleVariant_styleBound, + "end_StyleVariant": end_StyleVariant, + "begin_VerbType": begin_VerbType, + "VerbType_aux": VerbType_aux, + "VerbType_cop": VerbType_cop, + "VerbType_mod": VerbType_mod, + "VerbType_light": VerbType_light, + "end_VerbType": end_VerbType, +} + + +FIELD_SIZES = [get_field_size(field) for field in FIELDS] NAMES = {value: key for key, value in IDS.items()} # Unfortunate hack here, to work around problem with long cpdef enum From 3b6b018904b9ef8dbabd336027df7e2a9fc7424b Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 26 Sep 2018 21:01:48 +0200 Subject: [PATCH 025/287] Fix loading of gold morphology --- spacy/gold.pyx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/spacy/gold.pyx b/spacy/gold.pyx index 65a3932be..77c5944ca 100644 --- a/spacy/gold.pyx +++ b/spacy/gold.pyx @@ -429,17 +429,17 @@ cdef class GoldParse: if words is None: words = [token.text for token in doc] if tags is None: - tags = [None for _ in doc] + tags = [None for _ in words] if heads is None: - heads = [None for token in doc] + heads = [None for token in words] if deps is None: - deps = [None for _ in doc] + deps = [None for _ in words] if entities is None: - entities = [None for _ in doc] + entities = [None for _ in words] if morphology is None: - morphology = [None for _ in doc] + morphology = [None for _ in words] elif len(entities) == 0: - entities = ['O' for _ in doc] + entities = ['O' for _ in words] elif not isinstance(entities[0], basestring): # Assume we have entities specified by character offset. entities = biluo_tags_from_offsets(doc, entities) @@ -532,6 +532,7 @@ cdef class GoldParse: else: self.words[i] = words[gold_i] self.tags[i] = tags[gold_i] + self.morphology[i] = morphology[gold_i] if heads[gold_i] is None: self.heads[i] = None else: From 1f9f834dc08bdb2d1bcd2eb63eb953336f1b4b78 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 26 Sep 2018 21:02:13 +0200 Subject: [PATCH 026/287] Fix morphologizer --- spacy/_morphologizer.pyx | 65 ++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/spacy/_morphologizer.pyx b/spacy/_morphologizer.pyx index a3d3a301a..10baec8f5 100644 --- a/spacy/_morphologizer.pyx +++ b/spacy/_morphologizer.pyx @@ -20,7 +20,7 @@ from .compat import json_dumps, basestring_ from .tokens.doc cimport Doc from .vocab cimport Vocab from .morphology cimport Morphology -from .morphology import parse_feature +from .morphology import parse_feature, IDS, FIELDS, FIELD_SIZES, NAMES from .pipeline import Pipe @@ -28,9 +28,11 @@ class Morphologizer(Pipe): name = 'morphologizer' @classmethod - def Model(cls, attr_nums, **cfg): + def Model(cls, attr_nums=None, **cfg): if cfg.get('pretrained_dims') and not cfg.get('pretrained_vectors'): raise ValueError(TempErrors.T008) + if attr_nums is None: + attr_nums = list(FIELD_SIZES) return build_morphologizer_model(attr_nums, **cfg) def __init__(self, vocab, model=True, **cfg): @@ -71,29 +73,34 @@ class Morphologizer(Pipe): return guesses, tokvecs tokvecs = self.model.tok2vec(docs) scores = self.model.softmax(tokvecs) - guesses = [] - # Resolve multisoftmax into guesses - for doc_scores in scores: - guesses.append(scores_to_guesses(doc_scores, self.model.softmax.out_sizes)) - return guesses, tokvecs + return scores, tokvecs - def set_annotations(self, docs, batch_feature_ids, tensors=None): + def set_annotations(self, docs, batch_scores, tensors=None): if isinstance(docs, Doc): docs = [docs] cdef Doc doc cdef Vocab vocab = self.vocab + field_names = list(FIELDS) + offsets = [IDS['begin_%s' % field] for field in field_names] for i, doc in enumerate(docs): - doc_feat_ids = batch_feature_ids[i] - if hasattr(doc_feat_ids, 'get'): - doc_feat_ids = doc_feat_ids.get() + doc_scores = batch_scores[i] + doc_guesses = scores_to_guesses(doc_scores, self.model.softmax.out_sizes) # Convert the neuron indices into feature IDs. - offset = self.vocab.morphology.first_feature - for j, nr_feat in enumerate(self.model.softmax.out_sizes): - doc_feat_ids[:, j] += offset - offset += nr_feat - # Now add the analysis, and set the hash. - for j in range(doc_feat_ids.shape[0]): - doc.c[j].morph = self.vocab.morphology.add(doc_feat_ids[j]) + doc_feat_ids = self.model.ops.allocate((len(doc), len(field_names)), dtype='i') + for j in range(len(doc)): + for k, offset in enumerate(offsets): + if doc_guesses[j, k] == 0: + doc_feat_ids[j, k] = 0 + else: + doc_feat_ids[j, k] = offset + doc_guesses[j, k] + # Now add the analysis, and set the hash. + try: + doc.c[j].morph = self.vocab.morphology.add(doc_feat_ids[j]) + except: + print(offsets) + print(doc_guesses[j]) + print(doc_feat_ids[j]) + raise def update(self, docs, golds, drop=0., sgd=None, losses=None): if losses is not None and self.name not in losses: @@ -110,17 +117,27 @@ class Morphologizer(Pipe): guesses = [] for doc_scores in scores: guesses.append(scores_to_guesses(doc_scores, self.model.softmax.out_sizes)) - guesses = self.model.ops.flatten(guesses) + guesses = self.model.ops.xp.vstack(guesses) + scores = self.model.ops.xp.vstack(scores) cdef int idx = 0 target = numpy.zeros(scores.shape, dtype='f') + field_sizes = self.model.softmax.out_sizes for gold in golds: for features in gold.morphology: if features is None: - target[idx] = guesses[idx] + target[idx] = scores[idx] else: + by_field = {} for feature in features: - _, column = parse_feature(feature) - target[idx, column] = 1 + field, column = parse_feature(feature) + by_field[field] = column + col_offset = 0 + for field, field_size in enumerate(field_sizes): + if field in by_field: + target[idx, col_offset + by_field[field]] = 1. + else: + target[idx, col_offset] = 1. + col_offset += field_size idx += 1 target = self.model.ops.xp.array(target, dtype='f') d_scores = scores - target @@ -137,6 +154,8 @@ def scores_to_guesses(scores, out_sizes): guesses = xp.zeros((scores.shape[0], len(out_sizes)), dtype='i') offset = 0 for i, size in enumerate(out_sizes): - guesses[:, i] = scores[:, offset : offset + size].argmax(axis=1) + slice_ = scores[:, offset : offset + size] + col_guesses = slice_.argmax(axis=1) + guesses[:, i] = col_guesses offset += size return guesses From f03640b41ff94f42c38b21d1247e29a53a0dbb1a Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 26 Sep 2018 21:02:42 +0200 Subject: [PATCH 027/287] Fix morphology task in ud-train --- spacy/cli/ud_train.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/spacy/cli/ud_train.py b/spacy/cli/ud_train.py index 9a0b5e10c..c310c6616 100644 --- a/spacy/cli/ud_train.py +++ b/spacy/cli/ud_train.py @@ -84,10 +84,12 @@ def read_data(nlp, conllu_file, text_file, raw_text=True, oracle_segments=False, if oracle_segments: docs.append(Doc(nlp.vocab, words=sent['words'], spaces=sent['spaces'])) golds.append(GoldParse(docs[-1], **sent)) + assert golds[-1].morphology is not None sent_annots.append(sent) if raw_text and max_doc_length and len(sent_annots) >= max_doc_length: doc, gold = _make_gold(nlp, None, sent_annots) + assert gold.morphology is not None sent_annots = [] docs.append(doc) golds.append(gold) @@ -104,12 +106,13 @@ def read_data(nlp, conllu_file, text_file, raw_text=True, oracle_segments=False, def _parse_morph_string(morph_string): if morph_string == '_': - return None + return set() output = [] replacements = {'1': 'one', '2': 'two', '3': 'three'} for feature in morph_string.split('|'): key, value = feature.split('=') value = replacements.get(value, value) + value = value.split(',')[0] output.append('%s_%s' % (key, value.lower())) return set(output) @@ -146,7 +149,7 @@ def _make_gold(nlp, text, sent_annots, drop_deps=0.0): sent_starts = [] for sent in sent_annots: flat['heads'].extend(len(flat['words'])+head for head in sent['heads']) - for field in ['words', 'tags', 'deps', 'entities', 'spaces']: + for field in ['words', 'tags', 'deps', 'morphology', 'entities', 'spaces']: flat[field].extend(sent[field]) sent_starts.append(True) sent_starts.extend([False] * (len(sent['words'])-1)) @@ -238,22 +241,26 @@ def write_conllu(docs, file_): def print_progress(itn, losses, ud_scores): fields = { 'dep_loss': losses.get('parser', 0.0), + 'morph_loss': losses.get('morphologizer', 0.0), 'tag_loss': losses.get('tagger', 0.0), 'words': ud_scores['Words'].f1 * 100, 'sents': ud_scores['Sentences'].f1 * 100, 'tags': ud_scores['XPOS'].f1 * 100, 'uas': ud_scores['UAS'].f1 * 100, 'las': ud_scores['LAS'].f1 * 100, + 'morph': ud_scores['Feats'].f1 * 100, } - header = ['Epoch', 'Loss', 'LAS', 'UAS', 'TAG', 'SENT', 'WORD'] + header = ['Epoch', 'P.Loss', 'M.Loss', 'LAS', 'UAS', 'TAG', 'MORPH', 'SENT', 'WORD'] if itn == 0: print('\t'.join(header)) tpl = '\t'.join(( '{:d}', '{dep_loss:.1f}', + '{morph_loss:.1f}', '{las:.1f}', '{uas:.1f}', '{tags:.1f}', + '{morph:.1f}', '{sents:.1f}', '{words:.1f}', )) @@ -275,7 +282,19 @@ def get_token_conllu(token, i): head = 0 else: head = i + (token.head.i - token.i) + 1 - fields = [str(i+1), token.text, token.lemma_, token.pos_, token.tag_, '_', + features = token.vocab.morphology.get(token.morph_key) + feat_str = [] + replacements = {'one': '1', 'two': '2', 'three': '3'} + for feat in features: + if not feat.startswith('begin') and not feat.startswith('end'): + key, value = feat.split('_') + value = replacements.get(value, value) + feat_str.append('%s=%s' % (key, value.title())) + if not feat_str: + feat_str = '_' + else: + feat_str = '|'.join(feat_str) + fields = [str(i+1), token.text, token.lemma_, token.pos_, token.tag_, feat_str, str(head), token.dep_.lower(), '_', '_'] lines.append('\t'.join(fields)) return '\n'.join(lines) @@ -305,6 +324,7 @@ def load_nlp(corpus, config, vectors=None): def initialize_pipeline(nlp, docs, golds, config, device): nlp.add_pipe(nlp.create_pipe('tagger')) + nlp.add_pipe(nlp.create_pipe('morphologizer')) nlp.add_pipe(nlp.create_pipe('parser')) if config.multitask_tag: nlp.parser.add_multitask_objective('tag') @@ -437,11 +457,11 @@ def main(ud_dir, parses_dir, corpus, config=None, limit=0, gpu_device=-1, vector with nlp.use_params(optimizer.averages): if use_oracle_segments: parsed_docs, scores = evaluate(nlp, paths.dev.conllu, - paths.dev.conllu, out_path) + paths.dev.conllu, out_path) else: parsed_docs, scores = evaluate(nlp, paths.dev.text, - paths.dev.conllu, out_path) - print_progress(i, losses, scores) + paths.dev.conllu, out_path) + print_progress(i, losses, scores) def _render_parses(i, to_render): From 6f983132544e1d20684930b7157d92a70f16a32c Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 26 Sep 2018 21:03:03 +0200 Subject: [PATCH 028/287] Fix disjunctive features in English tag map --- spacy/lang/en/tag_map.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spacy/lang/en/tag_map.py b/spacy/lang/en/tag_map.py index fc3d2cc93..ffc1739cb 100644 --- a/spacy/lang/en/tag_map.py +++ b/spacy/lang/en/tag_map.py @@ -52,10 +52,10 @@ TAG_MAP = { "VBN": {POS: VERB, "VerbForm": "part", "Tense": "past", "Aspect": "perf"}, "VBP": {POS: VERB, "VerbForm": "fin", "Tense": "pres"}, "VBZ": {POS: VERB, "VerbForm": "fin", "Tense": "pres", "Number": "sing", "Person": 3}, - "WDT": {POS: ADJ, "PronType": "int|rel"}, - "WP": {POS: NOUN, "PronType": "int|rel"}, - "WP$": {POS: ADJ, "Poss": "yes", "PronType": "int|rel"}, - "WRB": {POS: ADV, "PronType": "int|rel"}, + "WDT": {POS: ADJ, "PronType": "int,rel"}, + "WP": {POS: NOUN, "PronType": "int,rel"}, + "WP$": {POS: ADJ, "Poss": "yes", "PronType": "int,rel"}, + "WRB": {POS: ADV, "PronType": "int,rel"}, "ADD": {POS: X}, "NFP": {POS: PUNCT}, "GW": {POS: X}, From 63502349294e0ee34a3a76ec201eefd702c7583a Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 26 Sep 2018 21:03:20 +0200 Subject: [PATCH 029/287] Add morphologizer pipeline component to Language --- spacy/language.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spacy/language.py b/spacy/language.py index e64768d05..8ba169833 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -19,6 +19,7 @@ from .pipeline import DependencyParser, Tensorizer, Tagger, EntityRecognizer from .pipeline import SimilarityHook, TextCategorizer, SentenceSegmenter from .pipeline import merge_noun_chunks, merge_entities, merge_subtokens from .pipeline import EntityRuler +from ._morphologizer import Morphologizer from .compat import json_dumps, izip, basestring_ from .gold import GoldParse from .scorer import Scorer @@ -103,6 +104,7 @@ class Language(object): 'tokenizer': lambda nlp: nlp.Defaults.create_tokenizer(nlp), 'tensorizer': lambda nlp, **cfg: Tensorizer(nlp.vocab, **cfg), 'tagger': lambda nlp, **cfg: Tagger(nlp.vocab, **cfg), + 'morphologizer': lambda nlp, **cfg: Morphologizer(nlp.vocab, **cfg), 'parser': lambda nlp, **cfg: DependencyParser(nlp.vocab, **cfg), 'ner': lambda nlp, **cfg: EntityRecognizer(nlp.vocab, **cfg), 'similarity': lambda nlp, **cfg: SimilarityHook(nlp.vocab, **cfg), From 022dcda9643c3d07b0e8bfa824bf873ffd2247b4 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 26 Sep 2018 21:03:44 +0200 Subject: [PATCH 030/287] Fix morphology enum --- spacy/morphology.pxd | 319 +++++++++++++++++++++++++------------------ 1 file changed, 183 insertions(+), 136 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index bc8c44417..adc5e5574 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -32,18 +32,22 @@ cdef class Morphology: cdef int _assign_tag_from_exceptions(self, TokenC* token, int tag_id) except -1 + cdef enum univ_morph_t: NIL = 0 + begin_Abbr - Abbr_yes # cz, fi, sl, U + Abbr_yes end_Abbr + begin_AdpType - AdpType_circ # U - AdpType_comprep # cz - AdpType_prep # cz, U - AdpType_post # U - AdpType_voc # cz + AdpType_circ + AdpType_comprep + AdpType_prep + AdpType_post + AdpType_voc end_AdpType + begin_AdvType AdvType_adadj AdvType_cau @@ -55,12 +59,14 @@ cdef enum univ_morph_t: AdvType_sta AdvType_tim end_AdvType + begin_Animacy - Animacy_anim = symbols.Animacy_anim + Animacy_anim Animacy_hum Animacy_inan Animacy_nhum end_Animacy + begin_Aspect Aspect_freq Aspect_imp @@ -68,6 +74,7 @@ cdef enum univ_morph_t: Aspect_none Aspect_perf end_Aspect + begin_Case Case_abe Case_abl @@ -97,6 +104,7 @@ cdef enum univ_morph_t: Case_tra Case_voc end_Case + begin_ConjType ConjType_comp # cz, U ConjType_oper # cz, U @@ -104,6 +112,7 @@ cdef enum univ_morph_t: begin_Connegative Connegative_yes # fi end_Connegative + begin_Definite Definite_cons # U20 Definite_def @@ -111,6 +120,7 @@ cdef enum univ_morph_t: Definite_red Definite_two end_Definite + begin_Degree Degree_abs Degree_cmp @@ -121,6 +131,31 @@ cdef enum univ_morph_t: Degree_com Degree_dim # du end_Degree + + begin_Derivation + Derivation_minen # fi + Derivation_sti # fi + Derivation_inen # fi + Derivation_lainen # fi + Derivation_ja # fi + Derivation_ton # fi + Derivation_vs # fi + Derivation_ttain # fi + Derivation_ttaa # fi + end_Derivation + + begin_Echo + Echo_rdp # U + Echo_ech # U + end_Echo + + begin_Foreign + Foreign_foreign # cz, fi, U + Foreign_fscript # cz, fi, U + Foreign_tscript # cz, U + Foreign_yes # sl + end_Foreign + begin_Gender Gender_com Gender_fem @@ -133,8 +168,18 @@ cdef enum univ_morph_t: Gender_psor_masc # cz, sl, U Gender_psor_fem # cz, sl, U Gender_psor_neut # sl - end_Gender + + begin_Hyph + Hyph_yes # cz, U + end_Hyph + + begin_InfForm + InfForm_one # fi + InfForm_two # fi + InfForm_three # fi + end_InfForm + begin_Mood Mood_cnd Mood_imp @@ -144,15 +189,30 @@ cdef enum univ_morph_t: Mood_sub Mood_opt end_Mood + + begin_NameType + NameType_geo # U, cz + NameType_prs # U, cz + NameType_giv # U, cz + NameType_sur # U, cz + NameType_nat # U, cz + NameType_com # U, cz + NameType_pro # U, cz + NameType_oth # U, cz + end_NameType + begin_Negative Negative_neg Negative_pos Negative_yes end_Negative - begin_Polarity - Polarity_neg # U20 - Polarity_pos # U20 - end_Polarity + + begin_NounType + NounType_com # U + NounType_prop # U + NounType_class # U + end_NounType + begin_Number Number_com Number_dual @@ -171,8 +231,14 @@ cdef enum univ_morph_t: Number_psee_plur # U Number_psor_sing # cz, fi, sl, U Number_psor_plur # cz, fi, sl, U - end_Number + + begin_NumForm + NumForm_digit # cz, sl, U + NumForm_roman # cz, sl, U + NumForm_word # cz, sl, U + end_NumForm + begin_NumType NumType_card NumType_dist @@ -183,7 +249,29 @@ cdef enum univ_morph_t: NumType_ord NumType_sets end_NumType - begin_Person + + begin_NumValue + NumValue_one # cz, U + NumValue_two # cz, U + NumValue_three # cz, U + end_NumValue + + begin_PartForm + PartForm_pres # fi + PartForm_past # fi + PartForm_agt # fi + PartForm_neg # fi + end_PartForm + + begin_PartType + PartType_mod # U + PartType_emp # U + PartType_res # U + PartType_inf # U + PartType_vbp # U + end_PartType + + begin_Person Person_one Person_two Person_three @@ -201,9 +289,36 @@ cdef enum univ_morph_t: Person_psor_two # fi, U Person_psor_three # fi, U end_Person + + begin_Polarity + Polarity_neg # U20 + Polarity_pos # U20 + end_Polarity + + begin_Polite + Polite_inf # bq, U + Polite_pol # bq, U + Polite_abs_inf # bq, U + Polite_abs_pol # bq, U + Polite_erg_inf # bq, U + Polite_erg_pol # bq, U + Polite_dat_inf # bq, U + Polite_dat_pol # bq, U + end_Polite + begin_Poss Poss_yes end_Poss + + begin_Prefix + Prefix_yes # U + end_Prefix + + begin_PrepCase + PrepCase_npr # cz + PrepCase_pre # U + end_PrepCase + begin_PronType PronType_advPart PronType_art @@ -219,15 +334,58 @@ cdef enum univ_morph_t: PronType_clit PronType_exc # es, ca, it, fa end_PronType + + begin_PunctSide + PunctSide_ini # U + PunctSide_fin # U + end_PunctSide + + begin_PunctType + PunctType_peri # U + PunctType_qest # U + PunctType_excl # U + PunctType_quot # U + PunctType_brck # U + PunctType_comm # U + PunctType_colo # U + PunctType_semi # U + PunctType_dash # U + end_PunctType + begin_Reflex Reflex_yes end_Reflex + + begin_Style + Style_arch # cz, fi, U + Style_rare # cz, fi, U + Style_poet # cz, U + Style_norm # cz, U + Style_coll # cz, U + Style_vrnc # cz, U + Style_sing # cz, U + Style_expr # cz, U + Style_derg # cz, U + Style_vulg # cz, U + Style_yes # fi, U + end_Style + + begin_StyleVariant + StyleVariant_styleShort # cz + StyleVariant_styleBound # cz, sl + end_StyleVariant + begin_Tense Tense_fut Tense_imp Tense_past Tense_pres end_Tense + + begin_Typo + Typo_yes + end_Typo + begin_VerbForm VerbForm_fin VerbForm_ger @@ -242,6 +400,14 @@ cdef enum univ_morph_t: VerbForm_conv # U20 VerbForm_gdv # la end_VerbForm + + begin_VerbType + VerbType_aux # U + VerbType_cop # U + VerbType_mod # U + VerbType_light # U + end_VerbType + begin_Voice Voice_act Voice_cau @@ -249,128 +415,7 @@ cdef enum univ_morph_t: Voice_mid # gkc Voice_int # hb end_Voice - begin_Derivation - Derivation_minen # fi - Derivation_sti # fi - Derivation_inen # fi - Derivation_lainen # fi - Derivation_ja # fi - Derivation_ton # fi - Derivation_vs # fi - Derivation_ttain # fi - Derivation_ttaa # fi - end_Derivation - begin_Echo - Echo_rdp # U - Echo_ech # U - end_Echo - begin_Foreign - Foreign_foreign # cz, fi, U - Foreign_fscript # cz, fi, U - Foreign_tscript # cz, U - Foreign_yes # sl - end_Foreign - begin_Hyph - Hyph_yes # cz, U - end_Hyph - begin_InfForm - InfForm_one # fi - InfForm_two # fi - InfForm_three # fi - end_InfForm - begin_NameType - NameType_geo # U, cz - NameType_prs # U, cz - NameType_giv # U, cz - NameType_sur # U, cz - NameType_nat # U, cz - NameType_com # U, cz - NameType_pro # U, cz - NameType_oth # U, cz - end_NameType - begin_NounType - NounType_com # U - NounType_prop # U - NounType_class # U - end_NounType - begin_NumForm - NumForm_digit # cz, sl, U - NumForm_roman # cz, sl, U - NumForm_word # cz, sl, U - end_NumForm - begin_NumValue - NumValue_one # cz, U - NumValue_two # cz, U - NumValue_three # cz, U - end_NumValue - begin_PartForm - PartForm_pres # fi - PartForm_past # fi - PartForm_agt # fi - PartForm_neg # fi - end_PartForm - begin_PartType - PartType_mod # U - PartType_emp # U - PartType_res # U - PartType_inf # U - PartType_vbp # U - end_PartType - begin_Polite - Polite_inf # bq, U - Polite_pol # bq, U - Polite_abs_inf # bq, U - Polite_abs_pol # bq, U - Polite_erg_inf # bq, U - Polite_erg_pol # bq, U - Polite_dat_inf # bq, U - Polite_dat_pol # bq, U - end_Polite - begin_Prefix - Prefix_yes # U - end_Prefix - begin_PrepCase - PrepCase_npr # cz - PrepCase_pre # U - end_PrepCase - begin_PunctSide - PunctSide_ini # U - PunctSide_fin # U - end_PunctSide - begin_PunctType - PunctType_peri # U - PunctType_qest # U - PunctType_excl # U - PunctType_quot # U - PunctType_brck # U - PunctType_comm # U - PunctType_colo # U - PunctType_semi # U - PunctType_dash # U - end_PunctType - begin_Style - Style_arch # cz, fi, U - Style_rare # cz, fi, U - Style_poet # cz, U - Style_norm # cz, U - Style_coll # cz, U - Style_vrnc # cz, U - Style_sing # cz, U - Style_expr # cz, U - Style_derg # cz, U - Style_vulg # cz, U - Style_yes # fi, U - end_Style - begin_StyleVariant - StyleVariant_styleShort # cz - StyleVariant_styleBound # cz, sl - end_StyleVariant - begin_VerbType - VerbType_aux # U - VerbType_cop # U - VerbType_mod # U - VerbType_light # U - end_VerbType + cdef struct RichTagC: univ_pos_t pos @@ -395,6 +440,7 @@ cdef struct RichTagC: univ_morph_t negative univ_morph_t number univ_morph_t name_type + univ_morph_t noun_type univ_morph_t num_form univ_morph_t num_type univ_morph_t num_value @@ -413,6 +459,7 @@ cdef struct RichTagC: univ_morph_t style univ_morph_t style_variant univ_morph_t tense + univ_morph_t typo univ_morph_t verb_form univ_morph_t voice univ_morph_t verb_type From 2b8a53ebdcfd6704f8f943faa4bbf2118247971a Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 26 Sep 2018 21:03:57 +0200 Subject: [PATCH 031/287] Fix morphology functions --- spacy/morphology.pyx | 511 ++++++++++++++++++++++++++----------------- 1 file changed, 309 insertions(+), 202 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 870f05a87..ee747cf3c 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -18,19 +18,8 @@ from .errors import Errors def _normalize_props(props): """Transform deprecated string keys to correct names.""" out = {} - morph_keys = [ - 'PunctType', 'PunctSide', 'Other', 'Degree', 'AdvType', 'Number', - 'VerbForm', 'PronType', 'Aspect', 'Tense', 'PartType', 'Poss', - 'Hyph', 'ConjType', 'NumType', 'Foreign', 'VerbType', 'NounType', - 'Gender', 'Mood', 'Negative', 'Tense', 'Voice', 'Abbr', - 'Derivation', 'Echo', 'Foreign', 'NameType', 'NounType', 'NumForm', - 'NumValue', 'PartType', 'Polite', 'StyleVariant', - 'PronType', 'AdjType', 'Person', 'Variant', 'AdpType', - 'Reflex', 'Negative', 'Mood', 'Aspect', 'Case', - 'Polarity', 'PrepCase', 'Animacy' # U20 - ] props = dict(props) - for key in morph_keys: + for key in FIELDS: if key in props: attr = '%s_%s' % (key, props[key]) if attr in IDS: @@ -57,6 +46,7 @@ def parse_feature(feature): feature = NAMES[feature] key, value = feature.split('_') begin = 'begin_%s' % key + # Note that this includes a 0 offset for the field, for no entry offset = IDS[feature] - IDS[begin] field_id = FIELDS[key] return (field_id, offset) @@ -65,7 +55,8 @@ def parse_feature(feature): def get_field_size(field): begin = 'begin_%s' % field end = 'end_%s' % field - return (IDS[end] - IDS[begin]) - 1 + # Extra field for no entry -- always 0 + return IDS[end] - IDS[begin] cdef class Morphology: @@ -113,9 +104,23 @@ cdef class Morphology: present. Returns the hash of the new analysis. """ features = intify_features(features) - cdef RichTagC tag = create_rich_tag(features) + cdef univ_morph_t feature + for feature in features: + if feature != 0 and feature not in NAMES: + print(list(NAMES.keys())[:10]) + print(NAMES.get(feature-1), NAMES.get(feature+1)) + raise KeyError("Unknown feature: %d" % feature) + cdef RichTagC tag + tag = create_rich_tag(features) cdef hash_t key = self.insert(tag) return key + + def get(self, hash_t morph): + tag = self.tags.get(morph) + if tag == NULL: + return [] + else: + return tag_to_json(tag[0]) cpdef update(self, hash_t morph, features): """Update a morphological analysis with new feature values.""" @@ -127,8 +132,6 @@ cdef class Morphology: morph = self.insert(tag) return morph - - def lemmatize(self, const univ_pos_t univ_pos, attr_t orth, morphology): if orth not in self.strings: return orth @@ -205,7 +208,8 @@ cdef class Morphology: token.lemma = lemma token.pos = pos token.tag = self.strings[tag_str] - token.morph = self.add(features) + #token.morph = self.add(features) + token.morph = 0 if (self.tag_names[tag_id], token.lex.orth) in self.exc: self._assign_tag_from_exceptions(token, tag_id) @@ -228,7 +232,7 @@ cdef class Morphology: tag_ptr = self.tags.get(key) if tag_ptr != NULL: json_tags.append(tag_to_json(tag_ptr[0])) - raise json.dumps(json_tags) + return json.dumps(json_tags) def from_bytes(self, byte_string): raise NotImplementedError @@ -249,7 +253,7 @@ cpdef intify_features(features): cdef hash_t hash_tag(RichTagC tag) nogil: return mrmr.hash64(&tag, sizeof(tag), 0) -cdef RichTagC create_rich_tag(features): +cdef RichTagC create_rich_tag(features) except *: cdef RichTagC tag cdef univ_morph_t feature memset(&tag, 0, sizeof(tag)) @@ -258,20 +262,105 @@ cdef RichTagC create_rich_tag(features): return tag cdef tag_to_json(RichTagC tag): - return {} + features = [] + if tag.abbr != 0: + features.append(NAMES[tag.abbr]) + if tag.adp_type != 0: + features.append(NAMES[tag.adp_type]) + if tag.adv_type != 0: + features.append(NAMES[tag.adv_type]) + if tag.animacy != 0: + features.append(NAMES[tag.animacy]) + if tag.aspect != 0: + features.append(NAMES[tag.aspect]) + if tag.case != 0: + features.append(NAMES[tag.case]) + if tag.conj_type != 0: + features.append(NAMES[tag.conj_type]) + if tag.connegative != 0: + features.append(NAMES[tag.connegative]) + if tag.definite != 0: + features.append(NAMES[tag.definite]) + if tag.degree != 0: + features.append(NAMES[tag.degree]) + if tag.derivation != 0: + features.append(NAMES[tag.derivation]) + if tag.echo != 0: + features.append(NAMES[tag.echo]) + if tag.foreign != 0: + features.append(NAMES[tag.foreign]) + if tag.gender != 0: + features.append(NAMES[tag.gender]) + if tag.hyph != 0: + features.append(NAMES[tag.hyph]) + if tag.inf_form != 0: + features.append(NAMES[tag.inf_form]) + if tag.mood != 0: + features.append(NAMES[tag.mood]) + if tag.negative != 0: + features.append(NAMES[tag.negative]) + if tag.number != 0: + features.append(NAMES[tag.number]) + if tag.name_type != 0: + features.append(NAMES[tag.name_type]) + if tag.noun_type != 0: + features.append(NAMES[tag.noun_type]) + if tag.num_form != 0: + features.append(NAMES[tag.num_form]) + if tag.num_type != 0: + features.append(NAMES[tag.num_type]) + if tag.num_value != 0: + features.append(NAMES[tag.num_value]) + if tag.part_form != 0: + features.append(NAMES[tag.part_form]) + if tag.part_type != 0: + features.append(NAMES[tag.part_type]) + if tag.person != 0: + features.append(NAMES[tag.person]) + if tag.polite != 0: + features.append(NAMES[tag.polite]) + if tag.polarity != 0: + features.append(NAMES[tag.polarity]) + if tag.poss != 0: + features.append(NAMES[tag.poss]) + if tag.prefix != 0: + features.append(NAMES[tag.prefix]) + if tag.prep_case != 0: + features.append(NAMES[tag.prep_case]) + if tag.pron_type != 0: + features.append(NAMES[tag.pron_type]) + if tag.punct_side != 0: + features.append(NAMES[tag.punct_side]) + if tag.punct_type != 0: + features.append(NAMES[tag.punct_type]) + if tag.reflex != 0: + features.append(NAMES[tag.reflex]) + if tag.style != 0: + features.append(NAMES[tag.style]) + if tag.style_variant != 0: + features.append(NAMES[tag.style_variant]) + if tag.tense != 0: + features.append(NAMES[tag.tense]) + if tag.verb_form != 0: + features.append(NAMES[tag.verb_form]) + if tag.voice != 0: + features.append(NAMES[tag.voice]) + if tag.verb_type != 0: + features.append(NAMES[tag.verb_type]) + return features cdef RichTagC tag_from_json(json_tag): cdef RichTagC tag return tag -cdef int set_feature(RichTagC* tag, univ_morph_t feature, int value) nogil: +cdef int set_feature(RichTagC* tag, univ_morph_t feature, int value) except -1: if value == True: value_ = feature else: value_ = NIL if feature == NIL: pass - if is_abbr_feature(feature): + elif is_abbr_feature(feature): tag.abbr = value_ elif is_adp_type_feature(feature): tag.adp_type = value_ @@ -311,8 +400,12 @@ cdef int set_feature(RichTagC* tag, univ_morph_t feature, int value) nogil: tag.number = value_ elif is_name_type_feature(feature): tag.name_type = value_ + elif is_noun_type_feature(feature): + tag.noun_type = value_ elif is_num_form_feature(feature): tag.num_form = value_ + elif is_num_type_feature(feature): + tag.num_type = value_ elif is_num_value_feature(feature): tag.num_value = value_ elif is_part_form_feature(feature): @@ -334,6 +427,8 @@ cdef int set_feature(RichTagC* tag, univ_morph_t feature, int value) nogil: elif is_pron_type_feature(feature): tag.pron_type = value_ elif is_punct_side_feature(feature): + tag.punct_side = value_ + elif is_punct_type_feature(feature): tag.punct_type = value_ elif is_reflex_feature(feature): tag.reflex = value_ @@ -343,6 +438,8 @@ cdef int set_feature(RichTagC* tag, univ_morph_t feature, int value) nogil: tag.style_variant = value_ elif is_tense_feature(feature): tag.tense = value_ + elif is_typo_feature(feature): + tag.typo = value_ elif is_verb_form_feature(feature): tag.verb_form = value_ elif is_voice_feature(feature): @@ -350,131 +447,136 @@ cdef int set_feature(RichTagC* tag, univ_morph_t feature, int value) nogil: elif is_verb_type_feature(feature): tag.verb_type = value_ else: - with gil: - raise ValueError("Unknown feature: %d" % feature) + raise ValueError("Unknown feature: %s (%d)" % (NAMES.get(feature), feature)) cdef int is_abbr_feature(univ_morph_t feature) nogil: - return feature > begin_Abbr and feature < end_Abbr + return feature >= begin_Abbr and feature <= end_Abbr cdef int is_adp_type_feature(univ_morph_t feature) nogil: - return feature > begin_AdpType and feature < end_AdpType + return feature >= begin_AdpType and feature <= end_AdpType cdef int is_adv_type_feature(univ_morph_t feature) nogil: - return feature > begin_AdvType and feature < end_AdvType + return feature >= begin_AdvType and feature <= end_AdvType cdef int is_animacy_feature(univ_morph_t feature) nogil: - return feature > begin_Animacy and feature < end_Animacy + return feature >= begin_Animacy and feature <= end_Animacy cdef int is_aspect_feature(univ_morph_t feature) nogil: - return feature > begin_Aspect and feature < end_Aspect + return feature >= begin_Aspect and feature <= end_Aspect cdef int is_case_feature(univ_morph_t feature) nogil: - return feature > begin_Case and feature < end_Case + return feature >= begin_Case and feature <= end_Case cdef int is_conj_type_feature(univ_morph_t feature) nogil: - return feature > begin_ConjType and feature < end_ConjType + return feature >= begin_ConjType and feature <= end_ConjType cdef int is_connegative_feature(univ_morph_t feature) nogil: - return feature > begin_Connegative and feature < end_Connegative + return feature >= begin_Connegative and feature <= end_Connegative cdef int is_definite_feature(univ_morph_t feature) nogil: - return feature > begin_Definite and feature < end_Definite + return feature >= begin_Definite and feature <= end_Definite cdef int is_degree_feature(univ_morph_t feature) nogil: - return feature > begin_Degree and feature < end_Degree + return feature >= begin_Degree and feature <= end_Degree cdef int is_derivation_feature(univ_morph_t feature) nogil: - return feature > begin_Derivation and feature < end_Derivation + return feature >= begin_Derivation and feature <= end_Derivation cdef int is_echo_feature(univ_morph_t feature) nogil: - return feature > begin_Echo and feature < end_Echo + return feature >= begin_Echo and feature <= end_Echo cdef int is_foreign_feature(univ_morph_t feature) nogil: - return feature > begin_Foreign and feature < end_Foreign + return feature >= begin_Foreign and feature <= end_Foreign cdef int is_gender_feature(univ_morph_t feature) nogil: - return feature > begin_Gender and feature < end_Gender + return feature >= begin_Gender and feature <= end_Gender cdef int is_hyph_feature(univ_morph_t feature) nogil: - return feature > begin_Hyph and feature < begin_Hyph + return feature >= begin_Hyph and feature <= end_Hyph cdef int is_inf_form_feature(univ_morph_t feature) nogil: - return feature > begin_InfForm and feature < end_InfForm + return feature >= begin_InfForm and feature <= end_InfForm cdef int is_mood_feature(univ_morph_t feature) nogil: - return feature > begin_Mood and feature < end_Mood - -cdef int is_negative_feature(univ_morph_t feature) nogil: - return feature > begin_Negative and feature < end_Negative - -cdef int is_number_feature(univ_morph_t feature) nogil: - return feature > begin_Number and feature < end_Number + return feature >= begin_Mood and feature <= end_Mood cdef int is_name_type_feature(univ_morph_t feature) nogil: - return feature > begin_NameType and feature < end_NameType + return feature >= begin_NameType and feature < end_NameType + +cdef int is_negative_feature(univ_morph_t feature) nogil: + return feature >= begin_Negative and feature <= end_Negative + +cdef int is_noun_type_feature(univ_morph_t feature) nogil: + return feature >= begin_NounType and feature <= end_NounType + +cdef int is_number_feature(univ_morph_t feature) nogil: + return feature >= begin_Number and feature <= end_Number cdef int is_num_form_feature(univ_morph_t feature) nogil: - return feature > begin_NumForm and feature < end_NumForm + return feature >= begin_NumForm and feature <= end_NumForm cdef int is_num_type_feature(univ_morph_t feature) nogil: - return feature > begin_NumType and feature < end_NumType + return feature >= begin_NumType and feature <= end_NumType cdef int is_num_value_feature(univ_morph_t feature) nogil: - return feature > begin_NumValue and feature < end_NumValue + return feature >= begin_NumValue and feature <= end_NumValue cdef int is_part_form_feature(univ_morph_t feature) nogil: - return feature > begin_PartForm and feature < end_PartForm + return feature >= begin_PartForm and feature <= end_PartForm cdef int is_part_type_feature(univ_morph_t feature) nogil: - return feature > begin_PartType and feature < end_PartType + return feature >= begin_PartType and feature <= end_PartType cdef int is_person_feature(univ_morph_t feature) nogil: - return feature > begin_Person and feature < end_Person + return feature >= begin_Person and feature <= end_Person cdef int is_polite_feature(univ_morph_t feature) nogil: - return feature > begin_Polite and feature < end_Polite + return feature >= begin_Polite and feature <= end_Polite cdef int is_polarity_feature(univ_morph_t feature) nogil: - return feature > begin_Polarity and feature < end_Polarity + return feature >= begin_Polarity and feature <= end_Polarity cdef int is_poss_feature(univ_morph_t feature) nogil: - return feature > begin_Poss and feature < end_Poss + return feature >= begin_Poss and feature <= end_Poss cdef int is_prefix_feature(univ_morph_t feature) nogil: - return feature > begin_Prefix and feature < end_Prefix + return feature >= begin_Prefix and feature <= end_Prefix cdef int is_prep_case_feature(univ_morph_t feature) nogil: - return feature > begin_PrepCase and feature < end_PrepCase + return feature >= begin_PrepCase and feature <= end_PrepCase cdef int is_pron_type_feature(univ_morph_t feature) nogil: - return feature > begin_PronType and feature < end_PronType + return feature >= begin_PronType and feature <= end_PronType cdef int is_punct_side_feature(univ_morph_t feature) nogil: - return feature > begin_PunctSide and feature < end_PunctSide + return feature >= begin_PunctSide and feature <= end_PunctSide cdef int is_punct_type_feature(univ_morph_t feature) nogil: - return feature > begin_PunctType and feature < end_PunctType + return feature >= begin_PunctType and feature <= end_PunctType cdef int is_reflex_feature(univ_morph_t feature) nogil: - return feature > begin_Reflex and feature < end_Reflex + return feature >= begin_Reflex and feature <= end_Reflex cdef int is_style_feature(univ_morph_t feature) nogil: - return feature > begin_Style and feature < end_Style + return feature >= begin_Style and feature <= end_Style cdef int is_style_variant_feature(univ_morph_t feature) nogil: - return feature > begin_StyleVariant and feature < end_StyleVariant + return feature >= begin_StyleVariant and feature <= end_StyleVariant cdef int is_tense_feature(univ_morph_t feature) nogil: - return feature > begin_Tense and feature < end_Tense + return feature >= begin_Tense and feature <= end_Tense + +cdef int is_typo_feature(univ_morph_t feature) nogil: + return feature >= begin_Typo and feature <= end_Typo cdef int is_verb_form_feature(univ_morph_t feature) nogil: - return feature > begin_VerbForm and feature < end_VerbForm + return feature >= begin_VerbForm and feature <= end_VerbForm cdef int is_voice_feature(univ_morph_t feature) nogil: - return feature > begin_Voice and feature < end_Voice + return feature >= begin_Voice and feature <= end_Voice cdef int is_verb_type_feature(univ_morph_t feature) nogil: - return feature > begin_VerbType and feature < end_VerbType + return feature >= begin_VerbType and feature <= end_VerbType FIELDS = { @@ -495,9 +597,9 @@ FIELDS = { 'Hyph': 14, 'InfForm': 15, 'Mood': 16, - 'Negative': 17, - 'Number': 18, - 'NameType': 19, + 'NameType': 17, + 'Negative': 18, + 'Number': 19, 'NumForm': 20, 'NumType': 21, 'NumValue': 22, @@ -516,14 +618,15 @@ FIELDS = { 'Style': 35, 'StyleVariant': 36, 'Tense': 37, - 'VerbForm': 38, - 'Voice': 39, - 'VerbType': 40 + 'Typo': 38, + 'VerbForm': 39, + 'Voice': 40, + 'VerbType': 41 } IDS = { "begin_Abbr": begin_Abbr, - "Abbr_yes ": Abbr_yes , + "Abbr_yes": Abbr_yes , "end_Abbr": end_Abbr, "begin_AdpType": begin_AdpType, "AdpType_circ": AdpType_circ, @@ -609,132 +712,6 @@ IDS = { "Degree_com": Degree_com, "Degree_dim": Degree_dim, "end_Degree": end_Degree, - "begin_Gender": begin_Gender, - "Gender_com": Gender_com, - "Gender_fem": Gender_fem, - "Gender_masc": Gender_masc, - "Gender_neut": Gender_neut, - "Gender_dat_masc": Gender_dat_masc, - "Gender_dat_fem": Gender_dat_fem, - "Gender_erg_masc": Gender_erg_masc, - "Gender_erg_fem": Gender_erg_fem, - "Gender_psor_masc": Gender_psor_masc, - "Gender_psor_fem": Gender_psor_fem, - "Gender_psor_neut": Gender_psor_neut, - "end_Gender": end_Gender, - "begin_Mood": begin_Mood, - "Mood_cnd": Mood_cnd, - "Mood_imp": Mood_imp, - "Mood_ind": Mood_ind, - "Mood_n": Mood_n, - "Mood_pot": Mood_pot, - "Mood_sub": Mood_sub, - "Mood_opt": Mood_opt, - "end_Mood": end_Mood, - "begin_Negative": begin_Negative, - "Negative_neg": Negative_neg, - "Negative_pos": Negative_pos, - "Negative_yes": Negative_yes, - "end_Negative": end_Negative, - "begin_Polarity": begin_Polarity, - "Polarity_neg": Polarity_neg, - "Polarity_pos": Polarity_pos, - "end_Polarity": end_Polarity, - "begin_Number": begin_Number, - "Number_com": Number_com, - "Number_dual": Number_dual, - "Number_none": Number_none, - "Number_plur": Number_plur, - "Number_sing": Number_sing, - "Number_ptan": Number_ptan, - "Number_count": Number_count, - "Number_abs_sing": Number_abs_sing, - "Number_abs_plur": Number_abs_plur, - "Number_dat_sing": Number_dat_sing, - "Number_dat_plur": Number_dat_plur, - "Number_erg_sing": Number_erg_sing, - "Number_erg_plur": Number_erg_plur, - "Number_psee_sing": Number_psee_sing, - "Number_psee_plur": Number_psee_plur, - "Number_psor_sing": Number_psor_sing, - "Number_psor_plur": Number_psor_plur, - "end_Number": end_Number, - "begin_NumType": begin_NumType, - "NumType_card": NumType_card, - "NumType_dist": NumType_dist, - "NumType_frac": NumType_frac, - "NumType_gen": NumType_gen, - "NumType_mult": NumType_mult, - "NumType_none": NumType_none, - "NumType_ord": NumType_ord, - "NumType_sets": NumType_sets, - "end_NumType": end_NumType, - "begin_Person": begin_Person, - "Person_one": Person_one, - "Person_two": Person_two, - "Person_three": Person_three, - "Person_none": Person_none, - "Person_abs_one": Person_abs_one, - "Person_abs_two": Person_abs_two, - "Person_abs_three": Person_abs_three, - "Person_dat_one": Person_dat_one, - "Person_dat_two": Person_dat_two, - "Person_dat_three": Person_dat_three, - "Person_erg_one": Person_erg_one, - "Person_erg_two": Person_erg_two, - "Person_erg_three": Person_erg_three, - "Person_psor_one": Person_psor_one, - "Person_psor_two": Person_psor_two, - "Person_psor_three": Person_psor_three, - "end_Person": end_Person, - "begin_Poss": begin_Poss, - "Poss_yes": Poss_yes, - "end_Poss": end_Poss, - "begin_PronType": begin_PronType, - "PronType_advPart": PronType_advPart, - "PronType_art": PronType_art, - "PronType_default": PronType_default, - "PronType_dem": PronType_dem, - "PronType_ind": PronType_ind, - "PronType_int": PronType_int, - "PronType_neg": PronType_neg, - "PronType_prs": PronType_prs, - "PronType_rcp": PronType_rcp, - "PronType_rel": PronType_rel, - "PronType_tot": PronType_tot, - "PronType_clit": PronType_clit, - "PronType_exc": PronType_exc, - "end_PronType": end_PronType, - "begin_Reflex": begin_Reflex, - "Reflex_yes": Reflex_yes, - "end_Reflex": end_Reflex, - "begin_Tense": begin_Tense, - "Tense_fut": Tense_fut, - "Tense_imp": Tense_imp, - "Tense_past": Tense_past, - "Tense_pres": Tense_pres, - "end_Tense": end_Tense, - "begin_VerbForm": begin_VerbForm, - "VerbForm_fin": VerbForm_fin, - "VerbForm_ger": VerbForm_ger, - "VerbForm_inf": VerbForm_inf, - "VerbForm_none": VerbForm_none, - "VerbForm_part": VerbForm_part, - "VerbForm_partFut": VerbForm_partFut, - "VerbForm_partPast": VerbForm_partPast, - "VerbForm_partPres": VerbForm_partPres, - "VerbForm_sup": VerbForm_sup, - "VerbForm_trans": VerbForm_trans, - "VerbForm_conv": VerbForm_conv, - "VerbForm_gdv": VerbForm_gdv, - "end_VerbForm": end_VerbForm, - "begin_Voice": begin_Voice, - "Voice_act": Voice_act, - "Voice_cau": Voice_cau, - "Voice_pass": Voice_pass, - "Voice_mid": Voice_mid, - "Voice_int": Voice_int, - "end_Voice": end_Voice, "begin_Derivation": begin_Derivation, "Derivation_minen": Derivation_minen, "Derivation_sti": Derivation_sti, @@ -756,6 +733,19 @@ IDS = { "Foreign_tscript": Foreign_tscript, "Foreign_yes": Foreign_yes, "end_Foreign": end_Foreign, + "begin_Gender": begin_Gender, + "Gender_com": Gender_com, + "Gender_fem": Gender_fem, + "Gender_masc": Gender_masc, + "Gender_neut": Gender_neut, + "Gender_dat_masc": Gender_dat_masc, + "Gender_dat_fem": Gender_dat_fem, + "Gender_erg_masc": Gender_erg_masc, + "Gender_erg_fem": Gender_erg_fem, + "Gender_psor_masc": Gender_psor_masc, + "Gender_psor_fem": Gender_psor_fem, + "Gender_psor_neut": Gender_psor_neut, + "end_Gender": end_Gender, "begin_Hyph": begin_Hyph, "Hyph_yes": Hyph_yes, "end_Hyph": end_Hyph, @@ -764,6 +754,15 @@ IDS = { "InfForm_two": InfForm_two, "InfForm_three": InfForm_three, "end_InfForm": end_InfForm, + "begin_Mood": begin_Mood, + "Mood_cnd": Mood_cnd, + "Mood_imp": Mood_imp, + "Mood_ind": Mood_ind, + "Mood_n": Mood_n, + "Mood_pot": Mood_pot, + "Mood_sub": Mood_sub, + "Mood_opt": Mood_opt, + "end_Mood": end_Mood, "begin_NameType": begin_NameType, "NameType_geo": NameType_geo, "NameType_prs": NameType_prs, @@ -774,16 +773,50 @@ IDS = { "NameType_pro": NameType_pro, "NameType_oth": NameType_oth, "end_NameType": end_NameType, + "begin_Negative": begin_Negative, + "Negative_neg": Negative_neg, + "Negative_pos": Negative_pos, + "Negative_yes": Negative_yes, + "end_Negative": end_Negative, "begin_NounType": begin_NounType, "NounType_com": NounType_com, "NounType_prop": NounType_prop, "NounType_class": NounType_class, "end_NounType": end_NounType, + "begin_Number": begin_Number, + "Number_com": Number_com, + "Number_dual": Number_dual, + "Number_none": Number_none, + "Number_plur": Number_plur, + "Number_sing": Number_sing, + "Number_ptan": Number_ptan, + "Number_count": Number_count, + "Number_abs_sing": Number_abs_sing, + "Number_abs_plur": Number_abs_plur, + "Number_dat_sing": Number_dat_sing, + "Number_dat_plur": Number_dat_plur, + "Number_erg_sing": Number_erg_sing, + "Number_erg_plur": Number_erg_plur, + "Number_psee_sing": Number_psee_sing, + "Number_psee_plur": Number_psee_plur, + "Number_psor_sing": Number_psor_sing, + "Number_psor_plur": Number_psor_plur, + "end_Number": end_Number, "begin_NumForm": begin_NumForm, "NumForm_digit": NumForm_digit, "NumForm_roman": NumForm_roman, "NumForm_word": NumForm_word, "end_NumForm": end_NumForm, + "begin_NumType": begin_NumType, + "NumType_card": NumType_card, + "NumType_dist": NumType_dist, + "NumType_frac": NumType_frac, + "NumType_gen": NumType_gen, + "NumType_mult": NumType_mult, + "NumType_none": NumType_none, + "NumType_ord": NumType_ord, + "NumType_sets": NumType_sets, + "end_NumType": end_NumType, "begin_NumValue": begin_NumValue, "NumValue_one": NumValue_one, "NumValue_two": NumValue_two, @@ -802,6 +835,29 @@ IDS = { "PartType_inf": PartType_inf, "PartType_vbp": PartType_vbp, "end_PartType": end_PartType, + + "begin_Person": begin_Person, + "Person_one": Person_one, + "Person_two": Person_two, + "Person_three": Person_three, + "Person_none": Person_none, + "Person_abs_one": Person_abs_one, + "Person_abs_two": Person_abs_two, + "Person_abs_three": Person_abs_three, + "Person_dat_one": Person_dat_one, + "Person_dat_two": Person_dat_two, + "Person_dat_three": Person_dat_three, + "Person_erg_one": Person_erg_one, + "Person_erg_two": Person_erg_two, + "Person_erg_three": Person_erg_three, + "Person_psor_one": Person_psor_one, + "Person_psor_two": Person_psor_two, + "Person_psor_three": Person_psor_three, + "end_Person": end_Person, + "begin_Polarity": begin_Polarity, + "Polarity_neg": Polarity_neg, + "Polarity_pos": Polarity_pos, + "end_Polarity": end_Polarity, "begin_Polite": begin_Polite, "Polite_inf": Polite_inf, "Polite_pol": Polite_pol, @@ -812,6 +868,9 @@ IDS = { "Polite_dat_inf": Polite_dat_inf, "Polite_dat_pol": Polite_dat_pol, "end_Polite": end_Polite, + "begin_Poss": begin_Poss, + "Poss_yes": Poss_yes, + "end_Poss": end_Poss, "begin_Prefix": begin_Prefix, "Prefix_yes": Prefix_yes, "end_Prefix": end_Prefix, @@ -819,6 +878,21 @@ IDS = { "PrepCase_npr": PrepCase_npr, "PrepCase_pre": PrepCase_pre, "end_PrepCase": end_PrepCase, + "begin_PronType": begin_PronType, + "PronType_advPart": PronType_advPart, + "PronType_art": PronType_art, + "PronType_default": PronType_default, + "PronType_dem": PronType_dem, + "PronType_ind": PronType_ind, + "PronType_int": PronType_int, + "PronType_neg": PronType_neg, + "PronType_prs": PronType_prs, + "PronType_rcp": PronType_rcp, + "PronType_rel": PronType_rel, + "PronType_tot": PronType_tot, + "PronType_clit": PronType_clit, + "PronType_exc": PronType_exc, + "end_PronType": end_PronType, "begin_PunctSide": begin_PunctSide, "PunctSide_ini": PunctSide_ini, "PunctSide_fin": PunctSide_fin, @@ -834,6 +908,9 @@ IDS = { "PunctType_semi": PunctType_semi, "PunctType_dash": PunctType_dash, "end_PunctType": end_PunctType, + "begin_Reflex": begin_Reflex, + "Reflex_yes": Reflex_yes, + "end_Reflex": end_Reflex, "begin_Style": begin_Style, "Style_arch": Style_arch, "Style_rare": Style_rare, @@ -851,12 +928,42 @@ IDS = { "StyleVariant_styleShort": StyleVariant_styleShort, "StyleVariant_styleBound": StyleVariant_styleBound, "end_StyleVariant": end_StyleVariant, + "begin_Tense": begin_Tense, + "Tense_fut": Tense_fut, + "Tense_imp": Tense_imp, + "Tense_past": Tense_past, + "Tense_pres": Tense_pres, + "end_Tense": end_Tense, + "begin_Typo": begin_Typo, + "Typo_yes": Typo_yes, + "end_Typo": end_Typo, + "begin_VerbForm": begin_VerbForm, + "VerbForm_fin": VerbForm_fin, + "VerbForm_ger": VerbForm_ger, + "VerbForm_inf": VerbForm_inf, + "VerbForm_none": VerbForm_none, + "VerbForm_part": VerbForm_part, + "VerbForm_partFut": VerbForm_partFut, + "VerbForm_partPast": VerbForm_partPast, + "VerbForm_partPres": VerbForm_partPres, + "VerbForm_sup": VerbForm_sup, + "VerbForm_trans": VerbForm_trans, + "VerbForm_conv": VerbForm_conv, + "VerbForm_gdv": VerbForm_gdv, + "end_VerbForm": end_VerbForm, "begin_VerbType": begin_VerbType, "VerbType_aux": VerbType_aux, "VerbType_cop": VerbType_cop, "VerbType_mod": VerbType_mod, "VerbType_light": VerbType_light, "end_VerbType": end_VerbType, + "begin_Voice": begin_Voice, + "Voice_act": Voice_act, + "Voice_cau": Voice_cau, + "Voice_pass": Voice_pass, + "Voice_mid": Voice_mid, + "Voice_int": Voice_int, + "end_Voice": end_Voice, } From 823cc4127ac8191235280406957a1f8694a4c9b5 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 26 Sep 2018 21:04:13 +0200 Subject: [PATCH 032/287] Update morphology tests --- spacy/tests/morphology/test_morph_features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/tests/morphology/test_morph_features.py b/spacy/tests/morphology/test_morph_features.py index 391fd1337..32cc665af 100644 --- a/spacy/tests/morphology/test_morph_features.py +++ b/spacy/tests/morphology/test_morph_features.py @@ -4,7 +4,7 @@ import pytest from ...morphology import Morphology from ...strings import StringStore from ...lemmatizer import Lemmatizer -from ...symbols import * +from ...morphology import * @pytest.fixture def morphology(): From c8a28413083d2acf13c15c1b22ff3ab3c94918d7 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 26 Sep 2018 21:04:29 +0200 Subject: [PATCH 033/287] Add property to get morph key on token --- spacy/tokens/token.pyx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spacy/tokens/token.pyx b/spacy/tokens/token.pyx index 6da93a726..3af5071d2 100644 --- a/spacy/tokens/token.pyx +++ b/spacy/tokens/token.pyx @@ -169,6 +169,10 @@ cdef class Token: return (numpy.dot(self.vector, other.vector) / (self.vector_norm * other.vector_norm)) + property morph_key: + def __get__(self): + return self.c.morph + property lex_id: """RETURNS (int): Sequential ID of the token's lexical type.""" def __get__(self): From b9ef8ac61657b5c681170f0a8ff1c15ca85b2b71 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 27 Sep 2018 15:14:27 +0200 Subject: [PATCH 034/287] Fix GoldParse class when no entities --- spacy/gold.pyx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/spacy/gold.pyx b/spacy/gold.pyx index 77c5944ca..cd8d6dab4 100644 --- a/spacy/gold.pyx +++ b/spacy/gold.pyx @@ -431,19 +431,18 @@ cdef class GoldParse: if tags is None: tags = [None for _ in words] if heads is None: - heads = [None for token in words] + heads = [None for _ in words] if deps is None: deps = [None for _ in words] - if entities is None: - entities = [None for _ in words] if morphology is None: morphology = [None for _ in words] + if entities is None: + entities = [None for _ in words] elif len(entities) == 0: entities = ['O' for _ in words] elif not isinstance(entities[0], basestring): # Assume we have entities specified by character offset. entities = biluo_tags_from_offsets(doc, entities) - self.mem = Pool() self.loss = 0 self.length = len(doc) From 010f846d5f5e893cd2251c54577e7e7d54827314 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 00:16:51 +0100 Subject: [PATCH 035/287] Fix dependencies in morphologizer --- spacy/_morphologizer.pyx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/spacy/_morphologizer.pyx b/spacy/_morphologizer.pyx index 10baec8f5..db0a0ff1b 100644 --- a/spacy/_morphologizer.pyx +++ b/spacy/_morphologizer.pyx @@ -1,12 +1,8 @@ from __future__ import unicode_literals from collections import OrderedDict, defaultdict -import cytoolz -import ujson import numpy cimport numpy as np -from .util import msgpack -from .util import msgpack_numpy from thinc.api import chain from thinc.neural.util import to_categorical, copy_array, get_array_module @@ -16,7 +12,7 @@ from ._ml import Tok2Vec, build_morphologizer_model from ._ml import link_vectors_to_models, zero_init, flatten from ._ml import create_default_optimizer from .errors import Errors, TempErrors -from .compat import json_dumps, basestring_ +from .compat import basestring_ from .tokens.doc cimport Doc from .vocab cimport Vocab from .morphology cimport Morphology @@ -58,7 +54,7 @@ class Morphologizer(Pipe): return doc def pipe(self, stream, batch_size=128, n_threads=-1): - for docs in cytoolz.partition_all(batch_size, stream): + for docs in util.minibatch(stream, size=batch_size): docs = list(docs) features, tokvecs = self.predict(docs) self.set_annotations(docs, features, tensors=tokvecs) From ae7c728c5f76d09f77981132f93702ecdfbeab1f Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 01:17:19 +0100 Subject: [PATCH 036/287] Fix json dependency --- spacy/morphology.pyx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index ee747cf3c..a4759e4ab 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -3,8 +3,9 @@ from __future__ import unicode_literals from libc.string cimport memset -import ujson as json +import srsly +from .strings import get_string_id from . import symbols from .attrs cimport POS, IS_SPACE from .attrs import LEMMA, intify_attrs @@ -232,7 +233,7 @@ cdef class Morphology: tag_ptr = self.tags.get(key) if tag_ptr != NULL: json_tags.append(tag_to_json(tag_ptr[0])) - return json.dumps(json_tags) + return srsly.json_dumps(json_tags) def from_bytes(self, byte_string): raise NotImplementedError From 98dfe5e433bf14b1d5467e293b9cfc0efeac7dee Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 01:31:23 +0100 Subject: [PATCH 037/287] Fix ud_train.py --- spacy/cli/ud/ud_train.py | 116 ++++++++------------------------------- 1 file changed, 24 insertions(+), 92 deletions(-) diff --git a/spacy/cli/ud/ud_train.py b/spacy/cli/ud/ud_train.py index 68fd3b5a9..afef6c073 100644 --- a/spacy/cli/ud/ud_train.py +++ b/spacy/cli/ud/ud_train.py @@ -156,13 +156,8 @@ def _make_gold(nlp, text, sent_annots, drop_deps=0.0): flat = defaultdict(list) sent_starts = [] for sent in sent_annots: -<<<<<<< HEAD:spacy/cli/ud_train.py - flat['heads'].extend(len(flat['words'])+head for head in sent['heads']) - for field in ['words', 'tags', 'deps', 'morphology', 'entities', 'spaces']: -======= - flat["heads"].extend(len(flat["words"]) + head for head in sent["heads"]) - for field in ["words", "tags", "deps", "entities", "spaces"]: ->>>>>>> develop:spacy/cli/ud/ud_train.py + flat["heads"].extend(len(flat["words"])+head for head in sent["heads"]) + for field in ["words", "tags", "deps", "morphology", "entities", "spaces"]: flat[field].extend(sent[field]) sent_starts.append(True) sent_starts.extend([False] * (len(sent["words"]) - 1)) @@ -260,55 +255,30 @@ def write_conllu(docs, file_): def print_progress(itn, losses, ud_scores): fields = { -<<<<<<< HEAD:spacy/cli/ud_train.py - 'dep_loss': losses.get('parser', 0.0), - 'morph_loss': losses.get('morphologizer', 0.0), - 'tag_loss': losses.get('tagger', 0.0), - 'words': ud_scores['Words'].f1 * 100, - 'sents': ud_scores['Sentences'].f1 * 100, - 'tags': ud_scores['XPOS'].f1 * 100, - 'uas': ud_scores['UAS'].f1 * 100, - 'las': ud_scores['LAS'].f1 * 100, - 'morph': ud_scores['Feats'].f1 * 100, - } - header = ['Epoch', 'P.Loss', 'M.Loss', 'LAS', 'UAS', 'TAG', 'MORPH', 'SENT', 'WORD'] - if itn == 0: - print('\t'.join(header)) - tpl = '\t'.join(( - '{:d}', - '{dep_loss:.1f}', - '{morph_loss:.1f}', - '{las:.1f}', - '{uas:.1f}', - '{tags:.1f}', - '{morph:.1f}', - '{sents:.1f}', - '{words:.1f}', - )) -======= "dep_loss": losses.get("parser", 0.0), + "morph_loss": losses.get("morphologizer", 0.0), "tag_loss": losses.get("tagger", 0.0), "words": ud_scores["Words"].f1 * 100, "sents": ud_scores["Sentences"].f1 * 100, "tags": ud_scores["XPOS"].f1 * 100, "uas": ud_scores["UAS"].f1 * 100, "las": ud_scores["LAS"].f1 * 100, + "morph": ud_scores["Feats"].f1 * 100, } - header = ["Epoch", "Loss", "LAS", "UAS", "TAG", "SENT", "WORD"] + header = ["Epoch", "P.Loss", "M.Loss", "LAS", "UAS", "TAG", "MORPH", "SENT", "WORD"] if itn == 0: print("\t".join(header)) - tpl = "\t".join( - ( - "{:d}", - "{dep_loss:.1f}", - "{las:.1f}", - "{uas:.1f}", - "{tags:.1f}", - "{sents:.1f}", - "{words:.1f}", - ) - ) ->>>>>>> develop:spacy/cli/ud/ud_train.py + tpl = "\t".join(( + "{:d}", + "{dep_loss:.1f}", + "{morph_loss:.1f}", + "{las:.1f}", + "{uas:.1f}", + "{tags:.1f}", + "{morph:.1f}", + "{sents:.1f}", + "{words:.1f}", + )) print(tpl.format(itn, **fields)) @@ -329,48 +299,26 @@ def get_token_conllu(token, i): head = 0 else: head = i + (token.head.i - token.i) + 1 -<<<<<<< HEAD:spacy/cli/ud_train.py features = token.vocab.morphology.get(token.morph_key) feat_str = [] - replacements = {'one': '1', 'two': '2', 'three': '3'} + replacements = {"one": "1", "two": "2", "three": "3"} for feat in features: - if not feat.startswith('begin') and not feat.startswith('end'): - key, value = feat.split('_') + if not feat.startswith("begin") and not feat.startswith("end"): + key, value = feat.split("_") value = replacements.get(value, value) - feat_str.append('%s=%s' % (key, value.title())) + feat_str.append("%s=%s" % (key, value.title())) if not feat_str: - feat_str = '_' + feat_str = "_" else: - feat_str = '|'.join(feat_str) + feat_str = "|".join(feat_str) fields = [str(i+1), token.text, token.lemma_, token.pos_, token.tag_, feat_str, - str(head), token.dep_.lower(), '_', '_'] - lines.append('\t'.join(fields)) - return '\n'.join(lines) - -Token.set_extension('get_conllu_lines', method=get_token_conllu) -Token.set_extension('begins_fused', default=False) -Token.set_extension('inside_fused', default=False) -======= - fields = [ - str(i + 1), - token.text, - token.lemma_, - token.pos_, - token.tag_, - "_", - str(head), - token.dep_.lower(), - "_", - "_", - ] + str(head), token.dep_.lower(), "_", "_"] lines.append("\t".join(fields)) return "\n".join(lines) - Token.set_extension("get_conllu_lines", method=get_token_conllu) Token.set_extension("begins_fused", default=False) Token.set_extension("inside_fused", default=False) ->>>>>>> develop:spacy/cli/ud/ud_train.py ################## @@ -394,14 +342,9 @@ def load_nlp(corpus, config, vectors=None): def initialize_pipeline(nlp, docs, golds, config, device): -<<<<<<< HEAD:spacy/cli/ud_train.py - nlp.add_pipe(nlp.create_pipe('tagger')) - nlp.add_pipe(nlp.create_pipe('morphologizer')) - nlp.add_pipe(nlp.create_pipe('parser')) -======= nlp.add_pipe(nlp.create_pipe("tagger")) + nlp.add_pipe(nlp.create_pipe("morphologizer")) nlp.add_pipe(nlp.create_pipe("parser")) ->>>>>>> develop:spacy/cli/ud/ud_train.py if config.multitask_tag: nlp.parser.add_multitask_objective("tag") if config.multitask_sent: @@ -597,23 +540,12 @@ def main( out_path = parses_dir / corpus / "epoch-{i}.conllu".format(i=i) with nlp.use_params(optimizer.averages): if use_oracle_segments: -<<<<<<< HEAD:spacy/cli/ud_train.py parsed_docs, scores = evaluate(nlp, paths.dev.conllu, paths.dev.conllu, out_path) else: parsed_docs, scores = evaluate(nlp, paths.dev.text, paths.dev.conllu, out_path) print_progress(i, losses, scores) -======= - parsed_docs, scores = evaluate( - nlp, paths.dev.conllu, paths.dev.conllu, out_path - ) - else: - parsed_docs, scores = evaluate( - nlp, paths.dev.text, paths.dev.conllu, out_path - ) - print_progress(i, losses, scores) ->>>>>>> develop:spacy/cli/ud/ud_train.py def _render_parses(i, to_render): From bfa52d9d8a940f89f5abc6eaebd3cf380f67d199 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 01:34:32 +0100 Subject: [PATCH 038/287] Move morphologizer within spacy/pipes --- spacy/{_morphologizer.pyx => pipeline/morphologizer.pyx} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename spacy/{_morphologizer.pyx => pipeline/morphologizer.pyx} (100%) diff --git a/spacy/_morphologizer.pyx b/spacy/pipeline/morphologizer.pyx similarity index 100% rename from spacy/_morphologizer.pyx rename to spacy/pipeline/morphologizer.pyx From fc1cc4c529e08c887287a3f449fc181fee9a8b6d Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 01:36:04 +0100 Subject: [PATCH 039/287] Move morphologizer under spacy/pipes --- setup.py | 2 +- spacy/language.py | 2 +- spacy/pipeline/__init__.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ea98f0b54..47c82b014 100755 --- a/setup.py +++ b/setup.py @@ -41,8 +41,8 @@ MOD_NAMES = [ "spacy.vocab", "spacy.attrs", "spacy.morphology", - "spacy._morphologizer", "spacy.pipeline.pipes", + "spacy.pipelines.morphologizer", "spacy.syntax.stateclass", "spacy.syntax._state", "spacy.tokenizer", diff --git a/spacy/language.py b/spacy/language.py index ed6dc64dc..b90fa8486 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -18,7 +18,7 @@ from .pipeline import DependencyParser, Tensorizer, Tagger, EntityRecognizer from .pipeline import SimilarityHook, TextCategorizer, SentenceSegmenter from .pipeline import merge_noun_chunks, merge_entities, merge_subtokens from .pipeline import EntityRuler -from ._morphologizer import Morphologizer +from .pipeline import Morphologizer from .compat import izip, basestring_ from .gold import GoldParse from .scorer import Scorer diff --git a/spacy/pipeline/__init__.py b/spacy/pipeline/__init__.py index d683cc989..36b9b8d46 100644 --- a/spacy/pipeline/__init__.py +++ b/spacy/pipeline/__init__.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from .pipes import Tagger, DependencyParser, EntityRecognizer # noqa from .pipes import TextCategorizer, Tensorizer, Pipe # noqa +from .morphologizer import Morphologizer from .entityruler import EntityRuler # noqa from .hooks import SentenceSegmenter, SimilarityHook # noqa from .functions import merge_entities, merge_noun_chunks, merge_subtokens # noqa From 21008ad2d8cd82a72b08e2f9e3d3a75eb32360b5 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 10:45:24 +0100 Subject: [PATCH 040/287] Draft API for morphological analysis class --- spacy/tokens/morphanalysis.pyx | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 spacy/tokens/morphanalysis.pyx diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx new file mode 100644 index 000000000..df2d6ec20 --- /dev/null +++ b/spacy/tokens/morphanalysis.pyx @@ -0,0 +1,57 @@ +cdef class Morphanalysis: + """Control access to morphological features for a token.""" + def __init__(self, Vocab vocab, features=None): + pass + + @classmethod + def from_id(self, Vocab vocab, hash_t key): + pass + + def __contains__(self, feature): + pass + + def __iter__(self): + pass + + def __len__(self): + pass + + def __str__(self): + pass + + def __repr__(self): + pass + + def __hash__(self): + pass + + @property + def is_base_form(self): + pass + + @property + def pos(self): + pass + + @property + def pos_(self): + pass + + @property + def id(self): + pass + + def get(self, name): + pass + + def set(self, name, value): + pass + + def add(self, feature): + pass + + def remove(self, feature): + pass + + def to_json(self): + pass From ef3110a44478d1364b3906d119dfe596c406df6a Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 10:45:55 +0100 Subject: [PATCH 041/287] Fix compile error --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47c82b014..f193d0498 100755 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ MOD_NAMES = [ "spacy.attrs", "spacy.morphology", "spacy.pipeline.pipes", - "spacy.pipelines.morphologizer", + "spacy.pipeline.morphologizer", "spacy.syntax.stateclass", "spacy.syntax._state", "spacy.tokenizer", From 88059664609c332849f674ba4eb4c8554c3c115c Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 10:46:27 +0100 Subject: [PATCH 042/287] Fix moved Morphologizer class --- spacy/pipeline/morphologizer.pyx | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index db0a0ff1b..820567e71 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -6,18 +6,17 @@ cimport numpy as np from thinc.api import chain from thinc.neural.util import to_categorical, copy_array, get_array_module -from . import util -from .pipeline import Pipe -from ._ml import Tok2Vec, build_morphologizer_model -from ._ml import link_vectors_to_models, zero_init, flatten -from ._ml import create_default_optimizer -from .errors import Errors, TempErrors -from .compat import basestring_ -from .tokens.doc cimport Doc -from .vocab cimport Vocab -from .morphology cimport Morphology -from .morphology import parse_feature, IDS, FIELDS, FIELD_SIZES, NAMES -from .pipeline import Pipe +from .. import util +from .pipes import Pipe +from .._ml import Tok2Vec, build_morphologizer_model +from .._ml import link_vectors_to_models, zero_init, flatten +from .._ml import create_default_optimizer +from ..errors import Errors, TempErrors +from ..compat import basestring_ +from ..tokens.doc cimport Doc +from ..vocab cimport Vocab +from ..morphology cimport Morphology +from ..morphology import parse_feature, IDS, FIELDS, FIELD_SIZES, NAMES class Morphologizer(Pipe): From 34651c8ddf06b3087151167bd269d21b6b546225 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 12:13:47 +0100 Subject: [PATCH 043/287] Fix lemmatizer --- spacy/lemmatizer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/spacy/lemmatizer.py b/spacy/lemmatizer.py index 69f5c8d20..c708800e9 100644 --- a/spacy/lemmatizer.py +++ b/spacy/lemmatizer.py @@ -48,6 +48,11 @@ class Lemmatizer(object): avoid lemmatization entirely. """ morphology = {} if morphology is None else morphology + morphology = dict(morphology) + for key, value in list(morphology.items()): + if value is True: + feat, value = key.split('_') + morphology[feat] = value others = [ key for key in morphology @@ -68,13 +73,13 @@ class Lemmatizer(object): return True elif univ_pos == "adj" and morphology.get("Degree") == "pos": return True - elif VerbForm_inf in morphology or 'VerbForm_inf' in morphology: + elif morphology.get('VerbForm') == 'inf': return True - elif VerbForm_none in morphology or 'VerbForm_none' in morphology: + elif morphology.get('VerbForm') == 'none': return True - elif Number_sing in morphology or 'Number_sing' in morphology: + elif morphology.get('VerbForm') == 'inf': return True - elif Degree_pos in morphology or 'Degree_pos' in morphology: + elif morphology.get('Degree') == 'pos': return True else: return False From be5235369cc23ad6838c21f0980c90cc20dc4f00 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 12:14:23 +0100 Subject: [PATCH 044/287] Space out symbols enum, to make maintaining easier --- spacy/attrs.pxd | 13 +++++++------ spacy/symbols.pxd | 19 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/spacy/attrs.pxd b/spacy/attrs.pxd index 79a177ba9..a70fae04b 100644 --- a/spacy/attrs.pxd +++ b/spacy/attrs.pxd @@ -1,7 +1,9 @@ +from . cimport symbols + # Reserve 64 values for flag features cdef enum attr_id_t: - NULL_ATTR - IS_ALPHA + NULL_ATTR = 0 + IS_ALPHA = symbols.IS_ALPHA IS_ASCII IS_DIGIT IS_LOWER @@ -20,7 +22,7 @@ cdef enum attr_id_t: IS_RIGHT_PUNCT IS_CURRENCY - FLAG19 = 19 + FLAG19 = symbols.FLAG19 FLAG20 FLAG21 FLAG22 @@ -66,7 +68,7 @@ cdef enum attr_id_t: FLAG62 FLAG63 - ID + ID = symbols.ID ORTH LOWER NORM @@ -74,7 +76,7 @@ cdef enum attr_id_t: PREFIX SUFFIX - LENGTH + LENGTH = symbols.LENGTH CLUSTER LEMMA POS @@ -86,5 +88,4 @@ cdef enum attr_id_t: SENT_START SPACY PROB - LANG diff --git a/spacy/symbols.pxd b/spacy/symbols.pxd index 051b92edb..1cd1f7ef7 100644 --- a/spacy/symbols.pxd +++ b/spacy/symbols.pxd @@ -19,7 +19,7 @@ cdef enum symbol_t: IS_RIGHT_PUNCT IS_CURRENCY - FLAG19 = 19 + FLAG19 = 1000 FLAG20 FLAG21 FLAG22 @@ -65,7 +65,7 @@ cdef enum symbol_t: FLAG62 FLAG63 - ID + ID = 2000 ORTH LOWER NORM @@ -73,7 +73,7 @@ cdef enum symbol_t: PREFIX SUFFIX - LENGTH + LENGTH = 3000 CLUSTER LEMMA POS @@ -87,7 +87,7 @@ cdef enum symbol_t: PROB LANG - ADJ + ADJ = 4000 ADP ADV AUX @@ -108,7 +108,7 @@ cdef enum symbol_t: EOL SPACE - Animacy_anim + Animacy_anim = 5000 Animacy_inan Animacy_hum # U20 Animacy_nhum @@ -385,7 +385,7 @@ cdef enum symbol_t: VerbType_mod # U VerbType_light # U - PERSON + PERSON = 6000 NORP FACILITY ORG @@ -397,7 +397,7 @@ cdef enum symbol_t: LANGUAGE LAW - DATE + DATE = 7000 TIME PERCENT MONEY @@ -405,7 +405,8 @@ cdef enum symbol_t: ORDINAL CARDINAL - acomp + acl = 8000 + acomp advcl advmod agent @@ -458,5 +459,3 @@ cdef enum symbol_t: rcmod root xcomp - - acl From 6734cfec8881f280aeaf952a31feecac48a8b93d Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 12:14:37 +0100 Subject: [PATCH 045/287] Add comment --- spacy/morphology.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index a4759e4ab..585a004b4 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -204,6 +204,7 @@ cdef class Morphology: pos = 0 cdef attr_t lemma = self._cache.get(tag_id, token.lex.orth) if lemma == 0: + # Ugh, self.lemmatize has opposite arg order from self.lemmatizer :( lemma = self.lemmatize(pos, token.lex.orth, features) self._cache.set(tag_id, token.lex.orth, lemma) token.lemma = lemma From d0ca64bb07476894b35d2881c39bfa6f7a555bee Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 12:14:53 +0100 Subject: [PATCH 046/287] Fix imports in morphanalysis --- spacy/tokens/morphanalysis.pyx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx index df2d6ec20..09ab04d89 100644 --- a/spacy/tokens/morphanalysis.pyx +++ b/spacy/tokens/morphanalysis.pyx @@ -1,3 +1,6 @@ +from ..vocab cimport Vocab +from ..typedefs cimport hash_t + cdef class Morphanalysis: """Control access to morphological features for a token.""" def __init__(self, Vocab vocab, features=None): From bcfe3bd3122a61147d31425566060da90e997115 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 12:51:11 +0100 Subject: [PATCH 047/287] Fix StringStore after symbols changes --- spacy/strings.pyx | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/spacy/strings.pyx b/spacy/strings.pyx index 64954503f..0565b2a0a 100644 --- a/spacy/strings.pyx +++ b/spacy/strings.pyx @@ -11,11 +11,15 @@ import srsly from .compat import basestring_ from .symbols import IDS as SYMBOLS_BY_STR -from .symbols import NAMES as SYMBOLS_BY_INT +from . import symbols from .typedefs cimport hash_t from .errors import Errors from . import util +SYMBOLS_BY_INT = {} +for name in symbols.NAMES: + SYMBOLS_BY_INT[SYMBOLS_BY_STR[name]] = name +print(SYMBOLS_BY_INT[6005]) def get_string_id(key): """Get a string ID, handling the reserved symbols correctly. If the key is @@ -116,6 +120,8 @@ cdef class StringStore: return u'' elif string_or_id in SYMBOLS_BY_STR: return SYMBOLS_BY_STR[string_or_id] + elif string_or_id in SYMBOLS_BY_INT: + return SYMBOLS_BY_INT[string_or_id] cdef hash_t key if isinstance(string_or_id, unicode): key = hash_string(string_or_id) @@ -123,8 +129,6 @@ cdef class StringStore: elif isinstance(string_or_id, bytes): key = hash_utf8(string_or_id, len(string_or_id)) return key - elif string_or_id < len(SYMBOLS_BY_INT): - return SYMBOLS_BY_INT[string_or_id] else: key = string_or_id self.hits.insert(key) @@ -181,11 +185,14 @@ cdef class StringStore: string (unicode): The string to check. RETURNS (bool): Whether the store contains the string. """ + global SYMBOLS_BY_INT cdef hash_t key if isinstance(string, int) or isinstance(string, long): if string == 0: return True key = string + if key in SYMBOLS_BY_INT: + return True elif len(string) == 0: return True elif string in SYMBOLS_BY_STR: @@ -195,11 +202,8 @@ cdef class StringStore: else: string = string.encode('utf8') key = hash_utf8(string, len(string)) - if key < len(SYMBOLS_BY_INT): - return True - else: - self.hits.insert(key) - return self._map.get(key) is not NULL + self.hits.insert(key) + return self._map.get(key) is not NULL def __iter__(self): """Iterate over the strings in the store, in order. From c773b5011c9e1cbe7cc8b8f7e93bb73c56ba0266 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 12:52:15 +0100 Subject: [PATCH 048/287] Revert "Fix StringStore after symbols changes" This reverts commit bcfe3bd3122a61147d31425566060da90e997115. --- spacy/strings.pyx | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/spacy/strings.pyx b/spacy/strings.pyx index 0565b2a0a..64954503f 100644 --- a/spacy/strings.pyx +++ b/spacy/strings.pyx @@ -11,15 +11,11 @@ import srsly from .compat import basestring_ from .symbols import IDS as SYMBOLS_BY_STR -from . import symbols +from .symbols import NAMES as SYMBOLS_BY_INT from .typedefs cimport hash_t from .errors import Errors from . import util -SYMBOLS_BY_INT = {} -for name in symbols.NAMES: - SYMBOLS_BY_INT[SYMBOLS_BY_STR[name]] = name -print(SYMBOLS_BY_INT[6005]) def get_string_id(key): """Get a string ID, handling the reserved symbols correctly. If the key is @@ -120,8 +116,6 @@ cdef class StringStore: return u'' elif string_or_id in SYMBOLS_BY_STR: return SYMBOLS_BY_STR[string_or_id] - elif string_or_id in SYMBOLS_BY_INT: - return SYMBOLS_BY_INT[string_or_id] cdef hash_t key if isinstance(string_or_id, unicode): key = hash_string(string_or_id) @@ -129,6 +123,8 @@ cdef class StringStore: elif isinstance(string_or_id, bytes): key = hash_utf8(string_or_id, len(string_or_id)) return key + elif string_or_id < len(SYMBOLS_BY_INT): + return SYMBOLS_BY_INT[string_or_id] else: key = string_or_id self.hits.insert(key) @@ -185,14 +181,11 @@ cdef class StringStore: string (unicode): The string to check. RETURNS (bool): Whether the store contains the string. """ - global SYMBOLS_BY_INT cdef hash_t key if isinstance(string, int) or isinstance(string, long): if string == 0: return True key = string - if key in SYMBOLS_BY_INT: - return True elif len(string) == 0: return True elif string in SYMBOLS_BY_STR: @@ -202,8 +195,11 @@ cdef class StringStore: else: string = string.encode('utf8') key = hash_utf8(string, len(string)) - self.hits.insert(key) - return self._map.get(key) is not NULL + if key < len(SYMBOLS_BY_INT): + return True + else: + self.hits.insert(key) + return self._map.get(key) is not NULL def __iter__(self): """Iterate over the strings in the store, in order. From 74db1d9602c13feffbf1fd4fd03ecf6297e973f7 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 12:52:30 +0100 Subject: [PATCH 049/287] Revert "Space out symbols enum, to make maintaining easier" This reverts commit be5235369cc23ad6838c21f0980c90cc20dc4f00. --- spacy/attrs.pxd | 13 ++++++------- spacy/symbols.pxd | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/spacy/attrs.pxd b/spacy/attrs.pxd index a70fae04b..79a177ba9 100644 --- a/spacy/attrs.pxd +++ b/spacy/attrs.pxd @@ -1,9 +1,7 @@ -from . cimport symbols - # Reserve 64 values for flag features cdef enum attr_id_t: - NULL_ATTR = 0 - IS_ALPHA = symbols.IS_ALPHA + NULL_ATTR + IS_ALPHA IS_ASCII IS_DIGIT IS_LOWER @@ -22,7 +20,7 @@ cdef enum attr_id_t: IS_RIGHT_PUNCT IS_CURRENCY - FLAG19 = symbols.FLAG19 + FLAG19 = 19 FLAG20 FLAG21 FLAG22 @@ -68,7 +66,7 @@ cdef enum attr_id_t: FLAG62 FLAG63 - ID = symbols.ID + ID ORTH LOWER NORM @@ -76,7 +74,7 @@ cdef enum attr_id_t: PREFIX SUFFIX - LENGTH = symbols.LENGTH + LENGTH CLUSTER LEMMA POS @@ -88,4 +86,5 @@ cdef enum attr_id_t: SENT_START SPACY PROB + LANG diff --git a/spacy/symbols.pxd b/spacy/symbols.pxd index 1cd1f7ef7..051b92edb 100644 --- a/spacy/symbols.pxd +++ b/spacy/symbols.pxd @@ -19,7 +19,7 @@ cdef enum symbol_t: IS_RIGHT_PUNCT IS_CURRENCY - FLAG19 = 1000 + FLAG19 = 19 FLAG20 FLAG21 FLAG22 @@ -65,7 +65,7 @@ cdef enum symbol_t: FLAG62 FLAG63 - ID = 2000 + ID ORTH LOWER NORM @@ -73,7 +73,7 @@ cdef enum symbol_t: PREFIX SUFFIX - LENGTH = 3000 + LENGTH CLUSTER LEMMA POS @@ -87,7 +87,7 @@ cdef enum symbol_t: PROB LANG - ADJ = 4000 + ADJ ADP ADV AUX @@ -108,7 +108,7 @@ cdef enum symbol_t: EOL SPACE - Animacy_anim = 5000 + Animacy_anim Animacy_inan Animacy_hum # U20 Animacy_nhum @@ -385,7 +385,7 @@ cdef enum symbol_t: VerbType_mod # U VerbType_light # U - PERSON = 6000 + PERSON NORP FACILITY ORG @@ -397,7 +397,7 @@ cdef enum symbol_t: LANGUAGE LAW - DATE = 7000 + DATE TIME PERCENT MONEY @@ -405,8 +405,7 @@ cdef enum symbol_t: ORDINAL CARDINAL - acl = 8000 - acomp + acomp advcl advmod agent @@ -459,3 +458,5 @@ cdef enum symbol_t: rcmod root xcomp + + acl From b69013e2d7805d3c870f099fa32046e0d2dbe994 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 13:11:38 +0100 Subject: [PATCH 050/287] Fix passing of morphological features to lemmatizer --- spacy/lemmatizer.py | 12 ------------ spacy/morphology.pyx | 11 ++++++++++- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/spacy/lemmatizer.py b/spacy/lemmatizer.py index c708800e9..99f157e05 100644 --- a/spacy/lemmatizer.py +++ b/spacy/lemmatizer.py @@ -47,17 +47,6 @@ class Lemmatizer(object): Check whether we're dealing with an uninflected paradigm, so we can avoid lemmatization entirely. """ - morphology = {} if morphology is None else morphology - morphology = dict(morphology) - for key, value in list(morphology.items()): - if value is True: - feat, value = key.split('_') - morphology[feat] = value - others = [ - key - for key in morphology - if key not in (POS, "Number", "POS", "VerbForm", "Tense") - ] if univ_pos == "noun" and morphology.get("Number") == "sing": return True elif univ_pos == "verb" and morphology.get("VerbForm") == "inf": @@ -68,7 +57,6 @@ class Lemmatizer(object): morphology.get("VerbForm") == "fin" and morphology.get("Tense") == "pres" and morphology.get("Number") is None - and not others ): return True elif univ_pos == "adj" and morphology.get("Degree") == "pos": diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 585a004b4..40c7f66af 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -141,7 +141,16 @@ cdef class Morphology: return self.strings.add(py_string.lower()) cdef list lemma_strings cdef unicode lemma_string - lemma_strings = self.lemmatizer(py_string, univ_pos, morphology) + # Normalize features into a dict keyed by the field, to make life easier + # for the lemmatizer. Handles string-to-int conversion too. + string_feats = {} + for key, value in morphology.items(): + if value is True: + name, value = self.strings.as_string(key).split('_', 1) + string_feats[name] = value + else: + string_feats[self.strings.as_string(key)] = self.strings.as_string(value) + lemma_strings = self.lemmatizer(py_string, univ_pos, string_feats) lemma_string = lemma_strings[0] lemma = self.strings.add(lemma_string) return lemma From b9ade7d4e090a2bd20626aa96ffa310031871c23 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 14:03:07 +0100 Subject: [PATCH 051/287] Add MorphAnalysisC struct --- spacy/morphology.pxd | 51 +---- spacy/morphology.pyx | 26 +-- spacy/structs.pxd | 46 ++++ spacy/tokens/morphanalysis.pyx | 371 +++++++++++++++++++++++++++++++-- 4 files changed, 420 insertions(+), 74 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index adc5e5574..24e54bdee 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -3,7 +3,7 @@ from preshed.maps cimport PreshMap, PreshMapArray from libc.stdint cimport uint64_t from murmurhash cimport mrmr -from .structs cimport TokenC +from .structs cimport TokenC, MorphAnalysisC from .strings cimport StringStore from .typedefs cimport hash_t, attr_t, flags_t from .parts_of_speech cimport univ_pos_t @@ -24,7 +24,7 @@ cdef class Morphology: cdef readonly int n_tags cpdef update(self, hash_t morph, features) - cdef hash_t insert(self, RichTagC tag) except 0 + cdef hash_t insert(self, MorphAnalysisC tag) except 0 cdef int assign_untagged(self, TokenC* token) except -1 cdef int assign_tag(self, TokenC* token, tag) except -1 @@ -416,50 +416,3 @@ cdef enum univ_morph_t: Voice_int # hb end_Voice - -cdef struct RichTagC: - univ_pos_t pos - - univ_morph_t abbr - univ_morph_t adp_type - univ_morph_t adv_type - univ_morph_t animacy - univ_morph_t aspect - univ_morph_t case - univ_morph_t conj_type - univ_morph_t connegative - univ_morph_t definite - univ_morph_t degree - univ_morph_t derivation - univ_morph_t echo - univ_morph_t foreign - univ_morph_t gender - univ_morph_t hyph - univ_morph_t inf_form - univ_morph_t mood - univ_morph_t negative - univ_morph_t number - univ_morph_t name_type - univ_morph_t noun_type - univ_morph_t num_form - univ_morph_t num_type - univ_morph_t num_value - univ_morph_t part_form - univ_morph_t part_type - univ_morph_t person - univ_morph_t polite - univ_morph_t polarity - univ_morph_t poss - univ_morph_t prefix - univ_morph_t prep_case - univ_morph_t pron_type - univ_morph_t punct_side - univ_morph_t punct_type - univ_morph_t reflex - univ_morph_t style - univ_morph_t style_variant - univ_morph_t tense - univ_morph_t typo - univ_morph_t verb_form - univ_morph_t voice - univ_morph_t verb_type diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 40c7f66af..52acfedfb 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -111,13 +111,13 @@ cdef class Morphology: print(list(NAMES.keys())[:10]) print(NAMES.get(feature-1), NAMES.get(feature+1)) raise KeyError("Unknown feature: %d" % feature) - cdef RichTagC tag + cdef MorphAnalysisC tag tag = create_rich_tag(features) cdef hash_t key = self.insert(tag) return key def get(self, hash_t morph): - tag = self.tags.get(morph) + tag = self.tags.get(morph) if tag == NULL: return [] else: @@ -125,7 +125,7 @@ cdef class Morphology: cpdef update(self, hash_t morph, features): """Update a morphological analysis with new feature values.""" - tag = (self.tags.get(morph))[0] + tag = (self.tags.get(morph))[0] features = intify_features(features) cdef univ_morph_t feature for feature in features: @@ -168,10 +168,10 @@ cdef class Morphology: attrs = intify_attrs(attrs, self.strings, _do_deprecated=True) self.exc[(tag_str, self.strings.add(orth_str))] = attrs - cdef hash_t insert(self, RichTagC tag) except 0: + cdef hash_t insert(self, MorphAnalysisC tag) except 0: cdef hash_t key = hash_tag(tag) if self.tags.get(key) == NULL: - tag_ptr = self.mem.alloc(1, sizeof(RichTagC)) + tag_ptr = self.mem.alloc(1, sizeof(MorphAnalysisC)) tag_ptr[0] = tag self.tags.set(key, tag_ptr) return key @@ -240,7 +240,7 @@ cdef class Morphology: def to_bytes(self): json_tags = [] for key in self.tags: - tag_ptr = self.tags.get(key) + tag_ptr = self.tags.get(key) if tag_ptr != NULL: json_tags.append(tag_to_json(tag_ptr[0])) return srsly.json_dumps(json_tags) @@ -261,18 +261,18 @@ cpdef univ_pos_t get_int_tag(pos_): cpdef intify_features(features): return {IDS.get(feature, feature) for feature in features} -cdef hash_t hash_tag(RichTagC tag) nogil: +cdef hash_t hash_tag(MorphAnalysisC tag) nogil: return mrmr.hash64(&tag, sizeof(tag), 0) -cdef RichTagC create_rich_tag(features) except *: - cdef RichTagC tag +cdef MorphAnalysisC create_rich_tag(features) except *: + cdef MorphAnalysisC tag cdef univ_morph_t feature memset(&tag, 0, sizeof(tag)) for feature in features: set_feature(&tag, feature, 1) return tag -cdef tag_to_json(RichTagC tag): +cdef tag_to_json(MorphAnalysisC tag): features = [] if tag.abbr != 0: features.append(NAMES[tag.abbr]) @@ -360,11 +360,11 @@ cdef tag_to_json(RichTagC tag): features.append(NAMES[tag.verb_type]) return features -cdef RichTagC tag_from_json(json_tag): - cdef RichTagC tag +cdef MorphAnalysisC tag_from_json(json_tag): + cdef MorphAnalysisC tag return tag -cdef int set_feature(RichTagC* tag, univ_morph_t feature, int value) except -1: +cdef int set_feature(MorphAnalysisC* tag, univ_morph_t feature, int value) except -1: if value == True: value_ = feature else: diff --git a/spacy/structs.pxd b/spacy/structs.pxd index 9f7904919..7452123c0 100644 --- a/spacy/structs.pxd +++ b/spacy/structs.pxd @@ -74,4 +74,50 @@ cdef struct TokenC: hash_t ent_id +cdef struct MorphAnalysisC: + univ_pos_t pos + + attr_t abbr + attr_t adp_type + attr_t adv_type + attr_t animacy + attr_t aspect + attr_t case + attr_t conj_type + attr_t connegative + attr_t definite + attr_t degree + attr_t derivation + attr_t echo + attr_t foreign + attr_t gender + attr_t hyph + attr_t inf_form + attr_t mood + attr_t negative + attr_t number + attr_t name_type + attr_t noun_type + attr_t num_form + attr_t num_type + attr_t num_value + attr_t part_form + attr_t part_type + attr_t person + attr_t polite + attr_t polarity + attr_t poss + attr_t prefix + attr_t prep_case + attr_t pron_type + attr_t punct_side + attr_t punct_type + attr_t reflex + attr_t style + attr_t style_variant + attr_t tense + attr_t typo + attr_t verb_form + attr_t voice + attr_t verb_type diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx index 09ab04d89..722f97994 100644 --- a/spacy/tokens/morphanalysis.pyx +++ b/spacy/tokens/morphanalysis.pyx @@ -1,10 +1,14 @@ from ..vocab cimport Vocab from ..typedefs cimport hash_t + cdef class Morphanalysis: """Control access to morphological features for a token.""" - def __init__(self, Vocab vocab, features=None): - pass + def __init__(self, Vocab vocab, features=tuple()): + self.vocab = vocab + self.key = self.vocab.morphology.add(features) + analysis = self.vocab.morphology.tags.get(self.key) + self.c = analysis[0] @classmethod def from_id(self, Vocab vocab, hash_t key): @@ -28,6 +32,12 @@ cdef class Morphanalysis: def __hash__(self): pass + def get(self, name): + pass + + def to_json(self): + pass + @property def is_base_form(self): pass @@ -44,17 +54,354 @@ cdef class Morphanalysis: def id(self): pass - def get(self, name): - pass + property abbr: + def __get__(self): + pass - def set(self, name, value): - pass + property adp_type: + def __get__(self): + pass - def add(self, feature): - pass + property adv_type: + def __get__(self): + pass - def remove(self, feature): - pass + property animacy: + def __get__(self): + pass - def to_json(self): - pass + property aspect: + def __get__(self): + pass + + property case: + def __get__(self): + pass + + property conj_type: + def __get__(self): + pass + + property connegative: + def __get__(self): + pass + + property definite: + def __get__(self): + pass + + property degree: + def __get__(self): + pass + + property derivation: + def __get__(self): + pass + + property echo: + def __get__(self): + pass + + property foreign: + def __get__(self): + pass + + property gender: + def __get__(self): + pass + + property hyph: + def __get__(self): + pass + + property inf_form: + def __get__(self): + pass + + property name_type: + def __get__(self): + pass + + property negative: + def __get__(self): + pass + + property mood: + def __get__(self): + pass + + property name_type: + def __get__(self): + pass + + property negative: + def __get__(self): + pass + + property number: + def __get__(self): + pass + + property num_form: + def __get__(self): + pass + + property num_type: + def __get__(self): + pass + + property num_value: + def __get__(self): + pass + + property part_form: + def __get__(self): + pass + + property part_type: + def __get__(self): + pass + + property person: + def __get__(self): + pass + + property polite: + def __get__(self): + pass + + property polarity: + def __get__(self): + pass + + property poss: + def __get__(self): + pass + + property prefix: + def __get__(self): + pass + + property prep_case: + def __get__(self): + pass + + property pron_type: + def __get__(self): + pass + + property punct_side: + def __get__(self): + pass + + property punct_type: + def __get__(self): + pass + + property reflex: + def __get__(self): + pass + + property style: + def __get__(self): + pass + + property style_variant: + def __get__(self): + pass + + property tense: + def __get__(self): + pass + + property typo: + def __get__(self): + pass + + property verb_form: + def __get__(self): + pass + + property voice: + def __get__(self): + pass + + property verb_type: + def __get__(self): + pass + + property abbr_: + def __get__(self): + pass + + property adp_type_: + def __get__(self): + pass + + property adv_type_: + def __get__(self): + pass + + property animacy_: + def __get__(self): + pass + + property aspect_: + def __get__(self): + pass + + property case_: + def __get__(self): + pass + + property conj_type_: + def __get__(self): + pass + + property connegative_: + def __get__(self): + pass + + property definite_: + def __get__(self): + pass + + property degree_: + def __get__(self): + pass + + property derivation_: + def __get__(self): + pass + + property echo_: + def __get__(self): + pass + + property foreign_: + def __get__(self): + pass + + property gender_: + def __get__(self): + pass + + property hyph_: + def __get__(self): + pass + + property inf_form_: + def __get__(self): + pass + + property name_type_: + def __get__(self): + pass + + property negative_: + def __get__(self): + pass + + property mood_: + def __get__(self): + pass + + property name_type_: + def __get__(self): + pass + + property negative_: + def __get__(self): + pass + + property number_: + def __get__(self): + pass + + property num_form_: + def __get__(self): + pass + + property num_type_: + def __get__(self): + pass + + property num_value_: + def __get__(self): + pass + + property part_form_: + def __get__(self): + pass + + property part_type_: + def __get__(self): + pass + + property person_: + def __get__(self): + pass + + property polite_: + def __get__(self): + pass + + property polarity_: + def __get__(self): + pass + + property poss_: + def __get__(self): + pass + + property prefix_: + def __get__(self): + pass + + property prep_case_: + def __get__(self): + pass + + property pron_type_: + def __get__(self): + pass + + property punct_side_: + def __get__(self): + pass + + property punct_type_: + def __get__(self): + pass + + property reflex_: + def __get__(self): + pass + + property style_: + def __get__(self): + pass + + property style_variant_: + def __get__(self): + pass + + property tense_: + def __get__(self): + pass + + property typo_: + def __get__(self): + pass + + property verb_form_: + def __get__(self): + pass + + property voice_: + def __get__(self): + pass + + property verb_type_: + def __get__(self): + pass From 932d7dde1c5e549dd0b92397c8bd6b00aab9ab0f Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 14:34:54 +0100 Subject: [PATCH 052/287] Fix compile error --- setup.py | 1 + spacy/tokens/morphanalysis.pyx | 22 +++++----------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index f193d0498..b1b3785eb 100755 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ MOD_NAMES = [ "spacy.tokens.doc", "spacy.tokens.span", "spacy.tokens.token", + "spacy.tokens.morphanalysis", "spacy.tokens._retokenize", "spacy.matcher.matcher", "spacy.matcher.phrasematcher", diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx index 722f97994..01ecf458b 100644 --- a/spacy/tokens/morphanalysis.pyx +++ b/spacy/tokens/morphanalysis.pyx @@ -2,7 +2,7 @@ from ..vocab cimport Vocab from ..typedefs cimport hash_t -cdef class Morphanalysis: +cdef class MorphAnalysis: """Control access to morphological features for a token.""" def __init__(self, Vocab vocab, features=tuple()): self.vocab = vocab @@ -118,14 +118,6 @@ cdef class Morphanalysis: def __get__(self): pass - property name_type: - def __get__(self): - pass - - property negative: - def __get__(self): - pass - property mood: def __get__(self): pass @@ -138,6 +130,10 @@ cdef class Morphanalysis: def __get__(self): pass + property noun_type: + def __get__(self): + pass + property number: def __get__(self): pass @@ -306,14 +302,6 @@ cdef class Morphanalysis: def __get__(self): pass - property name_type_: - def __get__(self): - pass - - property negative_: - def __get__(self): - pass - property number_: def __get__(self): pass From fed0371db753765425521243b5325fd09296dd4a Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 17:14:57 +0100 Subject: [PATCH 053/287] Remove enums from morphology --- spacy/morphology.pxd | 385 ------ spacy/morphology.pyx | 1104 +++++++---------- spacy/pipeline/morphologizer.pyx | 6 +- spacy/structs.pxd | 1 - spacy/tests/doc/test_retokenize_merge.py | 1 - spacy/tests/morphology/test_morph_features.py | 8 +- spacy/tokens/token.pyx | 5 + 7 files changed, 487 insertions(+), 1023 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index 24e54bdee..a057e8ed8 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -31,388 +31,3 @@ cdef class Morphology: cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1 cdef int _assign_tag_from_exceptions(self, TokenC* token, int tag_id) except -1 - - -cdef enum univ_morph_t: - NIL = 0 - - begin_Abbr - Abbr_yes - end_Abbr - - begin_AdpType - AdpType_circ - AdpType_comprep - AdpType_prep - AdpType_post - AdpType_voc - end_AdpType - - begin_AdvType - AdvType_adadj - AdvType_cau - AdvType_deg - AdvType_ex - AdvType_loc - AdvType_man - AdvType_mod - AdvType_sta - AdvType_tim - end_AdvType - - begin_Animacy - Animacy_anim - Animacy_hum - Animacy_inan - Animacy_nhum - end_Animacy - - begin_Aspect - Aspect_freq - Aspect_imp - Aspect_mod - Aspect_none - Aspect_perf - end_Aspect - - begin_Case - Case_abe - Case_abl - Case_abs - Case_acc - Case_ade - Case_all - Case_cau - Case_com - Case_dat - Case_del - Case_dis - Case_ela - Case_ess - Case_gen - Case_ill - Case_ine - Case_ins - Case_loc - Case_lat - Case_nom - Case_par - Case_sub - Case_sup - Case_tem - Case_ter - Case_tra - Case_voc - end_Case - - begin_ConjType - ConjType_comp # cz, U - ConjType_oper # cz, U - end_ConjType - begin_Connegative - Connegative_yes # fi - end_Connegative - - begin_Definite - Definite_cons # U20 - Definite_def - Definite_ind - Definite_red - Definite_two - end_Definite - - begin_Degree - Degree_abs - Degree_cmp - Degree_comp - Degree_none - Degree_pos - Degree_sup - Degree_com - Degree_dim # du - end_Degree - - begin_Derivation - Derivation_minen # fi - Derivation_sti # fi - Derivation_inen # fi - Derivation_lainen # fi - Derivation_ja # fi - Derivation_ton # fi - Derivation_vs # fi - Derivation_ttain # fi - Derivation_ttaa # fi - end_Derivation - - begin_Echo - Echo_rdp # U - Echo_ech # U - end_Echo - - begin_Foreign - Foreign_foreign # cz, fi, U - Foreign_fscript # cz, fi, U - Foreign_tscript # cz, U - Foreign_yes # sl - end_Foreign - - begin_Gender - Gender_com - Gender_fem - Gender_masc - Gender_neut - Gender_dat_masc # bq, U - Gender_dat_fem # bq, U - Gender_erg_masc # bq - Gender_erg_fem # bq - Gender_psor_masc # cz, sl, U - Gender_psor_fem # cz, sl, U - Gender_psor_neut # sl - end_Gender - - begin_Hyph - Hyph_yes # cz, U - end_Hyph - - begin_InfForm - InfForm_one # fi - InfForm_two # fi - InfForm_three # fi - end_InfForm - - begin_Mood - Mood_cnd - Mood_imp - Mood_ind - Mood_n - Mood_pot - Mood_sub - Mood_opt - end_Mood - - begin_NameType - NameType_geo # U, cz - NameType_prs # U, cz - NameType_giv # U, cz - NameType_sur # U, cz - NameType_nat # U, cz - NameType_com # U, cz - NameType_pro # U, cz - NameType_oth # U, cz - end_NameType - - begin_Negative - Negative_neg - Negative_pos - Negative_yes - end_Negative - - begin_NounType - NounType_com # U - NounType_prop # U - NounType_class # U - end_NounType - - begin_Number - Number_com - Number_dual - Number_none - Number_plur - Number_sing - Number_ptan # bg - Number_count # bg - Number_abs_sing # bq, U - Number_abs_plur # bq, U - Number_dat_sing # bq, U - Number_dat_plur # bq, U - Number_erg_sing # bq, U - Number_erg_plur # bq, U - Number_psee_sing # U - Number_psee_plur # U - Number_psor_sing # cz, fi, sl, U - Number_psor_plur # cz, fi, sl, U - end_Number - - begin_NumForm - NumForm_digit # cz, sl, U - NumForm_roman # cz, sl, U - NumForm_word # cz, sl, U - end_NumForm - - begin_NumType - NumType_card - NumType_dist - NumType_frac - NumType_gen - NumType_mult - NumType_none - NumType_ord - NumType_sets - end_NumType - - begin_NumValue - NumValue_one # cz, U - NumValue_two # cz, U - NumValue_three # cz, U - end_NumValue - - begin_PartForm - PartForm_pres # fi - PartForm_past # fi - PartForm_agt # fi - PartForm_neg # fi - end_PartForm - - begin_PartType - PartType_mod # U - PartType_emp # U - PartType_res # U - PartType_inf # U - PartType_vbp # U - end_PartType - - begin_Person - Person_one - Person_two - Person_three - Person_none - Person_abs_one # bq, U - Person_abs_two # bq, U - Person_abs_three # bq, U - Person_dat_one # bq, U - Person_dat_two # bq, U - Person_dat_three # bq, U - Person_erg_one # bq, U - Person_erg_two # bq, U - Person_erg_three # bq, U - Person_psor_one # fi, U - Person_psor_two # fi, U - Person_psor_three # fi, U - end_Person - - begin_Polarity - Polarity_neg # U20 - Polarity_pos # U20 - end_Polarity - - begin_Polite - Polite_inf # bq, U - Polite_pol # bq, U - Polite_abs_inf # bq, U - Polite_abs_pol # bq, U - Polite_erg_inf # bq, U - Polite_erg_pol # bq, U - Polite_dat_inf # bq, U - Polite_dat_pol # bq, U - end_Polite - - begin_Poss - Poss_yes - end_Poss - - begin_Prefix - Prefix_yes # U - end_Prefix - - begin_PrepCase - PrepCase_npr # cz - PrepCase_pre # U - end_PrepCase - - begin_PronType - PronType_advPart - PronType_art - PronType_default - PronType_dem - PronType_ind - PronType_int - PronType_neg - PronType_prs - PronType_rcp - PronType_rel - PronType_tot - PronType_clit - PronType_exc # es, ca, it, fa - end_PronType - - begin_PunctSide - PunctSide_ini # U - PunctSide_fin # U - end_PunctSide - - begin_PunctType - PunctType_peri # U - PunctType_qest # U - PunctType_excl # U - PunctType_quot # U - PunctType_brck # U - PunctType_comm # U - PunctType_colo # U - PunctType_semi # U - PunctType_dash # U - end_PunctType - - begin_Reflex - Reflex_yes - end_Reflex - - begin_Style - Style_arch # cz, fi, U - Style_rare # cz, fi, U - Style_poet # cz, U - Style_norm # cz, U - Style_coll # cz, U - Style_vrnc # cz, U - Style_sing # cz, U - Style_expr # cz, U - Style_derg # cz, U - Style_vulg # cz, U - Style_yes # fi, U - end_Style - - begin_StyleVariant - StyleVariant_styleShort # cz - StyleVariant_styleBound # cz, sl - end_StyleVariant - - begin_Tense - Tense_fut - Tense_imp - Tense_past - Tense_pres - end_Tense - - begin_Typo - Typo_yes - end_Typo - - begin_VerbForm - VerbForm_fin - VerbForm_ger - VerbForm_inf - VerbForm_none - VerbForm_part - VerbForm_partFut - VerbForm_partPast - VerbForm_partPres - VerbForm_sup - VerbForm_trans - VerbForm_conv # U20 - VerbForm_gdv # la - end_VerbForm - - begin_VerbType - VerbType_aux # U - VerbType_cop # U - VerbType_mod # U - VerbType_light # U - end_VerbType - - begin_Voice - Voice_act - Voice_cau - Voice_pass - Voice_mid # gkc - Voice_int # hb - end_Voice - diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 52acfedfb..1157c2502 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -4,6 +4,7 @@ from __future__ import unicode_literals from libc.string cimport memset import srsly +from collections import Counter from .strings import get_string_id from . import symbols @@ -14,6 +15,50 @@ from .parts_of_speech import IDS as POS_IDS from .lexeme cimport Lexeme from .errors import Errors +cdef enum univ_field_t: + Field_Abbr + Field_AdpType + Field_AdvType + Field_Animacy + Field_Aspect + Field_Case + Field_ConjType + Field_Connegative + Field_Definite + Field_Degree + Field_Derivation + Field_Echo + Field_Foreign + Field_Gender + Field_Hyph + Field_InfForm + Field_Mood + Field_NameType + Field_Negative + Field_NounType + Field_Number + Field_NumForm + Field_NumType + Field_NumValue + Field_PartForm + Field_PartType + Field_Person + Field_Polite + Field_Polarity + Field_Poss + Field_Prefix + Field_PrepCase + Field_PronType + Field_PunctSide + Field_PunctType + Field_Reflex + Field_Style + Field_StyleVariant + Field_Tense + Field_Typo + Field_VerbForm + Field_Voice + Field_VerbType def _normalize_props(props): @@ -23,7 +68,7 @@ def _normalize_props(props): for key in FIELDS: if key in props: attr = '%s_%s' % (key, props[key]) - if attr in IDS: + if attr in FEATURES: props.pop(key) props[attr] = True for key, value in props.items(): @@ -43,21 +88,21 @@ def _normalize_props(props): def parse_feature(feature): - if not hasattr(feature, 'split'): - feature = NAMES[feature] - key, value = feature.split('_') - begin = 'begin_%s' % key - # Note that this includes a 0 offset for the field, for no entry - offset = IDS[feature] - IDS[begin] - field_id = FIELDS[key] - return (field_id, offset) + field = FEATURE_FIELDS[feature] + offset = FEATURE_OFFSETS[feature] + return (field, offset) + + +def get_field_id(feature): + return FEATURE_FIELDS[feature] def get_field_size(field): - begin = 'begin_%s' % field - end = 'end_%s' % field - # Extra field for no entry -- always 0 - return IDS[end] - IDS[begin] + return FIELD_SIZES[field] + + +def get_field_offset(field): + return FIELD_OFFSETS[field] cdef class Morphology: @@ -105,11 +150,9 @@ cdef class Morphology: present. Returns the hash of the new analysis. """ features = intify_features(features) - cdef univ_morph_t feature + cdef attr_t feature for feature in features: - if feature != 0 and feature not in NAMES: - print(list(NAMES.keys())[:10]) - print(NAMES.get(feature-1), NAMES.get(feature+1)) + if feature != 0 and feature not in FEATURE_NAMES: raise KeyError("Unknown feature: %d" % feature) cdef MorphAnalysisC tag tag = create_rich_tag(features) @@ -127,9 +170,10 @@ cdef class Morphology: """Update a morphological analysis with new feature values.""" tag = (self.tags.get(morph))[0] features = intify_features(features) - cdef univ_morph_t feature + cdef attr_t feature for feature in features: - set_feature(&tag, feature, 1) + field = get_field_id(feature) + set_feature(&tag, field, feature, 1) morph = self.insert(tag) return morph @@ -259,729 +303,531 @@ cpdef univ_pos_t get_int_tag(pos_): return 0 cpdef intify_features(features): - return {IDS.get(feature, feature) for feature in features} + return {get_string_id(feature) for feature in features} cdef hash_t hash_tag(MorphAnalysisC tag) nogil: return mrmr.hash64(&tag, sizeof(tag), 0) + +def get_feature_field(feature): + cdef attr_t key = get_string_id(feature) + return FEATURE_FIELDS[feature] + + cdef MorphAnalysisC create_rich_tag(features) except *: cdef MorphAnalysisC tag - cdef univ_morph_t feature + cdef attr_t feature memset(&tag, 0, sizeof(tag)) for feature in features: - set_feature(&tag, feature, 1) + field = get_field_id(feature) + set_feature(&tag, field, feature, 1) return tag + cdef tag_to_json(MorphAnalysisC tag): features = [] if tag.abbr != 0: - features.append(NAMES[tag.abbr]) + features.append(FEATURE_NAMES[tag.abbr]) if tag.adp_type != 0: - features.append(NAMES[tag.adp_type]) + features.append(FEATURE_NAMES[tag.adp_type]) if tag.adv_type != 0: - features.append(NAMES[tag.adv_type]) + features.append(FEATURE_NAMES[tag.adv_type]) if tag.animacy != 0: - features.append(NAMES[tag.animacy]) + features.append(FEATURE_NAMES[tag.animacy]) if tag.aspect != 0: - features.append(NAMES[tag.aspect]) + features.append(FEATURE_NAMES[tag.aspect]) if tag.case != 0: - features.append(NAMES[tag.case]) + features.append(FEATURE_NAMES[tag.case]) if tag.conj_type != 0: - features.append(NAMES[tag.conj_type]) + features.append(FEATURE_NAMES[tag.conj_type]) if tag.connegative != 0: - features.append(NAMES[tag.connegative]) + features.append(FEATURE_NAMES[tag.connegative]) if tag.definite != 0: - features.append(NAMES[tag.definite]) + features.append(FEATURE_NAMES[tag.definite]) if tag.degree != 0: - features.append(NAMES[tag.degree]) + features.append(FEATURE_NAMES[tag.degree]) if tag.derivation != 0: - features.append(NAMES[tag.derivation]) + features.append(FEATURE_NAMES[tag.derivation]) if tag.echo != 0: - features.append(NAMES[tag.echo]) + features.append(FEATURE_NAMES[tag.echo]) if tag.foreign != 0: - features.append(NAMES[tag.foreign]) + features.append(FEATURE_NAMES[tag.foreign]) if tag.gender != 0: - features.append(NAMES[tag.gender]) + features.append(FEATURE_NAMES[tag.gender]) if tag.hyph != 0: - features.append(NAMES[tag.hyph]) + features.append(FEATURE_NAMES[tag.hyph]) if tag.inf_form != 0: - features.append(NAMES[tag.inf_form]) + features.append(FEATURE_NAMES[tag.inf_form]) if tag.mood != 0: - features.append(NAMES[tag.mood]) + features.append(FEATURE_NAMES[tag.mood]) if tag.negative != 0: - features.append(NAMES[tag.negative]) + features.append(FEATURE_NAMES[tag.negative]) if tag.number != 0: - features.append(NAMES[tag.number]) + features.append(FEATURE_NAMES[tag.number]) if tag.name_type != 0: - features.append(NAMES[tag.name_type]) + features.append(FEATURE_NAMES[tag.name_type]) if tag.noun_type != 0: - features.append(NAMES[tag.noun_type]) + features.append(FEATURE_NAMES[tag.noun_type]) if tag.num_form != 0: - features.append(NAMES[tag.num_form]) + features.append(FEATURE_NAMES[tag.num_form]) if tag.num_type != 0: - features.append(NAMES[tag.num_type]) + features.append(FEATURE_NAMES[tag.num_type]) if tag.num_value != 0: - features.append(NAMES[tag.num_value]) + features.append(FEATURE_NAMES[tag.num_value]) if tag.part_form != 0: - features.append(NAMES[tag.part_form]) + features.append(FEATURE_NAMES[tag.part_form]) if tag.part_type != 0: - features.append(NAMES[tag.part_type]) + features.append(FEATURE_NAMES[tag.part_type]) if tag.person != 0: - features.append(NAMES[tag.person]) + features.append(FEATURE_NAMES[tag.person]) if tag.polite != 0: - features.append(NAMES[tag.polite]) + features.append(FEATURE_NAMES[tag.polite]) if tag.polarity != 0: - features.append(NAMES[tag.polarity]) + features.append(FEATURE_NAMES[tag.polarity]) if tag.poss != 0: - features.append(NAMES[tag.poss]) + features.append(FEATURE_NAMES[tag.poss]) if tag.prefix != 0: - features.append(NAMES[tag.prefix]) + features.append(FEATURE_NAMES[tag.prefix]) if tag.prep_case != 0: - features.append(NAMES[tag.prep_case]) + features.append(FEATURE_NAMES[tag.prep_case]) if tag.pron_type != 0: - features.append(NAMES[tag.pron_type]) + features.append(FEATURE_NAMES[tag.pron_type]) if tag.punct_side != 0: - features.append(NAMES[tag.punct_side]) + features.append(FEATURE_NAMES[tag.punct_side]) if tag.punct_type != 0: - features.append(NAMES[tag.punct_type]) + features.append(FEATURE_NAMES[tag.punct_type]) if tag.reflex != 0: - features.append(NAMES[tag.reflex]) + features.append(FEATURE_NAMES[tag.reflex]) if tag.style != 0: - features.append(NAMES[tag.style]) + features.append(FEATURE_NAMES[tag.style]) if tag.style_variant != 0: - features.append(NAMES[tag.style_variant]) + features.append(FEATURE_NAMES[tag.style_variant]) if tag.tense != 0: - features.append(NAMES[tag.tense]) + features.append(FEATURE_NAMES[tag.tense]) if tag.verb_form != 0: - features.append(NAMES[tag.verb_form]) + features.append(FEATURE_NAMES[tag.verb_form]) if tag.voice != 0: - features.append(NAMES[tag.voice]) + features.append(FEATURE_NAMES[tag.voice]) if tag.verb_type != 0: - features.append(NAMES[tag.verb_type]) + features.append(FEATURE_NAMES[tag.verb_type]) return features cdef MorphAnalysisC tag_from_json(json_tag): cdef MorphAnalysisC tag return tag -cdef int set_feature(MorphAnalysisC* tag, univ_morph_t feature, int value) except -1: +cdef int set_feature(MorphAnalysisC* tag, + univ_field_t field, attr_t feature, int value) except -1: if value == True: value_ = feature else: - value_ = NIL - if feature == NIL: + value_ = 0 + if feature == 0: pass - elif is_abbr_feature(feature): + elif field == Field_Abbr: tag.abbr = value_ - elif is_adp_type_feature(feature): + elif field == Field_AdpType: tag.adp_type = value_ - elif is_adv_type_feature(feature): + elif field == Field_AdvType: tag.adv_type = value_ - elif is_animacy_feature(feature): + elif field == Field_Animacy: tag.animacy = value_ - elif is_aspect_feature(feature): + elif field == Field_Aspect: tag.aspect = value_ - elif is_case_feature(feature): + elif field == Field_Case: tag.case = value_ - elif is_conj_type_feature(feature): + elif field == Field_ConjType: tag.conj_type = value_ - elif is_connegative_feature(feature): + elif field == Field_Connegative: tag.connegative = value_ - elif is_definite_feature(feature): + elif field == Field_Definite: tag.definite = value_ - elif is_degree_feature(feature): + elif field == Field_Degree: tag.degree = value_ - elif is_derivation_feature(feature): + elif field == Field_Derivation: tag.derivation = value_ - elif is_echo_feature(feature): + elif field == Field_Echo: tag.echo = value_ - elif is_foreign_feature(feature): + elif field == Field_Foreign: tag.foreign = value_ - elif is_gender_feature(feature): + elif field == Field_Gender: tag.gender = value_ - elif is_hyph_feature(feature): + elif field == Field_Hyph: tag.hyph = value_ - elif is_inf_form_feature(feature): + elif field == Field_InfForm: tag.inf_form = value_ - elif is_mood_feature(feature): + elif field == Field_Mood: tag.mood = value_ - elif is_negative_feature(feature): + elif field == Field_Negative: tag.negative = value_ - elif is_number_feature(feature): + elif field == Field_Number: tag.number = value_ - elif is_name_type_feature(feature): + elif field == Field_NameType: tag.name_type = value_ - elif is_noun_type_feature(feature): + elif field == Field_NounType: tag.noun_type = value_ - elif is_num_form_feature(feature): + elif field == Field_NumForm: tag.num_form = value_ - elif is_num_type_feature(feature): + elif field == Field_NumType: tag.num_type = value_ - elif is_num_value_feature(feature): + elif field == Field_NumValue: tag.num_value = value_ - elif is_part_form_feature(feature): + elif field == Field_PartForm: tag.part_form = value_ - elif is_part_type_feature(feature): + elif field == Field_PartType: tag.part_type = value_ - elif is_person_feature(feature): + elif field == Field_Person: tag.person = value_ - elif is_polite_feature(feature): + elif field == Field_Polite: tag.polite = value_ - elif is_polarity_feature(feature): + elif field == Field_Polarity: tag.polarity = value_ - elif is_poss_feature(feature): + elif field == Field_Poss: tag.poss = value_ - elif is_prefix_feature(feature): + elif field == Field_Prefix: tag.prefix = value_ - elif is_prep_case_feature(feature): + elif field == Field_PrepCase: tag.prep_case = value_ - elif is_pron_type_feature(feature): + elif field == Field_PronType: tag.pron_type = value_ - elif is_punct_side_feature(feature): + elif field == Field_PunctSide: tag.punct_side = value_ - elif is_punct_type_feature(feature): + elif field == Field_PunctType: tag.punct_type = value_ - elif is_reflex_feature(feature): + elif field == Field_Reflex: tag.reflex = value_ - elif is_style_feature(feature): + elif field == Field_Style: tag.style = value_ - elif is_style_variant_feature(feature): + elif field == Field_StyleVariant: tag.style_variant = value_ - elif is_tense_feature(feature): + elif field == Field_Tense: tag.tense = value_ - elif is_typo_feature(feature): + elif field == Field_Typo: tag.typo = value_ - elif is_verb_form_feature(feature): + elif field == Field_VerbForm: tag.verb_form = value_ - elif is_voice_feature(feature): + elif field == Field_Voice: tag.voice = value_ - elif is_verb_type_feature(feature): + elif field == Field_VerbType: tag.verb_type = value_ else: - raise ValueError("Unknown feature: %s (%d)" % (NAMES.get(feature), feature)) - -cdef int is_abbr_feature(univ_morph_t feature) nogil: - return feature >= begin_Abbr and feature <= end_Abbr - -cdef int is_adp_type_feature(univ_morph_t feature) nogil: - return feature >= begin_AdpType and feature <= end_AdpType - -cdef int is_adv_type_feature(univ_morph_t feature) nogil: - return feature >= begin_AdvType and feature <= end_AdvType - -cdef int is_animacy_feature(univ_morph_t feature) nogil: - return feature >= begin_Animacy and feature <= end_Animacy - -cdef int is_aspect_feature(univ_morph_t feature) nogil: - return feature >= begin_Aspect and feature <= end_Aspect - -cdef int is_case_feature(univ_morph_t feature) nogil: - return feature >= begin_Case and feature <= end_Case - -cdef int is_conj_type_feature(univ_morph_t feature) nogil: - return feature >= begin_ConjType and feature <= end_ConjType - -cdef int is_connegative_feature(univ_morph_t feature) nogil: - return feature >= begin_Connegative and feature <= end_Connegative - -cdef int is_definite_feature(univ_morph_t feature) nogil: - return feature >= begin_Definite and feature <= end_Definite - -cdef int is_degree_feature(univ_morph_t feature) nogil: - return feature >= begin_Degree and feature <= end_Degree - -cdef int is_derivation_feature(univ_morph_t feature) nogil: - return feature >= begin_Derivation and feature <= end_Derivation - -cdef int is_echo_feature(univ_morph_t feature) nogil: - return feature >= begin_Echo and feature <= end_Echo - -cdef int is_foreign_feature(univ_morph_t feature) nogil: - return feature >= begin_Foreign and feature <= end_Foreign - -cdef int is_gender_feature(univ_morph_t feature) nogil: - return feature >= begin_Gender and feature <= end_Gender - -cdef int is_hyph_feature(univ_morph_t feature) nogil: - return feature >= begin_Hyph and feature <= end_Hyph - -cdef int is_inf_form_feature(univ_morph_t feature) nogil: - return feature >= begin_InfForm and feature <= end_InfForm - -cdef int is_mood_feature(univ_morph_t feature) nogil: - return feature >= begin_Mood and feature <= end_Mood - -cdef int is_name_type_feature(univ_morph_t feature) nogil: - return feature >= begin_NameType and feature < end_NameType - -cdef int is_negative_feature(univ_morph_t feature) nogil: - return feature >= begin_Negative and feature <= end_Negative - -cdef int is_noun_type_feature(univ_morph_t feature) nogil: - return feature >= begin_NounType and feature <= end_NounType - -cdef int is_number_feature(univ_morph_t feature) nogil: - return feature >= begin_Number and feature <= end_Number - -cdef int is_num_form_feature(univ_morph_t feature) nogil: - return feature >= begin_NumForm and feature <= end_NumForm - -cdef int is_num_type_feature(univ_morph_t feature) nogil: - return feature >= begin_NumType and feature <= end_NumType - -cdef int is_num_value_feature(univ_morph_t feature) nogil: - return feature >= begin_NumValue and feature <= end_NumValue - -cdef int is_part_form_feature(univ_morph_t feature) nogil: - return feature >= begin_PartForm and feature <= end_PartForm - -cdef int is_part_type_feature(univ_morph_t feature) nogil: - return feature >= begin_PartType and feature <= end_PartType - -cdef int is_person_feature(univ_morph_t feature) nogil: - return feature >= begin_Person and feature <= end_Person - -cdef int is_polite_feature(univ_morph_t feature) nogil: - return feature >= begin_Polite and feature <= end_Polite - -cdef int is_polarity_feature(univ_morph_t feature) nogil: - return feature >= begin_Polarity and feature <= end_Polarity - -cdef int is_poss_feature(univ_morph_t feature) nogil: - return feature >= begin_Poss and feature <= end_Poss - -cdef int is_prefix_feature(univ_morph_t feature) nogil: - return feature >= begin_Prefix and feature <= end_Prefix - -cdef int is_prep_case_feature(univ_morph_t feature) nogil: - return feature >= begin_PrepCase and feature <= end_PrepCase - -cdef int is_pron_type_feature(univ_morph_t feature) nogil: - return feature >= begin_PronType and feature <= end_PronType - -cdef int is_punct_side_feature(univ_morph_t feature) nogil: - return feature >= begin_PunctSide and feature <= end_PunctSide - -cdef int is_punct_type_feature(univ_morph_t feature) nogil: - return feature >= begin_PunctType and feature <= end_PunctType - -cdef int is_reflex_feature(univ_morph_t feature) nogil: - return feature >= begin_Reflex and feature <= end_Reflex - -cdef int is_style_feature(univ_morph_t feature) nogil: - return feature >= begin_Style and feature <= end_Style - -cdef int is_style_variant_feature(univ_morph_t feature) nogil: - return feature >= begin_StyleVariant and feature <= end_StyleVariant - -cdef int is_tense_feature(univ_morph_t feature) nogil: - return feature >= begin_Tense and feature <= end_Tense - -cdef int is_typo_feature(univ_morph_t feature) nogil: - return feature >= begin_Typo and feature <= end_Typo - -cdef int is_verb_form_feature(univ_morph_t feature) nogil: - return feature >= begin_VerbForm and feature <= end_VerbForm - -cdef int is_voice_feature(univ_morph_t feature) nogil: - return feature >= begin_Voice and feature <= end_Voice - -cdef int is_verb_type_feature(univ_morph_t feature) nogil: - return feature >= begin_VerbType and feature <= end_VerbType + raise ValueError("Unknown feature: %s (%d)" % (FEATURE_NAMES.get(feature), feature)) FIELDS = { - 'Abbr': 0, - 'AdpType': 1, - 'AdvType': 2, - 'Animacy': 3, - 'Aspect': 4, - 'Case': 5, - 'ConjType': 6, - 'Connegative': 7, - 'Definite': 8, - 'Degree': 9, - 'Derivation': 10, - 'Echo': 11, - 'Foreign': 12, - 'Gender': 13, - 'Hyph': 14, - 'InfForm': 15, - 'Mood': 16, - 'NameType': 17, - 'Negative': 18, - 'Number': 19, - 'NumForm': 20, - 'NumType': 21, - 'NumValue': 22, - 'PartForm': 23, - 'PartType': 24, - 'Person': 25, - 'Polite': 26, - 'Polarity': 27, - 'Poss': 28, - 'Prefix': 29, - 'PrepCase': 30, - 'PronType': 31, - 'PunctSide': 32, - 'PunctType': 33, - 'Reflex': 34, - 'Style': 35, - 'StyleVariant': 36, - 'Tense': 37, - 'Typo': 38, - 'VerbForm': 39, - 'Voice': 40, - 'VerbType': 41 + 'Abbr': Field_Abbr, + 'AdpType': Field_AdpType, + 'AdvType': Field_AdvType, + 'Animacy': Field_Animacy, + 'Aspect': Field_Aspect, + 'Case': Field_Case, + 'ConjType': Field_ConjType, + 'Connegative': Field_Connegative, + 'Definite': Field_Definite, + 'Degree': Field_Degree, + 'Derivation': Field_Derivation, + 'Echo': Field_Echo, + 'Foreign': Field_Foreign, + 'Gender': Field_Gender, + 'Hyph': Field_Hyph, + 'InfForm': Field_InfForm, + 'Mood': Field_Mood, + 'NameType': Field_NameType, + 'Negative': Field_Negative, + 'NounType': Field_NounType, + 'Number': Field_Number, + 'NumForm': Field_NumForm, + 'NumType': Field_NumType, + 'NumValue': Field_NumValue, + 'PartForm': Field_PartForm, + 'PartType': Field_PartType, + 'Person': Field_Person, + 'Polite': Field_Polite, + 'Polarity': Field_Polarity, + 'Poss': Field_Poss, + 'Prefix': Field_Prefix, + 'PrepCase': Field_PrepCase, + 'PronType': Field_PronType, + 'PunctSide': Field_PunctSide, + 'PunctType': Field_PunctType, + 'Reflex': Field_Reflex, + 'Style': Field_Style, + 'StyleVariant': Field_StyleVariant, + 'Tense': Field_Tense, + 'Typo': Field_Typo, + 'VerbForm': Field_VerbForm, + 'Voice': Field_Voice, + 'VerbType': Field_VerbType } -IDS = { - "begin_Abbr": begin_Abbr, - "Abbr_yes": Abbr_yes , - "end_Abbr": end_Abbr, - "begin_AdpType": begin_AdpType, - "AdpType_circ": AdpType_circ, - "AdpType_comprep": AdpType_comprep, - "AdpType_prep ": AdpType_prep , - "AdpType_post": AdpType_post, - "AdpType_voc": AdpType_voc, - "end_AdpType": end_AdpType, - "begin_AdvType": begin_AdvType, - "AdvType_adadj": AdvType_adadj, - "AdvType_cau": AdvType_cau, - "AdvType_deg": AdvType_deg, - "AdvType_ex": AdvType_ex, - "AdvType_loc": AdvType_loc, - "AdvType_man": AdvType_man, - "AdvType_mod": AdvType_mod, - "AdvType_sta": AdvType_sta, - "AdvType_tim": AdvType_tim, - "end_AdvType": end_AdvType, - "begin_Animacy": begin_Animacy, - "Animacy_anim": Animacy_anim, - "Animacy_hum": Animacy_hum, - "Animacy_inan": Animacy_inan, - "Animacy_nhum": Animacy_nhum, - "end_Animacy": end_Animacy, - "begin_Aspect": begin_Aspect, - "Aspect_freq": Aspect_freq, - "Aspect_imp": Aspect_imp, - "Aspect_mod": Aspect_mod, - "Aspect_none": Aspect_none, - "Aspect_perf": Aspect_perf, - "end_Aspect": end_Aspect, - "begin_Case": begin_Case, - "Case_abe": Case_abe, - "Case_abl": Case_abl, - "Case_abs": Case_abs, - "Case_acc": Case_acc, - "Case_ade": Case_ade, - "Case_all": Case_all, - "Case_cau": Case_cau, - "Case_com": Case_com, - "Case_dat": Case_dat, - "Case_del": Case_del, - "Case_dis": Case_dis, - "Case_ela": Case_ela, - "Case_ess": Case_ess, - "Case_gen": Case_gen, - "Case_ill": Case_ill, - "Case_ine": Case_ine, - "Case_ins": Case_ins, - "Case_loc": Case_loc, - "Case_lat": Case_lat, - "Case_nom": Case_nom, - "Case_par": Case_par, - "Case_sub": Case_sub, - "Case_sup": Case_sup, - "Case_tem": Case_tem, - "Case_ter": Case_ter, - "Case_tra": Case_tra, - "Case_voc": Case_voc, - "end_Case": end_Case, - "begin_ConjType": begin_ConjType, - "ConjType_comp ": ConjType_comp , - "ConjType_oper": ConjType_oper, - "end_ConjType": end_ConjType, - "begin_Connegative": begin_Connegative, - "Connegative_yes": Connegative_yes, - "end_Connegative": end_Connegative, - "begin_Definite": begin_Definite, - "Definite_cons": Definite_cons, - "Definite_def": Definite_def, - "Definite_ind": Definite_ind, - "Definite_red": Definite_red, - "Definite_two": Definite_two, - "end_Definite": end_Definite, - "begin_Degree": begin_Degree, - "Degree_abs": Degree_abs, - "Degree_cmp": Degree_cmp, - "Degree_comp": Degree_comp, - "Degree_none": Degree_none, - "Degree_pos": Degree_pos, - "Degree_sup": Degree_sup, - "Degree_com": Degree_com, - "Degree_dim": Degree_dim, - "end_Degree": end_Degree, - "begin_Derivation": begin_Derivation, - "Derivation_minen": Derivation_minen, - "Derivation_sti": Derivation_sti, - "Derivation_inen": Derivation_inen, - "Derivation_lainen": Derivation_lainen, - "Derivation_ja": Derivation_ja, - "Derivation_ton": Derivation_ton, - "Derivation_vs": Derivation_vs, - "Derivation_ttain": Derivation_ttain, - "Derivation_ttaa": Derivation_ttaa, - "end_Derivation": end_Derivation, - "begin_Echo": begin_Echo, - "Echo_rdp": Echo_rdp, - "Echo_ech": Echo_ech, - "end_Echo": end_Echo, - "begin_Foreign": begin_Foreign, - "Foreign_foreign": Foreign_foreign, - "Foreign_fscript": Foreign_fscript, - "Foreign_tscript": Foreign_tscript, - "Foreign_yes": Foreign_yes, - "end_Foreign": end_Foreign, - "begin_Gender": begin_Gender, - "Gender_com": Gender_com, - "Gender_fem": Gender_fem, - "Gender_masc": Gender_masc, - "Gender_neut": Gender_neut, - "Gender_dat_masc": Gender_dat_masc, - "Gender_dat_fem": Gender_dat_fem, - "Gender_erg_masc": Gender_erg_masc, - "Gender_erg_fem": Gender_erg_fem, - "Gender_psor_masc": Gender_psor_masc, - "Gender_psor_fem": Gender_psor_fem, - "Gender_psor_neut": Gender_psor_neut, - "end_Gender": end_Gender, - "begin_Hyph": begin_Hyph, - "Hyph_yes": Hyph_yes, - "end_Hyph": end_Hyph, - "begin_InfForm": begin_InfForm, - "InfForm_one": InfForm_one, - "InfForm_two": InfForm_two, - "InfForm_three": InfForm_three, - "end_InfForm": end_InfForm, - "begin_Mood": begin_Mood, - "Mood_cnd": Mood_cnd, - "Mood_imp": Mood_imp, - "Mood_ind": Mood_ind, - "Mood_n": Mood_n, - "Mood_pot": Mood_pot, - "Mood_sub": Mood_sub, - "Mood_opt": Mood_opt, - "end_Mood": end_Mood, - "begin_NameType": begin_NameType, - "NameType_geo": NameType_geo, - "NameType_prs": NameType_prs, - "NameType_giv": NameType_giv, - "NameType_sur": NameType_sur, - "NameType_nat": NameType_nat, - "NameType_com": NameType_com, - "NameType_pro": NameType_pro, - "NameType_oth": NameType_oth, - "end_NameType": end_NameType, - "begin_Negative": begin_Negative, - "Negative_neg": Negative_neg, - "Negative_pos": Negative_pos, - "Negative_yes": Negative_yes, - "end_Negative": end_Negative, - "begin_NounType": begin_NounType, - "NounType_com": NounType_com, - "NounType_prop": NounType_prop, - "NounType_class": NounType_class, - "end_NounType": end_NounType, - "begin_Number": begin_Number, - "Number_com": Number_com, - "Number_dual": Number_dual, - "Number_none": Number_none, - "Number_plur": Number_plur, - "Number_sing": Number_sing, - "Number_ptan": Number_ptan, - "Number_count": Number_count, - "Number_abs_sing": Number_abs_sing, - "Number_abs_plur": Number_abs_plur, - "Number_dat_sing": Number_dat_sing, - "Number_dat_plur": Number_dat_plur, - "Number_erg_sing": Number_erg_sing, - "Number_erg_plur": Number_erg_plur, - "Number_psee_sing": Number_psee_sing, - "Number_psee_plur": Number_psee_plur, - "Number_psor_sing": Number_psor_sing, - "Number_psor_plur": Number_psor_plur, - "end_Number": end_Number, - "begin_NumForm": begin_NumForm, - "NumForm_digit": NumForm_digit, - "NumForm_roman": NumForm_roman, - "NumForm_word": NumForm_word, - "end_NumForm": end_NumForm, - "begin_NumType": begin_NumType, - "NumType_card": NumType_card, - "NumType_dist": NumType_dist, - "NumType_frac": NumType_frac, - "NumType_gen": NumType_gen, - "NumType_mult": NumType_mult, - "NumType_none": NumType_none, - "NumType_ord": NumType_ord, - "NumType_sets": NumType_sets, - "end_NumType": end_NumType, - "begin_NumValue": begin_NumValue, - "NumValue_one": NumValue_one, - "NumValue_two": NumValue_two, - "NumValue_three": NumValue_three, - "end_NumValue": end_NumValue, - "begin_PartForm": begin_PartForm, - "PartForm_pres": PartForm_pres, - "PartForm_past": PartForm_past, - "PartForm_agt": PartForm_agt, - "PartForm_neg": PartForm_neg, - "end_PartForm": end_PartForm, - "begin_PartType": begin_PartType, - "PartType_mod": PartType_mod, - "PartType_emp": PartType_emp, - "PartType_res": PartType_res, - "PartType_inf": PartType_inf, - "PartType_vbp": PartType_vbp, - "end_PartType": end_PartType, +FEATURES = [ + "Abbr_yes", + "AdpType_circ", + "AdpType_comprep", + "AdpType_prep ", + "AdpType_post", + "AdpType_voc", + "AdvType_adadj," + "AdvType_cau", + "AdvType_deg", + "AdvType_ex", + "AdvType_loc", + "AdvType_man", + "AdvType_mod", + "AdvType_sta", + "AdvType_tim", + "Animacy_anim", + "Animacy_hum", + "Animacy_inan", + "Animacy_nhum", + "Aspect_freq", + "Aspect_imp", + "Aspect_mod", + "Aspect_none", + "Aspect_perf", + "Case_abe", + "Case_abl", + "Case_abs", + "Case_acc", + "Case_ade", + "Case_all", + "Case_cau", + "Case_com", + "Case_dat", + "Case_del", + "Case_dis", + "Case_ela", + "Case_ess", + "Case_gen", + "Case_ill", + "Case_ine", + "Case_ins", + "Case_loc", + "Case_lat", + "Case_nom", + "Case_par", + "Case_sub", + "Case_sup", + "Case_tem", + "Case_ter", + "Case_tra", + "Case_voc", + "ConjType_comp", + "ConjType_oper", + "Connegative_yes", + "Definite_cons", + "Definite_def", + "Definite_ind", + "Definite_red", + "Definite_two", + "Degree_abs", + "Degree_cmp", + "Degree_comp", + "Degree_none", + "Degree_pos", + "Degree_sup", + "Degree_com", + "Degree_dim", + "Derivation_minen", + "Derivation_sti", + "Derivation_inen", + "Derivation_lainen", + "Derivation_ja", + "Derivation_ton", + "Derivation_vs", + "Derivation_ttain", + "Derivation_ttaa", + "Echo_rdp", + "Echo_ech", + "Foreign_foreign", + "Foreign_fscript", + "Foreign_tscript", + "Foreign_yes", + "Gender_com", + "Gender_fem", + "Gender_masc", + "Gender_neut", + "Gender_dat_masc", + "Gender_dat_fem", + "Gender_erg_masc", + "Gender_erg_fem", + "Gender_psor_masc", + "Gender_psor_fem", + "Gender_psor_neut", + "Hyph_yes", + "InfForm_one", + "InfForm_two", + "InfForm_three", + "Mood_cnd", + "Mood_imp", + "Mood_ind", + "Mood_n", + "Mood_pot", + "Mood_sub", + "Mood_opt", + "NameType_geo", + "NameType_prs", + "NameType_giv", + "NameType_sur", + "NameType_nat", + "NameType_com", + "NameType_pro", + "NameType_oth", + "Negative_neg", + "Negative_pos", + "Negative_yes", + "NounType_com", + "NounType_prop", + "NounType_class", + "Number_com", + "Number_dual", + "Number_none", + "Number_plur", + "Number_sing", + "Number_ptan", + "Number_count", + "Number_abs_sing", + "Number_abs_plur", + "Number_dat_sing", + "Number_dat_plur", + "Number_erg_sing", + "Number_erg_plur", + "Number_psee_sing", + "Number_psee_plur", + "Number_psor_sing", + "Number_psor_plur", + "NumForm_digit", + "NumForm_roman", + "NumForm_word", + "NumType_card", + "NumType_dist", + "NumType_frac", + "NumType_gen", + "NumType_mult", + "NumType_none", + "NumType_ord", + "NumType_sets", + "NumValue_one", + "NumValue_two", + "NumValue_three", + "PartForm_pres", + "PartForm_past", + "PartForm_agt", + "PartForm_neg", + "PartType_mod", + "PartType_emp", + "PartType_res", + "PartType_inf", + "PartType_vbp", + "Person_one", + "Person_two", + "Person_three", + "Person_none", + "Person_abs_one", + "Person_abs_two", + "Person_abs_three", + "Person_dat_one", + "Person_dat_two", + "Person_dat_three", + "Person_erg_one", + "Person_erg_two", + "Person_erg_three", + "Person_psor_one", + "Person_psor_two", + "Person_psor_three", + "Polarity_neg", + "Polarity_pos", + "Polite_inf", + "Polite_pol", + "Polite_abs_inf", + "Polite_abs_pol", + "Polite_erg_inf", + "Polite_erg_pol", + "Polite_dat_inf", + "Polite_dat_pol", + "Poss_yes", + "Prefix_yes", + "PrepCase_npr", + "PrepCase_pre", + "PronType_advPart", + "PronType_art", + "PronType_default", + "PronType_dem", + "PronType_ind", + "PronType_int", + "PronType_neg", + "PronType_prs", + "PronType_rcp", + "PronType_rel", + "PronType_tot", + "PronType_clit", + "PronType_exc", + "PunctSide_ini", + "PunctSide_fin", + "PunctType_peri", + "PunctType_qest", + "PunctType_excl", + "PunctType_quot", + "PunctType_brck", + "PunctType_comm", + "PunctType_colo", + "PunctType_semi", + "PunctType_dash", + "Reflex_yes", + "Style_arch", + "Style_rare", + "Style_poet", + "Style_norm", + "Style_coll", + "Style_vrnc", + "Style_sing", + "Style_expr", + "Style_derg", + "Style_vulg", + "Style_yes", + "StyleVariant_styleShort", + "StyleVariant_styleBound", + "Tense_fut", + "Tense_imp", + "Tense_past", + "Tense_pres", + "Typo_yes", + "VerbForm_fin", + "VerbForm_ger", + "VerbForm_inf", + "VerbForm_none", + "VerbForm_part", + "VerbForm_partFut", + "VerbForm_partPast", + "VerbForm_partPres", + "VerbForm_sup", + "VerbForm_trans", + "VerbForm_conv", + "VerbForm_gdv", + "VerbType_aux", + "VerbType_cop", + "VerbType_mod", + "VerbType_light", + "Voice_act", + "Voice_cau", + "Voice_pass", + "Voice_mid", + "Voice_int", +] - "begin_Person": begin_Person, - "Person_one": Person_one, - "Person_two": Person_two, - "Person_three": Person_three, - "Person_none": Person_none, - "Person_abs_one": Person_abs_one, - "Person_abs_two": Person_abs_two, - "Person_abs_three": Person_abs_three, - "Person_dat_one": Person_dat_one, - "Person_dat_two": Person_dat_two, - "Person_dat_three": Person_dat_three, - "Person_erg_one": Person_erg_one, - "Person_erg_two": Person_erg_two, - "Person_erg_three": Person_erg_three, - "Person_psor_one": Person_psor_one, - "Person_psor_two": Person_psor_two, - "Person_psor_three": Person_psor_three, - "end_Person": end_Person, - "begin_Polarity": begin_Polarity, - "Polarity_neg": Polarity_neg, - "Polarity_pos": Polarity_pos, - "end_Polarity": end_Polarity, - "begin_Polite": begin_Polite, - "Polite_inf": Polite_inf, - "Polite_pol": Polite_pol, - "Polite_abs_inf": Polite_abs_inf, - "Polite_abs_pol": Polite_abs_pol, - "Polite_erg_inf": Polite_erg_inf, - "Polite_erg_pol": Polite_erg_pol, - "Polite_dat_inf": Polite_dat_inf, - "Polite_dat_pol": Polite_dat_pol, - "end_Polite": end_Polite, - "begin_Poss": begin_Poss, - "Poss_yes": Poss_yes, - "end_Poss": end_Poss, - "begin_Prefix": begin_Prefix, - "Prefix_yes": Prefix_yes, - "end_Prefix": end_Prefix, - "begin_PrepCase": begin_PrepCase, - "PrepCase_npr": PrepCase_npr, - "PrepCase_pre": PrepCase_pre, - "end_PrepCase": end_PrepCase, - "begin_PronType": begin_PronType, - "PronType_advPart": PronType_advPart, - "PronType_art": PronType_art, - "PronType_default": PronType_default, - "PronType_dem": PronType_dem, - "PronType_ind": PronType_ind, - "PronType_int": PronType_int, - "PronType_neg": PronType_neg, - "PronType_prs": PronType_prs, - "PronType_rcp": PronType_rcp, - "PronType_rel": PronType_rel, - "PronType_tot": PronType_tot, - "PronType_clit": PronType_clit, - "PronType_exc": PronType_exc, - "end_PronType": end_PronType, - "begin_PunctSide": begin_PunctSide, - "PunctSide_ini": PunctSide_ini, - "PunctSide_fin": PunctSide_fin, - "end_PunctSide": end_PunctSide, - "begin_PunctType": begin_PunctType, - "PunctType_peri": PunctType_peri, - "PunctType_qest": PunctType_qest, - "PunctType_excl": PunctType_excl, - "PunctType_quot": PunctType_quot, - "PunctType_brck": PunctType_brck, - "PunctType_comm": PunctType_comm, - "PunctType_colo": PunctType_colo, - "PunctType_semi": PunctType_semi, - "PunctType_dash": PunctType_dash, - "end_PunctType": end_PunctType, - "begin_Reflex": begin_Reflex, - "Reflex_yes": Reflex_yes, - "end_Reflex": end_Reflex, - "begin_Style": begin_Style, - "Style_arch": Style_arch, - "Style_rare": Style_rare, - "Style_poet": Style_poet, - "Style_norm": Style_norm, - "Style_coll": Style_coll, - "Style_vrnc": Style_vrnc, - "Style_sing": Style_sing, - "Style_expr": Style_expr, - "Style_derg": Style_derg, - "Style_vulg": Style_vulg, - "Style_yes": Style_yes, - "end_Style": end_Style, - "begin_StyleVariant": begin_StyleVariant, - "StyleVariant_styleShort": StyleVariant_styleShort, - "StyleVariant_styleBound": StyleVariant_styleBound, - "end_StyleVariant": end_StyleVariant, - "begin_Tense": begin_Tense, - "Tense_fut": Tense_fut, - "Tense_imp": Tense_imp, - "Tense_past": Tense_past, - "Tense_pres": Tense_pres, - "end_Tense": end_Tense, - "begin_Typo": begin_Typo, - "Typo_yes": Typo_yes, - "end_Typo": end_Typo, - "begin_VerbForm": begin_VerbForm, - "VerbForm_fin": VerbForm_fin, - "VerbForm_ger": VerbForm_ger, - "VerbForm_inf": VerbForm_inf, - "VerbForm_none": VerbForm_none, - "VerbForm_part": VerbForm_part, - "VerbForm_partFut": VerbForm_partFut, - "VerbForm_partPast": VerbForm_partPast, - "VerbForm_partPres": VerbForm_partPres, - "VerbForm_sup": VerbForm_sup, - "VerbForm_trans": VerbForm_trans, - "VerbForm_conv": VerbForm_conv, - "VerbForm_gdv": VerbForm_gdv, - "end_VerbForm": end_VerbForm, - "begin_VerbType": begin_VerbType, - "VerbType_aux": VerbType_aux, - "VerbType_cop": VerbType_cop, - "VerbType_mod": VerbType_mod, - "VerbType_light": VerbType_light, - "end_VerbType": end_VerbType, - "begin_Voice": begin_Voice, - "Voice_act": Voice_act, - "Voice_cau": Voice_cau, - "Voice_pass": Voice_pass, - "Voice_mid": Voice_mid, - "Voice_int": Voice_int, - "end_Voice": end_Voice, -} +FEATURE_NAMES = {get_string_id(name): name for name in FEATURES} +FEATURE_FIELDS = {feature: FIELDS[feature.split('_', 1)[0]] for feature in FEATURES} +for feat_id, name in FEATURE_NAMES.items(): + FEATURE_FIELDS[feat_id] = FEATURE_FIELDS[name] -FIELD_SIZES = [get_field_size(field) for field in FIELDS] - -NAMES = {value: key for key, value in IDS.items()} -# Unfortunate hack here, to work around problem with long cpdef enum -# (which is generating an enormous amount of C++ in Cython 0.24+) -# We keep the enum cdef, and just make sure the names are available to Python -locals().update(IDS) +FIELD_SIZES = Counter(FEATURE_FIELDS.values()) +FEATURE_OFFSETS = {} +FIELD_OFFSETS = {} +_seen_fields = Counter() +for i, feature in enumerate(FEATURES): + field = FEATURE_FIELDS[feature] + FEATURE_OFFSETS[feature] = _seen_fields[field] + if _seen_fields == 0: + FIELD_OFFSETS[field] = i + _seen_fields[field] += 1 diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index 820567e71..9f25ba357 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -16,7 +16,7 @@ from ..compat import basestring_ from ..tokens.doc cimport Doc from ..vocab cimport Vocab from ..morphology cimport Morphology -from ..morphology import parse_feature, IDS, FIELDS, FIELD_SIZES, NAMES +from ..morphology import get_field_size, get_field_offset, parse_feature, FIELDS class Morphologizer(Pipe): @@ -27,7 +27,7 @@ class Morphologizer(Pipe): if cfg.get('pretrained_dims') and not cfg.get('pretrained_vectors'): raise ValueError(TempErrors.T008) if attr_nums is None: - attr_nums = list(FIELD_SIZES) + attr_nums = [get_field_size(name) for name in FIELDS] return build_morphologizer_model(attr_nums, **cfg) def __init__(self, vocab, model=True, **cfg): @@ -76,7 +76,7 @@ class Morphologizer(Pipe): cdef Doc doc cdef Vocab vocab = self.vocab field_names = list(FIELDS) - offsets = [IDS['begin_%s' % field] for field in field_names] + offsets = [get_field_offset(field) for field in field_names] for i, doc in enumerate(docs): doc_scores = batch_scores[i] doc_guesses = scores_to_guesses(doc_scores, self.model.softmax.out_sizes) diff --git a/spacy/structs.pxd b/spacy/structs.pxd index 7452123c0..a4daa9b94 100644 --- a/spacy/structs.pxd +++ b/spacy/structs.pxd @@ -2,7 +2,6 @@ from libc.stdint cimport uint8_t, uint32_t, int32_t, uint64_t from .typedefs cimport flags_t, attr_t, hash_t from .parts_of_speech cimport univ_pos_t -from .morphology cimport univ_morph_t cdef struct LexemeC: diff --git a/spacy/tests/doc/test_retokenize_merge.py b/spacy/tests/doc/test_retokenize_merge.py index 4d4a70e30..b62e69f6c 100644 --- a/spacy/tests/doc/test_retokenize_merge.py +++ b/spacy/tests/doc/test_retokenize_merge.py @@ -69,7 +69,6 @@ def test_doc_retokenize_retokenizer_attrs(en_tokenizer): assert doc[4].ent_type_ == "ORG" -@pytest.mark.xfail def test_doc_retokenize_lex_attrs(en_tokenizer): """Test that lexical attributes can be changed (see #2390).""" doc = en_tokenizer("WKRO played beach boys songs") diff --git a/spacy/tests/morphology/test_morph_features.py b/spacy/tests/morphology/test_morph_features.py index 32cc665af..dcb0b32ff 100644 --- a/spacy/tests/morphology/test_morph_features.py +++ b/spacy/tests/morphology/test_morph_features.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import pytest from ...morphology import Morphology -from ...strings import StringStore +from ...strings import StringStore, get_string_id from ...lemmatizer import Lemmatizer from ...morphology import * @@ -17,14 +17,14 @@ def test_add_morphology_with_string_names(morphology): morphology.add({"Case_gen", "Number_sing"}) def test_add_morphology_with_int_ids(morphology): - morphology.add({Case_gen, Number_sing}) + morphology.add({get_string_id("Case_gen"), get_string_id("Number_sing")}) def test_add_morphology_with_mix_strings_and_ints(morphology): - morphology.add({PunctSide_ini, 'VerbType_aux'}) + morphology.add({get_string_id("PunctSide_ini"), 'VerbType_aux'}) def test_morphology_tags_hash_distinctly(morphology): - tag1 = morphology.add({PunctSide_ini, 'VerbType_aux'}) + tag1 = morphology.add({"PunctSide_ini", 'VerbType_aux'}) tag2 = morphology.add({"Case_gen", 'Number_sing'}) assert tag1 != tag2 diff --git a/spacy/tokens/token.pyx b/spacy/tokens/token.pyx index df596ceb5..1b60a3271 100644 --- a/spacy/tokens/token.pyx +++ b/spacy/tokens/token.pyx @@ -22,6 +22,7 @@ from ..compat import is_config from ..errors import Errors, Warnings, user_warning, models_warning from .. import util from .underscore import Underscore, get_ext_args +from .morphanalysis cimport MorphAnalysis cdef class Token: @@ -176,6 +177,10 @@ cdef class Token: def __get__(self): return self.c.morph + property morph: + def __get__(self): + return MorphAnalysis.from_id(self.vocab, self.c.morph) + property lex_id: """RETURNS (int): Sequential ID of the token's lexical type.""" def __get__(self): From 0ad09b16ad7c5686bc22b02603c7bbe126947d81 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 17:24:57 +0100 Subject: [PATCH 054/287] Add header for morphanalysis --- spacy/tokens/morphanalysis.pxd | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 spacy/tokens/morphanalysis.pxd diff --git a/spacy/tokens/morphanalysis.pxd b/spacy/tokens/morphanalysis.pxd new file mode 100644 index 000000000..22844454a --- /dev/null +++ b/spacy/tokens/morphanalysis.pxd @@ -0,0 +1,9 @@ +from ..vocab cimport Vocab +from ..typedefs cimport hash_t +from ..structs cimport MorphAnalysisC + + +cdef class MorphAnalysis: + cdef readonly Vocab vocab + cdef hash_t key + cdef MorphAnalysisC c From e585b5045832d38ffab00ed4b5a39c7d8f2b6e9f Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 18:32:09 +0100 Subject: [PATCH 055/287] Fix features in English tag map --- spacy/lang/en/tag_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/lang/en/tag_map.py b/spacy/lang/en/tag_map.py index eda4fa1c2..151b42c0c 100644 --- a/spacy/lang/en/tag_map.py +++ b/spacy/lang/en/tag_map.py @@ -17,7 +17,7 @@ TAG_MAP = { "$": {POS: SYM, "Other": {"SymType": "currency"}}, "#": {POS: SYM, "Other": {"SymType": "numbersign"}}, "AFX": {POS: ADJ, "Hyph": "yes"}, - "CC": {POS: CCONJ, "ConjType": "coor"}, + "CC": {POS: CCONJ, "ConjType": "comp"}, "CD": {POS: NUM, "NumType": "card"}, "DT": {POS: DET}, "EX": {POS: ADV, "AdvType": "ex"}, @@ -56,7 +56,7 @@ TAG_MAP = { "VerbForm": "fin", "Tense": "pres", "Number": "sing", - "Person": 3, + "Person": "three", }, "WDT": {POS: ADJ, "PronType": "int,rel"}, "WP": {POS: NOUN, "PronType": "int,rel"}, From 2669190b858af297ffeb28273e8bbe5eb5516194 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 18:32:36 +0100 Subject: [PATCH 056/287] Normalize props for morph exceptions --- spacy/morphology.pyx | 101 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 1157c2502..4e3ec1cf8 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -15,6 +15,7 @@ from .parts_of_speech import IDS as POS_IDS from .lexeme cimport Lexeme from .errors import Errors + cdef enum univ_field_t: Field_Abbr Field_AdpType @@ -138,6 +139,7 @@ cdef class Morphology: self.exc = {} if exc is not None: for (tag, orth), attrs in exc.items(): + attrs = _normalize_props(attrs) self.add_special_case( self.strings.as_string(tag), self.strings.as_string(orth), attrs) @@ -149,11 +151,13 @@ cdef class Morphology: """Insert a morphological analysis in the morphology table, if not already present. Returns the hash of the new analysis. """ + for f in features: + self.strings.add(f) features = intify_features(features) cdef attr_t feature for feature in features: if feature != 0 and feature not in FEATURE_NAMES: - raise KeyError("Unknown feature: %d" % feature) + raise KeyError("Unknown feature: %s" % self.strings[feature]) cdef MorphAnalysisC tag tag = create_rich_tag(features) cdef hash_t key = self.insert(tag) @@ -263,8 +267,7 @@ cdef class Morphology: token.lemma = lemma token.pos = pos token.tag = self.strings[tag_str] - #token.morph = self.add(features) - token.morph = 0 + token.morph = self.add(features) if (self.tag_names[tag_id], token.lex.orth) in self.exc: self._assign_tag_from_exceptions(token, tag_id) @@ -412,9 +415,101 @@ cdef tag_to_json(MorphAnalysisC tag): features.append(FEATURE_NAMES[tag.verb_type]) return features + cdef MorphAnalysisC tag_from_json(json_tag): cdef MorphAnalysisC tag return tag + + +cdef int check_feature(const MorphAnalysisC* tag, attr_t feature) nogil: + if tag.abbr == feature: + return 1 + elif tag.adp_type == feature: + return 1 + elif tag.adv_type == feature: + return 1 + elif tag.animacy == feature: + return 1 + elif tag.aspect == feature: + return 1 + elif tag.case == feature: + return 1 + elif tag.conj_type == feature: + return 1 + elif tag.connegative == feature: + return 1 + elif tag.definite == feature: + return 1 + elif tag.degree == feature: + return 1 + elif tag.derivation == feature: + return 1 + elif tag.echo == feature: + return 1 + elif tag.foreign == feature: + return 1 + elif tag.gender == feature: + return 1 + elif tag.hyph == feature: + return 1 + elif tag.inf_form == feature: + return 1 + elif tag.mood == feature: + return 1 + elif tag.negative == feature: + return 1 + elif tag.number == feature: + return 1 + elif tag.name_type == feature: + return 1 + elif tag.noun_type == feature: + return 1 + elif tag.num_form == feature: + return 1 + elif tag.num_type == feature: + return 1 + elif tag.num_value == feature: + return 1 + elif tag.part_form == feature: + return 1 + elif tag.part_type == feature: + return 1 + elif tag.person == feature: + return 1 + elif tag.polite == feature: + return 1 + elif tag.polarity == feature: + return 1 + elif tag.poss == feature: + return 1 + elif tag.prefix == feature: + return 1 + elif tag.prep_case == feature: + return 1 + elif tag.pron_type == feature: + return 1 + elif tag.punct_side == feature: + return 1 + elif tag.punct_type == feature: + return 1 + elif tag.reflex == feature: + return 1 + elif tag.style == feature: + return 1 + elif tag.style_variant == feature: + return 1 + elif tag.tense == feature: + return 1 + elif tag.typo == feature: + return 1 + elif tag.verb_form == feature: + return 1 + elif tag.voice == feature: + return 1 + elif tag.verb_type == feature: + return 1 + else: + return 0 cdef int set_feature(MorphAnalysisC* tag, univ_field_t field, attr_t feature, int value) except -1: From 357066ee2f40d00369b53e793f2e36c4b5df5041 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 18:32:51 +0100 Subject: [PATCH 057/287] Work on morphanalysis class --- spacy/tokens/morphanalysis.pyx | 140 ++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 62 deletions(-) diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx index 01ecf458b..11e65c19f 100644 --- a/spacy/tokens/morphanalysis.pyx +++ b/spacy/tokens/morphanalysis.pyx @@ -1,5 +1,10 @@ +from libc.string cimport memset + from ..vocab cimport Vocab -from ..typedefs cimport hash_t +from ..typedefs cimport hash_t, attr_t +from ..morphology cimport check_feature, tag_to_json + +from ..strings import get_string_id cdef class MorphAnalysis: @@ -8,223 +13,234 @@ cdef class MorphAnalysis: self.vocab = vocab self.key = self.vocab.morphology.add(features) analysis = self.vocab.morphology.tags.get(self.key) - self.c = analysis[0] + if analysis is not NULL: + self.c = analysis[0] + else: + memset(&self.c, 0, sizeof(self.c)) @classmethod - def from_id(self, Vocab vocab, hash_t key): - pass + def from_id(cls, Vocab vocab, hash_t key): + cdef MorphAnalysis morph = MorphAnalysis.__new__(MorphAnalysis, vocab) + morph.key = key + analysis = vocab.morphology.tags.get(key) + if analysis is not NULL: + morph.c = analysis[0] + else: + memset(&morph.c, 0, sizeof(morph.c)) + return morph def __contains__(self, feature): - pass + cdef attr_t feat_id = get_string_id(feature) + return check_feature(&self.c, feat_id) def __iter__(self): - pass + raise NotImplementedError def __len__(self): - pass + raise NotImplementedError def __str__(self): - pass + raise NotImplementedError def __repr__(self): - pass + raise NotImplementedError def __hash__(self): - pass + raise NotImplementedError - def get(self, name): - pass + def get(self, field): + raise NotImplementedError def to_json(self): - pass + return tag_to_json(self.c) @property def is_base_form(self): - pass + raise NotImplementedError @property def pos(self): - pass + return self.c.pos @property def pos_(self): - pass + return self.vocab.strings[self.c.pos] - @property - def id(self): - pass + property id: + def __get__(self): + return self.key property abbr: def __get__(self): - pass + return self.c.abbr property adp_type: def __get__(self): - pass + return self.c.adp_type property adv_type: def __get__(self): - pass + return self.c.adv_type property animacy: def __get__(self): - pass + return self.c.animacy property aspect: def __get__(self): - pass + return self.c.aspect property case: def __get__(self): - pass + return self.c.case property conj_type: def __get__(self): - pass + return self.c.conj_type property connegative: def __get__(self): - pass + return self.c.connegative property definite: def __get__(self): - pass + return self.c.definite property degree: def __get__(self): - pass + return self.c.degree property derivation: def __get__(self): - pass + return self.c.derivation property echo: def __get__(self): - pass + return self.c.echo property foreign: def __get__(self): - pass + return self.c.foreign property gender: def __get__(self): - pass + return self.c.gender property hyph: def __get__(self): - pass + return self.c.hyph property inf_form: def __get__(self): - pass + return self.c.inf_form property mood: def __get__(self): - pass + return self.c.mood property name_type: def __get__(self): - pass + return self.c.name_type property negative: def __get__(self): - pass + return self.c.negative property noun_type: def __get__(self): - pass + return self.c.noun_type property number: def __get__(self): - pass + return self.c.number property num_form: def __get__(self): - pass + return self.c.num_form property num_type: def __get__(self): - pass + return self.c.num_type property num_value: def __get__(self): - pass + return self.c.num_value property part_form: def __get__(self): - pass + return self.c.part_form property part_type: def __get__(self): - pass + return self.c.part_type property person: def __get__(self): - pass + return self.c.person property polite: def __get__(self): - pass + return self.c.polite property polarity: def __get__(self): - pass + return self.c.polarity property poss: def __get__(self): - pass + return self.c.poss property prefix: def __get__(self): - pass + return self.c.prefix property prep_case: def __get__(self): - pass + return self.c.prep_case property pron_type: def __get__(self): - pass + return self.c.pron_type property punct_side: def __get__(self): - pass + return self.c.punct_side property punct_type: def __get__(self): - pass + return self.c.punct_type property reflex: def __get__(self): - pass + return self.c.reflex property style: def __get__(self): - pass + return self.c.style property style_variant: def __get__(self): - pass + return self.c.style_variant property tense: def __get__(self): - pass + return self.c.tense property typo: def __get__(self): - pass + return self.c.typo property verb_form: def __get__(self): - pass + return self.c.verb_form property voice: def __get__(self): - pass + return self.c.voice property verb_type: def __get__(self): - pass + return self.c.verb_type property abbr_: def __get__(self): From c1888b05d2c57430c5cce9a197f408cc3c250b33 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 18:33:06 +0100 Subject: [PATCH 058/287] Export helper functions for morphology --- spacy/morphology.pxd | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index a057e8ed8..30c29c1c7 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -31,3 +31,8 @@ cdef class Morphology: cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1 cdef int _assign_tag_from_exceptions(self, TokenC* token, int tag_id) except -1 + + +cdef int check_feature(const MorphAnalysisC* tag, attr_t feature) nogil + +cdef tag_to_json(MorphAnalysisC tag) From 1a10bf29bcafa8676a84c8178a16c8db87e8a6fb Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 18:33:17 +0100 Subject: [PATCH 059/287] Remove morph_key from token api --- spacy/tokens/token.pyx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spacy/tokens/token.pyx b/spacy/tokens/token.pyx index 1b60a3271..7249a2b60 100644 --- a/spacy/tokens/token.pyx +++ b/spacy/tokens/token.pyx @@ -173,10 +173,6 @@ cdef class Token: return (numpy.dot(self.vector, other.vector) / (self.vector_norm * other.vector_norm)) - property morph_key: - def __get__(self): - return self.c.morph - property morph: def __get__(self): return MorphAnalysis.from_id(self.vocab, self.c.morph) From 3a667833d1a37ffa826ebb3ca76b057c1ce712e3 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 21:57:43 +0100 Subject: [PATCH 060/287] Fix morphological features in de tag_map --- spacy/lang/de/tag_map.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spacy/lang/de/tag_map.py b/spacy/lang/de/tag_map.py index 3bb6247c4..394478145 100644 --- a/spacy/lang/de/tag_map.py +++ b/spacy/lang/de/tag_map.py @@ -10,7 +10,7 @@ TAG_MAP = { "$,": {POS: PUNCT, "PunctType": "comm"}, "$.": {POS: PUNCT, "PunctType": "peri"}, "ADJA": {POS: ADJ}, - "ADJD": {POS: ADJ, "Variant": "short"}, + "ADJD": {POS: ADJ}, "ADV": {POS: ADV}, "APPO": {POS: ADP, "AdpType": "post"}, "APPR": {POS: ADP, "AdpType": "prep"}, @@ -32,7 +32,7 @@ TAG_MAP = { "PDAT": {POS: DET, "PronType": "dem"}, "PDS": {POS: PRON, "PronType": "dem"}, "PIAT": {POS: DET, "PronType": "ind|neg|tot"}, - "PIDAT": {POS: DET, "AdjType": "pdt", "PronType": "ind|neg|tot"}, + "PIDAT": {POS: DET, "PronType": "ind|neg|tot"}, "PIS": {POS: PRON, "PronType": "ind|neg|tot"}, "PPER": {POS: PRON, "PronType": "prs"}, "PPOSAT": {POS: DET, "Poss": "yes", "PronType": "prs"}, @@ -42,7 +42,7 @@ TAG_MAP = { "PRF": {POS: PRON, "PronType": "prs", "Reflex": "yes"}, "PTKA": {POS: PART}, "PTKANT": {POS: PART, "PartType": "res"}, - "PTKNEG": {POS: PART, "Polarity": "Neg"}, + "PTKNEG": {POS: PART, "Polarity": "neg"}, "PTKVZ": {POS: PART, "PartType": "vbp"}, "PTKZU": {POS: PART, "PartType": "inf"}, "PWAT": {POS: DET, "PronType": "int"}, From 7afe56a3602e4a7876f5c87ac5b4f3531fd60487 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 21:57:56 +0100 Subject: [PATCH 061/287] Fix morphological features in en tag_map --- spacy/lang/en/tag_map.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spacy/lang/en/tag_map.py b/spacy/lang/en/tag_map.py index 151b42c0c..88fc2db3e 100644 --- a/spacy/lang/en/tag_map.py +++ b/spacy/lang/en/tag_map.py @@ -58,10 +58,10 @@ TAG_MAP = { "Number": "sing", "Person": "three", }, - "WDT": {POS: ADJ, "PronType": "int,rel"}, - "WP": {POS: NOUN, "PronType": "int,rel"}, - "WP$": {POS: ADJ, "Poss": "yes", "PronType": "int,rel"}, - "WRB": {POS: ADV, "PronType": "int,rel"}, + "WDT": {POS: ADJ, "PronType": "rel"}, + "WP": {POS: NOUN, "PronType": "rel"}, + "WP$": {POS: ADJ, "Poss": "yes", "PronType": "rel"}, + "WRB": {POS: ADV, "PronType": "rel"}, "ADD": {POS: X}, "NFP": {POS: PUNCT}, "GW": {POS: X}, From 00cfadbf63ca59354c2f57a5b28e85d00097a190 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 21:58:16 +0100 Subject: [PATCH 062/287] Fix obsolete data in English tokenizer exceptions --- spacy/lang/en/tokenizer_exceptions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/spacy/lang/en/tokenizer_exceptions.py b/spacy/lang/en/tokenizer_exceptions.py index 5063319a6..3c750d627 100644 --- a/spacy/lang/en/tokenizer_exceptions.py +++ b/spacy/lang/en/tokenizer_exceptions.py @@ -35,8 +35,6 @@ for pron in ["i"]: LEMMA: "be", NORM: "am", TAG: "VBP", - "tenspect": 1, - "number": 1, }, ] From 987ee6e884c21d129d8cc66b04c167fa243ca77b Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 21:58:43 +0100 Subject: [PATCH 063/287] Fix data reading in morphology --- spacy/morphology.pyx | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 4e3ec1cf8..d169c6d31 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -6,6 +6,7 @@ from libc.string cimport memset import srsly from collections import Counter +from .compat import basestring_ from .strings import get_string_id from . import symbols from .attrs cimport POS, IS_SPACE @@ -68,7 +69,8 @@ def _normalize_props(props): props = dict(props) for key in FIELDS: if key in props: - attr = '%s_%s' % (key, props[key]) + value = str(props[key]).lower() + attr = '%s_%s' % (key, value) if attr in FEATURES: props.pop(key) props[attr] = True @@ -81,9 +83,11 @@ def _normalize_props(props): out[key] = value elif isinstance(key, int): out[key] = value + elif value is True: + out[key] = value elif key.lower() == 'pos': out[POS] = POS_IDS[value.upper()] - else: + elif key.lower() != 'morph': out[key] = value return out @@ -132,6 +136,7 @@ cdef class Morphology: self.reverse_index = {} for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())): attrs = _normalize_props(attrs) + self.add({FEATURE_NAMES[feat] for feat in attrs if feat in FEATURE_NAMES}) self.tag_map[tag_str] = dict(attrs) self.reverse_index[self.strings.add(tag_str)] = i @@ -152,7 +157,8 @@ cdef class Morphology: present. Returns the hash of the new analysis. """ for f in features: - self.strings.add(f) + if isinstance(f, basestring_): + self.strings.add(f) features = intify_features(features) cdef attr_t feature for feature in features: @@ -213,6 +219,7 @@ cdef class Morphology: """ attrs = dict(attrs) attrs = _normalize_props(attrs) + self.add({FEATURE_NAMES[feat] for feat in attrs if feat in FEATURE_NAMES}) attrs = intify_attrs(attrs, self.strings, _do_deprecated=True) self.exc[(tag_str, self.strings.add(orth_str))] = attrs @@ -659,7 +666,7 @@ FEATURES = [ "Abbr_yes", "AdpType_circ", "AdpType_comprep", - "AdpType_prep ", + "AdpType_prep", "AdpType_post", "AdpType_voc", "AdvType_adadj," From dd9ea478c5f4661d2e058e01a91b3471c701de41 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 21:59:03 +0100 Subject: [PATCH 064/287] Fix intify_attrs function for obsolete data --- spacy/attrs.pyx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spacy/attrs.pyx b/spacy/attrs.pyx index ed1f39a3f..f441007fe 100644 --- a/spacy/attrs.pyx +++ b/spacy/attrs.pyx @@ -143,8 +143,12 @@ def intify_attrs(stringy_attrs, strings_map=None, _do_deprecated=False): for name, value in stringy_attrs.items(): if isinstance(name, int): int_key = name - else: + elif name in IDS: + int_key = IDS[name] + elif name.upper() in IDS: int_key = IDS[name.upper()] + else: + continue if strings_map is not None and isinstance(value, basestring): if hasattr(strings_map, 'add'): value = strings_map.add(value) From a40d73cb2ad2320b3ef06ded55f997bca31877be Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 7 Mar 2019 21:59:25 +0100 Subject: [PATCH 065/287] Build out morphological analysis API --- spacy/tokens/morphanalysis.pyx | 84 +++++++++++++++++----------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx index 11e65c19f..59ce6daa0 100644 --- a/spacy/tokens/morphanalysis.pyx +++ b/spacy/tokens/morphanalysis.pyx @@ -244,168 +244,168 @@ cdef class MorphAnalysis: property abbr_: def __get__(self): - pass + return self.vocab.strings[self.c.abbr_] property adp_type_: def __get__(self): - pass + return self.vocab.strings[self.c.adp_type_] property adv_type_: def __get__(self): - pass + return self.vocab.strings[self.c.adv_type_] property animacy_: def __get__(self): - pass + return self.vocab.strings[self.c.animacy_] property aspect_: def __get__(self): - pass + return self.vocab.strings[self.c.aspect_] property case_: def __get__(self): - pass + return self.vocab.strings[self.c.case_] property conj_type_: def __get__(self): - pass + return self.vocab.strings[self.c.conj_type_] property connegative_: def __get__(self): - pass + return self.vocab.strings[self.c.connegative_] property definite_: def __get__(self): - pass + return self.vocab.strings[self.c.definite_] property degree_: def __get__(self): - pass + return self.vocab.strings[self.c.degree_] property derivation_: def __get__(self): - pass + return self.vocab.strings[self.c.derivation_] property echo_: def __get__(self): - pass + return self.vocab.strings[self.c.echo_] property foreign_: def __get__(self): - pass + return self.vocab.strings[self.c.foreign_] property gender_: def __get__(self): - pass + return self.vocab.strings[self.c.gender_] property hyph_: def __get__(self): - pass + return self.vocab.strings[self.c.hyph_] property inf_form_: def __get__(self): - pass + return self.vocab.strings[self.c.inf_form_] property name_type_: def __get__(self): - pass + return self.vocab.strings[self.c.name_type_] property negative_: def __get__(self): - pass + return self.vocab.strings[self.c.negative_] property mood_: def __get__(self): - pass + return self.vocab.strings[self.c.mood_] property number_: def __get__(self): - pass + return self.vocab.strings[self.c.number_] property num_form_: def __get__(self): - pass + return self.vocab.strings[self.c.num_form_] property num_type_: def __get__(self): - pass + return self.vocab.strings[self.c.num_type_] property num_value_: def __get__(self): - pass + return self.vocab.strings[self.c.num_value_] property part_form_: def __get__(self): - pass + return self.vocab.strings[self.c.part_form_] property part_type_: def __get__(self): - pass + return self.vocab.strings[self.c.part_type_] property person_: def __get__(self): - pass + return self.vocab.strings[self.c.person_] property polite_: def __get__(self): - pass + return self.vocab.strings[self.c.polite_] property polarity_: def __get__(self): - pass + return self.vocab.strings[self.c.polarity_] property poss_: def __get__(self): - pass + return self.vocab.strings[self.c.poss_] property prefix_: def __get__(self): - pass + return self.vocab.strings[self.c.prefix_] property prep_case_: def __get__(self): - pass + return self.vocab.strings[self.c.prep_case_] property pron_type_: def __get__(self): - pass + return self.vocab.strings[self.c.pron_type_] property punct_side_: def __get__(self): - pass + return self.vocab.strings[self.c.punct_side_] property punct_type_: def __get__(self): - pass + return self.vocab.strings[self.c.punct_type_] property reflex_: def __get__(self): - pass + return self.vocab.strings[self.c.reflex_] property style_: def __get__(self): - pass + return self.vocab.strings[self.c.style_] property style_variant_: def __get__(self): - pass + return self.vocab.strings[self.c.style_variant_] property tense_: def __get__(self): - pass + return self.vocab.strings[self.c.tense_] property typo_: def __get__(self): - pass + return self.vocab.strings[self.c.typo_] property verb_form_: def __get__(self): - pass + return self.vocab.strings[self.c.verb_form_] property voice_: def __get__(self): - pass + return self.vocab.strings[self.c.voice_] property verb_type_: def __get__(self): - pass + return self.vocab.strings[self.c.verb_type_] From b5f2b7b454604cf1fe6bdaf11f977cb57a8a9e11 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 00:08:35 +0100 Subject: [PATCH 066/287] Add list_features() helper, clean up --- spacy/morphology.pxd | 4 +- spacy/morphology.pyx | 276 +++++++++++++++++++++++++++++-------------- 2 files changed, 188 insertions(+), 92 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index 30c29c1c7..0001b9eb9 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -34,5 +34,7 @@ cdef class Morphology: cdef int check_feature(const MorphAnalysisC* tag, attr_t feature) nogil +cdef attr_t get_field(const MorphAnalysisC* tag, int field) nogil +cdef list list_features(const MorphAnalysisC* tag) -cdef tag_to_json(MorphAnalysisC tag) +cdef tag_to_json(const MorphAnalysisC* tag) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index d169c6d31..fa8245f47 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -174,7 +174,7 @@ cdef class Morphology: if tag == NULL: return [] else: - return tag_to_json(tag[0]) + return tag_to_json(tag) cpdef update(self, hash_t morph, features): """Update a morphological analysis with new feature values.""" @@ -296,7 +296,7 @@ cdef class Morphology: for key in self.tags: tag_ptr = self.tags.get(key) if tag_ptr != NULL: - json_tags.append(tag_to_json(tag_ptr[0])) + json_tags.append(tag_to_json(tag_ptr)) return srsly.json_dumps(json_tags) def from_bytes(self, byte_string): @@ -334,98 +334,186 @@ cdef MorphAnalysisC create_rich_tag(features) except *: return tag -cdef tag_to_json(MorphAnalysisC tag): - features = [] - if tag.abbr != 0: - features.append(FEATURE_NAMES[tag.abbr]) - if tag.adp_type != 0: - features.append(FEATURE_NAMES[tag.adp_type]) - if tag.adv_type != 0: - features.append(FEATURE_NAMES[tag.adv_type]) - if tag.animacy != 0: - features.append(FEATURE_NAMES[tag.animacy]) - if tag.aspect != 0: - features.append(FEATURE_NAMES[tag.aspect]) - if tag.case != 0: - features.append(FEATURE_NAMES[tag.case]) - if tag.conj_type != 0: - features.append(FEATURE_NAMES[tag.conj_type]) - if tag.connegative != 0: - features.append(FEATURE_NAMES[tag.connegative]) - if tag.definite != 0: - features.append(FEATURE_NAMES[tag.definite]) - if tag.degree != 0: - features.append(FEATURE_NAMES[tag.degree]) - if tag.derivation != 0: - features.append(FEATURE_NAMES[tag.derivation]) - if tag.echo != 0: - features.append(FEATURE_NAMES[tag.echo]) - if tag.foreign != 0: - features.append(FEATURE_NAMES[tag.foreign]) - if tag.gender != 0: - features.append(FEATURE_NAMES[tag.gender]) - if tag.hyph != 0: - features.append(FEATURE_NAMES[tag.hyph]) - if tag.inf_form != 0: - features.append(FEATURE_NAMES[tag.inf_form]) - if tag.mood != 0: - features.append(FEATURE_NAMES[tag.mood]) - if tag.negative != 0: - features.append(FEATURE_NAMES[tag.negative]) - if tag.number != 0: - features.append(FEATURE_NAMES[tag.number]) - if tag.name_type != 0: - features.append(FEATURE_NAMES[tag.name_type]) - if tag.noun_type != 0: - features.append(FEATURE_NAMES[tag.noun_type]) - if tag.num_form != 0: - features.append(FEATURE_NAMES[tag.num_form]) - if tag.num_type != 0: - features.append(FEATURE_NAMES[tag.num_type]) - if tag.num_value != 0: - features.append(FEATURE_NAMES[tag.num_value]) - if tag.part_form != 0: - features.append(FEATURE_NAMES[tag.part_form]) - if tag.part_type != 0: - features.append(FEATURE_NAMES[tag.part_type]) - if tag.person != 0: - features.append(FEATURE_NAMES[tag.person]) - if tag.polite != 0: - features.append(FEATURE_NAMES[tag.polite]) - if tag.polarity != 0: - features.append(FEATURE_NAMES[tag.polarity]) - if tag.poss != 0: - features.append(FEATURE_NAMES[tag.poss]) - if tag.prefix != 0: - features.append(FEATURE_NAMES[tag.prefix]) - if tag.prep_case != 0: - features.append(FEATURE_NAMES[tag.prep_case]) - if tag.pron_type != 0: - features.append(FEATURE_NAMES[tag.pron_type]) - if tag.punct_side != 0: - features.append(FEATURE_NAMES[tag.punct_side]) - if tag.punct_type != 0: - features.append(FEATURE_NAMES[tag.punct_type]) - if tag.reflex != 0: - features.append(FEATURE_NAMES[tag.reflex]) - if tag.style != 0: - features.append(FEATURE_NAMES[tag.style]) - if tag.style_variant != 0: - features.append(FEATURE_NAMES[tag.style_variant]) - if tag.tense != 0: - features.append(FEATURE_NAMES[tag.tense]) - if tag.verb_form != 0: - features.append(FEATURE_NAMES[tag.verb_form]) - if tag.voice != 0: - features.append(FEATURE_NAMES[tag.voice]) - if tag.verb_type != 0: - features.append(FEATURE_NAMES[tag.verb_type]) - return features +cdef tag_to_json(const MorphAnalysisC* tag): + return [FEATURE_NAMES[f] for f in list_features(tag)] cdef MorphAnalysisC tag_from_json(json_tag): - cdef MorphAnalysisC tag - return tag + raise NotImplementedError + + +cdef list list_features(const MorphAnalysisC* tag): + output = [] + if tag.abbr != 0: + output.append(tag.abbr) + if tag.adp_type != 0: + output.append(tag.adp_type) + if tag.adv_type != 0: + output.append(tag.adv_type) + if tag.animacy != 0: + output.append(tag.animacy) + if tag.aspect != 0: + output.append(tag.aspect) + if tag.case != 0: + output.append(tag.case) + if tag.conj_type != 0: + output.append(tag.conj_type) + if tag.connegative != 0: + output.append(tag.connegative) + if tag.definite != 0: + output.append(tag.definite) + if tag.degree != 0: + output.append(tag.degree) + if tag.derivation != 0: + output.append(tag.derivation) + if tag.echo != 0: + output.append(tag.echo) + if tag.foreign != 0: + output.append(tag.foreign) + if tag.gender != 0: + output.append(tag.gender) + if tag.hyph != 0: + output.append(tag.hyph) + if tag.inf_form != 0: + output.append(tag.inf_form) + if tag.mood != 0: + output.append(tag.mood) + if tag.negative != 0: + output.append(tag.negative) + if tag.number != 0: + output.append(tag.number) + if tag.name_type != 0: + output.append(tag.name_type) + if tag.noun_type != 0: + output.append(tag.noun_type) + if tag.part_form != 0: + output.append(tag.part_form) + if tag.part_type != 0: + output.append(tag.part_type) + if tag.person != 0: + output.append(tag.person) + if tag.polite != 0: + output.append(tag.polite) + if tag.polarity != 0: + output.append(tag.polarity) + if tag.poss != 0: + output.append(tag.poss) + if tag.prefix != 0: + output.append(tag.prefix) + if tag.prep_case != 0: + output.append(tag.prep_case) + if tag.pron_type != 0: + output.append(tag.pron_type) + if tag.punct_type != 0: + output.append(tag.punct_type) + if tag.reflex != 0: + output.append(tag.reflex) + if tag.style != 0: + output.append(tag.style) + if tag.style_variant != 0: + output.append(tag.style_variant) + if tag.typo != 0: + output.append(tag.typo) + if tag.verb_form != 0: + output.append(tag.verb_form) + if tag.voice != 0: + output.append(tag.voice) + if tag.verb_type != 0: + output.append(tag.verb_type) + return output + + +cdef attr_t get_field(const MorphAnalysisC* tag, int field_id) nogil: + field = field_id + if field == Field_Abbr: + return tag.abbr + elif field == Field_AdpType: + return tag.adp_type + elif field == Field_AdvType: + return tag.adv_type + elif field == Field_Animacy: + return tag.animacy + elif field == Field_Aspect: + return tag.aspect + elif field == Field_Case: + return tag.case + elif field == Field_ConjType: + return tag.conj_type + elif field == Field_Connegative: + return tag.connegative + elif field == Field_Definite: + return tag.definite + elif field == Field_Degree: + return tag.degree + elif field == Field_Derivation: + return tag.derivation + elif field == Field_Echo: + return tag.echo + elif field == Field_Foreign: + return tag.foreign + elif field == Field_Gender: + return tag.gender + elif field == Field_Hyph: + return tag.hyph + elif field == Field_InfForm: + return tag.inf_form + elif field == Field_Mood: + return tag.mood + elif field == Field_Negative: + return tag.negative + elif field == Field_Number: + return tag.number + elif field == Field_NameType: + return tag.name_type + elif field == Field_NounType: + return tag.noun_type + elif field == Field_NumForm: + return tag.num_form + elif field == Field_NumType: + return tag.num_type + elif field == Field_NumValue: + return tag.num_value + elif field == Field_PartForm: + return tag.part_form + elif field == Field_PartType: + return tag.part_type + elif field == Field_Person: + return tag.person + elif field == Field_Polite: + return tag.polite + elif field == Field_Polarity: + return tag.polarity + elif field == Field_Poss: + return tag.poss + elif field == Field_Prefix: + return tag.prefix + elif field == Field_PrepCase: + return tag.prep_case + elif field == Field_PronType: + return tag.pron_type + elif field == Field_PunctSide: + return tag.punct_side + elif field == Field_PunctType: + return tag.punct_type + elif field == Field_Reflex: + return tag.reflex + elif field == Field_Style: + return tag.style + elif field == Field_StyleVariant: + return tag.style_variant + elif field == Field_Tense: + return tag.tense + elif field == Field_Typo: + return tag.typo + elif field == Field_VerbForm: + return tag.verb_form + elif field == Field_Voice: + return tag.voice + elif field == Field_VerbType: + return tag.verb_type + else: + raise ValueError("Unknown feature: %s (%d)" % (FEATURE_NAMES.get(feature), feature)) + cdef int check_feature(const MorphAnalysisC* tag, attr_t feature) nogil: @@ -524,6 +612,11 @@ cdef int set_feature(MorphAnalysisC* tag, value_ = feature else: value_ = 0 + prev_value = get_field(tag, field) + if prev_value != 0 and value_ == 0: + tag.length -= 1 + elif prev_value == 0 and value_ != 0: + tag.length += 1 if feature == 0: pass elif field == Field_Abbr: @@ -616,6 +709,7 @@ cdef int set_feature(MorphAnalysisC* tag, raise ValueError("Unknown feature: %s (%d)" % (FEATURE_NAMES.get(feature), feature)) + FIELDS = { 'Abbr': Field_Abbr, 'AdpType': Field_AdpType, From 9a2d1cc6e05f9d6fbd431b99d9f46d786db4b153 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 00:08:57 +0100 Subject: [PATCH 067/287] Add length attribute to MorphAnalysisC --- spacy/structs.pxd | 1 + 1 file changed, 1 insertion(+) diff --git a/spacy/structs.pxd b/spacy/structs.pxd index a4daa9b94..bf7dc0d7a 100644 --- a/spacy/structs.pxd +++ b/spacy/structs.pxd @@ -75,6 +75,7 @@ cdef struct TokenC: cdef struct MorphAnalysisC: univ_pos_t pos + int length attr_t abbr attr_t adp_type From 3300e3d7abbd22f09f55104aba46fa47dbd52054 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 00:09:16 +0100 Subject: [PATCH 068/287] Implement more MorphAnalysis API --- spacy/tokens/morphanalysis.pyx | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx index 59ce6daa0..c9d915599 100644 --- a/spacy/tokens/morphanalysis.pyx +++ b/spacy/tokens/morphanalysis.pyx @@ -2,7 +2,7 @@ from libc.string cimport memset from ..vocab cimport Vocab from ..typedefs cimport hash_t, attr_t -from ..morphology cimport check_feature, tag_to_json +from ..morphology cimport list_features, check_feature, tag_to_json from ..strings import get_string_id @@ -21,6 +21,7 @@ cdef class MorphAnalysis: @classmethod def from_id(cls, Vocab vocab, hash_t key): cdef MorphAnalysis morph = MorphAnalysis.__new__(MorphAnalysis, vocab) + morph.vocab = vocab morph.key = key analysis = vocab.morphology.tags.get(key) if analysis is not NULL: @@ -34,25 +35,27 @@ cdef class MorphAnalysis: return check_feature(&self.c, feat_id) def __iter__(self): - raise NotImplementedError + cdef attr_t feature + for feature in list_features(&self.c): + yield self.vocab.strings[feature] def __len__(self): - raise NotImplementedError + return self.c.length def __str__(self): - raise NotImplementedError + return self.to_json() def __repr__(self): - raise NotImplementedError + return self.to_json() def __hash__(self): - raise NotImplementedError + return self.key def get(self, field): raise NotImplementedError def to_json(self): - return tag_to_json(self.c) + return tag_to_json(&self.c) @property def is_base_form(self): From 3c3259024310e3ed28bc71c8bb6fa60d608a049c Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 00:10:07 +0100 Subject: [PATCH 069/287] Add test for morph analysis --- spacy/tests/doc/test_morphanalysis.py | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 spacy/tests/doc/test_morphanalysis.py diff --git a/spacy/tests/doc/test_morphanalysis.py b/spacy/tests/doc/test_morphanalysis.py new file mode 100644 index 000000000..31c765e32 --- /dev/null +++ b/spacy/tests/doc/test_morphanalysis.py @@ -0,0 +1,30 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import pytest +import numpy +from spacy.attrs import IS_ALPHA, IS_DIGIT, IS_LOWER, IS_PUNCT, IS_TITLE, IS_STOP +from spacy.symbols import VERB +from spacy.vocab import Vocab +from spacy.tokens import Doc + +@pytest.fixture +def i_has(en_tokenizer): + doc = en_tokenizer("I has") + doc[0].tag_ = "PRP" + doc[1].tag_ = "VBZ" + return doc + +def test_token_morph_id(i_has): + assert i_has[0].morph.id + assert i_has[1].morph.id != 0 + assert i_has[0].morph.id != i_has[1].morph.id + +def test_morph_props(i_has): + assert i_has[0].morph.pron_type == i_has.vocab.strings["PronType_prs"] + assert i_has[1].morph.pron_type == 0 + + +def test_morph_iter(i_has): + assert list(i_has[0].morph) == ["PronType_prs"] + assert list(i_has[1].morph) == ["Number_sing", "Person_three", "VerbForm_fin"] From 322b64dca08964f97c1d618012921b89dcc1f355 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 01:38:15 +0100 Subject: [PATCH 070/287] Allow lookup of morphology by attribute name --- spacy/morphology.pxd | 1 + spacy/morphology.pyx | 52 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index 0001b9eb9..cb708166c 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -36,5 +36,6 @@ cdef class Morphology: cdef int check_feature(const MorphAnalysisC* tag, attr_t feature) nogil cdef attr_t get_field(const MorphAnalysisC* tag, int field) nogil cdef list list_features(const MorphAnalysisC* tag) +cdef int attribute_to_field(unicode attribute) cdef tag_to_json(const MorphAnalysisC* tag) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index fa8245f47..63d0291ff 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -98,6 +98,10 @@ def parse_feature(feature): return (field, offset) +cdef int attribute_to_field(unicode attribute_name): + return LOWER_FIELDS[attribute_name] + + def get_field_id(feature): return FEATURE_FIELDS[feature] @@ -709,7 +713,6 @@ cdef int set_feature(MorphAnalysisC* tag, raise ValueError("Unknown feature: %s (%d)" % (FEATURE_NAMES.get(feature), feature)) - FIELDS = { 'Abbr': Field_Abbr, 'AdpType': Field_AdpType, @@ -756,6 +759,53 @@ FIELDS = { 'VerbType': Field_VerbType } +LOWER_FIELDS = { + 'abbr': Field_Abbr, + 'adp_type': Field_AdpType, + 'adv_type': Field_AdvType, + 'animacy': Field_Animacy, + 'aspect': Field_Aspect, + 'case': Field_Case, + 'conj_type': Field_ConjType, + 'connegative': Field_Connegative, + 'definite': Field_Definite, + 'degree': Field_Degree, + 'derivation': Field_Derivation, + 'echo': Field_Echo, + 'foreign': Field_Foreign, + 'gender': Field_Gender, + 'hyph': Field_Hyph, + 'inf_form': Field_InfForm, + 'mood': Field_Mood, + 'name_type': Field_NameType, + 'negative': Field_Negative, + 'noun_type': Field_NounType, + 'number': Field_Number, + 'num_form': Field_NumForm, + 'num_type': Field_NumType, + 'num_value': Field_NumValue, + 'part_form': Field_PartForm, + 'part_type': Field_PartType, + 'person': Field_Person, + 'polite': Field_Polite, + 'polarity': Field_Polarity, + 'poss': Field_Poss, + 'prefix': Field_Prefix, + 'prep_case': Field_PrepCase, + 'pron_type': Field_PronType, + 'punct_side': Field_PunctSide, + 'punct_type': Field_PunctType, + 'reflex': Field_Reflex, + 'style': Field_Style, + 'style_variant': Field_StyleVariant, + 'tense': Field_Tense, + 'typo': Field_Typo, + 'verb_form': Field_VerbForm, + 'voice': Field_Voice, + 'verb_type': Field_VerbType +} + + FEATURES = [ "Abbr_yes", "AdpType_circ", From 9dceb97570812e5e82d380f5948a472f53e04699 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 01:38:34 +0100 Subject: [PATCH 071/287] Extend morphanalysis API --- spacy/tokens/morphanalysis.pyx | 92 +++++++++++++++++----------------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx index c9d915599..b727e2c3f 100644 --- a/spacy/tokens/morphanalysis.pyx +++ b/spacy/tokens/morphanalysis.pyx @@ -2,7 +2,8 @@ from libc.string cimport memset from ..vocab cimport Vocab from ..typedefs cimport hash_t, attr_t -from ..morphology cimport list_features, check_feature, tag_to_json +from ..morphology cimport list_features, check_feature, get_field, tag_to_json +from ..morphology cimport attribute_to_field from ..strings import get_string_id @@ -51,8 +52,9 @@ cdef class MorphAnalysis: def __hash__(self): return self.key - def get(self, field): - raise NotImplementedError + def get(self, unicode field): + cdef int field_id = attribute_to_field(field) + return self.vocab.strings[get_field(&self.c, field_id)] def to_json(self): return tag_to_json(&self.c) @@ -247,168 +249,168 @@ cdef class MorphAnalysis: property abbr_: def __get__(self): - return self.vocab.strings[self.c.abbr_] + return self.vocab.strings[self.c.abbr] property adp_type_: def __get__(self): - return self.vocab.strings[self.c.adp_type_] + return self.vocab.strings[self.c.adp_type] property adv_type_: def __get__(self): - return self.vocab.strings[self.c.adv_type_] + return self.vocab.strings[self.c.adv_type] property animacy_: def __get__(self): - return self.vocab.strings[self.c.animacy_] + return self.vocab.strings[self.c.animacy] property aspect_: def __get__(self): - return self.vocab.strings[self.c.aspect_] + return self.vocab.strings[self.c.aspect] property case_: def __get__(self): - return self.vocab.strings[self.c.case_] + return self.vocab.strings[self.c.case] property conj_type_: def __get__(self): - return self.vocab.strings[self.c.conj_type_] + return self.vocab.strings[self.c.conj_type] property connegative_: def __get__(self): - return self.vocab.strings[self.c.connegative_] + return self.vocab.strings[self.c.connegative] property definite_: def __get__(self): - return self.vocab.strings[self.c.definite_] + return self.vocab.strings[self.c.definite] property degree_: def __get__(self): - return self.vocab.strings[self.c.degree_] + return self.vocab.strings[self.c.degree] property derivation_: def __get__(self): - return self.vocab.strings[self.c.derivation_] + return self.vocab.strings[self.c.derivation] property echo_: def __get__(self): - return self.vocab.strings[self.c.echo_] + return self.vocab.strings[self.c.echo] property foreign_: def __get__(self): - return self.vocab.strings[self.c.foreign_] + return self.vocab.strings[self.c.foreign] property gender_: def __get__(self): - return self.vocab.strings[self.c.gender_] + return self.vocab.strings[self.c.gender] property hyph_: def __get__(self): - return self.vocab.strings[self.c.hyph_] + return self.vocab.strings[self.c.hyph] property inf_form_: def __get__(self): - return self.vocab.strings[self.c.inf_form_] + return self.vocab.strings[self.c.inf_form] property name_type_: def __get__(self): - return self.vocab.strings[self.c.name_type_] + return self.vocab.strings[self.c.name_type] property negative_: def __get__(self): - return self.vocab.strings[self.c.negative_] + return self.vocab.strings[self.c.negative] property mood_: def __get__(self): - return self.vocab.strings[self.c.mood_] + return self.vocab.strings[self.c.mood] property number_: def __get__(self): - return self.vocab.strings[self.c.number_] + return self.vocab.strings[self.c.number] property num_form_: def __get__(self): - return self.vocab.strings[self.c.num_form_] + return self.vocab.strings[self.c.num_form] property num_type_: def __get__(self): - return self.vocab.strings[self.c.num_type_] + return self.vocab.strings[self.c.num_type] property num_value_: def __get__(self): - return self.vocab.strings[self.c.num_value_] + return self.vocab.strings[self.c.num_value] property part_form_: def __get__(self): - return self.vocab.strings[self.c.part_form_] + return self.vocab.strings[self.c.part_form] property part_type_: def __get__(self): - return self.vocab.strings[self.c.part_type_] + return self.vocab.strings[self.c.part_type] property person_: def __get__(self): - return self.vocab.strings[self.c.person_] + return self.vocab.strings[self.c.person] property polite_: def __get__(self): - return self.vocab.strings[self.c.polite_] + return self.vocab.strings[self.c.polite] property polarity_: def __get__(self): - return self.vocab.strings[self.c.polarity_] + return self.vocab.strings[self.c.polarity] property poss_: def __get__(self): - return self.vocab.strings[self.c.poss_] + return self.vocab.strings[self.c.poss] property prefix_: def __get__(self): - return self.vocab.strings[self.c.prefix_] + return self.vocab.strings[self.c.prefix] property prep_case_: def __get__(self): - return self.vocab.strings[self.c.prep_case_] + return self.vocab.strings[self.c.prep_case] property pron_type_: def __get__(self): - return self.vocab.strings[self.c.pron_type_] + return self.vocab.strings[self.c.pron_type] property punct_side_: def __get__(self): - return self.vocab.strings[self.c.punct_side_] + return self.vocab.strings[self.c.punct_side] property punct_type_: def __get__(self): - return self.vocab.strings[self.c.punct_type_] + return self.vocab.strings[self.c.punct_type] property reflex_: def __get__(self): - return self.vocab.strings[self.c.reflex_] + return self.vocab.strings[self.c.reflex] property style_: def __get__(self): - return self.vocab.strings[self.c.style_] + return self.vocab.strings[self.c.style] property style_variant_: def __get__(self): - return self.vocab.strings[self.c.style_variant_] + return self.vocab.strings[self.c.style_variant] property tense_: def __get__(self): - return self.vocab.strings[self.c.tense_] + return self.vocab.strings[self.c.tense] property typo_: def __get__(self): - return self.vocab.strings[self.c.typo_] + return self.vocab.strings[self.c.typo] property verb_form_: def __get__(self): - return self.vocab.strings[self.c.verb_form_] + return self.vocab.strings[self.c.verb_form] property voice_: def __get__(self): - return self.vocab.strings[self.c.voice_] + return self.vocab.strings[self.c.voice] property verb_type_: def __get__(self): - return self.vocab.strings[self.c.verb_type_] + return self.vocab.strings[self.c.verb_type] From 19e6b39786964d5c4c401fe9bf8c03dacda11dd5 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 01:38:54 +0100 Subject: [PATCH 072/287] Test morphological features --- spacy/tests/doc/test_morphanalysis.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spacy/tests/doc/test_morphanalysis.py b/spacy/tests/doc/test_morphanalysis.py index 31c765e32..ffee5694a 100644 --- a/spacy/tests/doc/test_morphanalysis.py +++ b/spacy/tests/doc/test_morphanalysis.py @@ -22,9 +22,14 @@ def test_token_morph_id(i_has): def test_morph_props(i_has): assert i_has[0].morph.pron_type == i_has.vocab.strings["PronType_prs"] + assert i_has[0].morph.pron_type_ == "PronType_prs" assert i_has[1].morph.pron_type == 0 def test_morph_iter(i_has): assert list(i_has[0].morph) == ["PronType_prs"] assert list(i_has[1].morph) == ["Number_sing", "Person_three", "VerbForm_fin"] + + +def test_morph_get(i_has): + assert i_has[0].morph.get("pron_type") == "PronType_prs" From ad834be494f8074e93a6cb9fb76e6215b4e35307 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Fri, 8 Mar 2019 13:28:53 +0100 Subject: [PATCH 073/287] Tidy up and auto-format --- spacy/_ml.py | 81 +++++++++---------- spacy/errors.py | 1 - spacy/tests/doc/test_morphanalysis.py | 8 +- spacy/tests/morphology/test_morph_features.py | 29 ++++--- 4 files changed, 61 insertions(+), 58 deletions(-) diff --git a/spacy/_ml.py b/spacy/_ml.py index 422bbe66a..68dedc0b3 100644 --- a/spacy/_ml.py +++ b/spacy/_ml.py @@ -465,17 +465,16 @@ def getitem(i): @describe.attributes( - W=Synapses("Weights matrix", - lambda obj: (obj.nO, obj.nI), - lambda W, ops: None) + W=Synapses("Weights matrix", lambda obj: (obj.nO, obj.nI), lambda W, ops: None) ) class MultiSoftmax(Affine): - '''Neural network layer that predicts several multi-class attributes at once. + """Neural network layer that predicts several multi-class attributes at once. For instance, we might predict one class with 6 variables, and another with 5. We predict the 11 neurons required for this, and then softmax them such that columns 0-6 make a probability distribution and coumns 6-11 make another. - ''' - name = 'multisoftmax' + """ + + name = "multisoftmax" def __init__(self, out_sizes, nI=None, **kwargs): Model.__init__(self, **kwargs) @@ -487,12 +486,13 @@ class MultiSoftmax(Affine): output__BO = self.ops.affine(self.W, self.b, input__BI) i = 0 for out_size in self.out_sizes: - self.ops.softmax(output__BO[:, i : i+out_size], inplace=True) + self.ops.softmax(output__BO[:, i : i + out_size], inplace=True) i += out_size return output__BO - def begin_update(self, input__BI, drop=0.): + def begin_update(self, input__BI, drop=0.0): output__BO = self.predict(input__BI) + def finish_update(grad__BO, sgd=None): self.d_W += self.ops.gemm(grad__BO, input__BI, trans1=True) self.d_b += grad__BO.sum(axis=0) @@ -500,6 +500,7 @@ class MultiSoftmax(Affine): if sgd is not None: sgd(self._mem.weights, self._mem.gradient, key=self.id) return grad__BI + return output__BO, finish_update @@ -515,41 +516,41 @@ def build_tagger_model(nr_class, **cfg): if "tok2vec" in cfg: tok2vec = cfg["tok2vec"] else: - tok2vec = Tok2Vec(token_vector_width, embed_size, - subword_features=subword_features, - pretrained_vectors=pretrained_vectors) - softmax = with_flatten( - Softmax(nr_class, token_vector_width)) - model = ( - tok2vec - >> softmax - ) + tok2vec = Tok2Vec( + token_vector_width, + embed_size, + subword_features=subword_features, + pretrained_vectors=pretrained_vectors, + ) + softmax = with_flatten(Softmax(nr_class, token_vector_width)) + model = tok2vec >> softmax model.nI = None model.tok2vec = tok2vec model.softmax = softmax return model + def build_morphologizer_model(class_nums, **cfg): - embed_size = util.env_opt('embed_size', 7000) - if 'token_vector_width' in cfg: - token_vector_width = cfg['token_vector_width'] + embed_size = util.env_opt("embed_size", 7000) + if "token_vector_width" in cfg: + token_vector_width = cfg["token_vector_width"] else: - token_vector_width = util.env_opt('token_vector_width', 128) - pretrained_vectors = cfg.get('pretrained_vectors') - subword_features = cfg.get('subword_features', True) - with Model.define_operators({'>>': chain, '+': add}): - if 'tok2vec' in cfg: - tok2vec = cfg['tok2vec'] + token_vector_width = util.env_opt("token_vector_width", 128) + pretrained_vectors = cfg.get("pretrained_vectors") + subword_features = cfg.get("subword_features", True) + with Model.define_operators({">>": chain, "+": add}): + if "tok2vec" in cfg: + tok2vec = cfg["tok2vec"] else: - tok2vec = Tok2Vec(token_vector_width, embed_size, - subword_features=subword_features, - pretrained_vectors=pretrained_vectors) + tok2vec = Tok2Vec( + token_vector_width, + embed_size, + subword_features=subword_features, + pretrained_vectors=pretrained_vectors, + ) softmax = with_flatten(MultiSoftmax(class_nums, token_vector_width)) softmax.out_sizes = class_nums - model = ( - tok2vec - >> softmax - ) + model = tok2vec >> softmax model.nI = None model.tok2vec = tok2vec model.softmax = softmax @@ -630,17 +631,13 @@ def build_text_classifier(nr_class, width=64, **cfg): ) linear_model = _preprocess_doc >> LinearModel(nr_class) - if cfg.get('exclusive_classes'): + if cfg.get("exclusive_classes"): output_layer = Softmax(nr_class, nr_class * 2) else: output_layer = ( - zero_init(Affine(nr_class, nr_class * 2, drop_factor=0.0)) - >> logistic + zero_init(Affine(nr_class, nr_class * 2, drop_factor=0.0)) >> logistic ) - model = ( - (linear_model | cnn_model) - >> output_layer - ) + model = (linear_model | cnn_model) >> output_layer model.tok2vec = chain(tok2vec, flatten) model.nO = nr_class model.lsuv = False @@ -658,7 +655,9 @@ def build_simple_cnn_text_classifier(tok2vec, nr_class, exclusive_classes=False, if exclusive_classes: output_layer = Softmax(nr_class, tok2vec.nO) else: - output_layer = zero_init(Affine(nr_class, tok2vec.nO, drop_factor=0.0)) >> logistic + output_layer = ( + zero_init(Affine(nr_class, tok2vec.nO, drop_factor=0.0)) >> logistic + ) model = tok2vec >> flatten_add_lengths >> Pooling(mean_pool) >> output_layer model.tok2vec = chain(tok2vec, flatten) model.nO = nr_class diff --git a/spacy/errors.py b/spacy/errors.py index 13382d146..f9dd8535e 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -350,7 +350,6 @@ class Errors(object): "is likely a bug in spaCy.") - @add_codes class TempErrors(object): T003 = ("Resizing pre-trained Tagger models is not currently supported.") diff --git a/spacy/tests/doc/test_morphanalysis.py b/spacy/tests/doc/test_morphanalysis.py index ffee5694a..5d570af53 100644 --- a/spacy/tests/doc/test_morphanalysis.py +++ b/spacy/tests/doc/test_morphanalysis.py @@ -2,11 +2,7 @@ from __future__ import unicode_literals import pytest -import numpy -from spacy.attrs import IS_ALPHA, IS_DIGIT, IS_LOWER, IS_PUNCT, IS_TITLE, IS_STOP -from spacy.symbols import VERB -from spacy.vocab import Vocab -from spacy.tokens import Doc + @pytest.fixture def i_has(en_tokenizer): @@ -15,11 +11,13 @@ def i_has(en_tokenizer): doc[1].tag_ = "VBZ" return doc + def test_token_morph_id(i_has): assert i_has[0].morph.id assert i_has[1].morph.id != 0 assert i_has[0].morph.id != i_has[1].morph.id + def test_morph_props(i_has): assert i_has[0].morph.pron_type == i_has.vocab.strings["PronType_prs"] assert i_has[0].morph.pron_type_ == "PronType_prs" diff --git a/spacy/tests/morphology/test_morph_features.py b/spacy/tests/morphology/test_morph_features.py index dcb0b32ff..4b8f0d754 100644 --- a/spacy/tests/morphology/test_morph_features.py +++ b/spacy/tests/morphology/test_morph_features.py @@ -1,41 +1,48 @@ +# coding: utf-8 from __future__ import unicode_literals -import pytest -from ...morphology import Morphology -from ...strings import StringStore, get_string_id -from ...lemmatizer import Lemmatizer -from ...morphology import * +import pytest +from spacy.morphology import Morphology +from spacy.strings import StringStore, get_string_id +from spacy.lemmatizer import Lemmatizer + @pytest.fixture def morphology(): return Morphology(StringStore(), {}, Lemmatizer()) + def test_init(morphology): pass + def test_add_morphology_with_string_names(morphology): morphology.add({"Case_gen", "Number_sing"}) + def test_add_morphology_with_int_ids(morphology): morphology.add({get_string_id("Case_gen"), get_string_id("Number_sing")}) + def test_add_morphology_with_mix_strings_and_ints(morphology): - morphology.add({get_string_id("PunctSide_ini"), 'VerbType_aux'}) + morphology.add({get_string_id("PunctSide_ini"), "VerbType_aux"}) def test_morphology_tags_hash_distinctly(morphology): - tag1 = morphology.add({"PunctSide_ini", 'VerbType_aux'}) - tag2 = morphology.add({"Case_gen", 'Number_sing'}) + tag1 = morphology.add({"PunctSide_ini", "VerbType_aux"}) + tag2 = morphology.add({"Case_gen", "Number_sing"}) assert tag1 != tag2 + def test_morphology_tags_hash_independent_of_order(morphology): - tag1 = morphology.add({"Case_gen", 'Number_sing'}) - tag2 = morphology.add({"Number_sing", "Case_gen"}) + tag1 = morphology.add({"Case_gen", "Number_sing"}) + tag2 = morphology.add({"Number_sing", "Case_gen"}) assert tag1 == tag2 + def test_update_morphology_tag(morphology): tag1 = morphology.add({"Case_gen"}) tag2 = morphology.update(tag1, {"Number_sing"}) assert tag1 != tag2 - tag3 = morphology.add({"Number_sing", "Case_gen"}) + tag3 = morphology.add({"Number_sing", "Case_gen"}) assert tag2 == tag3 From 3908911da477a85073eb8ac607bc9ef7f60ccd7e Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 17:04:14 +0100 Subject: [PATCH 074/287] Fix import --- spacy/pipeline/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spacy/pipeline/__init__.py b/spacy/pipeline/__init__.py index 686743a6a..83c0c65d0 100644 --- a/spacy/pipeline/__init__.py +++ b/spacy/pipeline/__init__.py @@ -1,8 +1,9 @@ # coding: utf8 from __future__ import unicode_literals -from .pipes import Tagger, DependencyParser, EntityRecognizer, Morphologizer +from .pipes import Tagger, DependencyParser, EntityRecognizer from .pipes import TextCategorizer, Tensorizer, Pipe +from .morphologizer import Morphologizer from .entityruler import EntityRuler from .hooks import SentenceSegmenter, SimilarityHook from .functions import merge_entities, merge_noun_chunks, merge_subtokens From d7ec1d62cb6c711d94c440492aa329047f6068d4 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 18:54:25 +0100 Subject: [PATCH 075/287] Fix Morphologizer --- spacy/cli/ud/ud_train.py | 2 +- spacy/morphology.pyx | 8 +++++--- spacy/pipeline/morphologizer.pyx | 14 +++++++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/spacy/cli/ud/ud_train.py b/spacy/cli/ud/ud_train.py index afef6c073..44ecababe 100644 --- a/spacy/cli/ud/ud_train.py +++ b/spacy/cli/ud/ud_train.py @@ -299,7 +299,7 @@ def get_token_conllu(token, i): head = 0 else: head = i + (token.head.i - token.i) + 1 - features = token.vocab.morphology.get(token.morph_key) + features = list(token.morph) feat_str = [] replacements = {"one": "1", "two": "2", "three": "3"} for feat in features: diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 63d0291ff..2f3e8d1fa 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -107,11 +107,11 @@ def get_field_id(feature): def get_field_size(field): - return FIELD_SIZES[field] + return FIELD_SIZES[FIELDS[field]] def get_field_offset(field): - return FIELD_OFFSETS[field] + return FIELD_OFFSETS[FIELDS[]]]]field] cdef class Morphology: @@ -831,6 +831,8 @@ FEATURES = [ "Aspect_mod", "Aspect_none", "Aspect_perf", + "Aspect_prof", + "Aspect_prosp", "Case_abe", "Case_abl", "Case_abs", @@ -1074,6 +1076,6 @@ _seen_fields = Counter() for i, feature in enumerate(FEATURES): field = FEATURE_FIELDS[feature] FEATURE_OFFSETS[feature] = _seen_fields[field] - if _seen_fields == 0: + if _seen_fields[field] == 0: FIELD_OFFSETS[field] = i _seen_fields[field] += 1 diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index 9f25ba357..223bb6ec5 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -81,16 +81,18 @@ class Morphologizer(Pipe): doc_scores = batch_scores[i] doc_guesses = scores_to_guesses(doc_scores, self.model.softmax.out_sizes) # Convert the neuron indices into feature IDs. - doc_feat_ids = self.model.ops.allocate((len(doc), len(field_names)), dtype='i') + doc_feat_ids = numpy.zeros((len(doc), len(field_names)), dtype='i') for j in range(len(doc)): for k, offset in enumerate(offsets): if doc_guesses[j, k] == 0: doc_feat_ids[j, k] = 0 else: doc_feat_ids[j, k] = offset + doc_guesses[j, k] + # Get the set of feature names. + feats = {FEATURES[f] for f in doc_feat_ids[j] if f != 0} # Now add the analysis, and set the hash. try: - doc.c[j].morph = self.vocab.morphology.add(doc_feat_ids[j]) + doc.c[j].morph = self.vocab.morphology.add(feats) except: print(offsets) print(doc_guesses[j]) @@ -114,7 +116,12 @@ class Morphologizer(Pipe): guesses.append(scores_to_guesses(doc_scores, self.model.softmax.out_sizes)) guesses = self.model.ops.xp.vstack(guesses) scores = self.model.ops.xp.vstack(scores) + if not isinstance(scores, numpy.ndarray): + scores = scores.get() + if not isinstance(guesses, numpy.ndarray): + guesses = guesses.get() cdef int idx = 0 + # Do this on CPU, as we can't vectorize easily. target = numpy.zeros(scores.shape, dtype='f') field_sizes = self.model.softmax.out_sizes for gold in golds: @@ -134,7 +141,8 @@ class Morphologizer(Pipe): target[idx, col_offset] = 1. col_offset += field_size idx += 1 - target = self.model.ops.xp.array(target, dtype='f') + target = self.model.ops.asarray(target, dtype='f') + scores = self.model.ops.asarray(scores, dtype='f') d_scores = scores - target loss = (d_scores**2).sum() d_scores = self.model.ops.unflatten(d_scores, [len(d) for d in docs]) From 09b26f5e2e72923fbc4b8d14dbdb0a51d0793931 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 18:58:26 +0100 Subject: [PATCH 076/287] Fix compile error --- spacy/morphology.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 2f3e8d1fa..cead0756b 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -111,7 +111,7 @@ def get_field_size(field): def get_field_offset(field): - return FIELD_OFFSETS[FIELDS[]]]]field] + return FIELD_OFFSETS[FIELDS[field]] cdef class Morphology: From 49cf002ac440cd07c3d41ba5f318a5441fa60394 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 18:59:25 +0100 Subject: [PATCH 077/287] Add missing import --- spacy/pipeline/morphologizer.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index 223bb6ec5..7d0ad42cf 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -17,6 +17,7 @@ from ..tokens.doc cimport Doc from ..vocab cimport Vocab from ..morphology cimport Morphology from ..morphology import get_field_size, get_field_offset, parse_feature, FIELDS +from ..morphology import FEATURES class Morphologizer(Pipe): From c91577db028e3343e7280f0614f7bd89451f93f0 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 19:03:17 +0100 Subject: [PATCH 078/287] Add set_morphology cfg option for Tagger --- spacy/pipeline/pipes.pyx | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/spacy/pipeline/pipes.pyx b/spacy/pipeline/pipes.pyx index b3c3db04d..237d36a12 100644 --- a/spacy/pipeline/pipes.pyx +++ b/spacy/pipeline/pipes.pyx @@ -357,6 +357,14 @@ class Tagger(Pipe): self.cfg = OrderedDict(sorted(cfg.items())) self.cfg.setdefault("cnn_maxout_pieces", 2) + @property + def set_morphology(self): + return self.cfg.get("set_morphology", True) + + @property.setter + def set_morphology(self, value): + return self.cfg["set_morphology"] = True + @property def labels(self): return tuple(self.vocab.morphology.tag_names) @@ -410,12 +418,13 @@ class Tagger(Pipe): doc_tag_ids = doc_tag_ids.get() for j, tag_id in enumerate(doc_tag_ids): # Don't clobber preset POS tags - if doc.c[j].tag == 0 and doc.c[j].pos == 0: - # Don't clobber preset lemmas - lemma = doc.c[j].lemma - vocab.morphology.assign_tag_id(&doc.c[j], tag_id) - if lemma != 0 and lemma != doc.c[j].lex.orth: - doc.c[j].lemma = lemma + if doc.c[j].tag == 0: + if doc.c[j].pos == 0 and self.set_morphology: + # Don't clobber preset lemmas + lemma = doc.c[j].lemma + vocab.morphology.assign_tag_id(&doc.c[j], tag_id) + if lemma != 0 and lemma != doc.c[j].lex.orth: + doc.c[j].lemma = lemma idx += 1 if tensors is not None and len(tensors): if isinstance(doc.tensor, numpy.ndarray) \ From 27886d626f37eddca95a84b3f05138bfa3a56c8f Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 19:03:31 +0100 Subject: [PATCH 079/287] Dont set morphology in Tagger for ud_train --- spacy/cli/ud/ud_train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spacy/cli/ud/ud_train.py b/spacy/cli/ud/ud_train.py index 44ecababe..f172c8d78 100644 --- a/spacy/cli/ud/ud_train.py +++ b/spacy/cli/ud/ud_train.py @@ -342,9 +342,10 @@ def load_nlp(corpus, config, vectors=None): def initialize_pipeline(nlp, docs, golds, config, device): - nlp.add_pipe(nlp.create_pipe("tagger")) + nlp.add_pipe(nlp.create_pipe("tagger", set_morphology=False)) nlp.add_pipe(nlp.create_pipe("morphologizer")) nlp.add_pipe(nlp.create_pipe("parser")) + assert not nlp.get_pipe("tagger").set_morphology if config.multitask_tag: nlp.parser.add_multitask_objective("tag") if config.multitask_sent: From b27bd42613aa98a9a9625f6b44b98fce94f630f2 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 19:06:02 +0100 Subject: [PATCH 080/287] Fix compile error --- spacy/pipeline/pipes.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/pipeline/pipes.pyx b/spacy/pipeline/pipes.pyx index 237d36a12..1d428731e 100644 --- a/spacy/pipeline/pipes.pyx +++ b/spacy/pipeline/pipes.pyx @@ -363,7 +363,7 @@ class Tagger(Pipe): @property.setter def set_morphology(self, value): - return self.cfg["set_morphology"] = True + self.cfg["set_morphology"] = True @property def labels(self): From afa227e25b64213de7ee465c2a76b5fedc2985ef Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 19:10:01 +0100 Subject: [PATCH 081/287] Fix setter --- spacy/pipeline/pipes.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/pipeline/pipes.pyx b/spacy/pipeline/pipes.pyx index 1d428731e..52cfe0b44 100644 --- a/spacy/pipeline/pipes.pyx +++ b/spacy/pipeline/pipes.pyx @@ -361,9 +361,9 @@ class Tagger(Pipe): def set_morphology(self): return self.cfg.get("set_morphology", True) - @property.setter + @set_morphology.setter def set_morphology(self, value): - self.cfg["set_morphology"] = True + self.cfg["set_morphology"] = value @property def labels(self): From cc2b2dba146d1bb51d6d555041dc89ff172f6485 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 8 Mar 2019 19:16:02 +0100 Subject: [PATCH 082/287] Neaten set_morphology option on Tagger --- spacy/cli/ud/ud_train.py | 1 - spacy/pipeline/pipes.pyx | 11 ++--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/spacy/cli/ud/ud_train.py b/spacy/cli/ud/ud_train.py index f172c8d78..b10b50403 100644 --- a/spacy/cli/ud/ud_train.py +++ b/spacy/cli/ud/ud_train.py @@ -345,7 +345,6 @@ def initialize_pipeline(nlp, docs, golds, config, device): nlp.add_pipe(nlp.create_pipe("tagger", set_morphology=False)) nlp.add_pipe(nlp.create_pipe("morphologizer")) nlp.add_pipe(nlp.create_pipe("parser")) - assert not nlp.get_pipe("tagger").set_morphology if config.multitask_tag: nlp.parser.add_multitask_objective("tag") if config.multitask_sent: diff --git a/spacy/pipeline/pipes.pyx b/spacy/pipeline/pipes.pyx index 52cfe0b44..fa90603bc 100644 --- a/spacy/pipeline/pipes.pyx +++ b/spacy/pipeline/pipes.pyx @@ -357,14 +357,6 @@ class Tagger(Pipe): self.cfg = OrderedDict(sorted(cfg.items())) self.cfg.setdefault("cnn_maxout_pieces", 2) - @property - def set_morphology(self): - return self.cfg.get("set_morphology", True) - - @set_morphology.setter - def set_morphology(self, value): - self.cfg["set_morphology"] = value - @property def labels(self): return tuple(self.vocab.morphology.tag_names) @@ -412,6 +404,7 @@ class Tagger(Pipe): cdef Doc doc cdef int idx = 0 cdef Vocab vocab = self.vocab + assign_morphology = self.cfg.get("set_morphology", True) for i, doc in enumerate(docs): doc_tag_ids = batch_tag_ids[i] if hasattr(doc_tag_ids, "get"): @@ -419,7 +412,7 @@ class Tagger(Pipe): for j, tag_id in enumerate(doc_tag_ids): # Don't clobber preset POS tags if doc.c[j].tag == 0: - if doc.c[j].pos == 0 and self.set_morphology: + if doc.c[j].pos == 0 and assign_morphology: # Don't clobber preset lemmas lemma = doc.c[j].lemma vocab.morphology.assign_tag_id(&doc.c[j], tag_id) From c4df89ab908d0b46e7b5c735d854a54011e5cf30 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 00:20:11 +0000 Subject: [PATCH 083/287] Fixes for morphologizer --- spacy/cli/ud/ud_train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/cli/ud/ud_train.py b/spacy/cli/ud/ud_train.py index f172c8d78..d94d05755 100644 --- a/spacy/cli/ud/ud_train.py +++ b/spacy/cli/ud/ud_train.py @@ -304,7 +304,7 @@ def get_token_conllu(token, i): replacements = {"one": "1", "two": "2", "three": "3"} for feat in features: if not feat.startswith("begin") and not feat.startswith("end"): - key, value = feat.split("_") + key, value = feat.split("_", 1) value = replacements.get(value, value) feat_str.append("%s=%s" % (key, value.title())) if not feat_str: @@ -342,7 +342,7 @@ def load_nlp(corpus, config, vectors=None): def initialize_pipeline(nlp, docs, golds, config, device): - nlp.add_pipe(nlp.create_pipe("tagger", set_morphology=False)) + nlp.add_pipe(nlp.create_pipe("tagger", config={"set_morphology": False})) nlp.add_pipe(nlp.create_pipe("morphologizer")) nlp.add_pipe(nlp.create_pipe("parser")) assert not nlp.get_pipe("tagger").set_morphology From 42bc3ad73bbf4434b799e341903dcea7d9cd8401 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 00:20:29 +0000 Subject: [PATCH 084/287] Fix class mapping for morphologizer --- spacy/morphology.pyx | 10 +++++++--- spacy/pipeline/morphologizer.pyx | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index cead0756b..eab7c9e31 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -1066,16 +1066,20 @@ FEATURES = [ FEATURE_NAMES = {get_string_id(name): name for name in FEATURES} FEATURE_FIELDS = {feature: FIELDS[feature.split('_', 1)[0]] for feature in FEATURES} +FIELD_SIZES = Counter(FEATURE_FIELDS.values()) +for field in FIELD_SIZES: + FIELD_SIZES[field] += 1 for feat_id, name in FEATURE_NAMES.items(): FEATURE_FIELDS[feat_id] = FEATURE_FIELDS[name] - -FIELD_SIZES = Counter(FEATURE_FIELDS.values()) +# Mapping of feature names to their position in total vector FEATURE_OFFSETS = {} +# Mapping of field names to their first position in total vector. FIELD_OFFSETS = {} _seen_fields = Counter() for i, feature in enumerate(FEATURES): field = FEATURE_FIELDS[feature] - FEATURE_OFFSETS[feature] = _seen_fields[field] + # Add 1 for the NIL class, on each field + FEATURE_OFFSETS[feature] = _seen_fields[field] + 1 if _seen_fields[field] == 0: FIELD_OFFSETS[field] = i _seen_fields[field] += 1 diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index 7d0ad42cf..589373f80 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -88,7 +88,7 @@ class Morphologizer(Pipe): if doc_guesses[j, k] == 0: doc_feat_ids[j, k] = 0 else: - doc_feat_ids[j, k] = offset + doc_guesses[j, k] + doc_feat_ids[j, k] = offset + (doc_guesses[j, k]-1) # Get the set of feature names. feats = {FEATURES[f] for f in doc_feat_ids[j] if f != 0} # Now add the analysis, and set the hash. From 4c8730526bd3d538db350b2f913c2510bf583b10 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 00:41:34 +0000 Subject: [PATCH 085/287] Filter bad retokenizations --- spacy/cli/ud/ud_train.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/spacy/cli/ud/ud_train.py b/spacy/cli/ud/ud_train.py index d94d05755..fb2003d31 100644 --- a/spacy/cli/ud/ud_train.py +++ b/spacy/cli/ud/ud_train.py @@ -231,9 +231,14 @@ def write_conllu(docs, file_): for i, doc in enumerate(docs): matches = merger(doc) spans = [doc[start : end + 1] for _, start, end in matches] + seen_tokens = set() with doc.retokenize() as retokenizer: for span in spans: - retokenizer.merge(span) + span_tokens = set(range(span.start, span.end)) + if not span_tokens.intersection(seen_tokens): + retokenizer.merge(span) + seen_tokens.update(span_tokens) + file_.write("# newdoc id = {i}\n".format(i=i)) for j, sent in enumerate(doc.sents): file_.write("# sent_id = {i}.{j}\n".format(i=i, j=j)) From eae384ebb2ac6c1b32f97b09a9118ae286659da2 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 11:49:44 +0000 Subject: [PATCH 086/287] Add POS to morphological fields --- spacy/morphology.pyx | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index eab7c9e31..6b1b7fc27 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -18,6 +18,7 @@ from .errors import Errors cdef enum univ_field_t: + Field_POS Field_Abbr Field_AdpType Field_AdvType @@ -429,6 +430,8 @@ cdef list list_features(const MorphAnalysisC* tag): cdef attr_t get_field(const MorphAnalysisC* tag, int field_id) nogil: field = field_id + if field == Field_POS: + return tag.pos if field == Field_Abbr: return tag.abbr elif field == Field_AdpType: @@ -617,12 +620,14 @@ cdef int set_feature(MorphAnalysisC* tag, else: value_ = 0 prev_value = get_field(tag, field) - if prev_value != 0 and value_ == 0: + if prev_value != 0 and value_ == 0 and field != Field_POS: tag.length -= 1 - elif prev_value == 0 and value_ != 0: + elif prev_value == 0 and value_ != 0 and field != Field_POS: tag.length += 1 if feature == 0: pass + elif field == Field_POS: + tag.pos = get_string_id(FEATURE_NAMES[value_].split('_')[1]) elif field == Field_Abbr: tag.abbr = value_ elif field == Field_AdpType: @@ -714,6 +719,7 @@ cdef int set_feature(MorphAnalysisC* tag, FIELDS = { + 'POS': Field_POS, 'Abbr': Field_Abbr, 'AdpType': Field_AdpType, 'AdvType': Field_AdvType, @@ -760,6 +766,7 @@ FIELDS = { } LOWER_FIELDS = { + 'pos': Field_POS, 'abbr': Field_Abbr, 'adp_type': Field_AdpType, 'adv_type': Field_AdvType, @@ -807,6 +814,26 @@ LOWER_FIELDS = { FEATURES = [ + "POS_ADJ", + "POS_ADP", + "POS_ADV", + "POS_AUX", + "POS_CONJ", + "POS_CCONJ", + "POS_DET", + "POS_INTJ", + "POS_NOUN", + "POS_NUM", + "POS_PART", + "POS_PRON", + "POS_PROPN", + "POS_PUNCT", + "POS_SCONJ", + "POS_SYM", + "POS_VERB", + "POS_X", + "POS_EOL", + "POS_SPACE", "Abbr_yes", "AdpType_circ", "AdpType_comprep", @@ -1064,7 +1091,6 @@ FEATURES = [ ] FEATURE_NAMES = {get_string_id(name): name for name in FEATURES} - FEATURE_FIELDS = {feature: FIELDS[feature.split('_', 1)[0]] for feature in FEATURES} FIELD_SIZES = Counter(FEATURE_FIELDS.values()) for field in FIELD_SIZES: From e1a83d15ed53a0bc9779182bdf1732cd6f722918 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 11:50:08 +0000 Subject: [PATCH 087/287] Add support for character features to Tok2Vec --- spacy/_ml.py | 103 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 9 deletions(-) diff --git a/spacy/_ml.py b/spacy/_ml.py index 68dedc0b3..85d80c3f1 100644 --- a/spacy/_ml.py +++ b/spacy/_ml.py @@ -15,7 +15,7 @@ from thinc.api import uniqued, wrap, noop from thinc.api import with_square_sequences from thinc.linear.linear import LinearModel from thinc.neural.ops import NumpyOps, CupyOps -from thinc.neural.util import get_array_module +from thinc.neural.util import get_array_module, copy_array from thinc.neural.optimizers import Adam from thinc import describe @@ -273,6 +273,9 @@ def Tok2Vec(width, embed_size, **kwargs): pretrained_vectors = kwargs.get("pretrained_vectors", None) cnn_maxout_pieces = kwargs.get("cnn_maxout_pieces", 3) subword_features = kwargs.get("subword_features", True) + char_embed = kwargs.get("char_embed", False) + if char_embed: + subword_features = False conv_depth = kwargs.get("conv_depth", 4) bilstm_depth = kwargs.get("bilstm_depth", 0) cols = [ID, NORM, PREFIX, SUFFIX, SHAPE, ORTH] @@ -295,7 +298,7 @@ def Tok2Vec(width, embed_size, **kwargs): if pretrained_vectors is not None: glove = StaticVectors(pretrained_vectors, width, column=cols.index(ID)) - if subword_features: + if subword_features: embed = uniqued( (glove | norm | prefix | suffix | shape) >> LN(Maxout(width, width * 5, pieces=3)), @@ -310,8 +313,14 @@ def Tok2Vec(width, embed_size, **kwargs): embed = uniqued( (norm | prefix | suffix | shape) >> LN(Maxout(width, width * 4, pieces=3)), - column=cols.index(ORTH), + column=cols.index(ORTH) ) + elif char_embed: + embed = concatenate_lists( + CharacterEmbed(nM=64, nC=8), + FeatureExtracter(cols) >> with_flatten(norm) + ) + reduce_dimensions = LN(Maxout(width, 64*8+width, pieces=cnn_maxout_pieces)) else: embed = norm @@ -319,9 +328,23 @@ def Tok2Vec(width, embed_size, **kwargs): ExtractWindow(nW=1) >> LN(Maxout(width, width * 3, pieces=cnn_maxout_pieces)) ) - tok2vec = FeatureExtracter(cols) >> with_flatten( - embed >> convolution ** conv_depth, pad=conv_depth - ) + if char_embed: + tok2vec = ( + embed + >> with_flatten( + reduce_dimensions + >> convolution ** conv_depth, pad=conv_depth + ) + ) + else: + tok2vec = ( + FeatureExtracter(cols) + >> with_flatten( + embed + >> convolution ** conv_depth, pad=conv_depth + ) + ) + if bilstm_depth >= 1: tok2vec = tok2vec >> PyTorchBiLSTM(width, width, bilstm_depth) # Work around thinc API limitations :(. TODO: Revise in Thinc 7 @@ -537,7 +560,7 @@ def build_morphologizer_model(class_nums, **cfg): else: token_vector_width = util.env_opt("token_vector_width", 128) pretrained_vectors = cfg.get("pretrained_vectors") - subword_features = cfg.get("subword_features", True) + char_embed = cfg.get("char_embed", True) with Model.define_operators({">>": chain, "+": add}): if "tok2vec" in cfg: tok2vec = cfg["tok2vec"] @@ -545,7 +568,7 @@ def build_morphologizer_model(class_nums, **cfg): tok2vec = Tok2Vec( token_vector_width, embed_size, - subword_features=subword_features, + char_embed=char_embed, pretrained_vectors=pretrained_vectors, ) softmax = with_flatten(MultiSoftmax(class_nums, token_vector_width)) @@ -688,7 +711,8 @@ def concatenate_lists(*layers, **kwargs): # pragma: no cover concat = concatenate(*layers) def concatenate_lists_fwd(Xs, drop=0.0): - drop *= drop_factor + if drop is not None: + drop *= drop_factor lengths = ops.asarray([len(X) for X in Xs], dtype="i") flat_y, bp_flat_y = concat.begin_update(Xs, drop=drop) ys = ops.unflatten(flat_y, lengths) @@ -776,3 +800,64 @@ def _replace_word(word, random_words, mask="[MASK]"): return random_words.next() else: return word + + +def _uniform_init(lo, hi): + def wrapped(W, ops): + copy_array(W, ops.xp.random.uniform(lo, hi, W.shape)) + return wrapped + + +@describe.attributes( + nM=Dimension("Vector dimensions"), + nC=Dimension("Number of characters per word"), + vectors=Synapses("Embed matrix", + lambda obj: (obj.nC, obj.nV, obj.nM), + _uniform_init(-0.1, 0.1)), + d_vectors=Gradient("vectors") +) +class CharacterEmbed(Model): + def __init__(self, nM=None, nC=None, **kwargs): + Model.__init__(self, **kwargs) + self.nM = nM + self.nC = nC + + @property + def nO(self): + return self.nM * self.nC + + @property + def nV(self): + return 256 + + def begin_update(self, docs, drop=0.): + if not docs: + return [] + ids = [] + output = [] + weights = self.vectors + # This assists in indexing; it's like looping over this dimension. + # Still consider this weird witch craft...But thanks to Mark Neumann + # for the tip. + nCv = self.ops.xp.arange(self.nC) + for doc in docs: + doc_ids = doc.to_utf8_array(nr_char=self.nC) + doc_vectors = self.ops.allocate((len(doc), self.nC, self.nM)) + # Let's say I have a 2d array of indices, and a 3d table of data. What numpy + # incantation do I chant to get + # output[i, j, k] == data[j, ids[i, j], k]? + doc_vectors[:, nCv] = weights[nCv, doc_ids[:, nCv]] + output.append(doc_vectors.reshape((len(doc), self.nO))) + ids.append(doc_ids) + + def backprop_character_embed(d_vectors, sgd=None): + gradient = self.d_vectors + for doc_ids, d_doc_vectors in zip(ids, d_vectors): + d_doc_vectors = d_doc_vectors.reshape((len(doc_ids), self.nC, self.nM)) + gradient[nCv, doc_ids[:, nCv]] += d_doc_vectors[:, nCv] + if sgd is not None: + sgd(self._mem.weights, self._mem.gradient, key=self.id) + return None + return output, backprop_character_embed + + From bba5f57f91635a254d60c6f517fc7933d21dad6e Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 11:50:27 +0000 Subject: [PATCH 088/287] Add method to export utf8 array to Doc --- spacy/tokens/doc.pyx | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index 1dfcd1687..378921f3c 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -1022,6 +1022,37 @@ cdef class Doc: data["_"][attr] = value return data + def to_utf8_array(self, int nr_char=-1): + """Encode word strings to utf8, and export to a fixed-width array + of characters. Characters are placed into the array in the order: + 0, -1, 1, -2, etc + For example, if the array is sliced array[:, :8], the array will + contain the first 4 characters and last 4 characters of each word --- + with the middle characters clipped out. The value 255 is used as a pad + value. + """ + byte_strings = [token.orth_.encode('utf8') for token in self] + if nr_char == -1: + nr_char = max(len(bs) for bs in byte_strings) + cdef np.ndarray output = numpy.zeros((len(byte_strings), nr_char), dtype='uint8') + output.fill(255) + cdef int i, j, start_idx, end_idx + cdef bytes byte_string + cdef unsigned char utf8_char + for i, byte_string in enumerate(byte_strings): + j = 0 + start_idx = 0 + end_idx = len(byte_string) - 1 + while j < nr_char and start_idx <= end_idx: + output[i, j] = byte_string[start_idx] + start_idx += 1 + j += 1 + if j < nr_char and start_idx <= end_idx: + output[i, j] = byte_string[end_idx] + end_idx -= 1 + j += 1 + return output + cdef int token_by_start(const TokenC* tokens, int length, int start_char) except -2: cdef int i From a6d153b0a0a9dcbefeca906cda37329ca15d7ce2 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 11:50:50 +0000 Subject: [PATCH 089/287] Add UPOS as morphological field in ud_train --- spacy/cli/ud/ud_train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spacy/cli/ud/ud_train.py b/spacy/cli/ud/ud_train.py index 23f5a6820..5dcaa1684 100644 --- a/spacy/cli/ud/ud_train.py +++ b/spacy/cli/ud/ud_train.py @@ -84,6 +84,7 @@ def read_data( sent["words"].append(word) sent["tags"].append(tag) sent["morphology"].append(_parse_morph_string(morph)) + sent["morphology"][-1].add("POS_%s" % pos) sent["heads"].append(head) sent["deps"].append("ROOT" if dep == "root" else dep) sent["spaces"].append(space_after == "_") From f742900f83296362bbca371b9b08785141752809 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 11:51:11 +0000 Subject: [PATCH 090/287] Set pos attribute in morphologizer --- spacy/pipeline/morphologizer.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index 589373f80..9cb384a03 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -94,6 +94,8 @@ class Morphologizer(Pipe): # Now add the analysis, and set the hash. try: doc.c[j].morph = self.vocab.morphology.add(feats) + if doc[j].morph.pos != 0: + doc.c[j].pos = doc[j].morph.pos except: print(offsets) print(doc_guesses[j]) From 41a3016019782d66620fe830bcd13a9f76fbf013 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 20:55:33 +0100 Subject: [PATCH 091/287] Refactor morphologizer class map --- spacy/morphology.pxd | 2 +- spacy/morphology.pyx | 90 +++++++++++++++----------------- spacy/pipeline/morphologizer.pyx | 30 +++++------ spacy/tokens/morphanalysis.pyx | 3 +- 4 files changed, 58 insertions(+), 67 deletions(-) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd index cb708166c..1a3cedf97 100644 --- a/spacy/morphology.pxd +++ b/spacy/morphology.pxd @@ -20,6 +20,7 @@ cdef class Morphology: cdef readonly object tag_names cdef readonly object reverse_index cdef readonly object exc + cdef readonly object _feat_map cdef readonly PreshMapArray _cache cdef readonly int n_tags @@ -36,6 +37,5 @@ cdef class Morphology: cdef int check_feature(const MorphAnalysisC* tag, attr_t feature) nogil cdef attr_t get_field(const MorphAnalysisC* tag, int field) nogil cdef list list_features(const MorphAnalysisC* tag) -cdef int attribute_to_field(unicode attribute) cdef tag_to_json(const MorphAnalysisC* tag) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 6b1b7fc27..fdaa44813 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -93,26 +93,34 @@ def _normalize_props(props): return out -def parse_feature(feature): - field = FEATURE_FIELDS[feature] - offset = FEATURE_OFFSETS[feature] - return (field, offset) +class MorphologyClassMap(object): + def __init__(self, features, fields): + self.features = tuple(features) + self.fields = tuple(fields) + self.id2feat = {get_string_id(name): name for name in features} + self.feat2field = {feature: fields[feature.split('_', 1)[0]] for feature in features} + self.field2feats = {} + self.col2info = [] + self.attr2field = dict(LOWER_FIELDS.items()) + for feature in features: + field = self.feat2field[feature] + if field not in self.field2feats: + self.col2info.append((field, 0, 'NIL')) + self.field2feats.setdefault(field, []).append(feature) + self.col2info.append((field, len(self.field2feats[field]), feature)) + @property + def field_sizes(self): + return [len(self.field2feats[field]) for field in self.fields] -cdef int attribute_to_field(unicode attribute_name): - return LOWER_FIELDS[attribute_name] - - -def get_field_id(feature): - return FEATURE_FIELDS[feature] - - -def get_field_size(field): - return FIELD_SIZES[FIELDS[field]] - - -def get_field_offset(field): - return FIELD_OFFSETS[FIELDS[field]] + def get_field_offset(self, field): + n = 0 + for f in self.fields: + if f == field: + return n + n += len(self.field2feats[f]) + else: + return -1 cdef class Morphology: @@ -139,9 +147,11 @@ cdef class Morphology: self.lemmatizer = lemmatizer self.n_tags = len(tag_map) self.reverse_index = {} + self._feat_map = MorphologyClassMap(FEATURES, FIELDS) for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())): attrs = _normalize_props(attrs) - self.add({FEATURE_NAMES[feat] for feat in attrs if feat in FEATURE_NAMES}) + self.add({self._feat_map.id2feat[feat] for feat in attrs + if feat in self._feat_map.id2feat}) self.tag_map[tag_str] = dict(attrs) self.reverse_index[self.strings.add(tag_str)] = i @@ -167,7 +177,7 @@ cdef class Morphology: features = intify_features(features) cdef attr_t feature for feature in features: - if feature != 0 and feature not in FEATURE_NAMES: + if feature != 0 and feature not in self._feat_map.id2feat: raise KeyError("Unknown feature: %s" % self.strings[feature]) cdef MorphAnalysisC tag tag = create_rich_tag(features) @@ -187,7 +197,7 @@ cdef class Morphology: features = intify_features(features) cdef attr_t feature for feature in features: - field = get_field_id(feature) + field = FEATURE_FIELDS[FEATURE_NAMES[feature]] set_feature(&tag, field, feature, 1) morph = self.insert(tag) return morph @@ -224,7 +234,8 @@ cdef class Morphology: """ attrs = dict(attrs) attrs = _normalize_props(attrs) - self.add({FEATURE_NAMES[feat] for feat in attrs if feat in FEATURE_NAMES}) + self.add({self._feat_map.id2feat[feat] for feat in attrs + if feat in self._feat_map.id2feat}) attrs = intify_attrs(attrs, self.strings, _do_deprecated=True) self.exc[(tag_str, self.strings.add(orth_str))] = attrs @@ -313,6 +324,10 @@ cdef class Morphology: def from_disk(self, path): raise NotImplementedError + @classmethod + def create_class_map(cls): + return MorphologyClassMap(FEATURES, FIELDS) + cpdef univ_pos_t get_int_tag(pos_): return 0 @@ -324,17 +339,12 @@ cdef hash_t hash_tag(MorphAnalysisC tag) nogil: return mrmr.hash64(&tag, sizeof(tag), 0) -def get_feature_field(feature): - cdef attr_t key = get_string_id(feature) - return FEATURE_FIELDS[feature] - - cdef MorphAnalysisC create_rich_tag(features) except *: cdef MorphAnalysisC tag cdef attr_t feature memset(&tag, 0, sizeof(tag)) for feature in features: - field = get_field_id(feature) + field = FEATURE_FIELDS[FEATURE_NAMES[feature]] set_feature(&tag, field, feature, 1) return tag @@ -519,8 +529,7 @@ cdef attr_t get_field(const MorphAnalysisC* tag, int field_id) nogil: elif field == Field_VerbType: return tag.verb_type else: - raise ValueError("Unknown feature: %s (%d)" % (FEATURE_NAMES.get(feature), feature)) - + raise ValueError("Unknown field: (%d)" % field_id) cdef int check_feature(const MorphAnalysisC* tag, attr_t feature) nogil: @@ -1090,22 +1099,5 @@ FEATURES = [ "Voice_int", ] -FEATURE_NAMES = {get_string_id(name): name for name in FEATURES} -FEATURE_FIELDS = {feature: FIELDS[feature.split('_', 1)[0]] for feature in FEATURES} -FIELD_SIZES = Counter(FEATURE_FIELDS.values()) -for field in FIELD_SIZES: - FIELD_SIZES[field] += 1 -for feat_id, name in FEATURE_NAMES.items(): - FEATURE_FIELDS[feat_id] = FEATURE_FIELDS[name] -# Mapping of feature names to their position in total vector -FEATURE_OFFSETS = {} -# Mapping of field names to their first position in total vector. -FIELD_OFFSETS = {} -_seen_fields = Counter() -for i, feature in enumerate(FEATURES): - field = FEATURE_FIELDS[feature] - # Add 1 for the NIL class, on each field - FEATURE_OFFSETS[feature] = _seen_fields[field] + 1 - if _seen_fields[field] == 0: - FIELD_OFFSETS[field] = i - _seen_fields[field] += 1 +FEATURE_NAMES = {get_string_id(f): f for f in FEATURES} +FEATURE_FIELDS = {f: FIELDS[f.split('_', 1)[0]] for f in FEATURES} diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index 9cb384a03..d3d850da0 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -16,26 +16,24 @@ from ..compat import basestring_ from ..tokens.doc cimport Doc from ..vocab cimport Vocab from ..morphology cimport Morphology -from ..morphology import get_field_size, get_field_offset, parse_feature, FIELDS -from ..morphology import FEATURES class Morphologizer(Pipe): name = 'morphologizer' @classmethod - def Model(cls, attr_nums=None, **cfg): + def Model(cls, **cfg): if cfg.get('pretrained_dims') and not cfg.get('pretrained_vectors'): raise ValueError(TempErrors.T008) - if attr_nums is None: - attr_nums = [get_field_size(name) for name in FIELDS] - return build_morphologizer_model(attr_nums, **cfg) + class_map = Morphology.create_class_map() + return build_morphologizer_model(class_map.field_sizes, **cfg) def __init__(self, vocab, model=True, **cfg): self.vocab = vocab self.model = model self.cfg = OrderedDict(sorted(cfg.items())) self.cfg.setdefault('cnn_maxout_pieces', 2) + self._class_map = self.vocab.morphology.create_class_map() @property def labels(self): @@ -76,13 +74,13 @@ class Morphologizer(Pipe): docs = [docs] cdef Doc doc cdef Vocab vocab = self.vocab - field_names = list(FIELDS) - offsets = [get_field_offset(field) for field in field_names] + offsets = [self._class_map.get_field_offset(field) + for field in self._class_map.fields] for i, doc in enumerate(docs): doc_scores = batch_scores[i] doc_guesses = scores_to_guesses(doc_scores, self.model.softmax.out_sizes) # Convert the neuron indices into feature IDs. - doc_feat_ids = numpy.zeros((len(doc), len(field_names)), dtype='i') + doc_feat_ids = numpy.zeros((len(doc), len(self._class_map.fields)), dtype='i') for j in range(len(doc)): for k, offset in enumerate(offsets): if doc_guesses[j, k] == 0: @@ -90,7 +88,8 @@ class Morphologizer(Pipe): else: doc_feat_ids[j, k] = offset + (doc_guesses[j, k]-1) # Get the set of feature names. - feats = {FEATURES[f] for f in doc_feat_ids[j] if f != 0} + feats = {self._class_map.col2info[f][2] for f in doc_feat_ids[j] + if f != 0} # Now add the analysis, and set the hash. try: doc.c[j].morph = self.vocab.morphology.add(feats) @@ -132,14 +131,15 @@ class Morphologizer(Pipe): if features is None: target[idx] = scores[idx] else: - by_field = {} + gold_fields = {} for feature in features: - field, column = parse_feature(feature) - by_field[field] = column + field = self.get_field(feature) + column = self.get_column(feature) + gold_fields[field] = column col_offset = 0 for field, field_size in enumerate(field_sizes): - if field in by_field: - target[idx, col_offset + by_field[field]] = 1. + if field in gold_fields: + target[idx, col_offset + gold_fields[field]] = 1. else: target[idx, col_offset] = 1. col_offset += field_size diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx index b727e2c3f..17b11d84f 100644 --- a/spacy/tokens/morphanalysis.pyx +++ b/spacy/tokens/morphanalysis.pyx @@ -3,7 +3,6 @@ from libc.string cimport memset from ..vocab cimport Vocab from ..typedefs cimport hash_t, attr_t from ..morphology cimport list_features, check_feature, get_field, tag_to_json -from ..morphology cimport attribute_to_field from ..strings import get_string_id @@ -53,7 +52,7 @@ cdef class MorphAnalysis: return self.key def get(self, unicode field): - cdef int field_id = attribute_to_field(field) + cdef int field_id = self.vocab.morphology._feat_map.attr2field[field] return self.vocab.strings[get_field(&self.c, field_id)] def to_json(self): From 0f120824657349f6c5b580d362912eb9cc971bd8 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 9 Mar 2019 22:54:59 +0000 Subject: [PATCH 092/287] Refactor morphologizer --- spacy/_ml.py | 6 ++-- spacy/morphology.pyx | 50 +++++++++++++++++++++----------- spacy/pipeline/morphologizer.pyx | 41 ++++++++++++-------------- spacy/pipeline/pipes.pyx | 2 ++ 4 files changed, 58 insertions(+), 41 deletions(-) diff --git a/spacy/_ml.py b/spacy/_ml.py index 85d80c3f1..2e4df843c 100644 --- a/spacy/_ml.py +++ b/spacy/_ml.py @@ -561,7 +561,7 @@ def build_morphologizer_model(class_nums, **cfg): token_vector_width = util.env_opt("token_vector_width", 128) pretrained_vectors = cfg.get("pretrained_vectors") char_embed = cfg.get("char_embed", True) - with Model.define_operators({">>": chain, "+": add}): + with Model.define_operators({">>": chain, "+": add, "**": clone}): if "tok2vec" in cfg: tok2vec = cfg["tok2vec"] else: @@ -571,7 +571,9 @@ def build_morphologizer_model(class_nums, **cfg): char_embed=char_embed, pretrained_vectors=pretrained_vectors, ) - softmax = with_flatten(MultiSoftmax(class_nums, token_vector_width)) + softmax = with_flatten( + MultiSoftmax(class_nums, token_vector_width) + ) softmax.out_sizes = class_nums model = tok2vec >> softmax model.nI = None diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index fdaa44813..6c9ecebdc 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -46,8 +46,8 @@ cdef enum univ_field_t: Field_PartForm Field_PartType Field_Person - Field_Polite Field_Polarity + Field_Polite Field_Poss Field_Prefix Field_PrepCase @@ -60,8 +60,8 @@ cdef enum univ_field_t: Field_Tense Field_Typo Field_VerbForm - Field_Voice Field_VerbType + Field_Voice def _normalize_props(props): @@ -94,20 +94,36 @@ def _normalize_props(props): class MorphologyClassMap(object): - def __init__(self, features, fields): + def __init__(self, features): self.features = tuple(features) - self.fields = tuple(fields) + self.fields = [] + self.feat2field = {} + seen_fields = set() + for feature in features: + field = feature.split("_", 1)[0] + if field not in seen_fields: + self.fields.append(field) + seen_fields.add(field) + self.feat2field[feature] = FIELDS[field] self.id2feat = {get_string_id(name): name for name in features} - self.feat2field = {feature: fields[feature.split('_', 1)[0]] for feature in features} - self.field2feats = {} + self.field2feats = {"POS": []} self.col2info = [] self.attr2field = dict(LOWER_FIELDS.items()) + self.feat2offset = {} + self.field2col = {} + self.field2id = dict(FIELDS.items()) + self.fieldid2field = {field_id: field for field, field_id in FIELDS.items()} for feature in features: - field = self.feat2field[feature] - if field not in self.field2feats: - self.col2info.append((field, 0, 'NIL')) - self.field2feats.setdefault(field, []).append(feature) - self.col2info.append((field, len(self.field2feats[field]), feature)) + field = self.fields[self.feat2field[feature]] + if field not in self.field2col: + self.field2col[field] = len(self.col2info) + if field != "POS" and field not in self.field2feats: + self.col2info.append((field, 0, "NIL")) + self.field2feats.setdefault(field, ["NIL"]) + offset = len(self.field2feats[field]) + self.field2feats[field].append(feature) + self.col2info.append((field, offset, feature)) + self.feat2offset[feature] = offset @property def field_sizes(self): @@ -147,7 +163,7 @@ cdef class Morphology: self.lemmatizer = lemmatizer self.n_tags = len(tag_map) self.reverse_index = {} - self._feat_map = MorphologyClassMap(FEATURES, FIELDS) + self._feat_map = MorphologyClassMap(FEATURES) for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())): attrs = _normalize_props(attrs) self.add({self._feat_map.id2feat[feat] for feat in attrs @@ -326,7 +342,7 @@ cdef class Morphology: @classmethod def create_class_map(cls): - return MorphologyClassMap(FEATURES, FIELDS) + return MorphologyClassMap(FEATURES) cpdef univ_pos_t get_int_tag(pos_): @@ -770,8 +786,8 @@ FIELDS = { 'Tense': Field_Tense, 'Typo': Field_Typo, 'VerbForm': Field_VerbForm, + 'VerbType': Field_VerbType, 'Voice': Field_Voice, - 'VerbType': Field_VerbType } LOWER_FIELDS = { @@ -803,8 +819,8 @@ LOWER_FIELDS = { 'part_form': Field_PartForm, 'part_type': Field_PartType, 'person': Field_Person, - 'polite': Field_Polite, 'polarity': Field_Polarity, + 'polite': Field_Polite, 'poss': Field_Poss, 'prefix': Field_Prefix, 'prep_case': Field_PrepCase, @@ -817,8 +833,8 @@ LOWER_FIELDS = { 'tense': Field_Tense, 'typo': Field_Typo, 'verb_form': Field_VerbForm, + 'verb_type': Field_VerbType, 'voice': Field_Voice, - 'verb_type': Field_VerbType } @@ -849,7 +865,7 @@ FEATURES = [ "AdpType_prep", "AdpType_post", "AdpType_voc", - "AdvType_adadj," + "AdvType_adadj", "AdvType_cau", "AdvType_deg", "AdvType_ex", diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx index d3d850da0..b14e2bec7 100644 --- a/spacy/pipeline/morphologizer.pyx +++ b/spacy/pipeline/morphologizer.pyx @@ -86,20 +86,15 @@ class Morphologizer(Pipe): if doc_guesses[j, k] == 0: doc_feat_ids[j, k] = 0 else: - doc_feat_ids[j, k] = offset + (doc_guesses[j, k]-1) + doc_feat_ids[j, k] = offset + doc_guesses[j, k] # Get the set of feature names. - feats = {self._class_map.col2info[f][2] for f in doc_feat_ids[j] - if f != 0} + feats = {self._class_map.col2info[f][2] for f in doc_feat_ids[j]} + if "NIL" in feats: + feats.remove("NIL") # Now add the analysis, and set the hash. - try: - doc.c[j].morph = self.vocab.morphology.add(feats) - if doc[j].morph.pos != 0: - doc.c[j].pos = doc[j].morph.pos - except: - print(offsets) - print(doc_guesses[j]) - print(doc_feat_ids[j]) - raise + doc.c[j].morph = self.vocab.morphology.add(feats) + if doc[j].morph.pos != 0: + doc.c[j].pos = doc[j].morph.pos def update(self, docs, golds, drop=0., sgd=None, losses=None): if losses is not None and self.name not in losses: @@ -126,23 +121,25 @@ class Morphologizer(Pipe): # Do this on CPU, as we can't vectorize easily. target = numpy.zeros(scores.shape, dtype='f') field_sizes = self.model.softmax.out_sizes - for gold in golds: - for features in gold.morphology: + for doc, gold in zip(docs, golds): + for t, features in enumerate(gold.morphology): if features is None: target[idx] = scores[idx] else: gold_fields = {} for feature in features: - field = self.get_field(feature) - column = self.get_column(feature) - gold_fields[field] = column - col_offset = 0 - for field, field_size in enumerate(field_sizes): - if field in gold_fields: - target[idx, col_offset + gold_fields[field]] = 1. + field = self._class_map.feat2field[feature] + gold_fields[field] = self._class_map.feat2offset[feature] + for field in self._class_map.fields: + field_id = self._class_map.field2id[field] + col_offset = self._class_map.field2col[field] + if field_id in gold_fields: + target[idx, col_offset + gold_fields[field_id]] = 1. else: target[idx, col_offset] = 1. - col_offset += field_size + #print(doc[t]) + #for col, info in enumerate(self._class_map.col2info): + # print(col, info, scores[idx, col], target[idx, col]) idx += 1 target = self.model.ops.asarray(target, dtype='f') scores = self.model.ops.asarray(scores, dtype='f') diff --git a/spacy/pipeline/pipes.pyx b/spacy/pipeline/pipes.pyx index fa90603bc..450497f3b 100644 --- a/spacy/pipeline/pipes.pyx +++ b/spacy/pipeline/pipes.pyx @@ -418,6 +418,8 @@ class Tagger(Pipe): vocab.morphology.assign_tag_id(&doc.c[j], tag_id) if lemma != 0 and lemma != doc.c[j].lex.orth: doc.c[j].lemma = lemma + else: + doc.c[j].tag = self.vocab.strings[self.labels[tag_id]] idx += 1 if tensors is not None and len(tensors): if isinstance(doc.tensor, numpy.ndarray) \ From 5431c47b915d821e231d9f5d1df4041fcd57fadb Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 10 Mar 2019 00:59:51 +0000 Subject: [PATCH 093/287] Refactor morphology slightly --- spacy/morphology.pyx | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 6c9ecebdc..c592ee674 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -130,13 +130,7 @@ class MorphologyClassMap(object): return [len(self.field2feats[field]) for field in self.fields] def get_field_offset(self, field): - n = 0 - for f in self.fields: - if f == field: - return n - n += len(self.field2feats[f]) - else: - return -1 + return self.field2col[field] cdef class Morphology: From 08e8267a595aea045a2c8bd0eaa9ab16ffb03e12 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 13:50:00 +0200 Subject: [PATCH 094/287] Set version to 2.2.0.dev0 --- spacy/about.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/about.py b/spacy/about.py index 9587c9071..bd500ed6c 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -4,13 +4,13 @@ # fmt: off __title__ = "spacy" -__version__ = "2.1.8" +__version__ = "2.2.0.dev0" __summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython" __uri__ = "https://spacy.io" __author__ = "Explosion AI" __email__ = "contact@explosion.ai" __license__ = "MIT" -__release__ = True +__release__ = False __download_url__ = "https://github.com/explosion/spacy-models/releases/download" __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json" From f9075a6fd1dcb46611d5b25f6cd569ddd1bd9256 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 13:50:47 +0200 Subject: [PATCH 095/287] Update to blis 0.4 and thinc 7.1 --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index a6d721e96..865288a86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # Our libraries cymem>=2.0.2,<2.1.0 preshed>=2.0.1,<2.1.0 -thinc>=7.0.8,<7.1.0 -blis>=0.2.2,<0.3.0 +thinc>=7.1.0,<7.2.0 +blis>=0.4.0,<0.5.0 murmurhash>=0.28.0,<1.1.0 wasabi>=0.2.0,<1.1.0 srsly>=0.1.0,<1.1.0 diff --git a/setup.py b/setup.py index 1d2aa084b..783433611 100755 --- a/setup.py +++ b/setup.py @@ -247,7 +247,7 @@ def setup_package(): "cymem>=2.0.2,<2.1.0", "preshed>=2.0.1,<2.1.0", "thinc>=7.0.8,<7.1.0", - "blis>=0.2.2,<0.3.0", + "blis>=0.4.0,<0.5.0", "plac<1.0.0,>=0.9.6", "requests>=2.13.0,<3.0.0", "wasabi>=0.2.0,<1.1.0", From b8edc8dffb7fb5651973181a46ced5da5ec1a7cf Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 14:54:09 +0200 Subject: [PATCH 096/287] Require thinc 7.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 783433611..1d13bfd10 100755 --- a/setup.py +++ b/setup.py @@ -246,7 +246,7 @@ def setup_package(): "murmurhash>=0.28.0,<1.1.0", "cymem>=2.0.2,<2.1.0", "preshed>=2.0.1,<2.1.0", - "thinc>=7.0.8,<7.1.0", + "thinc>=7.1.0,<7.2.0", "blis>=0.4.0,<0.5.0", "plac<1.0.0,>=0.9.6", "requests>=2.13.0,<3.0.0", From 7bc68913e38c54c49d867df8df57738987e32769 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 14:54:19 +0200 Subject: [PATCH 097/287] Improve pex building in Makefile --- Makefile | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 2834096b7..0f5c31ca6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,17 @@ SHELL := /bin/bash sha = $(shell "git" "rev-parse" "--short" "HEAD") +version = $(shell "bin/get-version.sh") +wheel = spacy-$(version)-cp36-cp36m-linux_x86_64.whl -dist/spacy.pex : spacy/*.py* spacy/*/*.py* +dist/spacy.pex : dist/spacy-$(sha).pex + cp dist/spacy-$(sha).pex dist/spacy.pex + chmod a+rx dist/spacy.pex + +dist/spacy-$(sha).pex : dist/$(wheel) + env3.6/bin/python -m pip install pex==1.5.3 + env3.6/bin/pex pytest dist/$(wheel) -e spacy -o dist/spacy-$(sha).pex + +dist/$(wheel) : setup.py spacy/*.py* spacy/*/*.py* python3.6 -m venv env3.6 source env3.6/bin/activate env3.6/bin/pip install wheel @@ -9,10 +19,6 @@ dist/spacy.pex : spacy/*.py* spacy/*/*.py* env3.6/bin/python setup.py build_ext --inplace env3.6/bin/python setup.py sdist env3.6/bin/python setup.py bdist_wheel - env3.6/bin/python -m pip install pex==1.5.3 - env3.6/bin/pex pytest dist/*.whl -e spacy -o dist/spacy-$(sha).pex - cp dist/spacy-$(sha).pex dist/spacy.pex - chmod a+rx dist/spacy.pex .PHONY : clean From 9b5c94fed9fecfb9fd078a9a4616ca1972b0a405 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 15:12:36 +0200 Subject: [PATCH 098/287] Add get-version script --- bin/get-version.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 bin/get-version.sh diff --git a/bin/get-version.sh b/bin/get-version.sh new file mode 100755 index 000000000..5a12ddd7a --- /dev/null +++ b/bin/get-version.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +version=$(grep "__version__ = " spacy/about.py) +version=${version/__version__ = } +version=${version/\'/} +version=${version/\'/} +version=${version/\"/} +version=${version/\"/} + +echo $version From 22250cf6b7cd3a92290f54256078606c1e5db5e5 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 21:54:26 +0200 Subject: [PATCH 099/287] Make regression test less sensitive to tag-map stuff --- spacy/tests/regression/test_issue3001-3500.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spacy/tests/regression/test_issue3001-3500.py b/spacy/tests/regression/test_issue3001-3500.py index 3b0c2f1ed..c430678d3 100644 --- a/spacy/tests/regression/test_issue3001-3500.py +++ b/spacy/tests/regression/test_issue3001-3500.py @@ -30,20 +30,20 @@ def test_issue3002(): def test_issue3009(en_vocab): """Test problem with matcher quantifiers""" patterns = [ - [{"LEMMA": "have"}, {"LOWER": "to"}, {"LOWER": "do"}, {"POS": "ADP"}], + [{"LEMMA": "have"}, {"LOWER": "to"}, {"LOWER": "do"}, {"TAG": "IN"}], [ {"LEMMA": "have"}, {"IS_ASCII": True, "IS_PUNCT": False, "OP": "*"}, {"LOWER": "to"}, {"LOWER": "do"}, - {"POS": "ADP"}, + {"TAG": "IN"}, ], [ {"LEMMA": "have"}, {"IS_ASCII": True, "IS_PUNCT": False, "OP": "?"}, {"LOWER": "to"}, {"LOWER": "do"}, - {"POS": "ADP"}, + {"TAG": "IN"}, ], ] words = ["also", "has", "to", "do", "with"] From 095c63c6b8e3fc8d1da2c914a996ddecba19864f Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 21:56:47 +0200 Subject: [PATCH 100/287] Avoid making prepositions get the tag SCONJ --- spacy/lang/en/morph_rules.py | 48 +++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/spacy/lang/en/morph_rules.py b/spacy/lang/en/morph_rules.py index 198182ff0..b00534cc5 100644 --- a/spacy/lang/en/morph_rules.py +++ b/spacy/lang/en/morph_rules.py @@ -3,55 +3,59 @@ from __future__ import unicode_literals from ...symbols import LEMMA, PRON_LEMMA +# Several entries here look pretty suspicious. These will get the POS SCONJ +# given the tag IN, when an adpositional reading seems much more likely for +# a lot of these prepositions. I'm not sure what I was running in 04395ffa4 +# when I did this? It doesn't seem right. _subordinating_conjunctions = [ "that", "if", "as", "because", - "of", - "for", - "before", - "in", + #"of", + #"for", + #"before", + #"in", "while", - "after", + #"after", "since", "like", - "with", + #"with", "so", - "to", - "by", - "on", - "about", + #"to", + #"by", + #"on", + #"about", "than", "whether", "although", - "from", + #"from", "though", - "until", + #"until", "unless", "once", - "without", - "at", - "into", + #"without", + #"at", + #"into", "cause", - "over", + #"over", "upon", "till", "whereas", - "beyond", + #"beyond", "whilst", "except", "despite", "wether", - "then", + #"then", "but", "becuse", "whie", - "below", - "against", + #"below", + #"against", "it", "w/out", - "toward", + #"toward", "albeit", "save", "besides", @@ -63,7 +67,7 @@ _subordinating_conjunctions = [ "out", "near", "seince", - "towards", + #"towards", "tho", "sice", "will", From 188a1cf297860197dc0253165d0969c7520acf66 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 21:57:02 +0200 Subject: [PATCH 101/287] Fix morphology for | features --- spacy/morphology.pyx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 2fd81bf8e..ccfb214bc 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -71,6 +71,10 @@ def _normalize_props(props): for key in FIELDS: if key in props: value = str(props[key]).lower() + # We don't have support for disjunctive int|rel features, so + # just take the first one :( + if "|" in value: + value = value.split("|")[0] attr = '%s_%s' % (key, value) if attr in FEATURES: props.pop(key) From 71c0321ecf03ecbd0566295980472f7f70b037bc Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 22:03:37 +0200 Subject: [PATCH 102/287] Fix test --- spacy/tests/test_displacy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/tests/test_displacy.py b/spacy/tests/test_displacy.py index 5e99d261a..2d1f1bd8f 100644 --- a/spacy/tests/test_displacy.py +++ b/spacy/tests/test_displacy.py @@ -32,7 +32,7 @@ def test_displacy_parse_deps(en_vocab): assert isinstance(deps, dict) assert deps["words"] == [ {"text": "This", "tag": "DET"}, - {"text": "is", "tag": "VERB"}, + {"text": "is", "tag": "AUX"}, {"text": "a", "tag": "DET"}, {"text": "sentence", "tag": "NOUN"}, ] From af7fad2c6d75ea3ce755058c1acaee3a827d77d2 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 25 Aug 2019 22:05:47 +0200 Subject: [PATCH 103/287] Set version to v2.2.0.dev1 --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index bd500ed6c..6a9784c54 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -4,7 +4,7 @@ # fmt: off __title__ = "spacy" -__version__ = "2.2.0.dev0" +__version__ = "2.2.0.dev1" __summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython" __uri__ = "https://spacy.io" __author__ = "Explosion AI" From aae05ff16bcf12e4f60bb5936c3bf5728a96b78c Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 28 Aug 2019 09:14:20 +0200 Subject: [PATCH 104/287] Add train_docs() option to add orth variants Filtering by orth and tag, create variants of training docs with alternate orth variants, e.g., unicode quotes, dashes, and ellipses. The variants can be single tokens (dashes) or paired tokens (quotes) with left and right versions. Currently restricted to only add variants to training documents without raw text provided, where only gold.words needs to be modified. --- spacy/gold.pyx | 61 +++++++++++++++++++++++++++++++++++++++++------ spacy/language.py | 2 ++ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/spacy/gold.pyx b/spacy/gold.pyx index f6ec8d3fa..1cd49814a 100644 --- a/spacy/gold.pyx +++ b/spacy/gold.pyx @@ -7,6 +7,7 @@ import random import numpy import tempfile import shutil +import itertools from pathlib import Path import srsly @@ -206,13 +207,14 @@ class GoldCorpus(object): return n def train_docs(self, nlp, gold_preproc=False, max_length=None, - noise_level=0.0): + noise_level=0.0, orth_variant_level=0.0): locs = list((self.tmp_dir / 'train').iterdir()) random.shuffle(locs) train_tuples = self.read_tuples(locs, limit=self.limit) gold_docs = self.iter_gold_docs(nlp, train_tuples, gold_preproc, max_length=max_length, noise_level=noise_level, + orth_variant_level=orth_variant_level, make_projective=True) yield from gold_docs @@ -226,27 +228,31 @@ class GoldCorpus(object): @classmethod def iter_gold_docs(cls, nlp, tuples, gold_preproc, max_length=None, - noise_level=0.0, make_projective=False): + noise_level=0.0, orth_variant_level=0.0, make_projective=False): for raw_text, paragraph_tuples in tuples: if gold_preproc: raw_text = None else: paragraph_tuples = merge_sents(paragraph_tuples) - docs = cls._make_docs(nlp, raw_text, paragraph_tuples, gold_preproc, - noise_level=noise_level) + docs, paragraph_tuples = cls._make_docs(nlp, raw_text, + paragraph_tuples, gold_preproc, noise_level=noise_level, + orth_variant_level=orth_variant_level) golds = cls._make_golds(docs, paragraph_tuples, make_projective) for doc, gold in zip(docs, golds): if (not max_length) or len(doc) < max_length: yield doc, gold @classmethod - def _make_docs(cls, nlp, raw_text, paragraph_tuples, gold_preproc, noise_level=0.0): + def _make_docs(cls, nlp, raw_text, paragraph_tuples, gold_preproc, noise_level=0.0, orth_variant_level=0.0): if raw_text is not None: raw_text = add_noise(raw_text, noise_level) - return [nlp.make_doc(raw_text)] + return [nlp.make_doc(raw_text)], paragraph_tuples else: + docs = [] + raw_text, paragraph_tuples = make_orth_variants(nlp, None, paragraph_tuples, orth_variant_level) return [Doc(nlp.vocab, words=add_noise(sent_tuples[1], noise_level)) - for (sent_tuples, brackets) in paragraph_tuples] + for (sent_tuples, brackets) in paragraph_tuples], paragraph_tuples + @classmethod def _make_golds(cls, docs, paragraph_tuples, make_projective): @@ -263,6 +269,47 @@ class GoldCorpus(object): in zip(docs, paragraph_tuples)] +def make_orth_variants(nlp, raw, paragraph_tuples, orth_variant_level=0.0): + if random.random() >= orth_variant_level: + return raw, paragraph_tuples + variant_paragraph_tuples = [] + for sent_tuples, brackets in paragraph_tuples: + ids, words, tags, heads, labels, ner = sent_tuples + # single variants + ndsv = nlp.Defaults.single_orth_variants + punct_choices = [random.choice(x["variants"]) for x in ndsv] + for word_idx in range(len(words)): + for punct_idx in range(len(ndsv)): + if tags[word_idx] in ndsv[punct_idx]["tags"] \ + and words[word_idx] in ndsv[punct_idx]["variants"]: + words[word_idx] = punct_choices[punct_idx] + # paired variants + ndpv = nlp.Defaults.paired_orth_variants + punct_choices = [random.choice(x["variants"]) for x in ndpv] + for word_idx in range(len(words)): + for punct_idx in range(len(ndpv)): + if tags[word_idx] in ndpv[punct_idx]["tags"] \ + and words[word_idx] in itertools.chain.from_iterable(ndpv[punct_idx]["variants"]): + # backup option: random left vs. right from pair + pair_idx = random.choice([0, 1]) + # best option: rely on paired POS tags like `` / '' + if len(ndpv[punct_idx]["tags"]) == 2: + pair_idx = ndpv[punct_idx]["tags"].index(tags[word_idx]) + # next best option: rely on position in variants + # (may not be unambiguous, so order of variants matters) + else: + for pair in ndpv[punct_idx]["variants"]: + if words[word_idx] in pair: + pair_idx = pair.index(words[word_idx]) + words[word_idx] = punct_choices[punct_idx][pair_idx] + + variant_paragraph_tuples.append(((ids, words, tags, heads, labels, ner), brackets)) + if raw is not None: + # TODO: modify raw text accordingly + return raw, paragraph_tuples + return raw, variant_paragraph_tuples + + def add_noise(orig, noise_level): if random.random() >= noise_level: return orig diff --git a/spacy/language.py b/spacy/language.py index 86acf0257..0cf3528a2 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -108,6 +108,8 @@ class BaseDefaults(object): syntax_iterators = {} resources = {} writing_system = {"direction": "ltr", "has_case": True, "has_letters": True} + single_orth_variants = [] + paired_orth_variants = [] class Language(object): From 56c38484a1a3ca1625a455985a3272057823abbd Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 28 Aug 2019 09:16:40 +0200 Subject: [PATCH 105/287] Single and paired orth variants for English --- spacy/lang/en/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spacy/lang/en/__init__.py b/spacy/lang/en/__init__.py index 7d00c749c..2f391de0b 100644 --- a/spacy/lang/en/__init__.py +++ b/spacy/lang/en/__init__.py @@ -38,6 +38,10 @@ class EnglishDefaults(Language.Defaults): "lemma_index": "lemmatizer/lemma_index.json", "lemma_exc": "lemmatizer/lemma_exc.json", } + single_orth_variants = [{"tags": ["NFP"], "variants": ["…", "..."]}, + {"tags": [":"], "variants": ["-", "—", "–", "--", "---", "——"]}] + paired_orth_variants = [{"tags": ["``", "''"], "variants": [("'", "'"), ("‘", "’")]}, + {"tags": ["``", "''"], "variants": [('"', '"'), ("“", "”")]}] class English(Language): From 47af3f676e54138bd83a971be0679f3cf93ff9a7 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 28 Aug 2019 09:16:54 +0200 Subject: [PATCH 106/287] Single and paired orth variants for German --- spacy/lang/de/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spacy/lang/de/__init__.py b/spacy/lang/de/__init__.py index 1b5aee6a8..ae972072f 100644 --- a/spacy/lang/de/__init__.py +++ b/spacy/lang/de/__init__.py @@ -27,6 +27,10 @@ class GermanDefaults(Language.Defaults): stop_words = STOP_WORDS syntax_iterators = SYNTAX_ITERATORS resources = {"lemma_lookup": "lemma_lookup.json"} + single_orth_variants = [{"tags": ["$("], "variants": ["…", "..."]}, + {"tags": ["$("], "variants": ["-", "—", "–", "--", "---", "——"]}] + paired_orth_variants = [{"tags": ["$("], "variants": [("'", "'"), (",", "'"), ("‚", "‘")]}, + {"tags": ["$("], "variants": [("``", "''"), ('"', '"'), ("„", "“")]}] class German(Language): From 0a26e94d02d0417c512d6d151c81b138f1a22484 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 28 Aug 2019 13:38:54 +0200 Subject: [PATCH 107/287] Modify raw to match orth variant annotation tuples If raw is available, attempt to modify raw to match the orth variants. If raw/words can't be aligned, abort and return unmodified raw/annotation. --- spacy/gold.pyx | 56 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/spacy/gold.pyx b/spacy/gold.pyx index 1cd49814a..1028d831d 100644 --- a/spacy/gold.pyx +++ b/spacy/gold.pyx @@ -245,11 +245,12 @@ class GoldCorpus(object): @classmethod def _make_docs(cls, nlp, raw_text, paragraph_tuples, gold_preproc, noise_level=0.0, orth_variant_level=0.0): if raw_text is not None: + raw_text, paragraph_tuples = make_orth_variants(nlp, raw_text, paragraph_tuples, orth_variant_level=orth_variant_level) raw_text = add_noise(raw_text, noise_level) return [nlp.make_doc(raw_text)], paragraph_tuples else: docs = [] - raw_text, paragraph_tuples = make_orth_variants(nlp, None, paragraph_tuples, orth_variant_level) + raw_text, paragraph_tuples = make_orth_variants(nlp, None, paragraph_tuples, orth_variant_level=orth_variant_level) return [Doc(nlp.vocab, words=add_noise(sent_tuples[1], noise_level)) for (sent_tuples, brackets) in paragraph_tuples], paragraph_tuples @@ -272,11 +273,13 @@ class GoldCorpus(object): def make_orth_variants(nlp, raw, paragraph_tuples, orth_variant_level=0.0): if random.random() >= orth_variant_level: return raw, paragraph_tuples + ndsv = nlp.Defaults.single_orth_variants + ndpv = nlp.Defaults.paired_orth_variants + # modify words in paragraph_tuples variant_paragraph_tuples = [] for sent_tuples, brackets in paragraph_tuples: ids, words, tags, heads, labels, ner = sent_tuples # single variants - ndsv = nlp.Defaults.single_orth_variants punct_choices = [random.choice(x["variants"]) for x in ndsv] for word_idx in range(len(words)): for punct_idx in range(len(ndsv)): @@ -284,7 +287,6 @@ def make_orth_variants(nlp, raw, paragraph_tuples, orth_variant_level=0.0): and words[word_idx] in ndsv[punct_idx]["variants"]: words[word_idx] = punct_choices[punct_idx] # paired variants - ndpv = nlp.Defaults.paired_orth_variants punct_choices = [random.choice(x["variants"]) for x in ndpv] for word_idx in range(len(words)): for punct_idx in range(len(ndpv)): @@ -304,10 +306,50 @@ def make_orth_variants(nlp, raw, paragraph_tuples, orth_variant_level=0.0): words[word_idx] = punct_choices[punct_idx][pair_idx] variant_paragraph_tuples.append(((ids, words, tags, heads, labels, ner), brackets)) - if raw is not None: - # TODO: modify raw text accordingly - return raw, paragraph_tuples - return raw, variant_paragraph_tuples + # modify raw to match variant_paragraph_tuples + if raw is not None: + variants = [] + for single_variants in ndsv: + variants.extend(single_variants["variants"]) + for paired_variants in ndpv: + variants.extend(list(itertools.chain.from_iterable(paired_variants["variants"]))) + # store variants in reverse length order to be able to prioritize + # longer matches (e.g., "---" before "--") + variants = sorted(variants, key=lambda x: len(x)) + variants.reverse() + variant_raw = "" + raw_idx = 0 + # add initial whitespace + while raw_idx < len(raw) and re.match("\s", raw[raw_idx]): + variant_raw += raw[raw_idx] + raw_idx += 1 + for sent_tuples, brackets in variant_paragraph_tuples: + ids, words, tags, heads, labels, ner = sent_tuples + for word in words: + match_found = False + # add identical word + if word not in variants and raw[raw_idx:].startswith(word): + variant_raw += word + raw_idx += len(word) + match_found = True + # add variant word + else: + for variant in variants: + if not match_found and \ + raw[raw_idx:].startswith(variant): + raw_idx += len(variant) + variant_raw += word + match_found = True + # something went wrong, abort + # (add a warning message?) + if not match_found: + return raw, paragraph_tuples + # add following whitespace + while raw_idx < len(raw) and re.match("\s", raw[raw_idx]): + variant_raw += raw[raw_idx] + raw_idx += 1 + return variant_raw, variant_paragraph_tuples + return raw, variant_paragraph_tuples def add_noise(orig, noise_level): From 782056d11722a5b3eb352cfdf29a4b5deabd49a8 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 28 Aug 2019 16:59:45 +0200 Subject: [PATCH 108/287] Fix morph rules --- spacy/lang/en/morph_rules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spacy/lang/en/morph_rules.py b/spacy/lang/en/morph_rules.py index b00534cc5..f910e42b8 100644 --- a/spacy/lang/en/morph_rules.py +++ b/spacy/lang/en/morph_rules.py @@ -73,10 +73,11 @@ _subordinating_conjunctions = [ "will", ] -_relative_pronouns = ["this", "that", "those", "these"] +# This seems kind of wrong too? +#_relative_pronouns = ["this", "that", "those", "these"] MORPH_RULES = { - "DT": {word: {"POS": "PRON"} for word in _relative_pronouns}, + #"DT": {word: {"POS": "PRON"} for word in _relative_pronouns}, "IN": {word: {"POS": "SCONJ"} for word in _subordinating_conjunctions}, "NN": { "something": {"POS": "PRON"}, From bc5ce498593a4f583225ddbf672d3703f1865928 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 28 Aug 2019 17:55:38 +0200 Subject: [PATCH 109/287] Fix 'noise_level' in train cmd --- spacy/cli/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/train.py b/spacy/cli/train.py index c4355f1a1..04e734068 100644 --- a/spacy/cli/train.py +++ b/spacy/cli/train.py @@ -240,7 +240,7 @@ def train( best_score = 0.0 for i in range(n_iter): train_docs = corpus.train_docs( - nlp, noise_level=noise_level, gold_preproc=gold_preproc, max_length=0 + nlp, orth_variant_level=noise_level, gold_preproc=gold_preproc, max_length=0 ) if raw_text: random.shuffle(raw_text) From 7d6d4385663222b0f517d859db6be330c747132a Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 28 Aug 2019 18:30:43 +0200 Subject: [PATCH 110/287] Set version to v2.2.0.dev2 --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index 6a9784c54..2ea1c5c9d 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -4,7 +4,7 @@ # fmt: off __title__ = "spacy" -__version__ = "2.2.0.dev1" +__version__ = "2.2.0.dev2" __summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython" __uri__ = "https://spacy.io" __author__ = "Explosion AI" From f3906950d3f17681bff3b1bede1b81d8ebfb1bec Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 29 Aug 2019 09:10:35 +0200 Subject: [PATCH 111/287] Add separate noise vs orth level to train CLI --- spacy/cli/train.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spacy/cli/train.py b/spacy/cli/train.py index 04e734068..46f4b8900 100644 --- a/spacy/cli/train.py +++ b/spacy/cli/train.py @@ -65,6 +65,7 @@ from .. import about str, ), noise_level=("Amount of corruption for data augmentation", "option", "nl", float), + orth_variant_level=("Amount of orthography variation for data augmentation", "option", "ovl", float), eval_beam_widths=("Beam widths to evaluate, e.g. 4,8", "option", "bw", str), gold_preproc=("Use gold preprocessing", "flag", "G", bool), learn_tokens=("Make parser learn gold-standard tokenization", "flag", "T", bool), @@ -90,6 +91,7 @@ def train( parser_multitasks="", entity_multitasks="", noise_level=0.0, + orth_variant_level=0.0, eval_beam_widths="", gold_preproc=False, learn_tokens=False, @@ -240,7 +242,7 @@ def train( best_score = 0.0 for i in range(n_iter): train_docs = corpus.train_docs( - nlp, orth_variant_level=noise_level, gold_preproc=gold_preproc, max_length=0 + nlp, noise_level=noise_level, orth_variant_level=orth_variant_level, gold_preproc=gold_preproc, max_length=0 ) if raw_text: random.shuffle(raw_text) From 6511e1d8d328e5e50dc3bf103a8c12434468fd57 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 29 Aug 2019 14:33:07 +0200 Subject: [PATCH 112/287] Fix NER gold-standard around whitespace --- spacy/gold.pyx | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/spacy/gold.pyx b/spacy/gold.pyx index 6d784d1bd..b8ae2e505 100644 --- a/spacy/gold.pyx +++ b/spacy/gold.pyx @@ -635,7 +635,7 @@ cdef class GoldParse: self.tags[i] = "_SP" self.heads[i] = None self.labels[i] = None - self.ner[i] = "O" + self.ner[i] = None self.morphology[i] = set() if gold_i is None: if i in i2j_multi: @@ -686,9 +686,20 @@ cdef class GoldParse: self.labels[i] = deps[gold_i] self.ner[i] = entities[gold_i] + # Prevent whitespace that isn't within entities from being tagged as + # an entity. + for i in range(len(self.ner)): + if self.tags[i] == "_SP": + prev_ner = self.ner[i-1] if i >= 1 else None + next_ner = self.ner[i+1] if (i+1) < len(self.ner) else None + if prev_ner == "O" or next_ner == "O": + self.ner[i] = "O" + cycle = nonproj.contains_cycle(self.heads) if cycle is not None: - raise ValueError(Errors.E069.format(cycle=cycle, cycle_tokens=" ".join(["'{}'".format(self.words[tok_id]) for tok_id in cycle]), doc_tokens=" ".join(words[:50]))) + raise ValueError(Errors.E069.format(cycle=cycle, + cycle_tokens=" ".join(["'{}'".format(self.words[tok_id]) for tok_id in cycle]), + doc_tokens=" ".join(words[:50]))) def __len__(self): """Get the number of gold-standard tokens. From 3c1c0ec18ec702afbe03e24519cdb0c3a513c945 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 29 Aug 2019 14:33:39 +0200 Subject: [PATCH 113/287] Add tests for NER oracle with whitespace --- spacy/tests/parser/test_ner.py | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/spacy/tests/parser/test_ner.py b/spacy/tests/parser/test_ner.py index 43c00a963..c39491ecf 100644 --- a/spacy/tests/parser/test_ner.py +++ b/spacy/tests/parser/test_ner.py @@ -91,3 +91,69 @@ def test_doc_add_entities_set_ents_iob(en_vocab): assert [w.ent_iob_ for w in doc] == ["", "", "", "B"] doc.ents = [(doc.vocab.strings["WORD"], 0, 2)] assert [w.ent_iob_ for w in doc] == ["B", "I", "", ""] + + +def test_oracle_moves_missing_B(en_vocab): + words = ["B", "52", "Bomber"] + biluo_tags = [None, None, "L-PRODUCT"] + + doc = Doc(en_vocab, words=words) + gold = GoldParse(doc, words=words, entities=biluo_tags) + + moves = BiluoPushDown(en_vocab.strings) + move_types = ("M", "B", "I", "L", "U", "O") + for tag in biluo_tags: + if tag is None: + continue + elif tag == "O": + moves.add_action(move_types.index("O"), "") + else: + action, label = tag.split("-") + moves.add_action(move_types.index("B"), label) + moves.add_action(move_types.index("I"), label) + moves.add_action(move_types.index("L"), label) + moves.add_action(move_types.index("U"), label) + moves.preprocess_gold(gold) + seq = moves.get_oracle_sequence(doc, gold) + print(seq) + + +def test_oracle_moves_whitespace(en_vocab): + words = [ + "production", + "\n", + "of", + "Northrop", + "\n", + "Corp.", + "\n", + "'s", + "radar", + ] + biluo_tags = [ + "O", + "O", + "O", + "B-ORG", + None, + "I-ORG", + "L-ORG", + "O", + "O", + ] + + doc = Doc(en_vocab, words=words) + gold = GoldParse(doc, words=words, entities=biluo_tags) + + moves = BiluoPushDown(en_vocab.strings) + move_types = ("M", "B", "I", "L", "U", "O") + for tag in biluo_tags: + if tag is None: + continue + elif tag == "O": + moves.add_action(move_types.index("O"), "") + else: + action, label = tag.split("-") + moves.add_action(move_types.index(action), label) + moves.preprocess_gold(gold) + seq = moves.get_oracle_sequence(doc, gold) From 32842a3cd45850201e067bc5d212e169aa14c045 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 29 Aug 2019 15:01:58 +0200 Subject: [PATCH 114/287] Disable whitespace corruption --- spacy/gold.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spacy/gold.pyx b/spacy/gold.pyx index b8ae2e505..b40bdb8a8 100644 --- a/spacy/gold.pyx +++ b/spacy/gold.pyx @@ -356,19 +356,19 @@ def add_noise(orig, noise_level): if random.random() >= noise_level: return orig elif type(orig) == list: - corrupted = [_corrupt(word, noise_level) for word in orig] + corrupted = [_corrupt(word, noise_level, replace_space=False) for word in orig] corrupted = [w for w in corrupted if w] return corrupted else: - return "".join(_corrupt(c, noise_level) for c in orig) + return "".join(_corrupt(c, noise_level, replace_space=False) for c in orig) -def _corrupt(c, noise_level): +def _corrupt(c, noise_level, replace_space=False): if random.random() >= noise_level: return c - elif c == " ": + elif replace_space and c == " ": return "\n" - elif c == "\n": + elif replace_space and c == "\n": return " " elif c in [".", "'", "!", "?", ","]: return "" From c94fc9edb9baf53b0e4a2f63bd561528045f6a4d Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 29 Aug 2019 15:39:32 +0200 Subject: [PATCH 115/287] Fix noise addition --- spacy/gold.pyx | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/spacy/gold.pyx b/spacy/gold.pyx index b40bdb8a8..af0588349 100644 --- a/spacy/gold.pyx +++ b/spacy/gold.pyx @@ -356,22 +356,18 @@ def add_noise(orig, noise_level): if random.random() >= noise_level: return orig elif type(orig) == list: - corrupted = [_corrupt(word, noise_level, replace_space=False) for word in orig] + corrupted = [_corrupt(word, noise_level) for word in orig] corrupted = [w for w in corrupted if w] return corrupted else: - return "".join(_corrupt(c, noise_level, replace_space=False) for c in orig) + return "".join(_corrupt(c, noise_level) for c in orig) -def _corrupt(c, noise_level, replace_space=False): +def _corrupt(c, noise_level): if random.random() >= noise_level: return c - elif replace_space and c == " ": - return "\n" - elif replace_space and c == "\n": - return " " elif c in [".", "'", "!", "?", ","]: - return "" + return "\n" else: return c.lower() From fc0a3c8c3877694c19ec5c4c5bab969e7ae2c93b Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 29 Aug 2019 21:17:34 +0200 Subject: [PATCH 116/287] Add morphology serialization --- spacy/morphology.pyx | 45 ++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index ccfb214bc..a7a1bee57 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -15,6 +15,7 @@ from .parts_of_speech cimport SPACE from .parts_of_speech import IDS as POS_IDS from .lexeme cimport Lexeme from .errors import Errors +from .util import ensure_path cdef enum univ_field_t: @@ -162,12 +163,7 @@ cdef class Morphology: self.n_tags = len(tag_map) self.reverse_index = {} self._feat_map = MorphologyClassMap(FEATURES) - for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())): - attrs = _normalize_props(attrs) - self.add({self._feat_map.id2feat[feat] for feat in attrs - if feat in self._feat_map.id2feat}) - self.tag_map[tag_str] = dict(attrs) - self.reverse_index[self.strings.add(tag_str)] = i + self._load_from_tag_map(tag_map) self._cache = PreshMapArray(self.n_tags) self.exc = {} @@ -177,6 +173,14 @@ cdef class Morphology: self.add_special_case( self.strings.as_string(tag), self.strings.as_string(orth), attrs) + def _load_from_tag_map(self, tag_map): + for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())): + attrs = _normalize_props(attrs) + self.add({self._feat_map.id2feat[feat] for feat in attrs + if feat in self._feat_map.id2feat}) + self.tag_map[tag_str] = dict(attrs) + self.reverse_index[self.strings.add(tag_str)] = i + def __reduce__(self): return (Morphology, (self.strings, self.tag_map, self.lemmatizer, self.exc), None, None) @@ -188,6 +192,7 @@ cdef class Morphology: for f in features: if isinstance(f, basestring_): self.strings.add(f) + string_features = features features = intify_features(features) cdef attr_t feature for feature in features: @@ -321,22 +326,34 @@ cdef class Morphology: for form_str, attrs in entries.items(): self.add_special_case(tag_str, form_str, attrs) - def to_bytes(self): - json_tags = [] + def to_bytes(self, exclude=tuple(), **kwargs): + tag_map = {} for key in self.tags: tag_ptr = self.tags.get(key) if tag_ptr != NULL: - json_tags.append(tag_to_json(tag_ptr)) - return srsly.json_dumps(json_tags) + tag_map[key] = tag_to_json(tag_ptr) + exceptions = {} + for (tag_str, orth_int), attrs in sorted(self.exc.items()): + exceptions.setdefault(tag_str, {}) + exceptions[tag_str][self.strings[orth_int]] = attrs + data = {"tag_map": tag_map, "exceptions": exceptions} + return srsly.msgpack_dumps(data) def from_bytes(self, byte_string): - raise NotImplementedError + msg = srsly.msgpack_loads(byte_string) + self._load_from_tag_map(msg["tag_map"]) + self.load_morph_exceptions(msg["exceptions"]) + return self - def to_disk(self, path): - raise NotImplementedError + def to_disk(self, path, exclude=tuple(), **kwargs): + path = ensure_path(path) + with path.open("wb") as file_: + file_.write(self.to_bytes()) def from_disk(self, path): - raise NotImplementedError + with path.open("rb") as file_: + byte_string = file_.read() + return self.from_bytes(byte_string) @classmethod def create_class_map(cls): From f3c3ce7f1ec5fec87cb5965efe1c937f4666afea Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 29 Aug 2019 21:19:54 +0200 Subject: [PATCH 117/287] Update vocab --- spacy/vocab.pyx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx index 02d5cbcff..b649fdded 100644 --- a/spacy/vocab.pyx +++ b/spacy/vocab.pyx @@ -433,6 +433,8 @@ cdef class Vocab: file_.write(self.lexemes_to_bytes()) if "vectors" not in "exclude" and self.vectors is not None: self.vectors.to_disk(path) + if "morphology" not in exclude: + self.morphology.to_disk(path / "morphology.bin") def from_disk(self, path, exclude=tuple(), **kwargs): """Loads state from a directory. Modifies the object in place and @@ -457,6 +459,8 @@ cdef class Vocab: self.vectors.from_disk(path, exclude=["strings"]) if self.vectors.name is not None: link_vectors_to_models(self) + if "morphology" not in exclude: + self.morphology.from_disk(path / "morphology.bin") return self def to_bytes(self, exclude=tuple(), **kwargs): @@ -476,7 +480,8 @@ cdef class Vocab: getters = OrderedDict(( ("strings", lambda: self.strings.to_bytes()), ("lexemes", lambda: self.lexemes_to_bytes()), - ("vectors", deserialize_vectors) + ("vectors", deserialize_vectors), + ("morphology", lambda: self.morphology.to_bytes()) )) exclude = util.get_serialization_exclude(getters, exclude, kwargs) return util.to_bytes(getters, exclude) @@ -499,7 +504,8 @@ cdef class Vocab: setters = OrderedDict(( ("strings", lambda b: self.strings.from_bytes(b)), ("lexemes", lambda b: self.lexemes_from_bytes(b)), - ("vectors", lambda b: serialize_vectors(b)) + ("vectors", lambda b: serialize_vectors(b)), + ("morphology", lambda b: self.morphology.from_bytes(b)) )) exclude = util.get_serialization_exclude(setters, exclude, kwargs) util.from_bytes(bytes_data, setters, exclude) From 02babf931793f4e2d372a6c89ef0ba3df9573140 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 30 Aug 2019 11:29:19 +0200 Subject: [PATCH 118/287] English tag map without unsupported features/values --- spacy/lang/en/tag_map.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/spacy/lang/en/tag_map.py b/spacy/lang/en/tag_map.py index 5c9a97786..9bd884a3a 100644 --- a/spacy/lang/en/tag_map.py +++ b/spacy/lang/en/tag_map.py @@ -14,8 +14,8 @@ TAG_MAP = { '""': {POS: PUNCT, "PunctType": "quot", "PunctSide": "fin"}, "''": {POS: PUNCT, "PunctType": "quot", "PunctSide": "fin"}, ":": {POS: PUNCT}, - "$": {POS: SYM, "Other": {"SymType": "currency"}}, - "#": {POS: SYM, "Other": {"SymType": "numbersign"}}, + "$": {POS: SYM}, + "#": {POS: SYM}, "AFX": {POS: ADJ, "Hyph": "yes"}, "CC": {POS: CCONJ, "ConjType": "comp"}, "CD": {POS: NUM, "NumType": "card"}, @@ -34,7 +34,7 @@ TAG_MAP = { "NNP": {POS: PROPN, "NounType": "prop", "Number": "sing"}, "NNPS": {POS: PROPN, "NounType": "prop", "Number": "plur"}, "NNS": {POS: NOUN, "Number": "plur"}, - "PDT": {POS: DET, "AdjType": "pdt", "PronType": "prn"}, + "PDT": {POS: DET}, "POS": {POS: PART, "Poss": "yes"}, "PRP": {POS: PRON, "PronType": "prs"}, "PRP$": {POS: PRON, "PronType": "prs", "Poss": "yes"}, @@ -58,10 +58,10 @@ TAG_MAP = { "Number": "sing", "Person": "three", }, - "WDT": {POS: PRON, "PronType": "int|rel"}, - "WP": {POS: PRON, "PronType": "int|rel"}, - "WP$": {POS: PRON, "Poss": "yes", "PronType": "int|rel"}, - "WRB": {POS: ADV, "PronType": "int|rel"}, + "WDT": {POS: PRON}, + "WP": {POS: PRON}, + "WP$": {POS: PRON, "Poss": "yes"}, + "WRB": {POS: ADV}, "ADD": {POS: X}, "NFP": {POS: PUNCT}, "GW": {POS: X}, From 893f11a9e38d4a29c608602e61798a29d4800d99 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 30 Aug 2019 11:30:03 +0200 Subject: [PATCH 119/287] Serialize tag_map directly Fix Aspect_prof typo --- spacy/morphology.pyx | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index a7a1bee57..f706fec7f 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -327,16 +327,11 @@ cdef class Morphology: self.add_special_case(tag_str, form_str, attrs) def to_bytes(self, exclude=tuple(), **kwargs): - tag_map = {} - for key in self.tags: - tag_ptr = self.tags.get(key) - if tag_ptr != NULL: - tag_map[key] = tag_to_json(tag_ptr) exceptions = {} for (tag_str, orth_int), attrs in sorted(self.exc.items()): exceptions.setdefault(tag_str, {}) exceptions[tag_str][self.strings[orth_int]] = attrs - data = {"tag_map": tag_map, "exceptions": exceptions} + data = {"tag_map": self.tag_map, "exceptions": exceptions} return srsly.msgpack_dumps(data) def from_bytes(self, byte_string): @@ -898,7 +893,7 @@ FEATURES = [ "Aspect_mod", "Aspect_none", "Aspect_perf", - "Aspect_prof", + "Aspect_prog", "Aspect_prosp", "Case_abe", "Case_abl", From 67c3d039055fd9a82bca67eda99c77284593f12f Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 30 Aug 2019 13:13:07 +0200 Subject: [PATCH 120/287] Revert morphology serialisation --- spacy/morphology.pyx | 24 ------------------------ spacy/vocab.pyx | 6 ------ 2 files changed, 30 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index f706fec7f..2d58b8f27 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -326,30 +326,6 @@ cdef class Morphology: for form_str, attrs in entries.items(): self.add_special_case(tag_str, form_str, attrs) - def to_bytes(self, exclude=tuple(), **kwargs): - exceptions = {} - for (tag_str, orth_int), attrs in sorted(self.exc.items()): - exceptions.setdefault(tag_str, {}) - exceptions[tag_str][self.strings[orth_int]] = attrs - data = {"tag_map": self.tag_map, "exceptions": exceptions} - return srsly.msgpack_dumps(data) - - def from_bytes(self, byte_string): - msg = srsly.msgpack_loads(byte_string) - self._load_from_tag_map(msg["tag_map"]) - self.load_morph_exceptions(msg["exceptions"]) - return self - - def to_disk(self, path, exclude=tuple(), **kwargs): - path = ensure_path(path) - with path.open("wb") as file_: - file_.write(self.to_bytes()) - - def from_disk(self, path): - with path.open("rb") as file_: - byte_string = file_.read() - return self.from_bytes(byte_string) - @classmethod def create_class_map(cls): return MorphologyClassMap(FEATURES) diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx index b649fdded..35d9374d0 100644 --- a/spacy/vocab.pyx +++ b/spacy/vocab.pyx @@ -433,8 +433,6 @@ cdef class Vocab: file_.write(self.lexemes_to_bytes()) if "vectors" not in "exclude" and self.vectors is not None: self.vectors.to_disk(path) - if "morphology" not in exclude: - self.morphology.to_disk(path / "morphology.bin") def from_disk(self, path, exclude=tuple(), **kwargs): """Loads state from a directory. Modifies the object in place and @@ -459,8 +457,6 @@ cdef class Vocab: self.vectors.from_disk(path, exclude=["strings"]) if self.vectors.name is not None: link_vectors_to_models(self) - if "morphology" not in exclude: - self.morphology.from_disk(path / "morphology.bin") return self def to_bytes(self, exclude=tuple(), **kwargs): @@ -481,7 +477,6 @@ cdef class Vocab: ("strings", lambda: self.strings.to_bytes()), ("lexemes", lambda: self.lexemes_to_bytes()), ("vectors", deserialize_vectors), - ("morphology", lambda: self.morphology.to_bytes()) )) exclude = util.get_serialization_exclude(getters, exclude, kwargs) return util.to_bytes(getters, exclude) @@ -505,7 +500,6 @@ cdef class Vocab: ("strings", lambda b: self.strings.from_bytes(b)), ("lexemes", lambda b: self.lexemes_from_bytes(b)), ("vectors", lambda b: serialize_vectors(b)), - ("morphology", lambda b: self.morphology.from_bytes(b)) )) exclude = util.get_serialization_exclude(setters, exclude, kwargs) util.from_bytes(bytes_data, setters, exclude) From c39c13f26b119ad3fcac3b3d3669c3f28a7d5504 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 4 Sep 2019 20:05:08 +0200 Subject: [PATCH 121/287] Add guillemets/chevrons to German orth variants Add guillemets/chevrons to German orth variants for both German/Austrian and Swiss conventions. --- spacy/lang/de/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/lang/de/__init__.py b/spacy/lang/de/__init__.py index ae972072f..1ddee54b3 100644 --- a/spacy/lang/de/__init__.py +++ b/spacy/lang/de/__init__.py @@ -29,8 +29,8 @@ class GermanDefaults(Language.Defaults): resources = {"lemma_lookup": "lemma_lookup.json"} single_orth_variants = [{"tags": ["$("], "variants": ["…", "..."]}, {"tags": ["$("], "variants": ["-", "—", "–", "--", "---", "——"]}] - paired_orth_variants = [{"tags": ["$("], "variants": [("'", "'"), (",", "'"), ("‚", "‘")]}, - {"tags": ["$("], "variants": [("``", "''"), ('"', '"'), ("„", "“")]}] + paired_orth_variants = [{"tags": ["$("], "variants": [("'", "'"), (",", "'"), ("‚", "‘"), ("›", "‹"), ("‹", "›")]}, + {"tags": ["$("], "variants": [("``", "''"), ('"', '"'), ("„", "“"), ("»", "«"), ("«", "»")]}] class German(Language): From fde4f8ac8e5f9678b8fe39efac2d94c5304dca54 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 8 Sep 2019 18:08:09 +0200 Subject: [PATCH 122/287] Create lookups if not passed in --- spacy/language.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spacy/language.py b/spacy/language.py index c10718e80..6a02a1fdd 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -39,6 +39,8 @@ from . import about class BaseDefaults(object): @classmethod def create_lemmatizer(cls, nlp=None, lookups=None): + if lookups is None: + lookups = cls.create_lookups(nlp=nlp) lemma_rules, lemma_index, lemma_exc, lemma_lookup = util.get_lemma_tables(lookups) return Lemmatizer(lemma_index, lemma_exc, lemma_rules, lemma_lookup) From aec6174ae6c032ca85d0ca10f2703f1fffa85cd1 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 8 Sep 2019 18:09:53 +0200 Subject: [PATCH 123/287] Fix lemmatizer --- spacy/lemmatizer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spacy/lemmatizer.py b/spacy/lemmatizer.py index f7a58aa9f..c9ccbcd0d 100644 --- a/spacy/lemmatizer.py +++ b/spacy/lemmatizer.py @@ -55,6 +55,8 @@ class Lemmatizer(object): Check whether we're dealing with an uninflected paradigm, so we can avoid lemmatization entirely. """ + if morphology is None: + morphology = {} if univ_pos == "noun" and morphology.get("Number") == "sing": return True elif univ_pos == "verb" and morphology.get("VerbForm") == "inf": From da8830d909e56344fc9a02d2548d66a886fbb70f Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 8 Sep 2019 18:22:03 +0200 Subject: [PATCH 124/287] Set version to v2.2.0.dev3 --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index 2ea1c5c9d..ed4781af1 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -4,7 +4,7 @@ # fmt: off __title__ = "spacy" -__version__ = "2.2.0.dev2" +__version__ = "2.2.0.dev3" __summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython" __uri__ = "https://spacy.io" __author__ = "Explosion AI" From 1653b818c5e8a7da0efb61ae500481661a141241 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 8 Sep 2019 20:57:58 +0200 Subject: [PATCH 125/287] Update Lithuanian tag map --- spacy/lang/lt/tag_map.py | 236 +++++++++++++++++++-------------------- 1 file changed, 118 insertions(+), 118 deletions(-) diff --git a/spacy/lang/lt/tag_map.py b/spacy/lang/lt/tag_map.py index eab231b2c..6ea4f8ae0 100644 --- a/spacy/lang/lt/tag_map.py +++ b/spacy/lang/lt/tag_map.py @@ -1605,7 +1605,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Pos", "VerbForm": "Fin", }, @@ -1613,7 +1613,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Pos", "VerbForm": "Fin", }, @@ -1621,7 +1621,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Reflex": "Yes", "VerbForm": "Fin", @@ -1630,7 +1630,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Neg", "VerbForm": "Fin", }, @@ -1638,7 +1638,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Reflex": "Yes", "VerbForm": "Fin", @@ -1647,7 +1647,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "VerbForm": "Fin", }, @@ -1655,7 +1655,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Reflex": "Yes", "VerbForm": "Fin", @@ -1664,7 +1664,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Neg", "VerbForm": "Fin", }, @@ -1672,7 +1672,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Reflex": "Yes", "VerbForm": "Fin", @@ -1681,7 +1681,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Pos", "VerbForm": "Fin", }, @@ -1689,7 +1689,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Pos", "VerbForm": "Fin", }, @@ -1697,7 +1697,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Reflex": "Yes", "VerbForm": "Fin", @@ -1706,7 +1706,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Neg", "VerbForm": "Fin", }, @@ -1714,7 +1714,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Neg", "Reflex": "Yes", "VerbForm": "Fin", @@ -1723,7 +1723,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Pos", "VerbForm": "Fin", }, @@ -1731,7 +1731,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Pos", "VerbForm": "Fin", }, @@ -1739,7 +1739,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Reflex": "Yes", "VerbForm": "Fin", @@ -1748,7 +1748,7 @@ TAG_MAP = { POS: VERB, "Mood": "Imp", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Neg", "VerbForm": "Fin", }, @@ -1756,21 +1756,21 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Neg", "VerbForm": "Fin", }, "Vgm-3---n--ns-": { POS: VERB, "Mood": "Cnd", - "Person": "3", + "Person": "three", "Polarity": "Pos", "VerbForm": "Fin", }, "Vgm-3---n--ys-": { POS: VERB, "Mood": "Cnd", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "VerbForm": "Fin", @@ -1778,14 +1778,14 @@ TAG_MAP = { "Vgm-3---y--ns-": { POS: VERB, "Mood": "Cnd", - "Person": "3", + "Person": "three", "Polarity": "Neg", "VerbForm": "Fin", }, "Vgm-3---y--ys-": { POS: VERB, "Mood": "Cnd", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Reflex": "Yes", "VerbForm": "Fin", @@ -1794,7 +1794,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "VerbForm": "Fin", }, @@ -1802,7 +1802,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "VerbForm": "Fin", @@ -1811,7 +1811,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Neg", "VerbForm": "Fin", }, @@ -1819,7 +1819,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "VerbForm": "Fin", }, @@ -1827,7 +1827,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "VerbForm": "Fin", @@ -1836,7 +1836,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Neg", "VerbForm": "Fin", }, @@ -1844,7 +1844,7 @@ TAG_MAP = { POS: VERB, "Mood": "Cnd", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Reflex": "Yes", "VerbForm": "Fin", @@ -1853,7 +1853,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -1862,7 +1862,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -1872,7 +1872,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Tense": "Past", "VerbForm": "Fin", @@ -1881,7 +1881,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Past", @@ -1891,7 +1891,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -1900,7 +1900,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -1910,7 +1910,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Tense": "Past", "VerbForm": "Fin", @@ -1919,7 +1919,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Past", @@ -1929,7 +1929,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -1938,7 +1938,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -1948,7 +1948,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Neg", "Tense": "Past", "VerbForm": "Fin", @@ -1957,7 +1957,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -1966,7 +1966,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Neg", "Tense": "Past", "VerbForm": "Fin", @@ -1974,7 +1974,7 @@ TAG_MAP = { "Vgma3---n--ni-": { POS: VERB, "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -1982,7 +1982,7 @@ TAG_MAP = { "Vgma3---n--yi-": { POS: VERB, "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -1991,7 +1991,7 @@ TAG_MAP = { "Vgma3---y--ni-": { POS: VERB, "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Past", "VerbForm": "Fin", @@ -1999,7 +1999,7 @@ TAG_MAP = { "Vgma3--y--ni-": { POS: VERB, "Case": "Nom", - "Person": "3", + "Person": "three", "Tense": "Past", "VerbForm": "Fin", }, @@ -2007,7 +2007,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -2016,7 +2016,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -2026,7 +2026,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Past", "VerbForm": "Fin", @@ -2035,7 +2035,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Past", @@ -2045,7 +2045,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -2054,7 +2054,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -2064,7 +2064,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -2074,7 +2074,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Past", "VerbForm": "Fin", @@ -2083,7 +2083,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Past", @@ -2093,7 +2093,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Tense": "Fut", "VerbForm": "Fin", @@ -2102,7 +2102,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Fut", @@ -2112,7 +2112,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Tense": "Fut", "VerbForm": "Fin", @@ -2121,7 +2121,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Tense": "Fut", "VerbForm": "Fin", @@ -2130,7 +2130,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Fut", @@ -2140,7 +2140,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Tense": "Fut", "VerbForm": "Fin", @@ -2149,7 +2149,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Tense": "Fut", "VerbForm": "Fin", @@ -2158,7 +2158,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Fut", @@ -2168,7 +2168,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Tense": "Fut", "VerbForm": "Fin", @@ -2177,7 +2177,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Fut", @@ -2187,7 +2187,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Neg", "Tense": "Fut", "VerbForm": "Fin", @@ -2196,7 +2196,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Fut", @@ -2205,7 +2205,7 @@ TAG_MAP = { "Vgmf3---n--ni-": { POS: VERB, "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Fut", "VerbForm": "Fin", @@ -2213,7 +2213,7 @@ TAG_MAP = { "Vgmf3---y--ni-": { POS: VERB, "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Fut", "VerbForm": "Fin", @@ -2222,7 +2222,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Fut", "VerbForm": "Fin", @@ -2231,7 +2231,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Fut", @@ -2241,7 +2241,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Fut", "VerbForm": "Fin", @@ -2250,7 +2250,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Fut", "VerbForm": "Fin", @@ -2259,7 +2259,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Fut", @@ -2269,7 +2269,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Fut", "VerbForm": "Fin", @@ -2278,7 +2278,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Fut", @@ -2288,7 +2288,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2297,7 +2297,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Pres", @@ -2307,7 +2307,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Tense": "Pres", "VerbForm": "Fin", @@ -2316,7 +2316,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Pres", @@ -2326,7 +2326,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2335,7 +2335,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2344,7 +2344,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Pres", @@ -2354,7 +2354,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Tense": "Pres", "VerbForm": "Fin", @@ -2363,7 +2363,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Pres", @@ -2373,7 +2373,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2382,7 +2382,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Pres", @@ -2392,7 +2392,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Neg", "Tense": "Pres", "VerbForm": "Fin", @@ -2401,7 +2401,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "2", + "Person": "two", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Pres", @@ -2411,7 +2411,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2420,7 +2420,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Pres", @@ -2430,7 +2430,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Neg", "Tense": "Pres", "VerbForm": "Fin", @@ -2438,7 +2438,7 @@ TAG_MAP = { "Vgmp3---n--ni-": { POS: VERB, "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2446,7 +2446,7 @@ TAG_MAP = { "Vgmp3---n--yi-": { POS: VERB, "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Pres", @@ -2455,7 +2455,7 @@ TAG_MAP = { "Vgmp3---y--ni-": { POS: VERB, "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Pres", "VerbForm": "Fin", @@ -2463,7 +2463,7 @@ TAG_MAP = { "Vgmp3---y--yi-": { POS: VERB, "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Pres", @@ -2473,7 +2473,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2482,7 +2482,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Pres", @@ -2492,7 +2492,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Pres", "VerbForm": "Fin", @@ -2501,7 +2501,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Pres", @@ -2511,7 +2511,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2520,7 +2520,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2529,7 +2529,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Pres", "VerbForm": "Fin", @@ -2538,7 +2538,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Pres", @@ -2548,7 +2548,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Pres", "VerbForm": "Fin", @@ -2557,7 +2557,7 @@ TAG_MAP = { POS: VERB, "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Reflex": "Yes", "Tense": "Pres", @@ -2568,7 +2568,7 @@ TAG_MAP = { "Aspect": "Hab", "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -2578,7 +2578,7 @@ TAG_MAP = { "Aspect": "Hab", "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -2589,7 +2589,7 @@ TAG_MAP = { "Aspect": "Hab", "Mood": "Ind", "Number": "Sing", - "Person": "1", + "Person": "one", "Polarity": "Neg", "Tense": "Past", "VerbForm": "Fin", @@ -2599,7 +2599,7 @@ TAG_MAP = { "Aspect": "Hab", "Mood": "Ind", "Number": "Sing", - "Person": "2", + "Person": "two", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -2608,7 +2608,7 @@ TAG_MAP = { POS: VERB, "Aspect": "Hab", "Mood": "Ind", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -2618,7 +2618,7 @@ TAG_MAP = { "Aspect": "Hab", "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -2628,7 +2628,7 @@ TAG_MAP = { "Aspect": "Hab", "Mood": "Ind", "Number": "Plur", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -2639,7 +2639,7 @@ TAG_MAP = { "Aspect": "Hab", "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", @@ -2649,7 +2649,7 @@ TAG_MAP = { "Aspect": "Hab", "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Reflex": "Yes", "Tense": "Past", @@ -2660,7 +2660,7 @@ TAG_MAP = { "Aspect": "Hab", "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Neg", "Tense": "Past", "VerbForm": "Fin", @@ -2670,7 +2670,7 @@ TAG_MAP = { "Aspect": "Perf", "Mood": "Ind", "Number": "Sing", - "Person": "3", + "Person": "three", "Polarity": "Pos", "Tense": "Past", "VerbForm": "Fin", From 28741ff5db7927d29fa498a6016249c74e21ecce Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 10 Sep 2019 19:13:07 +0200 Subject: [PATCH 126/287] Require preshed v3.0.0 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 865288a86..8fea3253b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Our libraries cymem>=2.0.2,<2.1.0 -preshed>=2.0.1,<2.1.0 +preshed>=3.0.0,<3.1.0 thinc>=7.1.0,<7.2.0 blis>=0.4.0,<0.5.0 murmurhash>=0.28.0,<1.1.0 diff --git a/setup.py b/setup.py index 29bdb96fa..ee60aa07f 100755 --- a/setup.py +++ b/setup.py @@ -247,7 +247,7 @@ def setup_package(): "numpy>=1.15.0", "murmurhash>=0.28.0,<1.1.0", "cymem>=2.0.2,<2.1.0", - "preshed>=2.0.1,<2.1.0", + "preshed>=3.0.0,<3.1.0", "thinc>=7.1.0,<7.2.0", "blis>=0.4.0,<0.5.0", "plac<1.0.0,>=0.9.6", From c181a94e7540d2c0155a41c508a87398f0973eb6 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 10 Sep 2019 20:12:24 +0200 Subject: [PATCH 127/287] Require thinc 7.1.1 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8fea3253b..5c3ed9981 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # Our libraries cymem>=2.0.2,<2.1.0 preshed>=3.0.0,<3.1.0 -thinc>=7.1.0,<7.2.0 +thinc>=7.1.1,<7.2.0 blis>=0.4.0,<0.5.0 murmurhash>=0.28.0,<1.1.0 wasabi>=0.2.0,<1.1.0 diff --git a/setup.py b/setup.py index ee60aa07f..f37c8783c 100755 --- a/setup.py +++ b/setup.py @@ -248,7 +248,7 @@ def setup_package(): "murmurhash>=0.28.0,<1.1.0", "cymem>=2.0.2,<2.1.0", "preshed>=3.0.0,<3.1.0", - "thinc>=7.1.0,<7.2.0", + "thinc>=7.1.1,<7.2.0", "blis>=0.4.0,<0.5.0", "plac<1.0.0,>=0.9.6", "requests>=2.13.0,<3.0.0", From 178d010b25c0dd438a85565e452dca31660efc11 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 11 Sep 2019 12:28:37 +0200 Subject: [PATCH 128/287] Set version to 2.2.0.dev4 --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index ed4781af1..944bcae77 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -4,7 +4,7 @@ # fmt: off __title__ = "spacy" -__version__ = "2.2.0.dev3" +__version__ = "2.2.0.dev4" __summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython" __uri__ = "https://spacy.io" __author__ = "Explosion AI" From af253236534a69ccdef1428cb0d8b6b7461e271c Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Wed, 11 Sep 2019 14:00:36 +0200 Subject: [PATCH 129/287] Tidy up and auto-format --- spacy/_ml.py | 50 ++++++++++++--------------- spacy/cli/train.py | 13 +++++-- spacy/errors.py | 1 + spacy/lang/de/__init__.py | 18 +++++++--- spacy/lang/en/__init__.py | 12 ++++--- spacy/lang/en/morph_rules.py | 48 ++++++++++++------------- spacy/lang/en/tokenizer_exceptions.py | 7 +--- spacy/lemmatizer.py | 11 +++--- spacy/lookups.py | 1 + spacy/tests/parser/test_ner.py | 26 ++------------ 10 files changed, 90 insertions(+), 97 deletions(-) diff --git a/spacy/_ml.py b/spacy/_ml.py index 97660f8f9..d81ceccc1 100644 --- a/spacy/_ml.py +++ b/spacy/_ml.py @@ -348,7 +348,7 @@ def Tok2Vec(width, embed_size, **kwargs): if pretrained_vectors is not None: glove = StaticVectors(pretrained_vectors, width, column=cols.index(ID)) - if subword_features: + if subword_features: embed = uniqued( (glove | norm | prefix | suffix | shape) >> LN(Maxout(width, width * 5, pieces=3)), @@ -363,14 +363,16 @@ def Tok2Vec(width, embed_size, **kwargs): embed = uniqued( (norm | prefix | suffix | shape) >> LN(Maxout(width, width * 4, pieces=3)), - column=cols.index(ORTH) + column=cols.index(ORTH), ) - elif char_embed: + elif char_embed: embed = concatenate_lists( CharacterEmbed(nM=64, nC=8), - FeatureExtracter(cols) >> with_flatten(norm) + FeatureExtracter(cols) >> with_flatten(norm), + ) + reduce_dimensions = LN( + Maxout(width, 64 * 8 + width, pieces=cnn_maxout_pieces) ) - reduce_dimensions = LN(Maxout(width, 64*8+width, pieces=cnn_maxout_pieces)) else: embed = norm @@ -379,22 +381,14 @@ def Tok2Vec(width, embed_size, **kwargs): >> LN(Maxout(width, width * 3, pieces=cnn_maxout_pieces)) ) if char_embed: - tok2vec = ( - embed - >> with_flatten( - reduce_dimensions - >> convolution ** conv_depth, pad=conv_depth - ) + tok2vec = embed >> with_flatten( + reduce_dimensions >> convolution ** conv_depth, pad=conv_depth ) else: - tok2vec = ( - FeatureExtracter(cols) - >> with_flatten( - embed - >> convolution ** conv_depth, pad=conv_depth - ) + tok2vec = FeatureExtracter(cols) >> with_flatten( + embed >> convolution ** conv_depth, pad=conv_depth ) - + if bilstm_depth >= 1: tok2vec = tok2vec >> PyTorchBiLSTM(width, width, bilstm_depth) # Work around thinc API limitations :(. TODO: Revise in Thinc 7 @@ -611,9 +605,7 @@ def build_morphologizer_model(class_nums, **cfg): char_embed=char_embed, pretrained_vectors=pretrained_vectors, ) - softmax = with_flatten( - MultiSoftmax(class_nums, token_vector_width) - ) + softmax = with_flatten(MultiSoftmax(class_nums, token_vector_width)) softmax.out_sizes = class_nums model = tok2vec >> softmax model.nI = None @@ -906,16 +898,17 @@ def _replace_word(word, random_words, mask="[MASK]"): def _uniform_init(lo, hi): def wrapped(W, ops): copy_array(W, ops.xp.random.uniform(lo, hi, W.shape)) + return wrapped @describe.attributes( nM=Dimension("Vector dimensions"), nC=Dimension("Number of characters per word"), - vectors=Synapses("Embed matrix", - lambda obj: (obj.nC, obj.nV, obj.nM), - _uniform_init(-0.1, 0.1)), - d_vectors=Gradient("vectors") + vectors=Synapses( + "Embed matrix", lambda obj: (obj.nC, obj.nV, obj.nM), _uniform_init(-0.1, 0.1) + ), + d_vectors=Gradient("vectors"), ) class CharacterEmbed(Model): def __init__(self, nM=None, nC=None, **kwargs): @@ -926,12 +919,12 @@ class CharacterEmbed(Model): @property def nO(self): return self.nM * self.nC - + @property def nV(self): return 256 - def begin_update(self, docs, drop=0.): + def begin_update(self, docs, drop=0.0): if not docs: return [] ids = [] @@ -959,6 +952,7 @@ class CharacterEmbed(Model): if sgd is not None: sgd(self._mem.weights, self._mem.gradient, key=self.id) return None + return output, backprop_character_embed @@ -974,4 +968,4 @@ def get_cossim_loss(yh, y): cosine = (yh * y).sum(axis=1, keepdims=True) / mul_norms d_yh = (y / mul_norms) - (cosine * (yh / norm_yh ** 2)) loss = xp.abs(cosine - 1).sum() - return loss, -d_yh \ No newline at end of file + return loss, -d_yh diff --git a/spacy/cli/train.py b/spacy/cli/train.py index 365e7ea44..8d162362c 100644 --- a/spacy/cli/train.py +++ b/spacy/cli/train.py @@ -64,7 +64,12 @@ from .. import about str, ), noise_level=("Amount of corruption for data augmentation", "option", "nl", float), - orth_variant_level=("Amount of orthography variation for data augmentation", "option", "ovl", float), + orth_variant_level=( + "Amount of orthography variation for data augmentation", + "option", + "ovl", + float, + ), eval_beam_widths=("Beam widths to evaluate, e.g. 4,8", "option", "bw", str), gold_preproc=("Use gold preprocessing", "flag", "G", bool), learn_tokens=("Make parser learn gold-standard tokenization", "flag", "T", bool), @@ -245,7 +250,11 @@ def train( best_score = 0.0 for i in range(n_iter): train_docs = corpus.train_docs( - nlp, noise_level=noise_level, orth_variant_level=orth_variant_level, gold_preproc=gold_preproc, max_length=0 + nlp, + noise_level=noise_level, + orth_variant_level=orth_variant_level, + gold_preproc=gold_preproc, + max_length=0, ) if raw_text: random.shuffle(raw_text) diff --git a/spacy/errors.py b/spacy/errors.py index c0868800d..b8a8dccba 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -456,6 +456,7 @@ class Errors(object): E159 = ("Can't find table '{name}' in lookups. Available tables: {tables}") E160 = ("Can't find language data file: {path}") + @add_codes class TempErrors(object): T003 = ("Resizing pre-trained Tagger models is not currently supported.") diff --git a/spacy/lang/de/__init__.py b/spacy/lang/de/__init__.py index 1ddee54b3..b96069235 100644 --- a/spacy/lang/de/__init__.py +++ b/spacy/lang/de/__init__.py @@ -27,10 +27,20 @@ class GermanDefaults(Language.Defaults): stop_words = STOP_WORDS syntax_iterators = SYNTAX_ITERATORS resources = {"lemma_lookup": "lemma_lookup.json"} - single_orth_variants = [{"tags": ["$("], "variants": ["…", "..."]}, - {"tags": ["$("], "variants": ["-", "—", "–", "--", "---", "——"]}] - paired_orth_variants = [{"tags": ["$("], "variants": [("'", "'"), (",", "'"), ("‚", "‘"), ("›", "‹"), ("‹", "›")]}, - {"tags": ["$("], "variants": [("``", "''"), ('"', '"'), ("„", "“"), ("»", "«"), ("«", "»")]}] + single_orth_variants = [ + {"tags": ["$("], "variants": ["…", "..."]}, + {"tags": ["$("], "variants": ["-", "—", "–", "--", "---", "——"]}, + ] + paired_orth_variants = [ + { + "tags": ["$("], + "variants": [("'", "'"), (",", "'"), ("‚", "‘"), ("›", "‹"), ("‹", "›")], + }, + { + "tags": ["$("], + "variants": [("``", "''"), ('"', '"'), ("„", "“"), ("»", "«"), ("«", "»")], + }, + ] class German(Language): diff --git a/spacy/lang/en/__init__.py b/spacy/lang/en/__init__.py index 2f391de0b..e4c745c83 100644 --- a/spacy/lang/en/__init__.py +++ b/spacy/lang/en/__init__.py @@ -38,10 +38,14 @@ class EnglishDefaults(Language.Defaults): "lemma_index": "lemmatizer/lemma_index.json", "lemma_exc": "lemmatizer/lemma_exc.json", } - single_orth_variants = [{"tags": ["NFP"], "variants": ["…", "..."]}, - {"tags": [":"], "variants": ["-", "—", "–", "--", "---", "——"]}] - paired_orth_variants = [{"tags": ["``", "''"], "variants": [("'", "'"), ("‘", "’")]}, - {"tags": ["``", "''"], "variants": [('"', '"'), ("“", "”")]}] + single_orth_variants = [ + {"tags": ["NFP"], "variants": ["…", "..."]}, + {"tags": [":"], "variants": ["-", "—", "–", "--", "---", "——"]}, + ] + paired_orth_variants = [ + {"tags": ["``", "''"], "variants": [("'", "'"), ("‘", "’")]}, + {"tags": ["``", "''"], "variants": [('"', '"'), ("“", "”")]}, + ] class English(Language): diff --git a/spacy/lang/en/morph_rules.py b/spacy/lang/en/morph_rules.py index f910e42b8..5ed4eac59 100644 --- a/spacy/lang/en/morph_rules.py +++ b/spacy/lang/en/morph_rules.py @@ -12,50 +12,50 @@ _subordinating_conjunctions = [ "if", "as", "because", - #"of", - #"for", - #"before", - #"in", + # "of", + # "for", + # "before", + # "in", "while", - #"after", + # "after", "since", "like", - #"with", + # "with", "so", - #"to", - #"by", - #"on", - #"about", + # "to", + # "by", + # "on", + # "about", "than", "whether", "although", - #"from", + # "from", "though", - #"until", + # "until", "unless", "once", - #"without", - #"at", - #"into", + # "without", + # "at", + # "into", "cause", - #"over", + # "over", "upon", "till", "whereas", - #"beyond", + # "beyond", "whilst", "except", "despite", "wether", - #"then", + # "then", "but", "becuse", "whie", - #"below", - #"against", + # "below", + # "against", "it", "w/out", - #"toward", + # "toward", "albeit", "save", "besides", @@ -67,17 +67,17 @@ _subordinating_conjunctions = [ "out", "near", "seince", - #"towards", + # "towards", "tho", "sice", "will", ] # This seems kind of wrong too? -#_relative_pronouns = ["this", "that", "those", "these"] +# _relative_pronouns = ["this", "that", "those", "these"] MORPH_RULES = { - #"DT": {word: {"POS": "PRON"} for word in _relative_pronouns}, + # "DT": {word: {"POS": "PRON"} for word in _relative_pronouns}, "IN": {word: {"POS": "SCONJ"} for word in _subordinating_conjunctions}, "NN": { "something": {"POS": "PRON"}, diff --git a/spacy/lang/en/tokenizer_exceptions.py b/spacy/lang/en/tokenizer_exceptions.py index 91c29c9e0..c45197771 100644 --- a/spacy/lang/en/tokenizer_exceptions.py +++ b/spacy/lang/en/tokenizer_exceptions.py @@ -30,12 +30,7 @@ for pron in ["i"]: for orth in [pron, pron.title()]: _exc[orth + "'m"] = [ {ORTH: orth, LEMMA: PRON_LEMMA, NORM: pron, TAG: "PRP"}, - { - ORTH: "'m", - LEMMA: "be", - NORM: "am", - TAG: "VBP", - }, + {ORTH: "'m", LEMMA: "be", NORM: "am", TAG: "VBP"}, ] _exc[orth + "m"] = [ diff --git a/spacy/lemmatizer.py b/spacy/lemmatizer.py index c9ccbcd0d..d14f5292e 100644 --- a/spacy/lemmatizer.py +++ b/spacy/lemmatizer.py @@ -2,8 +2,7 @@ from __future__ import unicode_literals from collections import OrderedDict -from .symbols import POS, NOUN, VERB, ADJ, PUNCT, PROPN -from .symbols import VerbForm_inf, VerbForm_none, Number_sing, Degree_pos +from .symbols import NOUN, VERB, ADJ, PUNCT, PROPN class Lemmatizer(object): @@ -71,13 +70,13 @@ class Lemmatizer(object): return True elif univ_pos == "adj" and morphology.get("Degree") == "pos": return True - elif morphology.get('VerbForm') == 'inf': + elif morphology.get("VerbForm") == "inf": return True - elif morphology.get('VerbForm') == 'none': + elif morphology.get("VerbForm") == "none": return True - elif morphology.get('VerbForm') == 'inf': + elif morphology.get("VerbForm") == "inf": return True - elif morphology.get('Degree') == 'pos': + elif morphology.get("Degree") == "pos": return True else: return False diff --git a/spacy/lookups.py b/spacy/lookups.py index 801b4d00d..741d40330 100644 --- a/spacy/lookups.py +++ b/spacy/lookups.py @@ -137,6 +137,7 @@ class Table(OrderedDict): """A table in the lookups. Subclass of builtin dict that implements a slightly more consistent and unified API. """ + @classmethod def from_dict(cls, data, name=None): self = cls(name=name) diff --git a/spacy/tests/parser/test_ner.py b/spacy/tests/parser/test_ner.py index c39491ecf..db911dba0 100644 --- a/spacy/tests/parser/test_ner.py +++ b/spacy/tests/parser/test_ner.py @@ -119,28 +119,8 @@ def test_oracle_moves_missing_B(en_vocab): def test_oracle_moves_whitespace(en_vocab): - words = [ - "production", - "\n", - "of", - "Northrop", - "\n", - "Corp.", - "\n", - "'s", - "radar", - ] - biluo_tags = [ - "O", - "O", - "O", - "B-ORG", - None, - "I-ORG", - "L-ORG", - "O", - "O", - ] + words = ["production", "\n", "of", "Northrop", "\n", "Corp.", "\n", "'s", "radar"] + biluo_tags = ["O", "O", "O", "B-ORG", None, "I-ORG", "L-ORG", "O", "O"] doc = Doc(en_vocab, words=words) gold = GoldParse(doc, words=words, entities=biluo_tags) @@ -156,4 +136,4 @@ def test_oracle_moves_whitespace(en_vocab): action, label = tag.split("-") moves.add_action(move_types.index(action), label) moves.preprocess_gold(gold) - seq = moves.get_oracle_sequence(doc, gold) + moves.get_oracle_sequence(doc, gold) From c47c0269b174653012a2961dcfb9a50c6f7d5576 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 11 Sep 2019 15:16:53 +0200 Subject: [PATCH 130/287] Update morphology features --- spacy/morphology.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 2d58b8f27..8fcf81ade 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -864,13 +864,13 @@ FEATURES = [ "Animacy_hum", "Animacy_inan", "Animacy_nhum", - "Aspect_freq", + "Aspect_hab", "Aspect_imp", - "Aspect_mod", - "Aspect_none", + "Aspect_iter", "Aspect_perf", "Aspect_prog", "Aspect_prosp", + "Aspect_none", "Case_abe", "Case_abl", "Case_abs", From f8ce9dde0fb07bdbcfa7699a91ad5cbe064e3fcd Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 11 Sep 2019 17:41:21 +0200 Subject: [PATCH 131/287] Set version to v2.2.0.dev5 --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index 944bcae77..dba60ffc8 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -4,7 +4,7 @@ # fmt: off __title__ = "spacy" -__version__ = "2.2.0.dev4" +__version__ = "2.2.0.dev5" __summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython" __uri__ = "https://spacy.io" __author__ = "Explosion AI" From f7a096b46257078a81b8219deea8c1c9c74b7209 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 11 Sep 2019 18:06:43 +0200 Subject: [PATCH 132/287] Update morphology --- spacy/morphology.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx index 8fcf81ade..bf7aaced0 100644 --- a/spacy/morphology.pyx +++ b/spacy/morphology.pyx @@ -985,6 +985,7 @@ FEATURES = [ "NumForm_digit", "NumForm_roman", "NumForm_word", + "NumForm_combi", "NumType_card", "NumType_dist", "NumType_frac", @@ -993,6 +994,7 @@ FEATURES = [ "NumType_none", "NumType_ord", "NumType_sets", + "NumType_dual", "NumValue_one", "NumValue_two", "NumValue_three", From 7fbb559045c8a2127059edc8a17e5722b0e423c6 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 11 Sep 2019 18:07:20 +0200 Subject: [PATCH 133/287] Set version to v2.2.0.dev6 --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index dba60ffc8..4d3de2d40 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -4,7 +4,7 @@ # fmt: off __title__ = "spacy" -__version__ = "2.2.0.dev5" +__version__ = "2.2.0.dev6" __summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython" __uri__ = "https://spacy.io" __author__ = "Explosion AI" From 0b4b4f1819df2b8a885a2ce6a89a2bfe88249aba Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Thu, 12 Sep 2019 11:38:34 +0200 Subject: [PATCH 134/287] Documentation for Entity Linking (#4065) * document token ent_kb_id * document span kb_id * update pipeline documentation * prior and context weights as bool's instead * entitylinker api documentation * drop for both models * finish entitylinker documentation * small fixes * documentation for KB * candidate documentation * links to api pages in code * small fix * frequency examples as counts for consistency * consistent documentation about tensors returned by predict * add entity linking to usage 101 * add entity linking infobox and KB section to 101 * entity-linking in linguistic features * small typo corrections * training example and docs for entity_linker * predefined nlp and kb * revert back to similarity encodings for simplicity (for now) * set prior probabilities to 0 when excluded * code clean up * bugfix: deleting kb ID from tokens when entities were removed * refactor train el example to use either model or vocab * pretrain_kb example for example kb generation * add to training docs for KB + EL example scripts * small fixes * error numbering * ensure the language of vocab and nlp stay consistent across serialization * equality with = * avoid conflict in errors file * add error 151 * final adjustements to the train scripts - consistency * update of goldparse documentation * small corrections * push commit * typo fix * add candidate API to kb documentation * update API sidebar with EntityLinker and KnowledgeBase * remove EL from 101 docs * remove entity linker from 101 pipelines / rephrase * custom el model instead of existing model * set version to 2.2 for EL functionality * update documentation for 2 CLI scripts --- examples/pipeline/dummy_entity_linking.py | 0 examples/pipeline/wikidata_entity_linking.py | 0 examples/training/pretrain_kb.py | 5 +- examples/training/train_entity_linker.py | 4 +- spacy/kb.pyx | 2 +- website/docs/api/cli.md | 2 +- website/docs/api/entitylinker.md | 297 +++++++++++++++++++ website/docs/api/entityrecognizer.md | 11 +- website/docs/api/goldparse.md | 22 +- website/docs/api/kb.md | 268 +++++++++++++++++ website/docs/api/span.md | 21 +- website/docs/api/tagger.md | 16 +- website/docs/api/textcategorizer.md | 11 +- website/docs/api/token.md | 4 +- website/docs/usage/101/_named-entities.md | 12 +- website/docs/usage/101/_pipelines.md | 20 +- website/docs/usage/101/_training.md | 2 +- website/docs/usage/facts-figures.md | 2 +- website/docs/usage/linguistic-features.md | 48 +++ website/docs/usage/processing-pipelines.md | 1 + website/docs/usage/spacy-101.md | 79 +++++ website/docs/usage/training.md | 83 +++++- website/meta/sidebars.json | 2 + 23 files changed, 847 insertions(+), 65 deletions(-) create mode 100644 examples/pipeline/dummy_entity_linking.py create mode 100644 examples/pipeline/wikidata_entity_linking.py create mode 100644 website/docs/api/entitylinker.md create mode 100644 website/docs/api/kb.md diff --git a/examples/pipeline/dummy_entity_linking.py b/examples/pipeline/dummy_entity_linking.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/pipeline/wikidata_entity_linking.py b/examples/pipeline/wikidata_entity_linking.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/training/pretrain_kb.py b/examples/training/pretrain_kb.py index d5281ad42..2c494d5c4 100644 --- a/examples/training/pretrain_kb.py +++ b/examples/training/pretrain_kb.py @@ -8,8 +8,8 @@ For more details, see the documentation: * Knowledge base: https://spacy.io/api/kb * Entity Linking: https://spacy.io/usage/linguistic-features#entity-linking -Compatible with: spaCy vX.X -Last tested with: vX.X +Compatible with: spaCy v2.2 +Last tested with: v2.2 """ from __future__ import unicode_literals, print_function @@ -73,7 +73,6 @@ def main(vocab_path=None, model=None, output_dir=None, n_iter=50): input_dim=INPUT_DIM, desc_width=DESC_WIDTH, epochs=n_iter, - threshold=0.001, ) encoder.train(description_list=descriptions, to_print=True) diff --git a/examples/training/train_entity_linker.py b/examples/training/train_entity_linker.py index 12ed531a6..d2b2c2417 100644 --- a/examples/training/train_entity_linker.py +++ b/examples/training/train_entity_linker.py @@ -8,8 +8,8 @@ For more details, see the documentation: * Training: https://spacy.io/usage/training * Entity Linking: https://spacy.io/usage/linguistic-features#entity-linking -Compatible with: spaCy vX.X -Last tested with: vX.X +Compatible with: spaCy v2.2 +Last tested with: v2.2 """ from __future__ import unicode_literals, print_function diff --git a/spacy/kb.pyx b/spacy/kb.pyx index 176ac17de..6cbc06e2c 100644 --- a/spacy/kb.pyx +++ b/spacy/kb.pyx @@ -24,7 +24,7 @@ cdef class Candidate: algorithm which will disambiguate the various candidates to the correct one. Each candidate (alias, entity) pair is assigned to a certain prior probability. - DOCS: https://spacy.io/api/candidate + DOCS: https://spacy.io/api/kb/#candidate_init """ def __init__(self, KnowledgeBase kb, entity_hash, entity_freq, entity_vector, alias_hash, prior_prob): diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index 32e3623b0..d01637925 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -226,7 +226,7 @@ $ python -m spacy train [lang] [output_path] [train_path] [dev_path] | `--entity-multitasks`, `-et` | option | Side objectives for NER CNN, e.g. `'dep'` or `'dep,tag'` | | `--noise-level`, `-nl` | option | Float indicating the amount of corruption for data augmentation. | | `--gold-preproc`, `-G` | flag | Use gold preprocessing. | -| `--learn-tokens`, `-T` | flag | Make parser learn gold-standard tokenization by merging ] subtokens. Typically used for languages like Chinese. | +| `--learn-tokens`, `-T` | flag | Make parser learn gold-standard tokenization by merging subtokens. Typically used for languages like Chinese. | | `--verbose`, `-VV` 2.0.13 | flag | Show more detailed messages during training. | | `--help`, `-h` | flag | Show help message and available arguments. | | **CREATES** | model, pickle | A spaCy model on each epoch. | diff --git a/website/docs/api/entitylinker.md b/website/docs/api/entitylinker.md new file mode 100644 index 000000000..64db50943 --- /dev/null +++ b/website/docs/api/entitylinker.md @@ -0,0 +1,297 @@ +--- +title: EntityLinker +teaser: Functionality to disambiguate a named entity in text to a unique knowledge base identifier. +tag: class +source: spacy/pipeline/pipes.pyx +new: 2.2 +--- + +This class is a subclass of `Pipe` and follows the same API. The pipeline +component is available in the [processing pipeline](/usage/processing-pipelines) +via the ID `"entity_linker"`. + +## EntityLinker.Model {#model tag="classmethod"} + +Initialize a model for the pipe. The model should implement the +`thinc.neural.Model` API, and should contain a field `tok2vec` that contains +the context encoder. Wrappers are under development for most major machine +learning libraries. + +| Name | Type | Description | +| ----------- | ------ | ------------------------------------- | +| `**kwargs` | - | Parameters for initializing the model | +| **RETURNS** | object | The initialized model. | + +## EntityLinker.\_\_init\_\_ {#init tag="method"} + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.create_pipe`](/api/language#create_pipe). + +> #### Example +> +> ```python +> # Construction via create_pipe +> entity_linker = nlp.create_pipe("entity_linker") +> +> # Construction from class +> from spacy.pipeline import EntityLinker +> entity_linker = EntityLinker(nlp.vocab) +> entity_linker.from_disk("/path/to/model") +> ``` + +| Name | Type | Description | +| --------------- | ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | `Vocab` | The shared vocabulary. | +| `model` | `thinc.neural.Model` / `True` | The model powering the pipeline component. If no model is supplied, the model is created when you call `begin_training`, `from_disk` or `from_bytes`. | +| `hidden_width` | int | Width of the hidden layer of the entity linking model, defaults to 128. | +| `incl_prior` | bool | Whether or not to include prior probabilities in the model. Defaults to True. | +| `incl_context` | bool | Whether or not to include the local context in the model (if not: only prior probabilites are used). Defaults to True. | +| **RETURNS** | `EntityLinker` | The newly constructed object. | + +## EntityLinker.\_\_call\_\_ {#call tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/entitylinker#call) and +[`pipe`](/api/entitylinker#pipe) delegate to the +[`predict`](/api/entitylinker#predict) and +[`set_annotations`](/api/entitylinker#set_annotations) methods. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> doc = nlp(u"This is a sentence.") +> # This usually happens under the hood +> processed = entity_linker(doc) +> ``` + +| Name | Type | Description | +| ----------- | ----- | ------------------------ | +| `doc` | `Doc` | The document to process. | +| **RETURNS** | `Doc` | The processed document. | + +## EntityLinker.pipe {#pipe tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/entitylinker#call) and +[`pipe`](/api/entitylinker#pipe) delegate to the +[`predict`](/api/entitylinker#predict) and +[`set_annotations`](/api/entitylinker#set_annotations) methods. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> for doc in entity_linker.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Type | Description | +| ------------ | -------- | ------------------------------------------------------ | +| `stream` | iterable | A stream of documents. | +| `batch_size` | int | The number of texts to buffer. Defaults to `128`. | +| **YIELDS** | `Doc` | Processed documents in the order of the original text. | + +## EntityLinker.predict {#predict tag="method"} + +Apply the pipeline's model to a batch of docs, without modifying them. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> kb_ids, tensors = entity_linker.predict([doc1, doc2]) +> ``` + +| Name | Type | Description | +| ----------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docs` | iterable | The documents to predict. | +| **RETURNS** | tuple | A `(kb_ids, tensors)` tuple where `kb_ids` are the model's predicted KB identifiers for the entities in the `docs`, and `tensors` are the token representations used to predict these identifiers. | + +## EntityLinker.set_annotations {#set_annotations tag="method"} + +Modify a batch of documents, using pre-computed entity IDs for a list of named entities. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> kb_ids, tensors = entity_linker.predict([doc1, doc2]) +> entity_linker.set_annotations([doc1, doc2], kb_ids, tensors) +> ``` + +| Name | Type | Description | +| ---------- | -------- | --------------------------------------------------------------------------------------------------- | +| `docs` | iterable | The documents to modify. | +| `kb_ids` | iterable | The knowledge base identifiers for the entities in the docs, predicted by `EntityLinker.predict`. | +| `tensors` | iterable | The token representations used to predict the identifiers. | + +## EntityLinker.update {#update tag="method"} + +Learn from a batch of documents and gold-standard information, updating both the +pipe's entity linking model and context encoder. Delegates to [`predict`](/api/entitylinker#predict) and +[`get_loss`](/api/entitylinker#get_loss). + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> losses = {} +> optimizer = nlp.begin_training() +> entity_linker.update([doc1, doc2], [gold1, gold2], losses=losses, sgd=optimizer) +> ``` + +| Name | Type | Description | +| -------- | -------- | ------------------------------------------------------------------------------------------------------------- | +| `docs` | iterable | A batch of documents to learn from. | +| `golds` | iterable | The gold-standard data. Must have the same length as `docs`. | +| `drop` | float | The dropout rate, used both for the EL model and the context encoder. | +| `sgd` | callable | The optimizer for the EL model. Should take two arguments `weights` and `gradient`, and an optional ID. | +| `losses` | dict | Optional record of the loss during training. The value keyed by the model's name is updated. | + +## EntityLinker.get_loss {#get_loss tag="method"} + +Find the loss and gradient of loss for the entities in a batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> kb_ids, tensors = entity_linker.predict(docs) +> loss, d_loss = entity_linker.get_loss(docs, [gold1, gold2], kb_ids, tensors) +> ``` + +| Name | Type | Description | +| --------------- | -------- | ------------------------------------------------------------ | +| `docs` | iterable | The batch of documents. | +| `golds` | iterable | The gold-standard data. Must have the same length as `docs`. | +| `kb_ids` | iterable | KB identifiers representing the model's predictions. | +| `tensors` | iterable | The token representations used to predict the identifiers | +| **RETURNS** | tuple | The loss and the gradient, i.e. `(loss, gradient)`. | + +## EntityLinker.set_kb {#set_kb tag="method"} + +Define the knowledge base (KB) used for disambiguating named entities to KB identifiers. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> entity_linker.set_kb(kb) +> ``` + +| Name | Type | Description | +| --------------- | --------------- | ------------------------------------------------------------ | +| `kb` | `KnowledgeBase` | The [`KnowledgeBase`](/api/kb). | + +## EntityLinker.begin_training {#begin_training tag="method"} + +Initialize the pipe for training, using data examples if available. If no model +has been initialized yet, the model is added. +Before calling this method, a knowledge base should have been defined with [`set_kb`](/api/entitylinker#set_kb). + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> entity_linker.set_kb(kb) +> nlp.add_pipe(entity_linker, last=True) +> optimizer = entity_linker.begin_training(pipeline=nlp.pipeline) +> ``` + +| Name | Type | Description | +| ------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `gold_tuples` | iterable | Optional gold-standard annotations from which to construct [`GoldParse`](/api/goldparse) objects. | +| `pipeline` | list | Optional list of pipeline components that this component is part of. | +| `sgd` | callable | An optional optimizer. Should take two arguments `weights` and `gradient`, and an optional ID. Will be created via [`EntityLinker`](/api/entitylinker#create_optimizer) if not set. | +| **RETURNS** | callable | An optimizer. | + +## EntityLinker.create_optimizer {#create_optimizer tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> optimizer = entity_linker.create_optimizer() +> ``` + +| Name | Type | Description | +| ----------- | -------- | -------------- | +| **RETURNS** | callable | The optimizer. | + +## EntityLinker.use_params {#use_params tag="method, contextmanager"} + +Modify the pipe's EL model, to use the given parameter values. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> with entity_linker.use_params(optimizer.averages): +> entity_linker.to_disk("/best_model") +> ``` + +| Name | Type | Description | +| -------- | ---- | ---------------------------------------------------------------------------------------------------------- | +| `params` | dict | The parameter values to use in the model. At the end of the context, the original parameters are restored. | + + +## EntityLinker.to_disk {#to_disk tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> entity_linker.to_disk("/path/to/entity_linker") +> ``` + +| Name | Type | Description | +| --------- | ---------------- | --------------------------------------------------------------------------------------------------------------------- | +| `path` | unicode / `Path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. | +| `exclude` | list | String names of [serialization fields](#serialization-fields) to exclude. | + +## EntityLinker.from_disk {#from_disk tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> entity_linker = EntityLinker(nlp.vocab) +> entity_linker.from_disk("/path/to/entity_linker") +> ``` + +| Name | Type | Description | +| ----------- | ------------------ | -------------------------------------------------------------------------- | +| `path` | unicode / `Path` | A path to a directory. Paths may be either strings or `Path`-like objects. | +| `exclude` | list | String names of [serialization fields](#serialization-fields) to exclude. | +| **RETURNS** | `EntityLinker` | The modified `EntityLinker` object. | + +## Serialization fields {#serialization-fields} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = entity_linker.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | +| `kb` | The knowledge base. You usually don't want to exclude this. | + diff --git a/website/docs/api/entityrecognizer.md b/website/docs/api/entityrecognizer.md index 7279a7f77..46e8b44ee 100644 --- a/website/docs/api/entityrecognizer.md +++ b/website/docs/api/entityrecognizer.md @@ -99,7 +99,7 @@ Apply the pipeline's model to a batch of docs, without modifying them. > > ```python > ner = EntityRecognizer(nlp.vocab) -> scores = ner.predict([doc1, doc2]) +> scores, tensors = ner.predict([doc1, doc2]) > ``` | Name | Type | Description | @@ -115,14 +115,15 @@ Modify a batch of documents, using pre-computed scores. > > ```python > ner = EntityRecognizer(nlp.vocab) -> scores = ner.predict([doc1, doc2]) -> ner.set_annotations([doc1, doc2], scores) +> scores, tensors = ner.predict([doc1, doc2]) +> ner.set_annotations([doc1, doc2], scores, tensors) > ``` | Name | Type | Description | | -------- | -------- | ---------------------------------------------------------- | | `docs` | iterable | The documents to modify. | | `scores` | - | The scores to set, produced by `EntityRecognizer.predict`. | +| `tensors`| iterable | The token representations used to predict the scores. | ## EntityRecognizer.update {#update tag="method"} @@ -210,13 +211,13 @@ Modify the pipe's model, to use the given parameter values. > > ```python > ner = EntityRecognizer(nlp.vocab) -> with ner.use_params(): +> with ner.use_params(optimizer.averages): > ner.to_disk("/best_model") > ``` | Name | Type | Description | | -------- | ---- | ---------------------------------------------------------------------------------------------------------- | -| `params` | - | The parameter values to use in the model. At the end of the context, the original parameters are restored. | +| `params` | dict | The parameter values to use in the model. At the end of the context, the original parameters are restored. | ## EntityRecognizer.add_label {#add_label tag="method"} diff --git a/website/docs/api/goldparse.md b/website/docs/api/goldparse.md index 5a2d8a110..db7d07795 100644 --- a/website/docs/api/goldparse.md +++ b/website/docs/api/goldparse.md @@ -23,6 +23,7 @@ gradient for those labels will be zero. | `deps` | iterable | A sequence of strings, representing the syntactic relation types. | | `entities` | iterable | A sequence of named entity annotations, either as BILUO tag strings, or as `(start_char, end_char, label)` tuples, representing the entity positions. If BILUO tag strings, you can specify missing values by setting the tag to None. | | `cats` | dict | Labels for text classification. Each key in the dictionary may be a string or an int, or a `(start_char, end_char, label)` tuple, indicating that the label is applied to only part of the document (usually a sentence). | +| `links` | dict | Labels for entity linking. A dict with `(start_char, end_char)` keys, and the values being dicts with `kb_id:value` entries, representing external KB IDs mapped to either 1.0 (positive) or 0.0 (negative). | | **RETURNS** | `GoldParse` | The newly constructed object. | ## GoldParse.\_\_len\_\_ {#len tag="method"} @@ -43,16 +44,17 @@ Whether the provided syntactic annotations form a projective dependency tree. ## Attributes {#attributes} -| Name | Type | Description | -| --------------------------------- | ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `words` | list | The words. | -| `tags` | list | The part-of-speech tag annotations. | -| `heads` | list | The syntactic head annotations. | -| `labels` | list | The syntactic relation-type annotations. | -| `ner` | list | The named entity annotations as BILUO tags. | -| `cand_to_gold` | list | The alignment from candidate tokenization to gold tokenization. | -| `gold_to_cand` | list | The alignment from gold tokenization to candidate tokenization. | -| `cats` 2 | list | Entries in the list should be either a label, or a `(start, end, label)` triple. The tuple form is used for categories applied to spans of the document. | +| Name | Type | Description | +| ------------------------------------ | ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `words` | list | The words. | +| `tags` | list | The part-of-speech tag annotations. | +| `heads` | list | The syntactic head annotations. | +| `labels` | list | The syntactic relation-type annotations. | +| `ner` | list | The named entity annotations as BILUO tags. | +| `cand_to_gold` | list | The alignment from candidate tokenization to gold tokenization. | +| `gold_to_cand` | list | The alignment from gold tokenization to candidate tokenization. | +| `cats` 2 | list | Entries in the list should be either a label, or a `(start, end, label)` triple. The tuple form is used for categories applied to spans of the document. | +| `links` 2.2 | dict | Keys in the dictionary are `(start_char, end_char)` triples, and the values are dictionaries with `kb_id:value` entries. | ## Utilities {#util} diff --git a/website/docs/api/kb.md b/website/docs/api/kb.md new file mode 100644 index 000000000..639ababb6 --- /dev/null +++ b/website/docs/api/kb.md @@ -0,0 +1,268 @@ +--- +title: KnowledgeBase +teaser: A storage class for entities and aliases of a specific knowledge base (ontology) +tag: class +source: spacy/kb.pyx +new: 2.2 +--- + +The `KnowledgeBase` object provides a method to generate [`Candidate`](/api/kb/#candidate_init) +objects, which are plausible external identifiers given a certain textual mention. +Each such `Candidate` holds information from the relevant KB entities, +such as its frequency in text and possible aliases. +Each entity in the knowledge base also has a pre-trained entity vector of a fixed size. + +## KnowledgeBase.\_\_init\_\_ {#init tag="method"} + +Create the knowledge base. + +> #### Example +> +> ```python +> from spacy.kb import KnowledgeBase +> vocab = nlp.vocab +> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> ``` + +| Name | Type | Description | +| ----------------------- | ---------------- | ----------------------------------------- | +| `vocab` | `Vocab` | A `Vocab` object. | +| `entity_vector_length` | int | Length of the fixed-size entity vectors. | +| **RETURNS** | `KnowledgeBase` | The newly constructed object. | + + +## KnowledgeBase.entity_vector_length {#entity_vector_length tag="property"} + +The length of the fixed-size entity vectors in the knowledge base. + +| Name | Type | Description | +| ----------- | ---- | ----------------------------------------- | +| **RETURNS** | int | Length of the fixed-size entity vectors. | + +## KnowledgeBase.add_entity {#add_entity tag="method"} + +Add an entity to the knowledge base, specifying its corpus frequency +and entity vector, which should be of length [`entity_vector_length`](/api/kb#entity_vector_length). + +> #### Example +> +> ```python +> kb.add_entity(entity="Q42", freq=32, entity_vector=vector1) +> kb.add_entity(entity="Q463035", freq=111, entity_vector=vector2) +> ``` + +| Name | Type | Description | +| --------------- | ------------- | ------------------------------------------------- | +| `entity` | unicode | The unique entity identifier | +| `freq` | float | The frequency of the entity in a typical corpus | +| `entity_vector` | vector | The pre-trained vector of the entity | + +## KnowledgeBase.set_entities {#set_entities tag="method"} + +Define the full list of entities in the knowledge base, specifying the corpus frequency +and entity vector for each entity. + +> #### Example +> +> ```python +> kb.set_entities(entity_list=["Q42", "Q463035"], freq_list=[32, 111], vector_list=[vector1, vector2]) +> ``` + +| Name | Type | Description | +| ------------- | ------------- | ------------------------------------------------- | +| `entity_list` | iterable | List of unique entity identifiers | +| `freq_list` | iterable | List of entity frequencies | +| `vector_list` | iterable | List of entity vectors | + +## KnowledgeBase.add_alias {#add_alias tag="method"} + +Add an alias or mention to the knowledge base, specifying its potential KB identifiers +and their prior probabilities. The entity identifiers should refer to entities previously +added with [`add_entity`](/api/kb#add_entity) or [`set_entities`](/api/kb#set_entities). +The sum of the prior probabilities should not exceed 1. + +> #### Example +> +> ```python +> kb.add_alias(alias="Douglas", entities=["Q42", "Q463035"], probabilities=[0.6, 0.3]) +> ``` + +| Name | Type | Description | +| -------------- | ------------- | -------------------------------------------------- | +| `alias` | unicode | The textual mention or alias | +| `entities` | iterable | The potential entities that the alias may refer to | +| `probabilities`| iterable | The prior probabilities of each entity | + +## KnowledgeBase.\_\_len\_\_ {#len tag="method"} + +Get the total number of entities in the knowledge base. + +> #### Example +> +> ```python +> total_entities = len(kb) +> ``` + +| Name | Type | Description | +| ----------- | ---- | --------------------------------------------- | +| **RETURNS** | int | The number of entities in the knowledge base. | + +## KnowledgeBase.get_entity_strings {#get_entity_strings tag="method"} + +Get a list of all entity IDs in the knowledge base. + +> #### Example +> +> ```python +> all_entities = kb.get_entity_strings() +> ``` + +| Name | Type | Description | +| ----------- | ---- | --------------------------------------------- | +| **RETURNS** | list | The list of entities in the knowledge base. | + +## KnowledgeBase.get_size_aliases {#get_size_aliases tag="method"} + +Get the total number of aliases in the knowledge base. + +> #### Example +> +> ```python +> total_aliases = kb.get_size_aliases() +> ``` + +| Name | Type | Description | +| ----------- | ---- | --------------------------------------------- | +| **RETURNS** | int | The number of aliases in the knowledge base. | + +## KnowledgeBase.get_alias_strings {#get_alias_strings tag="method"} + +Get a list of all aliases in the knowledge base. + +> #### Example +> +> ```python +> all_aliases = kb.get_alias_strings() +> ``` + +| Name | Type | Description | +| ----------- | ---- | --------------------------------------------- | +| **RETURNS** | list | The list of aliases in the knowledge base. | + +## KnowledgeBase.get_candidates {#get_candidates tag="method"} + +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb/#candidate_init). + +> #### Example +> +> ```python +> candidates = kb.get_candidates("Douglas") +> ``` + +| Name | Type | Description | +| ------------- | ------------- | -------------------------------------------------- | +| `alias` | unicode | The textual mention or alias | +| **RETURNS** | iterable | The list of relevant `Candidate` objects | + +## KnowledgeBase.get_vector {#get_vector tag="method"} + +Given a certain entity ID, retrieve its pre-trained entity vector. + +> #### Example +> +> ```python +> vector = kb.get_vector("Q42") +> ``` + +| Name | Type | Description | +| ------------- | ------------- | -------------------------------------------------- | +| `entity` | unicode | The entity ID | +| **RETURNS** | vector | The entity vector | + +## KnowledgeBase.get_prior_prob {#get_prior_prob tag="method"} + +Given a certain entity ID and a certain textual mention, retrieve +the prior probability of the fact that the mention links to the entity ID. + +> #### Example +> +> ```python +> probability = kb.get_prior_prob("Q42", "Douglas") +> ``` + +| Name | Type | Description | +| ------------- | ------------- | --------------------------------------------------------------- | +| `entity` | unicode | The entity ID | +| `alias` | unicode | The textual mention or alias | +| **RETURNS** | float | The prior probability of the `alias` referring to the `entity` | + +## KnowledgeBase.dump {#dump tag="method"} + +Save the current state of the knowledge base to a directory. + +> #### Example +> +> ```python +> kb.dump(loc) +> ``` + +| Name | Type | Description | +| ------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `loc` | unicode / `Path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. | + +## KnowledgeBase.load_bulk {#load_bulk tag="method"} + +Restore the state of the knowledge base from a given directory. Note that the [`Vocab`](/api/vocab) +should also be the same as the one used to create the KB. + +> #### Example +> +> ```python +> from spacy.kb import KnowledgeBase +> from spacy.vocab import Vocab +> vocab = Vocab().from_disk("/path/to/vocab") +> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> kb.load_bulk("/path/to/kb") +> ``` + + +| Name | Type | Description | +| ----------- | ---------------- | ----------------------------------------------------------------------------------------- | +| `loc` | unicode / `Path` | A path to a directory. Paths may be either strings or `Path`-like objects. | +| **RETURNS** | `KnowledgeBase` | The modified `KnowledgeBase` object. | + + +## Candidate.\_\_init\_\_ {#candidate_init tag="method"} + +Construct a `Candidate` object. Usually this constructor is not called directly, +but instead these objects are returned by the [`get_candidates`](/api/kb#get_candidates) method +of a `KnowledgeBase`. + +> #### Example +> +> ```python +> from spacy.kb import Candidate +> candidate = Candidate(kb, entity_hash, entity_freq, entity_vector, alias_hash, prior_prob) +> ``` + +| Name | Type | Description | +| ------------- | --------------- | -------------------------------------------------------------- | +| `kb` | `KnowledgeBase` | The knowledge base that defined this candidate. | +| `entity_hash` | int | The hash of the entity's KB ID. | +| `entity_freq` | float | The entity frequency as recorded in the KB. | +| `alias_hash` | int | The hash of the textual mention or alias. | +| `prior_prob` | float | The prior probability of the `alias` referring to the `entity` | +| **RETURNS** | `Candidate` | The newly constructed object. | + +## Candidate attributes {#candidate_attributes} + +| Name | Type | Description | +| ---------------------- | ------------ | ------------------------------------------------------------------ | +| `entity` | int | The entity's unique KB identifier | +| `entity_` | unicode | The entity's unique KB identifier | +| `alias` | int | The alias or textual mention | +| `alias_` | unicode | The alias or textual mention | +| `prior_prob` | long | The prior probability of the `alias` referring to the `entity` | +| `entity_freq` | long | The frequency of the entity in a typical corpus | +| `entity_vector` | vector | The pre-trained vector of the entity | diff --git a/website/docs/api/span.md b/website/docs/api/span.md index 0af305b37..79be81ef8 100644 --- a/website/docs/api/span.md +++ b/website/docs/api/span.md @@ -18,14 +18,15 @@ Create a Span object from the slice `doc[start : end]`. > assert [t.text for t in span] == [u"it", u"back", u"!"] > ``` -| Name | Type | Description | -| ----------- | ---------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `doc` | `Doc` | The parent document. | -| `start` | int | The index of the first token of the span. | -| `end` | int | The index of the first token after the span. | -| `label` | int / unicode | A label to attach to the span, e.g. for named entities. As of v2.1, the label can also be a unicode string. | -| `vector` | `numpy.ndarray[ndim=1, dtype='float32']` | A meaning representation of the span. | -| **RETURNS** | `Span` | The newly constructed object. | +| Name | Type | Description | +| ----------- | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------| +| `doc` | `Doc` | The parent document. | +| `start` | int | The index of the first token of the span. | +| `end` | int | The index of the first token after the span. | +| `label` | int / unicode | A label to attach to the span, e.g. for named entities. As of v2.1, the label can also be a unicode string. | +| `kb_id` | int / unicode | A knowledge base ID to attach to the span, e.g. for named entities. The ID can be an integer or a unicode string. | +| `vector` | `numpy.ndarray[ndim=1, dtype='float32']` | A meaning representation of the span. | +| **RETURNS** | `Span` | The newly constructed object. | ## Span.\_\_getitem\_\_ {#getitem tag="method"} @@ -477,9 +478,11 @@ The L2 norm of the span's vector representation. | `text_with_ws` | unicode | The text content of the span with a trailing whitespace character if the last token has one. | | `orth` | int | ID of the verbatim text content. | | `orth_` | unicode | Verbatim text content (identical to `Span.text`). Exists mostly for consistency with the other attributes. | -| `label` | int | The span's label. | +| `label` | int | The hash value of the span's label. | | `label_` | unicode | The span's label. | | `lemma_` | unicode | The span's lemma. | +| `kb_id` | int | The hash value of the knowledge base ID referred to by the span. | +| `kb_id_` | unicode | The knowledge base ID referred to by the span. | | `ent_id` | int | The hash value of the named entity the token is an instance of. | | `ent_id_` | unicode | The string ID of the named entity the token is an instance of. | | `sentiment` | float | A scalar value indicating the positivity or negativity of the span. | diff --git a/website/docs/api/tagger.md b/website/docs/api/tagger.md index a1d921b41..fc6fc67a6 100644 --- a/website/docs/api/tagger.md +++ b/website/docs/api/tagger.md @@ -97,7 +97,7 @@ Apply the pipeline's model to a batch of docs, without modifying them. > > ```python > tagger = Tagger(nlp.vocab) -> scores = tagger.predict([doc1, doc2]) +> scores, tensors = tagger.predict([doc1, doc2]) > ``` | Name | Type | Description | @@ -113,14 +113,16 @@ Modify a batch of documents, using pre-computed scores. > > ```python > tagger = Tagger(nlp.vocab) -> scores = tagger.predict([doc1, doc2]) -> tagger.set_annotations([doc1, doc2], scores) +> scores, tensors = tagger.predict([doc1, doc2]) +> tagger.set_annotations([doc1, doc2], scores, tensors) > ``` -| Name | Type | Description | -| -------- | -------- | ------------------------------------------------ | -| `docs` | iterable | The documents to modify. | -| `scores` | - | The scores to set, produced by `Tagger.predict`. | +| Name | Type | Description | +| -------- | -------- | ----------------------------------------------------- | +| `docs` | iterable | The documents to modify. | +| `scores` | - | The scores to set, produced by `Tagger.predict`. | +| `tensors`| iterable | The token representations used to predict the scores. | + ## Tagger.update {#update tag="method"} diff --git a/website/docs/api/textcategorizer.md b/website/docs/api/textcategorizer.md index 310122b9c..f7158541b 100644 --- a/website/docs/api/textcategorizer.md +++ b/website/docs/api/textcategorizer.md @@ -116,7 +116,7 @@ Apply the pipeline's model to a batch of docs, without modifying them. > > ```python > textcat = TextCategorizer(nlp.vocab) -> scores = textcat.predict([doc1, doc2]) +> scores, tensors = textcat.predict([doc1, doc2]) > ``` | Name | Type | Description | @@ -132,14 +132,15 @@ Modify a batch of documents, using pre-computed scores. > > ```python > textcat = TextCategorizer(nlp.vocab) -> scores = textcat.predict([doc1, doc2]) -> textcat.set_annotations([doc1, doc2], scores) +> scores, tensors = textcat.predict([doc1, doc2]) +> textcat.set_annotations([doc1, doc2], scores, tensors) > ``` | Name | Type | Description | | -------- | -------- | --------------------------------------------------------- | | `docs` | iterable | The documents to modify. | | `scores` | - | The scores to set, produced by `TextCategorizer.predict`. | +| `tensors`| iterable | The token representations used to predict the scores. | ## TextCategorizer.update {#update tag="method"} @@ -227,13 +228,13 @@ Modify the pipe's model, to use the given parameter values. > > ```python > textcat = TextCategorizer(nlp.vocab) -> with textcat.use_params(): +> with textcat.use_params(optimizer.averages): > textcat.to_disk("/best_model") > ``` | Name | Type | Description | | -------- | ---- | ---------------------------------------------------------------------------------------------------------- | -| `params` | - | The parameter values to use in the model. At the end of the context, the original parameters are restored. | +| `params` | dict | The parameter values to use in the model. At the end of the context, the original parameters are restored. | ## TextCategorizer.add_label {#add_label tag="method"} diff --git a/website/docs/api/token.md b/website/docs/api/token.md index 24816b401..8da13454b 100644 --- a/website/docs/api/token.md +++ b/website/docs/api/token.md @@ -425,8 +425,10 @@ The L2 norm of the token's vector representation. | `i` | int | The index of the token within the parent document. | | `ent_type` | int | Named entity type. | | `ent_type_` | unicode | Named entity type. | -| `ent_iob` | int | IOB code of named entity tag. `3` means the token begins an entity, `2` means it is outside an entity, `1` means it is inside an entity, and `0` means no entity tag is set. | | +| `ent_iob` | int | IOB code of named entity tag. `3` means the token begins an entity, `2` means it is outside an entity, `1` means it is inside an entity, and `0` means no entity tag is set. | | `ent_iob_` | unicode | IOB code of named entity tag. "B" means the token begins an entity, "I" means it is inside an entity, "O" means it is outside an entity, and "" means no entity tag is set. | +| `ent_kb_id` 2.2 | int | Knowledge base ID that refers to the named entity this token is a part of, if any. | +| `ent_kb_id_` 2.2 | unicode | Knowledge base ID that refers to the named entity this token is a part of, if any. | | `ent_id` | int | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. | | `ent_id_` | unicode | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. | | `lemma` | int | Base form of the token, with no inflectional suffixes. | diff --git a/website/docs/usage/101/_named-entities.md b/website/docs/usage/101/_named-entities.md index 54db6dbe8..a282ec370 100644 --- a/website/docs/usage/101/_named-entities.md +++ b/website/docs/usage/101/_named-entities.md @@ -1,5 +1,5 @@ A named entity is a "real-world object" that's assigned a name – for example, a -person, a country, a product or a book title. spaCy can **recognize** +person, a country, a product or a book title. spaCy can **recognize** [various types](/api/annotation#named-entities) of named entities in a document, by asking the model for a **prediction**. Because models are statistical and strongly depend on the examples they were trained on, this doesn't always work @@ -21,12 +21,12 @@ for ent in doc.ents: > - **Text:** The original entity text. > - **Start:** Index of start of entity in the `Doc`. > - **End:** Index of end of entity in the `Doc`. -> - **LabeL:** Entity label, i.e. type. +> - **Label:** Entity label, i.e. type. -| Text | Start | End | Label | Description | -| ----------- | :---: | :-: | ------- | ---------------------------------------------------- | -| Apple | 0 | 5 | `ORG` | Companies, agencies, institutions. | -| U.K. | 27 | 31 | `GPE` | Geopolitical entity, i.e. countries, cities, states. | +| Text | Start | End | Label | Description | +| ----------- | :---: | :-: | ------- | ---------------------------------------------------- | +| Apple | 0 | 5 | `ORG` | Companies, agencies, institutions. | +| U.K. | 27 | 31 | `GPE` | Geopolitical entity, i.e. countries, cities, states. | | \$1 billion | 44 | 54 | `MONEY` | Monetary values, including unit. | Using spaCy's built-in [displaCy visualizer](/usage/visualizers), here's what diff --git a/website/docs/usage/101/_pipelines.md b/website/docs/usage/101/_pipelines.md index 68308a381..d33ea45fd 100644 --- a/website/docs/usage/101/_pipelines.md +++ b/website/docs/usage/101/_pipelines.md @@ -12,14 +12,14 @@ passed on to the next component. > - **Creates:** Objects, attributes and properties modified and set by the > component. -| Name | Component | Creates | Description | -| ------------- | ------------------------------------------------------------------ | ----------------------------------------------------------- | ------------------------------------------------ | -| **tokenizer** | [`Tokenizer`](/api/tokenizer) | `Doc` | Segment text into tokens. | -| **tagger** | [`Tagger`](/api/tagger) | `Doc[i].tag` | Assign part-of-speech tags. | -| **parser** | [`DependencyParser`](/api/dependencyparser) | `Doc[i].head`, `Doc[i].dep`, `Doc.sents`, `Doc.noun_chunks` | Assign dependency labels. | -| **ner** | [`EntityRecognizer`](/api/entityrecognizer) | `Doc.ents`, `Doc[i].ent_iob`, `Doc[i].ent_type` | Detect and label named entities. | -| **textcat** | [`TextCategorizer`](/api/textcategorizer) | `Doc.cats` | Assign document labels. | -| ... | [custom components](/usage/processing-pipelines#custom-components) | `Doc._.xxx`, `Token._.xxx`, `Span._.xxx` | Assign custom attributes, methods or properties. | +| Name | Component | Creates | Description | +| ----------------- | ------------------------------------------------------------------ | ----------------------------------------------------------- | ------------------------------------------------ | +| **tokenizer** | [`Tokenizer`](/api/tokenizer) | `Doc` | Segment text into tokens. | +| **tagger** | [`Tagger`](/api/tagger) | `Doc[i].tag` | Assign part-of-speech tags. | +| **parser** | [`DependencyParser`](/api/dependencyparser) | `Doc[i].head`, `Doc[i].dep`, `Doc.sents`, `Doc.noun_chunks` | Assign dependency labels. | +| **ner** | [`EntityRecognizer`](/api/entityrecognizer) | `Doc.ents`, `Doc[i].ent_iob`, `Doc[i].ent_type` | Detect and label named entities. | +| **textcat** | [`TextCategorizer`](/api/textcategorizer) | `Doc.cats` | Assign document labels. | +| ... | [custom components](/usage/processing-pipelines#custom-components) | `Doc._.xxx`, `Token._.xxx`, `Span._.xxx` | Assign custom attributes, methods or properties. | The processing pipeline always **depends on the statistical model** and its capabilities. For example, a pipeline can only include an entity recognizer @@ -49,6 +49,10 @@ them, its dependency predictions may be different. Similarly, it matters if you add the [`EntityRuler`](/api/entityruler) before or after the statistical entity recognizer: if it's added before, the entity recognizer will take the existing entities into account when making predictions. +The [`EntityLinker`](/api/entitylinker), which resolves named entities to +knowledge base IDs, should be preceded by +a pipeline component that recognizes entities such as the +[`EntityRecognizer`](/api/entityrecognizer). diff --git a/website/docs/usage/101/_training.md b/website/docs/usage/101/_training.md index 61e047748..baf3a1891 100644 --- a/website/docs/usage/101/_training.md +++ b/website/docs/usage/101/_training.md @@ -20,7 +20,7 @@ difference, the more significant the gradient and the updates to our model. ![The training process](../../images/training.svg) When training a model, we don't just want it to memorize our examples – we want -it to come up with theory that can be **generalized across other examples**. +it to come up with a theory that can be **generalized across other examples**. After all, we don't just want the model to learn that this one instance of "Amazon" right here is a company – we want it to learn that "Amazon", in contexts _like this_, is most likely a company. That's why the training data diff --git a/website/docs/usage/facts-figures.md b/website/docs/usage/facts-figures.md index a3683b668..40b39d871 100644 --- a/website/docs/usage/facts-figures.md +++ b/website/docs/usage/facts-figures.md @@ -26,7 +26,7 @@ Here's a quick comparison of the functionalities offered by spaCy, | Sentence segmentation | ✅ | ✅ | ✅ | | Dependency parsing | ✅ | ❌ | ✅ | | Entity recognition | ✅ | ✅ | ✅ | -| Entity linking | ❌ | ❌ | ❌ | +| Entity linking | ✅ | ❌ | ❌ | | Coreference resolution | ❌ | ❌ | ✅ | ### When should I use what? {#comparison-usage} diff --git a/website/docs/usage/linguistic-features.md b/website/docs/usage/linguistic-features.md index 66ad816f5..fc1f159ce 100644 --- a/website/docs/usage/linguistic-features.md +++ b/website/docs/usage/linguistic-features.md @@ -576,6 +576,54 @@ import DisplacyEntHtml from 'images/displacy-ent2.html'