explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-13 10:46:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rename 'SP' special tag to '_SP'

Browse Source 
Renaming the tag with an underscore lets us add it to the tag map
without worrying that we'll change the sequence of tags, which throws
off the tag-to-ID mapping. For instance, if we inserted a 'SP' tag,
the "VERB" tag is pushed to a different class ID, and the model is all
messed up.
This commit is contained in:
Matthew Honnibal 2017-10-20 14:01:12 +02:00
parent 506cf2eb13
commit 49895fbef6
5 changed files with 54 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/lang/de/tag_map.py
View File

@ -62,5 +62,5 @@ TAG_MAP = {
"VVIZU": {POS: VERB, "VerbForm": "inf"}, "VVIZU": {POS: VERB, "VerbForm": "inf"},
"VVPP": {POS: VERB, "Aspect": "perf", "VerbForm": "part"}, "VVPP": {POS: VERB, "Aspect": "perf", "VerbForm": "part"},
"XY": {POS: X}, "XY": {POS: X},
"SP": {POS: SPACE} "_SP": {POS: SPACE}
} }

4
spacy/lang/en/tag_map.py
View File

@ -55,11 +55,11 @@ TAG_MAP = {
"WP": {POS: NOUN, "PronType": "int|rel"}, "WP": {POS: NOUN, "PronType": "int|rel"},
"WP$": {POS: ADJ, "Poss": "yes", "PronType": "int|rel"}, "WP$": {POS: ADJ, "Poss": "yes", "PronType": "int|rel"},
"WRB": {POS: ADV, "PronType": "int|rel"}, "WRB": {POS: ADV, "PronType": "int|rel"},
"SP": {POS: SPACE},
"ADD": {POS: X}, "ADD": {POS: X},
"NFP": {POS: PUNCT}, "NFP": {POS: PUNCT},
"GW": {POS: X}, "GW": {POS: X},
"XX": {POS: X}, "XX": {POS: X},
"BES": {POS: VERB}, "BES": {POS: VERB},
"HVS": {POS: VERB} "HVS": {POS: VERB},
"_SP": {POS: SPACE},
} }

2
spacy/lang/es/tag_map.py
View File

@ -303,5 +303,5 @@ TAG_MAP = {
"VERB__VerbForm=Ger": {"morph": "VerbForm=Ger", "pos": "VERB"}, "VERB__VerbForm=Ger": {"morph": "VerbForm=Ger", "pos": "VERB"},
"VERB__VerbForm=Inf": {"morph": "VerbForm=Inf", "pos": "VERB"}, "VERB__VerbForm=Inf": {"morph": "VerbForm=Inf", "pos": "VERB"},
"X___": {"morph": "_", "pos": "X"}, "X___": {"morph": "_", "pos": "X"},
"SP": {"morph": "_", "pos": "SPACE"}, "_SP": {"morph": "_", "pos": "SPACE"},
} }

3
spacy/lang/th/tag_map.py
View File

@ -77,5 +77,6 @@ TAG_MAP = {
"NEG": {POS: PART}, "NEG": {POS: PART},
# PUNCT # PUNCT
"PUNCT": {POS: PUNCT}, "PUNCT": {POS: PUNCT},
"PUNC": {POS: PUNCT} "PUNC": {POS: PUNCT},
"_SP": {POS: SPACE}
} }

14
spacy/morphology.pyx
View File

@ -4,7 +4,7 @@ from __future__ import unicode_literals
from libc.string cimport memset from libc.string cimport memset
from .parts_of_speech cimport ADJ, VERB, NOUN, PUNCT from .parts_of_speech cimport ADJ, VERB, NOUN, PUNCT, SPACE
from .attrs cimport POS, IS_SPACE from .attrs cimport POS, IS_SPACE
from .parts_of_speech import IDS as POS_IDS from .parts_of_speech import IDS as POS_IDS
from .lexeme cimport Lexeme from .lexeme cimport Lexeme
@ -36,14 +36,22 @@ cdef class Morphology:
def __init__(self, StringStore string_store, tag_map, lemmatizer, exc=None): def __init__(self, StringStore string_store, tag_map, lemmatizer, exc=None):
self.mem = Pool() self.mem = Pool()
self.strings = string_store self.strings = string_store
# Add special space symbol. We prefix with underscore, to make sure it
# always sorts to the end.
space_attrs = tag_map.pop('SP', {POS: SPACE})
if '_SP' not in tag_map:
self.strings.add('_SP')
tag_map = dict(tag_map)
tag_map['_SP'] = space_attrs
self.tag_names = tuple(sorted(tag_map.keys()))
self.tag_map = {} self.tag_map = {}
self.lemmatizer = lemmatizer self.lemmatizer = lemmatizer
self.n_tags = len(tag_map) self.n_tags = len(tag_map)
self.tag_names = tuple(sorted(tag_map.keys()))
self.reverse_index = {} self.reverse_index = {}
self.rich_tags = <RichTagC*>self.mem.alloc(self.n_tags+1, sizeof(RichTagC)) self.rich_tags = <RichTagC*>self.mem.alloc(self.n_tags+1, sizeof(RichTagC))
for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())): for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())):
self.strings.add(tag_str)
self.tag_map[tag_str] = dict(attrs) self.tag_map[tag_str] = dict(attrs)
attrs = _normalize_props(attrs) attrs = _normalize_props(attrs)
attrs = intify_attrs(attrs, self.strings, _do_deprecated=True) attrs = intify_attrs(attrs, self.strings, _do_deprecated=True)
@ -93,7 +101,7 @@ cdef class Morphology:
# the statistical model fails. # the statistical model fails.
# Related to Issue #220 # Related to Issue #220
if Lexeme.c_check_flag(token.lex, IS_SPACE): if Lexeme.c_check_flag(token.lex, IS_SPACE):
tag_id = self.reverse_index[self.strings.add('SP')] tag_id = self.reverse_index[self.strings.add('_SP')]
rich_tag = self.rich_tags[tag_id] rich_tag = self.rich_tags[tag_id]
analysis = <MorphAnalysisC*>self._cache.get(tag_id, token.lex.orth) analysis = <MorphAnalysisC*>self._cache.get(tag_id, token.lex.orth)
if analysis is NULL: if analysis is NULL: