mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 19:39:13 +03:00
229 lines
6.8 KiB
Cython
229 lines
6.8 KiB
Cython
# cython: profile=True
|
|
# cython: embedsignature=True
|
|
'''Tokenize English text, using a scheme that differs from the Penn Treebank 3
|
|
scheme in several important respects:
|
|
|
|
* Whitespace is added as tokens, except for single spaces. e.g.,
|
|
|
|
>>> [w.string for w in EN.tokenize(u'\\nHello \\tThere')]
|
|
[u'\\n', u'Hello', u' ', u'\\t', u'There']
|
|
|
|
* Contractions are normalized, e.g.
|
|
|
|
>>> [w.string for w in EN.tokenize(u"isn't ain't won't he's")]
|
|
[u'is', u'not', u'are', u'not', u'will', u'not', u'he', u"__s"]
|
|
|
|
* Hyphenated words are split, with the hyphen preserved, e.g.:
|
|
|
|
>>> [w.string for w in EN.tokenize(u'New York-based')]
|
|
[u'New', u'York', u'-', u'based']
|
|
|
|
Other improvements:
|
|
|
|
* Email addresses, URLs, European-formatted dates and other numeric entities not
|
|
found in the PTB are tokenized correctly
|
|
* Heuristic handling of word-final periods (PTB expects sentence boundary detection
|
|
as a pre-process before tokenization.)
|
|
|
|
Take care to ensure your training and run-time data is tokenized according to the
|
|
same scheme. Tokenization problems are a major cause of poor performance for
|
|
NLP tools. If you're using a pre-trained model, the :py:mod:`spacy.ptb3` module
|
|
provides a fully Penn Treebank 3-compliant tokenizer.
|
|
'''
|
|
from __future__ import unicode_literals
|
|
|
|
from murmurhash.mrmr cimport hash64
|
|
|
|
cimport lang
|
|
from .typedefs cimport hash_t, id_t, flags_t
|
|
import orth
|
|
from .morphology cimport NO_TAG, ADJ, ADV, ADP, CONJ, DET, NOUN, NUM, PRON, PRT, VERB
|
|
from .morphology cimport X, PUNCT, EOL
|
|
|
|
from .tokens cimport Morphology
|
|
|
|
|
|
DEF USE_POS_CACHE = True
|
|
|
|
|
|
POS_TAGS = {
|
|
'NULL': (NO_TAG, {}),
|
|
'EOL': (EOL, {}),
|
|
'CC': (CONJ, {}),
|
|
'CD': (NUM, {}),
|
|
'DT': (DET, {}),
|
|
'EX': (DET, {}),
|
|
'FW': (X, {}),
|
|
'IN': (ADP, {}),
|
|
'JJ': (ADJ, {}),
|
|
'JJR': (ADJ, {'misc': COMPARATIVE}),
|
|
'JJS': (ADJ, {'misc': SUPERLATIVE}),
|
|
'LS': (X, {}),
|
|
'MD': (VERB, {'tenspect': MODAL}),
|
|
'NN': (NOUN, {}),
|
|
'NNS': (NOUN, {'number': PLURAL}),
|
|
'NNP': (NOUN, {'misc': NAME}),
|
|
'NNPS': (NOUN, {'misc': NAME, 'number': PLURAL}),
|
|
'PDT': (DET, {}),
|
|
'POS': (PRT, {'case': GENITIVE}),
|
|
'PRP': (NOUN, {}),
|
|
'PRP$': (NOUN, {'case': GENITIVE}),
|
|
'RB': (ADV, {}),
|
|
'RBR': (ADV, {'misc': COMPARATIVE}),
|
|
'RBS': (ADV, {'misc': SUPERLATIVE}),
|
|
'RP': (PRT, {}),
|
|
'SYM': (X, {}),
|
|
'TO': (PRT, {}),
|
|
'UH': (X, {}),
|
|
'VB': (VERB, {}),
|
|
'VBD': (VERB, {'tenspect': PAST}),
|
|
'VBG': (VERB, {'tenspect': ING}),
|
|
'VBN': (VERB, {'tenspect': PASSIVE}),
|
|
'VBP': (VERB, {'tenspect': PRESENT}),
|
|
'VBZ': (VERB, {'tenspect': PRESENT, 'person': THIRD}),
|
|
'WDT': (DET, {'misc': RELATIVE}),
|
|
'WP': (PRON, {'misc': RELATIVE}),
|
|
'WP$': (PRON, {'misc': RELATIVE, 'case': GENITIVE}),
|
|
'WRB': (ADV, {'misc': RELATIVE}),
|
|
'!': (PUNCT, {}),
|
|
'#': (PUNCT, {}),
|
|
'$': (PUNCT, {}),
|
|
"''": (PUNCT, {}),
|
|
"(": (PUNCT, {}),
|
|
")": (PUNCT, {}),
|
|
"-LRB-": (PUNCT, {}),
|
|
"-RRB-": (PUNCT, {}),
|
|
".": (PUNCT, {}),
|
|
",": (PUNCT, {}),
|
|
"``": (PUNCT, {}),
|
|
":": (PUNCT, {}),
|
|
"?": (PUNCT, {}),
|
|
}
|
|
|
|
|
|
POS_TEMPLATES = (
|
|
(W_sic,),
|
|
(P1_lemma, P1_pos),
|
|
(P2_lemma, P2_pos),
|
|
(N1_sic,),
|
|
(N2_sic,),
|
|
|
|
(W_suffix,),
|
|
(W_prefix,),
|
|
|
|
(P1_pos,),
|
|
(P2_pos,),
|
|
(P1_pos, P2_pos),
|
|
(P1_pos, W_sic),
|
|
(P1_suffix,),
|
|
(N1_suffix,),
|
|
|
|
(W_shape,),
|
|
(W_cluster,),
|
|
(N1_cluster,),
|
|
(N2_cluster,),
|
|
(P1_cluster,),
|
|
(P2_cluster,),
|
|
|
|
(W_pos_type,),
|
|
(N1_pos_type,),
|
|
(N1_pos_type,),
|
|
(P1_pos, W_pos_type, N1_pos_type),
|
|
)
|
|
|
|
|
|
cdef class English(Language):
|
|
"""English tokenizer, tightly coupled to lexicon.
|
|
|
|
Attributes:
|
|
name (unicode): The two letter code used by Wikipedia for the language.
|
|
lexicon (Lexicon): The lexicon. Exposes the lookup method.
|
|
"""
|
|
def load_pos_cache(self, loc):
|
|
cdef int i = 0
|
|
cdef hash_t key
|
|
cdef int pos
|
|
with open(loc) as file_:
|
|
for line in file_:
|
|
pieces = line.split()
|
|
if i >= 500000:
|
|
break
|
|
i += 1
|
|
key = int(pieces[1])
|
|
pos = int(pieces[2])
|
|
self._pos_cache.set(key, <void*>pos)
|
|
|
|
def get_props(self, unicode string):
|
|
return {'flags': self.set_flags(string), 'dense': orth.word_shape(string)}
|
|
|
|
def set_flags(self, unicode string):
|
|
cdef flags_t flags = 0
|
|
flags |= orth.is_alpha(string) << IS_ALPHA
|
|
flags |= orth.is_ascii(string) << IS_ASCII
|
|
flags |= orth.is_digit(string) << IS_DIGIT
|
|
flags |= orth.is_lower(string) << IS_LOWER
|
|
flags |= orth.is_punct(string) << IS_PUNCT
|
|
flags |= orth.is_space(string) << IS_SPACE
|
|
flags |= orth.is_title(string) << IS_TITLE
|
|
flags |= orth.is_upper(string) << IS_UPPER
|
|
|
|
flags |= orth.like_url(string) << LIKE_URL
|
|
flags |= orth.like_number(string) << LIKE_NUMBER
|
|
return flags
|
|
|
|
def set_pos(self, Tokens tokens):
|
|
cdef int i
|
|
cdef atom_t[N_CONTEXT_FIELDS] context
|
|
cdef TokenC* t = tokens.data
|
|
cdef id_t[2] bigram
|
|
cdef hash_t cache_key
|
|
cdef void* cached = NULL
|
|
assert self.morphologizer is not None
|
|
cdef dict tagdict = self.pos_tagger.tagdict
|
|
for i in range(tokens.length):
|
|
if USE_POS_CACHE:
|
|
bigram[0] = tokens.data[i].lex.sic
|
|
bigram[1] = tokens.data[i-1].lex.sic
|
|
cache_key = hash64(bigram, sizeof(id_t) * 2, 0)
|
|
cached = self._pos_cache.get(cache_key)
|
|
if cached != NULL:
|
|
t[i].pos = <int><size_t>cached
|
|
else:
|
|
fill_pos_context(context, i, t)
|
|
t[i].pos = self.pos_tagger.predict(context)
|
|
self.morphologizer.set_morph(i, t)
|
|
|
|
def train_pos(self, Tokens tokens, golds):
|
|
cdef int i
|
|
cdef atom_t[N_CONTEXT_FIELDS] context
|
|
c = 0
|
|
cdef TokenC* t = tokens.data
|
|
for i in range(tokens.length):
|
|
fill_pos_context(context, i, t)
|
|
t[i].pos = self.pos_tagger.predict(context, [golds[i]])
|
|
self.morphologizer.set_morph(i, t)
|
|
c += t[i].pos == golds[i]
|
|
return c
|
|
|
|
|
|
cdef int fill_pos_context(atom_t* context, const int i, const TokenC* tokens) except -1:
|
|
_fill_from_token(&context[P2_sic], &tokens[i-2])
|
|
_fill_from_token(&context[P1_sic], &tokens[i-1])
|
|
_fill_from_token(&context[W_sic], &tokens[i])
|
|
_fill_from_token(&context[N1_sic], &tokens[i+1])
|
|
_fill_from_token(&context[N2_sic], &tokens[i+2])
|
|
|
|
|
|
cdef inline void _fill_from_token(atom_t* context, const TokenC* t) nogil:
|
|
context[0] = t.lex.sic
|
|
context[1] = t.lex.cluster
|
|
context[2] = t.lex.shape
|
|
context[3] = t.lex.prefix
|
|
context[4] = t.lex.suffix
|
|
context[5] = t.pos
|
|
context[6] = t.lemma
|
|
context[7] = t.lex.pos_type
|
|
|
|
|
|
EN = English('en')
|