mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-22 03:49:17 +03:00
123 lines
3.8 KiB
Cython
123 lines
3.8 KiB
Cython
# cython: profile=True
|
|
# cython: embedsignature=True
|
|
'''Tokenize English text, using a scheme that differs from the Penn Treebank 3
|
|
scheme in several important respects:
|
|
|
|
* Whitespace is added as tokens, except for single spaces. e.g.,
|
|
|
|
>>> [w.string for w in EN.tokenize(u'\\nHello \\tThere')]
|
|
[u'\\n', u'Hello', u' ', u'\\t', u'There']
|
|
|
|
* Contractions are normalized, e.g.
|
|
|
|
>>> [w.string for w in EN.tokenize(u"isn't ain't won't he's")]
|
|
[u'is', u'not', u'are', u'not', u'will', u'not', u'he', u"__s"]
|
|
|
|
* Hyphenated words are split, with the hyphen preserved, e.g.:
|
|
|
|
>>> [w.string for w in EN.tokenize(u'New York-based')]
|
|
[u'New', u'York', u'-', u'based']
|
|
|
|
Other improvements:
|
|
|
|
* Email addresses, URLs, European-formatted dates and other numeric entities not
|
|
found in the PTB are tokenized correctly
|
|
* Heuristic handling of word-final periods (PTB expects sentence boundary detection
|
|
as a pre-process before tokenization.)
|
|
|
|
Take care to ensure your training and run-time data is tokenized according to the
|
|
same scheme. Tokenization problems are a major cause of poor performance for
|
|
NLP tools. If you're using a pre-trained model, the :py:mod:`spacy.ptb3` module
|
|
provides a fully Penn Treebank 3-compliant tokenizer.
|
|
'''
|
|
# TODO
|
|
#The script translate_treebank_tokenization can be used to transform a treebank's
|
|
#annotation to use one of the spacy tokenization schemes.
|
|
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
from libc.stdlib cimport malloc, calloc, free
|
|
from libc.stdint cimport uint64_t
|
|
|
|
cimport lang
|
|
from spacy.lexeme cimport lexeme_check_flag
|
|
from spacy.lexeme cimport lexeme_string_view
|
|
from spacy._hashing cimport PointerHash
|
|
|
|
|
|
from spacy import orth
|
|
|
|
|
|
cdef class English(Language):
|
|
"""English tokenizer, tightly coupled to lexicon.
|
|
|
|
Attributes:
|
|
name (unicode): The two letter code used by Wikipedia for the language.
|
|
lexicon (Lexicon): The lexicon. Exposes the lookup method.
|
|
"""
|
|
cdef int _find_prefix(self, Py_UNICODE* chars, size_t length) except -1:
|
|
cdef Py_UNICODE c0 = chars[0]
|
|
cdef Py_UNICODE c1 = chars[1]
|
|
if c0 == ",":
|
|
return 1
|
|
elif c0 == '"':
|
|
return 1
|
|
elif c0 == "(":
|
|
return 1
|
|
elif c0 == "[":
|
|
return 1
|
|
elif c0 == "{":
|
|
return 1
|
|
elif c0 == "*":
|
|
return 1
|
|
elif c0 == "<":
|
|
return 1
|
|
elif c0 == "$":
|
|
return 1
|
|
elif c0 == "£":
|
|
return 1
|
|
elif c0 == "€":
|
|
return 1
|
|
elif c0 == "\u201c":
|
|
return 1
|
|
elif c0 == "'":
|
|
if c1 == "s":
|
|
return 2
|
|
elif c1 == "S":
|
|
return 2
|
|
elif c1 == "'":
|
|
return 2
|
|
else:
|
|
return 1
|
|
elif c0 == "`":
|
|
if c1 == "`":
|
|
return 2
|
|
else:
|
|
return 1
|
|
else:
|
|
return 0
|
|
|
|
abbreviations = set(['U.S', 'u.s', 'U.N', 'Ms', 'Mr', 'P'])
|
|
cdef bint _check_punct(Py_UNICODE* characters, size_t i, size_t length):
|
|
cdef unicode char_i = characters[i]
|
|
cdef unicode char_i1 = characters[i+1]
|
|
# Don't count appostrophes as punct if the next char is a letter
|
|
if characters[i] == "'" and i < (length - 1) and char_i1.isalpha():
|
|
return i == 0
|
|
if characters[i] == "-":
|
|
return False
|
|
#and i < (length - 1) and characters[i+1] == '-':
|
|
#return False
|
|
# Don't count commas as punct if the next char is a number
|
|
if characters[i] == "," and i < (length - 1) and char_i1.isdigit():
|
|
return False
|
|
if characters[i] == "." and i < (length - 1):
|
|
return False
|
|
if characters[i] == "." and characters[:i] in abbreviations:
|
|
return False
|
|
return not char_i.isalnum()
|
|
|
|
|
|
EN = English('en', [], [])
|