Merge pull request #275 from henningpeters/unidecode

remove text-unidecode dependency
This commit is contained in:
Matthew Honnibal 2016-02-25 04:10:45 +11:00
commit 1ba31f6229
4 changed files with 1 additions and 33 deletions

View File

@ -4,7 +4,6 @@ pathlib
preshed>=0.46.1,<0.47.0
thinc>=5.0.0,<5.1.0
murmurhash>=0.26,<0.27
text-unidecode
numpy
plac
six

View File

@ -174,7 +174,7 @@ def setup_package():
license=about['__license__'],
ext_modules=ext_modules,
install_requires=['numpy', 'murmurhash>=0.26,<0.27', 'cymem>=1.30,<1.32.0', 'preshed>=0.46.1,<0.47',
'thinc>=5.0.0,<5.1.0', 'text_unidecode', 'plac', 'six',
'thinc>=5.0.0,<5.1.0', 'plac', 'six',
'ujson', 'cloudpickle', 'sputnik>=0.9.0,<0.10.0'],
cmdclass = {
'build_ext': build_ext_subclass},

View File

@ -10,4 +10,3 @@ cpdef bint like_email(unicode string)
cpdef bint like_url(unicode string)
cpdef bint like_number(unicode string)
cpdef unicode word_shape(unicode string)
cpdef bytes asciied(unicode string)

View File

@ -2,21 +2,8 @@
# cython: infer_types=True
from __future__ import unicode_literals
import unicodedata
# If your license is not GPL compatible, use text_unidecode. But if your code
# is, you should use the unidecode library, because its performance is better.
# spaCy does not list unidecode as a dependency, in case your license is not
# GPL compatible.
try:
from unidecode import unidecode
except ImportError:
from text_unidecode import unidecode
import re
import math
TAGS = 'adj adp adv conj det noun num pdt pos pron prt punct verb'.upper().split()
@ -171,23 +158,6 @@ cpdef unicode word_shape(unicode string):
return ''.join(shape)
cpdef unicode norm1(unicode string, lower_pc=0.0, upper_pc=0.0, title_pc=0.0):
"""Apply level 1 normalization:
* Case is canonicalized, using frequency statistics
* Unicode mapped to ascii, via unidecode
* Regional spelling variations are normalized
"""
pass
cpdef bytes asciied(unicode string):
stripped = unidecode(string)
if not stripped:
return b'???'
return stripped.encode('ascii')
# Exceptions --- do not convert these
_uk_us_except = set([
'our',