mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-10 19:57:17 +03:00
remove text-unidecode dependency
This commit is contained in:
parent
63deae47fe
commit
12d58a7099
|
@ -4,7 +4,6 @@ pathlib
|
|||
preshed>=0.46.1,<0.47.0
|
||||
thinc>=5.0.0,<5.1.0
|
||||
murmurhash>=0.26,<0.27
|
||||
text-unidecode
|
||||
numpy
|
||||
plac
|
||||
six
|
||||
|
|
2
setup.py
2
setup.py
|
@ -174,7 +174,7 @@ def setup_package():
|
|||
license=about['__license__'],
|
||||
ext_modules=ext_modules,
|
||||
install_requires=['numpy', 'murmurhash>=0.26,<0.27', 'cymem>=1.30,<1.32.0', 'preshed>=0.46.1,<0.47',
|
||||
'thinc>=5.0.0,<5.1.0', 'text_unidecode', 'plac', 'six',
|
||||
'thinc>=5.0.0,<5.1.0', 'plac', 'six',
|
||||
'ujson', 'cloudpickle', 'sputnik>=0.9.0,<0.10.0'],
|
||||
cmdclass = {
|
||||
'build_ext': build_ext_subclass},
|
||||
|
|
|
@ -10,4 +10,3 @@ cpdef bint like_email(unicode string)
|
|||
cpdef bint like_url(unicode string)
|
||||
cpdef bint like_number(unicode string)
|
||||
cpdef unicode word_shape(unicode string)
|
||||
cpdef bytes asciied(unicode string)
|
||||
|
|
|
@ -2,21 +2,8 @@
|
|||
# cython: infer_types=True
|
||||
from __future__ import unicode_literals
|
||||
import unicodedata
|
||||
|
||||
# If your license is not GPL compatible, use text_unidecode. But if your code
|
||||
# is, you should use the unidecode library, because its performance is better.
|
||||
# spaCy does not list unidecode as a dependency, in case your license is not
|
||||
# GPL compatible.
|
||||
try:
|
||||
from unidecode import unidecode
|
||||
except ImportError:
|
||||
from text_unidecode import unidecode
|
||||
|
||||
|
||||
import re
|
||||
|
||||
import math
|
||||
|
||||
|
||||
TAGS = 'adj adp adv conj det noun num pdt pos pron prt punct verb'.upper().split()
|
||||
|
||||
|
@ -171,23 +158,6 @@ cpdef unicode word_shape(unicode string):
|
|||
return ''.join(shape)
|
||||
|
||||
|
||||
cpdef unicode norm1(unicode string, lower_pc=0.0, upper_pc=0.0, title_pc=0.0):
|
||||
"""Apply level 1 normalization:
|
||||
|
||||
* Case is canonicalized, using frequency statistics
|
||||
* Unicode mapped to ascii, via unidecode
|
||||
* Regional spelling variations are normalized
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
cpdef bytes asciied(unicode string):
|
||||
stripped = unidecode(string)
|
||||
if not stripped:
|
||||
return b'???'
|
||||
return stripped.encode('ascii')
|
||||
|
||||
|
||||
# Exceptions --- do not convert these
|
||||
_uk_us_except = set([
|
||||
'our',
|
||||
|
|
Loading…
Reference in New Issue
Block a user