explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 01:46:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

remove text-unidecode dependency

Browse Source
This commit is contained in:
Henning Peters 2016-02-24 08:01:59 +01:00
parent 63deae47fe
commit 12d58a7099
4 changed files with 1 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@ -4,7 +4,6 @@ pathlib
preshed>=0.46.1,<0.47.0 preshed>=0.46.1,<0.47.0
thinc>=5.0.0,<5.1.0 thinc>=5.0.0,<5.1.0
murmurhash>=0.26,<0.27 murmurhash>=0.26,<0.27
text-unidecode
numpy numpy
plac plac
six six

2
setup.py
View File

@ -174,7 +174,7 @@ def setup_package():
license=about['__license__'], license=about['__license__'],
ext_modules=ext_modules, ext_modules=ext_modules,
install_requires=['numpy', 'murmurhash>=0.26,<0.27', 'cymem>=1.30,<1.32.0', 'preshed>=0.46.1,<0.47', install_requires=['numpy', 'murmurhash>=0.26,<0.27', 'cymem>=1.30,<1.32.0', 'preshed>=0.46.1,<0.47',
'thinc>=5.0.0,<5.1.0', 'text_unidecode', 'plac', 'six', 'thinc>=5.0.0,<5.1.0', 'plac', 'six',
'ujson', 'cloudpickle', 'sputnik>=0.9.0,<0.10.0'], 'ujson', 'cloudpickle', 'sputnik>=0.9.0,<0.10.0'],
cmdclass = { cmdclass = {
'build_ext': build_ext_subclass}, 'build_ext': build_ext_subclass},

1
spacy/orth.pxd
View File

@ -10,4 +10,3 @@ cpdef bint like_email(unicode string)
cpdef bint like_url(unicode string) cpdef bint like_url(unicode string)
cpdef bint like_number(unicode string) cpdef bint like_number(unicode string)
cpdef unicode word_shape(unicode string) cpdef unicode word_shape(unicode string)
cpdef bytes asciied(unicode string)

30
spacy/orth.pyx
View File

@ -2,21 +2,8 @@
# cython: infer_types=True # cython: infer_types=True
from __future__ import unicode_literals from __future__ import unicode_literals
import unicodedata import unicodedata
# If your license is not GPL compatible, use text_unidecode. But if your code
# is, you should use the unidecode library, because its performance is better.
# spaCy does not list unidecode as a dependency, in case your license is not
# GPL compatible.
try:
from unidecode import unidecode
except ImportError:
from text_unidecode import unidecode
import re import re
import math
TAGS = 'adj adp adv conj det noun num pdt pos pron prt punct verb'.upper().split() TAGS = 'adj adp adv conj det noun num pdt pos pron prt punct verb'.upper().split()
@ -171,23 +158,6 @@ cpdef unicode word_shape(unicode string):
return ''.join(shape) return ''.join(shape)
cpdef unicode norm1(unicode string, lower_pc=0.0, upper_pc=0.0, title_pc=0.0):
"""Apply level 1 normalization:
* Case is canonicalized, using frequency statistics
* Unicode mapped to ascii, via unidecode
* Regional spelling variations are normalized
"""
pass
cpdef bytes asciied(unicode string):
stripped = unidecode(string)
if not stripped:
return b'???'
return stripped.encode('ascii')
# Exceptions --- do not convert these # Exceptions --- do not convert these
_uk_us_except = set([ _uk_us_except = set([
'our', 'our',