mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-24 17:06:29 +03:00
Reorganise lex_attrs and add dict
This commit is contained in:
ines
parent
e666f14d20
commit
2216e5f326
|
|
@ -4,16 +4,23 @@ from __future__ import unicode_literals
|
||||||
import unicodedata
|
import unicodedata
|
||||||
import regex as re
|
import regex as re
|
||||||
|
|
||||||
|
from .. import attrs
|
||||||
|
|
||||||
|
|
||||||
_like_email = re.compile(r'([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)').match
|
_like_email = re.compile(r'([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)').match
|
||||||
|
_tlds = set(
|
||||||
|
"com|org|edu|gov|net|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|"
|
||||||
def is_alpha(text):
|
"name|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|"
|
||||||
return text.isalpha()
|
"ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|"
|
||||||
|
"cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|"
|
||||||
|
"ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|"
|
||||||
def is_digit(text):
|
"gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|"
|
||||||
return text.isdigit()
|
"je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|"
|
||||||
|
"lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|"
|
||||||
|
"na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|"
|
||||||
|
"pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|"
|
||||||
|
"ss|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|"
|
||||||
|
"ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw".split('|'))
|
||||||
|
|
||||||
|
|
||||||
def is_punct(text):
|
def is_punct(text):
|
||||||
|
|
@ -54,37 +61,7 @@ def is_right_punct(text):
|
||||||
return text in right_punct
|
return text in right_punct
|
||||||
|
|
||||||
|
|
||||||
def is_title(text):
|
|
||||||
return text.istitle()
|
|
||||||
|
|
||||||
|
|
||||||
def is_lower(text):
|
|
||||||
return text.islower()
|
|
||||||
|
|
||||||
|
|
||||||
def is_upper(text):
|
|
||||||
return text.isupper()
|
|
||||||
|
|
||||||
|
|
||||||
def like_email(text):
|
|
||||||
return _like_email(text)
|
|
||||||
|
|
||||||
|
|
||||||
def like_url(text):
|
def like_url(text):
|
||||||
TLDs = set('com|org|edu|gov|net|mil|aero|asia|biz|cat|coop|info|int|jobs|'
|
|
||||||
'mobi|museum|name|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|'
|
|
||||||
'ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|'
|
|
||||||
'br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|'
|
|
||||||
'cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|'
|
|
||||||
'fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|'
|
|
||||||
'gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|'
|
|
||||||
'jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|'
|
|
||||||
'lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|'
|
|
||||||
'mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|'
|
|
||||||
'ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|'
|
|
||||||
'sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sy|sz|tc|td|tf|tg|th|'
|
|
||||||
'tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|'
|
|
||||||
'vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw'.split('|'))
|
|
||||||
# We're looking for things that function in text like URLs. So, valid URL
|
# We're looking for things that function in text like URLs. So, valid URL
|
||||||
# or not, anything they say http:// is going to be good.
|
# or not, anything they say http:// is going to be good.
|
||||||
if text.startswith('http://') or text.startswith('https://'):
|
if text.startswith('http://') or text.startswith('https://'):
|
||||||
|
|
@ -101,7 +78,7 @@ def like_url(text):
|
||||||
tld = string.rsplit('.', 1)[1].split(':', 1)[0]
|
tld = string.rsplit('.', 1)[1].split(':', 1)[0]
|
||||||
if tld.endswith('/'):
|
if tld.endswith('/'):
|
||||||
return True
|
return True
|
||||||
if tld.isalpha() and tld in TLDs:
|
if tld.isalpha() and tld in _tlds:
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -132,3 +109,30 @@ def word_shape(text):
|
||||||
if seq < 4:
|
if seq < 4:
|
||||||
shape.append(shape_char)
|
shape.append(shape_char)
|
||||||
return ''.join(shape)
|
return ''.join(shape)
|
||||||
|
|
||||||
|
|
||||||
|
LEX_ATTRS = {
|
||||||
|
attrs.LOWER: lambda string: string.lower(),
|
||||||
|
attrs.NORM: lambda string: string,
|
||||||
|
attrs.PREFIX: lambda string: string[0],
|
||||||
|
attrs.SUFFIX: lambda string: string[-3:],
|
||||||
|
attrs.CLUSTER: lambda string: 0,
|
||||||
|
attrs.IS_ALPHA: lambda string: string.isalpha(),
|
||||||
|
attrs.IS_DIGIT: lambda string: string.isdigit(),
|
||||||
|
attrs.IS_LOWER: lambda string: string.islower(),
|
||||||
|
attrs.IS_SPACE: lambda string: string.isspace(),
|
||||||
|
attrs.IS_TITLE: lambda string: string.istitle(),
|
||||||
|
attrs.IS_UPPER: lambda string: string.isupper(),
|
||||||
|
attrs.LIKE_EMAIL: lambda string: _like_email(string)
|
||||||
|
attrs.IS_STOP: lambda string: False,
|
||||||
|
attrs.IS_OOV: lambda string: True,
|
||||||
|
attrs.LIKE_NUM: lambda string: False,
|
||||||
|
attrs.IS_PUNCT: is_punct,
|
||||||
|
attrs.IS_ASCII: is_ascii,
|
||||||
|
attrs.SHAPE: word_shape,
|
||||||
|
attrs.IS_BRACKET:is_bracket,
|
||||||
|
attrs.IS_QUOTE: is_quote,
|
||||||
|
attrs.IS_LEFT_PUNCT: is_left_punct,
|
||||||
|
attrs.IS_RIGHT_PUNCT: is_right_punct,
|
||||||
|
attrs.LIKE_URL: like_url
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user