From 2216e5f3265356856d72117c4907e1104bad79b9 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 9 May 2017 00:57:54 +0200 Subject: [PATCH] Reorganise lex_attrs and add dict --- spacy/lang/lex_attrs.py | 82 +++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/spacy/lang/lex_attrs.py b/spacy/lang/lex_attrs.py index df50623fd..ae6d935ea 100644 --- a/spacy/lang/lex_attrs.py +++ b/spacy/lang/lex_attrs.py @@ -4,16 +4,23 @@ from __future__ import unicode_literals import unicodedata import regex as re +from .. import attrs + _like_email = re.compile(r'([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)').match - - -def is_alpha(text): - return text.isalpha() - - -def is_digit(text): - return text.isdigit() +_tlds = set( + "com|org|edu|gov|net|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|" + "name|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|" + "ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|" + "cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|" + "ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|" + "gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|" + "je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|" + "lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|" + "na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|" + "pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|" + "ss|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|" + "ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw".split('|')) def is_punct(text): @@ -54,37 +61,7 @@ def is_right_punct(text): return text in right_punct -def is_title(text): - return text.istitle() - - -def is_lower(text): - return text.islower() - - -def is_upper(text): - return text.isupper() - - -def like_email(text): - return _like_email(text) - - def like_url(text): - TLDs = set('com|org|edu|gov|net|mil|aero|asia|biz|cat|coop|info|int|jobs|' - 'mobi|museum|name|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|' - 'ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|' - 'br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|' - 'cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|' - 'fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|' - 'gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|' - 'jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|' - 'lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|' - 'mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|' - 'ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|' - 'sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sy|sz|tc|td|tf|tg|th|' - 'tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|' - 'vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw'.split('|')) # We're looking for things that function in text like URLs. So, valid URL # or not, anything they say http:// is going to be good. if text.startswith('http://') or text.startswith('https://'): @@ -101,7 +78,7 @@ def like_url(text): tld = string.rsplit('.', 1)[1].split(':', 1)[0] if tld.endswith('/'): return True - if tld.isalpha() and tld in TLDs: + if tld.isalpha() and tld in _tlds: return True return False @@ -132,3 +109,30 @@ def word_shape(text): if seq < 4: shape.append(shape_char) return ''.join(shape) + + +LEX_ATTRS = { + attrs.LOWER: lambda string: string.lower(), + attrs.NORM: lambda string: string, + attrs.PREFIX: lambda string: string[0], + attrs.SUFFIX: lambda string: string[-3:], + attrs.CLUSTER: lambda string: 0, + attrs.IS_ALPHA: lambda string: string.isalpha(), + attrs.IS_DIGIT: lambda string: string.isdigit(), + attrs.IS_LOWER: lambda string: string.islower(), + attrs.IS_SPACE: lambda string: string.isspace(), + attrs.IS_TITLE: lambda string: string.istitle(), + attrs.IS_UPPER: lambda string: string.isupper(), + attrs.LIKE_EMAIL: lambda string: _like_email(string) + attrs.IS_STOP: lambda string: False, + attrs.IS_OOV: lambda string: True, + attrs.LIKE_NUM: lambda string: False, + attrs.IS_PUNCT: is_punct, + attrs.IS_ASCII: is_ascii, + attrs.SHAPE: word_shape, + attrs.IS_BRACKET:is_bracket, + attrs.IS_QUOTE: is_quote, + attrs.IS_LEFT_PUNCT: is_left_punct, + attrs.IS_RIGHT_PUNCT: is_right_punct, + attrs.LIKE_URL: like_url +}