mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-14 21:57:15 +03:00
1448ad100c
* Improved stop words list * Removed some wrong stop words form list * Improved stop words list * Removed some wrong stop words form list * Improved Polish Tokenizer (#38) * Add tests for polish tokenizer * Add polish tokenizer exceptions * Don't split any words containing hyphens * Fix test case with wrong model answer * Remove commented out line of code until better solution is found * Add source srx' license * Rename exception_list.py to match spaCy conventionality * Add a brief explanation of where the exception list comes from * Add newline after reach exception * Rename COPYING.txt to LICENSE * Delete old files * Add header to the license * Agreements signed * Stanisław Giziński agreement * Krzysztof Kowalczyk - signed agreement * Mateusz Olko agreement * Add DoomCoder's contributor agreement * Improve like number checking in polish lang * like num tests added * all from SI system added * Final licence and removed splitting exceptions * Added polish stop words to LEX_ATTRA * Add encoding info to pl tokenizer exceptions
35 lines
1.2 KiB
Python
35 lines
1.2 KiB
Python
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
from ...attrs import LIKE_NUM
|
|
|
|
|
|
_num_words = ['zero', 'jeden', 'dwa', 'trzy', 'cztery', 'pięć', 'sześć',
|
|
'siedem', 'osiem', 'dziewięć', 'dziesięć', 'jedenaście',
|
|
'dwanaście', 'trzynaście', 'czternaście',
|
|
'pietnaście', 'szesnaście', 'siedemnaście', 'osiemnaście',
|
|
'dziewiętnaście', 'dwadzieścia', 'trzydzieści', 'czterdzieści',
|
|
'pięćdziesiąt', 'szcześćdziesiąt', 'siedemdziesiąt',
|
|
'osiemdziesiąt', 'dziewięćdziesiąt', 'sto',
|
|
'dwieście', 'trzysta', 'czterysta', 'pięćset', 'sześćset',
|
|
'siedemset', 'osiemset', 'dziewięćset', 'tysiąc', 'milion',
|
|
'miliard', 'bilion', 'biliard', 'trylion', 'tryliard', 'kwadrylion']
|
|
|
|
|
|
def like_num(text):
|
|
text = text.replace(',', '').replace('.', '')
|
|
if text.isdigit():
|
|
return True
|
|
if text.count('/') == 1:
|
|
num, denom = text.split('/')
|
|
if num.isdigit() and denom.isdigit():
|
|
return True
|
|
if text.lower() in _num_words:
|
|
return True
|
|
return False
|
|
|
|
|
|
LEX_ATTRS = {
|
|
LIKE_NUM: like_num
|
|
}
|