mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-13 13:17:06 +03:00
1448ad100c
* Improved stop words list * Removed some wrong stop words form list * Improved stop words list * Removed some wrong stop words form list * Improved Polish Tokenizer (#38) * Add tests for polish tokenizer * Add polish tokenizer exceptions * Don't split any words containing hyphens * Fix test case with wrong model answer * Remove commented out line of code until better solution is found * Add source srx' license * Rename exception_list.py to match spaCy conventionality * Add a brief explanation of where the exception list comes from * Add newline after reach exception * Rename COPYING.txt to LICENSE * Delete old files * Add header to the license * Agreements signed * Stanisław Giziński agreement * Krzysztof Kowalczyk - signed agreement * Mateusz Olko agreement * Add DoomCoder's contributor agreement * Improve like number checking in polish lang * like num tests added * all from SI system added * Final licence and removed splitting exceptions * Added polish stop words to LEX_ATTRA * Add encoding info to pl tokenizer exceptions
15 lines
658 B
Python
15 lines
658 B
Python
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
from ..char_classes import LIST_ELLIPSES, LIST_ICONS
|
|
from ..char_classes import QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
|
_quotes = QUOTES.replace("'", '')
|
|
_infixes = (LIST_ELLIPSES + LIST_ICONS +
|
|
[r'(?<=[{}])\.(?=[{}])'.format(ALPHA_LOWER, ALPHA_UPPER),
|
|
r'(?<=[{a}])[,!?](?=[{a}])'.format(a=ALPHA),
|
|
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
|
|
r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),
|
|
r'(?<=[{a}])([{q}\)\]\(\[])(?=[\{a}])'.format(a=ALPHA, q=_quotes),
|
|
r'(?<=[{a}])--(?=[{a}])'.format(a=ALPHA)])
|
|
|
|
TOKENIZER_INFIXES = _infixes
|