mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-13 21:26:58 +03:00
b892b446cc
* Added the same punctuation rules as danish language. * Added abbreviations and also the possibility to have capitalized abbreviations on some. Added a few specific cases too * Added test for long texts in swedish * Added morph rules, infixes and suffixes to __init__.py for swedish * Added some tests for prefixes, infixes and suffixes * Added tests for lemma * Renamed files to follow convention * [sv] Removed ambigious abbreviations * Added more tests for tokenizer exceptions * Added test for problem with punctuation in issue #2578 * Contributor agreement * Removed faulty lemmatization of 'jag' ('I') as it was lemmatized to 'jaga' ('hunt')
26 lines
919 B
Python
26 lines
919 B
Python
# coding: utf8
|
||
"""Punctuation stolen from Danish"""
|
||
from __future__ import unicode_literals
|
||
|
||
from ..char_classes import LIST_ELLIPSES, LIST_ICONS
|
||
from ..char_classes import QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||
from ..punctuation import TOKENIZER_SUFFIXES
|
||
|
||
|
||
_quotes = QUOTES.replace("'", '')
|
||
|
||
_infixes = (LIST_ELLIPSES + LIST_ICONS +
|
||
[r'(?<=[{}])\.(?=[{}])'.format(ALPHA_LOWER, ALPHA_UPPER),
|
||
r'(?<=[{a}])[,!?](?=[{a}])'.format(a=ALPHA),
|
||
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||
r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),
|
||
r'(?<=[{a}])([{q}\)\]\(\[])(?=[\{a}])'.format(a=ALPHA, q=_quotes),
|
||
r'(?<=[{a}])--(?=[{a}])'.format(a=ALPHA)])
|
||
|
||
_suffixes = [suffix for suffix in TOKENIZER_SUFFIXES if suffix not in ["'s", "'S", "’s", "’S", r"\'"]]
|
||
_suffixes += [r"(?<=[^sSxXzZ])\'"]
|
||
|
||
|
||
TOKENIZER_INFIXES = _infixes
|
||
TOKENIZER_SUFFIXES = _suffixes
|