spaCy/spacy/fr/punctuation.py

38 lines
1.0 KiB
Python
Raw Normal View History

2017-03-12 15:07:28 +03:00
# coding: utf8
2017-01-24 11:47:13 +03:00
from __future__ import unicode_literals
from ..language_data.punctuation import ALPHA, TOKENIZER_INFIXES, LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, CURRENCY,\
UNITS, ALPHA_LOWER, QUOTES, ALPHA_UPPER
2017-01-24 11:47:13 +03:00
_ELISION = " ' "
ELISION = _ELISION.strip().replace(' ', '').replace('\n', '')
HYPHENS = r"""- """.strip().replace(' ', '').replace('\n', '')
TOKENIZER_SUFFIXES = (
LIST_PUNCT +
LIST_ELLIPSES +
LIST_QUOTES +
[
r'(?<=[0-9])\+',
r'(?<=°[FfCcKk])\.', # 4°C. -> ["4°C", "."]
r'(?<=[0-9])°[FfCcKk]', # 4°C -> ["4", "°C"]
r'(?<=[0-9])%', # 4% -> ["4", "%"]
r'(?<=[0-9])(?:{c})'.format(c=CURRENCY),
r'(?<=[0-9])(?:{u})'.format(u=UNITS),
r'(?<=[0-9{al}{p}(?:{q})])\.'.format(al=ALPHA_LOWER, p=r'%²\-\)\]\+', q=QUOTES),
r'(?<=[{au}][{au}])\.'.format(au=ALPHA_UPPER),
]
)
2017-01-24 11:47:13 +03:00
TOKENIZER_INFIXES += [
r'(?<=[{a}][{el}])(?=[{a}])'.format(a=ALPHA, el=ELISION),
]
__all__ = ["TOKENIZER_SUFFIXES", "TOKENIZER_INFIXES"]