explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 18:06:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

punctution rules

Browse Source
This commit is contained in:
Jim Geovedi 2017-07-24 06:17:18 +07:00
parent 082e94ac1c
commit 37fa2c8c80
1 changed files with 36 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
spacy/lang/id/punctuation.py Normal file
View File

@ -0,0 +1,36 @@
# coding: utf8
from __future__ import unicode_literals
from ..punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
from ..char_classes import merge_chars, split_chars, _currency
from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES
from ..char_classes import QUOTES, UNITS, ALPHA, ALPHA_LOWER, ALPHA_UPPER
_units = ('s bit Gbps Mbps mbps Kbps kbps ƒ '
'kHz MHz GHz mAh ')
_currency = r'Rp IDR RMB SGD S\$' + _currency
UNITS = merge_chars(_units)
CURRENCY = merge_chars(_currency)
HTML_PREFIX = r'<(b|strong|i|em|p|span|div|br)\s?/>'
HTML_SUFFIX = r'</(b|strong|i|em|p|span|div)>'
_prefixes = TOKENIZER_PREFIXES + split_chars(_currency) + [HTML_PREFIX] + ['ke-']
_suffixes = TOKENIZER_SUFFIXES + [
r'(?<=[0-9])(?:{})'.format(CURRENCY),
r'(?<=[0-9])(?:{})'.format(UNITS),
r'(?<=[0-9])%',
r'(?<=[0-9{a}]{h})(?:[\.,:-])'.format(a=ALPHA, h=HTML_SUFFIX),
r'(?<=[0-9{a}])(?:{h})'.format(a=ALPHA, h=HTML_SUFFIX),
]
_infixes = TOKENIZER_INFIXES + [
r'(?<=[0-9])([\\/])(?=[0-9%-])',
r'(?<=[0-9])(%)(?=[{a}0-9/])'.format(a=ALPHA),
r'(?<=[0-9]{u})\/(?=[0-9])'.format(u=UNITS),
]
TOKENIZER_PREFIXES = _prefixes
TOKENIZER_SUFFIXES = _suffixes
TOKENIZER_INFIXES = _infixes