spaCy/spacy/lang/el/punctuation.py

# -*- coding: utf-8 -*-

from __future__ import unicode_literals

from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY
from ..char_classes import LIST_ICONS, ALPHA_LOWER, ALPHA_UPPER, ALPHA, HYPHENS
from ..char_classes import QUOTES, CURRENCY

_units = ('km km² km³ m m² m³ dm dm² dm³ cm cm² cm³ mm mm² mm³ ha µm nm yd in ft '
          'kg g mg µg t lb oz m/s km/h kmh mph hPa Pa mbar mb MB kb KB gb GB tb '
          'TB T G M K км км² км³ м м² м³ дм дм² дм³ см см² см³ мм мм² мм³ нм '
          'кг г мг м/с км/ч кПа Па мбар Кб КБ кб Мб МБ мб Гб ГБ гб Тб ТБ тб')


def merge_chars(char): return char.strip().replace(' ', '|')


UNITS = merge_chars(_units)

_prefixes = (['\'\'', '§', '%', '=', r'\+[0-9]+%',  # 90%
              r'\'([0-9]){2}([\-]\'([0-9]){2})*',  # '12'-13
              r'\-([0-9]){1,9}\.([0-9]){1,9}',  # -12.13
              r'\'([Α-Ωα-ωίϊΐόάέύϋΰήώ]+)\'',  # 'αβγ'
              r'([Α-Ωα-ωίϊΐόάέύϋΰήώ]){1,3}\'',  # αβγ'
              r'http://www.[A-Za-z]+\-[A-Za-z]+(\.[A-Za-z]+)+(\/[A-Za-z]+)*(\.[A-Za-z]+)*',
              r'[ΈΆΊΑ-Ωα-ωίϊΐόάέύϋΰήώ]+\*',  # όνομα*
              r'\$([0-9])+([\,\.]([0-9])+){0,1}',
              ] + LIST_PUNCT + LIST_ELLIPSES + LIST_QUOTES +
             LIST_CURRENCY + LIST_ICONS)

_suffixes = (LIST_PUNCT + LIST_ELLIPSES + LIST_QUOTES + LIST_ICONS +
             [r'(?<=[0-9])\+',  # 12+
              r'([0-9])+\'',  # 12'
              r'([A-Za-z])?\'',  # a'
              r'^([0-9]){1,2}\.',  # 12.
              r' ([0-9]){1,2}\.',  # 12.
              r'([0-9]){1}\) ',  # 12)
              r'^([0-9]){1}\)$',  # 12)
              r'(?<=°[FfCcKk])\.',
              r'([0-9])+\&',  # 12&
              r'(?<=[0-9])(?:{})'.format(CURRENCY),
              r'(?<=[0-9])(?:{})'.format(UNITS),
              r'(?<=[0-9{}{}(?:{})])\.'.format(ALPHA_LOWER, r'²\-\)\]\+', QUOTES),
              r'(?<=[{a}][{a}])\.'.format(a=ALPHA_UPPER),
              r'(?<=[Α-Ωα-ωίϊΐόάέύϋΰήώ])\-',  # όνομα-
              r'(?<=[Α-Ωα-ωίϊΐόάέύϋΰήώ])\.',
              r'^[Α-Ω]{1}\.',
              r'\ [Α-Ω]{1}\.',
              # πρώτος-δεύτερος , πρώτος-δεύτερος-τρίτος
              r'[ΈΆΊΑΌ-Ωα-ωίϊΐόάέύϋΰήώ]+([\-]([ΈΆΊΑΌ-Ωα-ωίϊΐόάέύϋΰήώ]+))+',
              r'([0-9]+)mg',  # 13mg
              r'([0-9]+)\.([0-9]+)m'  # 1.2m
              ])

_infixes = (LIST_ELLIPSES + LIST_ICONS +
            [r'(?<=[0-9])[+\/\-\*^](?=[0-9])',  # 1/2 , 1-2 , 1*2
             r'([a-zA-Z]+)\/([a-zA-Z]+)\/([a-zA-Z]+)',  # name1/name2/name3
             r'([0-9])+(\.([0-9]+))*([\-]([0-9])+)+',  # 10.9 , 10.9.9 , 10.9-6
             r'([0-9])+[,]([0-9])+[\-]([0-9])+[,]([0-9])+',  # 10,11,12
             r'([0-9])+[ης]+([\-]([0-9])+)+',  # 1ης-2
             # 15/2 , 15/2/17 , 2017/2/15
             r'([0-9]){1,4}[\/]([0-9]){1,2}([\/]([0-9]){0,4}){0,1}',
             r'[A-Za-z]+\@[A-Za-z]+(\-[A-Za-z]+)*\.[A-Za-z]+',  # abc@cde-fgh.a
             r'([a-zA-Z]+)(\-([a-zA-Z]+))+',  # abc-abc
             r'(?<=[{}])\.(?=[{}])'.format(ALPHA_LOWER, ALPHA_UPPER),
             r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),
             r'(?<=[{a}])[?";:=,.]*(?:{h})(?=[{a}])'.format(a=ALPHA, h=HYPHENS),
             r'(?<=[{a}"])[:<>=/](?=[{a}])'.format(a=ALPHA)])

TOKENIZER_PREFIXES = _prefixes
TOKENIZER_SUFFIXES = _suffixes
TOKENIZER_INFIXES = _infixes