spaCy/spacy/lang/ko/punctuation.py

from ..char_classes import LIST_QUOTES
from ..punctuation import TOKENIZER_INFIXES as BASE_TOKENIZER_INFIXES


_infixes = (
    ["·", "ㆍ", r"\(", r"\)"]
    + [r"(?<=[0-9])~(?=[0-9-])"]
    + LIST_QUOTES
    + BASE_TOKENIZER_INFIXES
)

TOKENIZER_INFIXES = _infixes
Update Korean defaults for Tokenizer (#10322) Update Korean defaults for `Tokenizer` for tokenization following UD Korean Kaist. 2022-02-21 12:26:19 +03:00			`from ..char_classes import LIST_QUOTES`
			`from ..punctuation import TOKENIZER_INFIXES as BASE_TOKENIZER_INFIXES`


			`_infixes = (`
Fix regex invalid escape sequences (#11276) 2022-08-09 11:59:36 +03:00			`["·", "ㆍ", r"\(", r"\)"]`
Update Korean defaults for Tokenizer (#10322) Update Korean defaults for `Tokenizer` for tokenization following UD Korean Kaist. 2022-02-21 12:26:19 +03:00			`+ [r"(?<=[0-9])~(?=[0-9-])"]`
			`+ LIST_QUOTES`
			`+ BASE_TOKENIZER_INFIXES`
			`)`

			`TOKENIZER_INFIXES = _infixes`