spaCy/spacy/lang/lb/punctuation.py

# coding: utf8
from __future__ import unicode_literals

from ..punctuation import TOKENIZER_INFIXES
from ..char_classes import ALPHA


ELISION = " ' ’ ".strip().replace(" ", "")
HYPHENS = r"- – — ‐ ‑".strip().replace(" ", "")


_infixes = TOKENIZER_INFIXES + [
    r"(?<=[{a}][{el}])(?=[{a}])".format(a=ALPHA, el=ELISION)
]

TOKENIZER_INFIXES = _infixes
-												Fix basic language support for Luxembourgish (by adding punctuation.py) (#4648)

* Update __init__.py

* Create punctuation.py

* Update tokenizer_exceptions.py

* Create questoph.md

* Update questoph.md

* Update test_text.py

* Update test_text.py

* Update test_text.py

* Update test_text.py

											
										
										
											2019-11-15 18:16:47 +03:00
+								# coding: utf8
 								from __future__ import unicode_literals
 								from ..punctuation import TOKENIZER_INFIXES
 								from ..char_classes import ALPHA
-												Remove unnecessary newline replace

											
										
										
											2019-11-15 18:19:01 +03:00
+								ELISION = " ' ’ ".strip().replace(" ", "")
 								HYPHENS = r"- – — ‐ ‑".strip().replace(" ", "")
-												Fix basic language support for Luxembourgish (by adding punctuation.py) (#4648)

* Update __init__.py

* Create punctuation.py

* Update tokenizer_exceptions.py

* Create questoph.md

* Update questoph.md

* Update test_text.py

* Update test_text.py

* Update test_text.py

* Update test_text.py

											
										
										
											2019-11-15 18:16:47 +03:00
 								_infixes = TOKENIZER_INFIXES + [
 								    r"(?<=[{a}][{el}])(?=[{a}])".format(a=ALPHA, el=ELISION)
 								]
 								TOKENIZER_INFIXES = _infixes