explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-08-05 04:40:20 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update punctuation.py

Browse Source 
Add mathematical left and right angle brackets as punctuation for ancient Greek for better tokenization.
This commit is contained in:
Jacobo Myerston 2023-07-15 15:43:03 -07:00
parent 8933b37021
commit e8c23a04f3
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/lang/grc/punctuation.py
View File

@ -6,6 +6,7 @@ _prefixes = (
[ [
"†", "†",
"⸏", "⸏",
"〈",
] ]
+ LIST_PUNCT + LIST_PUNCT
+ LIST_ELLIPSES + LIST_ELLIPSES
@ -22,6 +23,7 @@ _suffixes = (
+ [ + [
"†", "†",
"⸎", "⸎",
"〉",
r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])[\-\.⸏]", r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])[\-\.⸏]",
] ]
) )