explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-26 09:14:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

Use inline flags in token_match patterns (#5257)

Browse Source 
* Use inline flags in token_match patterns

Use inline flags in `token_match` patterns so that serializing does not
lose the flag information.

* Modify inline flag

* Modify inline flag
This commit is contained in:
adrianeboyd 2020-04-06 13:19:04 +02:00 committed by GitHub
parent e8be15e9b7
commit c981aa6684
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/lang/fr/tokenizer_exceptions.py
View File

@ -461,5 +461,5 @@ _regular_exp.append(URL_PATTERN)
TOKENIZER_EXCEPTIONS = _exc
TOKEN_MATCH = re.compile(
"|".join("(?:{})".format(m) for m in _regular_exp), re.IGNORECASE | re.UNICODE
"(?iu)" + "|".join("(?:{})".format(m) for m in _regular_exp)
).match

2
spacy/lang/tokenizer_exceptions.py
View File

@ -58,7 +58,7 @@ URL_PATTERN = (
# fmt: on
).strip()
TOKEN_MATCH = re.compile(URL_PATTERN, re.UNICODE).match
TOKEN_MATCH = re.compile("(?u)" + URL_PATTERN).match
BASE_EXCEPTIONS = {}

2
spacy/tokenizer.pyx
View File

@ -567,7 +567,7 @@ cdef class Tokenizer:
))
exclude = util.get_serialization_exclude(deserializers, exclude, kwargs)
msg = util.from_bytes(bytes_data, deserializers, exclude)
for key in ["prefix_search", "suffix_search", "infix_finditer"]:
for key in ["prefix_search", "suffix_search", "infix_finditer", "token_match"]:
if key in data:
data[key] = unescape_unicode(data[key])
if "prefix_search" in data and isinstance(data["prefix_search"], basestring_):