explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-13 02:36:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

Handle deprecated tokenizer prefix data

Browse Source
This commit is contained in:
Ines Montani 2017-01-08 20:33:28 +01:00
parent abb09782f9
commit de5aa92bc2
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
spacy/util.py
View File

@ -94,8 +94,13 @@ def read_regex(path):
def compile_prefix_regex(entries): def compile_prefix_regex(entries):
expression = '|'.join(['^' + re.escape(piece) for piece in entries if piece.strip()]) if '(' in entries:
return re.compile(expression) # Handle deprecated data
expression = '|'.join(['^' + re.escape(piece) for piece in entries if piece.strip()])
return re.compile(expression)
else:
expression = '|'.join(['^' + piece for piece in entries if piece.strip()])
return re.compile(expression)
def compile_suffix_regex(entries): def compile_suffix_regex(entries):