explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-27 10:26:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Add function to read detokenization rules

Browse Source
This commit is contained in:
Matthew Honnibal 2014-10-22 12:54:59 +11:00
parent 077885637d
commit 43d5964e13
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
spacy/util.py
View File

@ -70,6 +70,15 @@ def read_tokenization(lang):
return entries return entries
def read_detoken_rules(lang):
loc = path.join(DATA_DIR, lang, 'detokenize')
entries = []
with utf8open(loc) as file_:
for line in file_:
entries.append(line.strip())
return entries
def align_tokens(ref, indices): def align_tokens(ref, indices):
start = 0 start = 0
queue = list(indices) queue = list(indices)