explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 09:26:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add locale_escape compat function

Browse Source
This commit is contained in:
Matthew Honnibal 2017-10-12 22:22:04 +02:00
parent e72603f39f
commit 27b927259a
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
spacy/compat.py
View File

@ -6,6 +6,7 @@ import ftfy
import sys import sys
import ujson import ujson
import itertools import itertools
import locale
from thinc.neural.util import copy_array from thinc.neural.util import copy_array
@ -113,3 +114,12 @@ def import_file(name, loc):
module = importlib.util.module_from_spec(spec) module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) spec.loader.exec_module(module)
return module return module
def locale_escape(string, errors='replace'):
'''
Mangle non-supported characters, for savages with ascii terminals.
'''
encoding = locale.getpreferredencoding()
string = string.encode(encoding, errors).decode('utf8')
return string