Add locale_escape compat function

This commit is contained in:
Matthew Honnibal 2017-10-12 22:22:04 +02:00
parent e72603f39f
commit 27b927259a

View File

@ -6,6 +6,7 @@ import ftfy
import sys
import ujson
import itertools
import locale
from thinc.neural.util import copy_array
@ -113,3 +114,12 @@ def import_file(name, loc):
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def locale_escape(string, errors='replace'):
'''
Mangle non-supported characters, for savages with ascii terminals.
'''
encoding = locale.getpreferredencoding()
string = string.encode(encoding, errors).decode('utf8')
return string