explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 17:36:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Fix strings i/o, removing use of ujson library in favour of plain text file. Allows better control of codecs.

Browse Source
This commit is contained in:
Matthew Honnibal 2014-11-02 13:20:37 +11:00
parent 3352e89e21
commit 437cd2217d
1 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
spacy/utf8string.pyx
View File

@ -1,9 +1,9 @@
from libc.string cimport memcpy from libc.string cimport memcpy
from murmurhash.mrmr cimport hash64 from murmurhash.mrmr cimport hash64
import codecs
import ujson SEPARATOR = '\n|-SEP-|\n'
cdef class StringStore: cdef class StringStore:
def __init__(self): def __init__(self):
@ -61,12 +61,15 @@ cdef class StringStore:
for i in range(self.size): for i in range(self.size):
string = &self.strings[i] string = &self.strings[i]
py_string = string.chars[:string.length] py_string = string.chars[:string.length]
strings.append(py_string) strings.append(py_string.decode('utf8'))
with open(loc, 'w') as file_: with codecs.open(loc, 'w', 'utf8') as file_:
ujson.dump(strings, file_, ensure_ascii=False) file_.write(SEPARATOR.join(strings))
def load(self, loc): def load(self, loc):
with open(loc) as file_: with codecs.open(loc, 'r', 'utf8') as file_:
strings = ujson.load(file_) strings = file_.read().split(SEPARATOR)
cdef unicode string
cdef bytes byte_string
for string in strings[1:]: for string in strings[1:]:
self.intern(string, len(string)) byte_string = string.encode('utf8')
self.intern(byte_string, len(byte_string))