explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-14 13:47:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Add cautionary note to vocab about encoding

Browse Source
This commit is contained in:
Matthew Honnibal 2016-02-22 00:13:20 +01:00
parent 92f62bcb84
commit 7a519ea5af
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/vocab.pyx
View File

@ -418,6 +418,8 @@ def write_binary_vectors(in_loc, out_loc):
with bz2.BZ2File(in_loc, 'r') as file_: with bz2.BZ2File(in_loc, 'r') as file_:
for line in file_: for line in file_:
pieces = line.split() pieces = line.split()
# TODO: This should be explicit about the encoding to utf8,
# and the fact that len() refers to number of utf8 characters
word = pieces.pop(0) word = pieces.pop(0)
mem = Address(len(pieces), sizeof(float)) mem = Address(len(pieces), sizeof(float))
vec = <float*>mem.ptr vec = <float*>mem.ptr