explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-10-30 23:47:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

Make fasttext reader accommodate whitespace

Browse Source
This commit is contained in:
Matthew Honnibal 2017-11-12 12:07:13 +01:00 committed by GitHub
parent 94d8b711a3
commit f0e28e8ae5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/vectors_fast_text.py
View File

@ -30,7 +30,7 @@ def main(vectors_loc, lang=None):
nlp.vocab.reset_vectors(width=int(nr_dim)) nlp.vocab.reset_vectors(width=int(nr_dim))
for line in file_: for line in file_:
line = line.decode('utf8') line = line.decode('utf8')
pieces = line.split() pieces = line.rsplit(' ', nr_dim)
word = pieces[0] word = pieces[0]
vector = numpy.asarray([float(v) for v in pieces[1:]], dtype='f') vector = numpy.asarray([float(v) for v in pieces[1:]], dtype='f')
nlp.vocab.set_vector(word, vector) # add the vectors to the vocab nlp.vocab.set_vector(word, vector) # add the vectors to the vocab