mirror of
https://github.com/explosion/spaCy.git
synced 2025-07-05 20:33:10 +03:00
Make fasttext reader accommodate whitespace
This commit is contained in:
parent
94d8b711a3
commit
f0e28e8ae5
|
@ -30,7 +30,7 @@ def main(vectors_loc, lang=None):
|
||||||
nlp.vocab.reset_vectors(width=int(nr_dim))
|
nlp.vocab.reset_vectors(width=int(nr_dim))
|
||||||
for line in file_:
|
for line in file_:
|
||||||
line = line.decode('utf8')
|
line = line.decode('utf8')
|
||||||
pieces = line.split()
|
pieces = line.rsplit(' ', nr_dim)
|
||||||
word = pieces[0]
|
word = pieces[0]
|
||||||
vector = numpy.asarray([float(v) for v in pieces[1:]], dtype='f')
|
vector = numpy.asarray([float(v) for v in pieces[1:]], dtype='f')
|
||||||
nlp.vocab.set_vector(word, vector) # add the vectors to the vocab
|
nlp.vocab.set_vector(word, vector) # add the vectors to the vocab
|
||||||
|
|
Loading…
Reference in New Issue
Block a user