spaCy/examples/vectors_fast_text.py

33 lines
940 B
Python
Raw Normal View History

2017-10-26 18:32:59 +03:00
#!/usr/bin/env python
# coding: utf8
"""Load vectors for a language trained using FastText
2017-10-02 00:40:02 +03:00
https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md
2017-10-26 18:32:59 +03:00
"""
2017-10-02 00:40:02 +03:00
from __future__ import unicode_literals
import plac
import numpy
2017-10-26 18:32:59 +03:00
import from spacy.language import Language
2017-10-02 00:40:02 +03:00
2017-10-26 18:32:59 +03:00
@plac.annotations(vectors_loc=("Path to vectors", "positional", None, str))
2017-10-02 00:40:02 +03:00
def main(vectors_loc):
2017-10-26 18:32:59 +03:00
nlp = Language()
2017-10-02 00:40:02 +03:00
with open(vectors_loc, 'rb') as file_:
header = file_.readline()
nr_row, nr_dim = header.split()
nlp.vocab.clear_vectors(int(nr_dim))
for line in file_:
line = line.decode('utf8')
2017-10-26 18:32:59 +03:00
pieces = line.split()
2017-10-02 00:40:02 +03:00
word = pieces[0]
vector = numpy.asarray([float(v) for v in pieces[1:]], dtype='f')
nlp.vocab.set_vector(word, vector)
doc = nlp(u'class colspan')
print(doc[0].similarity(doc[1]))
if __name__ == '__main__':
plac.call(main)