mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 11:29:13 +03:00
289 lines
10 KiB
Cython
289 lines
10 KiB
Cython
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
import numpy
|
|
from collections import OrderedDict
|
|
import msgpack
|
|
import msgpack_numpy
|
|
msgpack_numpy.patch()
|
|
cimport numpy as np
|
|
from thinc.neural.util import get_array_module
|
|
from thinc.neural._classes.model import Model
|
|
|
|
from .strings cimport StringStore
|
|
from .compat import basestring_, path2str
|
|
from . import util
|
|
|
|
|
|
cdef class Vectors:
|
|
"""Store, save and load word vectors.
|
|
|
|
Vectors data is kept in the vectors.data attribute, which should be an
|
|
instance of numpy.ndarray (for CPU vectors) or cupy.ndarray
|
|
(for GPU vectors). `vectors.key2row` is a dictionary mapping word hashes to
|
|
rows in the vectors.data table.
|
|
|
|
Multiple keys can be mapped to the same vector, so len(keys) may be greater
|
|
(but not smaller) than data.shape[0].
|
|
"""
|
|
cdef public object data
|
|
cdef readonly StringStore strings
|
|
cdef public object key2row
|
|
cdef public object keys
|
|
cdef public int i
|
|
|
|
def __init__(self, strings, width=0, data=None):
|
|
"""Create a new vector store. To keep the vector table empty, pass
|
|
`width=0`. You can also create the vector table and add vectors one by
|
|
one, or set the vector values directly on initialisation.
|
|
|
|
strings (StringStore or list): List of strings or StringStore that maps
|
|
strings to hash values, and vice versa.
|
|
width (int): Number of dimensions.
|
|
data (numpy.ndarray): The vector data.
|
|
RETURNS (Vectors): The newly created object.
|
|
"""
|
|
if isinstance(strings, StringStore):
|
|
self.strings = strings
|
|
else:
|
|
self.strings = StringStore()
|
|
for string in strings:
|
|
self.strings.add(string)
|
|
if data is not None:
|
|
self.data = numpy.asarray(data, dtype='f')
|
|
else:
|
|
self.data = numpy.zeros((len(self.strings), width), dtype='f')
|
|
self.i = 0
|
|
self.key2row = {}
|
|
self.keys = numpy.zeros((self.data.shape[0],), dtype='uint64')
|
|
for i, string in enumerate(self.strings):
|
|
if i >= self.data.shape[0]:
|
|
break
|
|
self.add(self.strings[string], vector=self.data[i])
|
|
|
|
def __reduce__(self):
|
|
return (Vectors, (self.strings, self.data))
|
|
|
|
def __getitem__(self, key):
|
|
"""Get a vector by key. If key is a string, it is hashed to an integer
|
|
ID using the vectors.strings table. If the integer key is not found in
|
|
the table, a KeyError is raised.
|
|
|
|
key (unicode / int): The key to get the vector for.
|
|
RETURNS (numpy.ndarray): The vector for the key.
|
|
"""
|
|
if isinstance(key, basestring):
|
|
key = self.strings[key]
|
|
i = self.key2row[key]
|
|
if i is None:
|
|
raise KeyError(key)
|
|
else:
|
|
return self.data[i]
|
|
|
|
def __setitem__(self, key, vector):
|
|
"""Set a vector for the given key. If key is a string, it is hashed
|
|
to an integer ID using the vectors.strings table.
|
|
|
|
key (unicode / int): The key to set the vector for.
|
|
vector (numpy.ndarray): The vector to set.
|
|
"""
|
|
if isinstance(key, basestring):
|
|
key = self.strings.add(key)
|
|
i = self.key2row[key]
|
|
self.data[i] = vector
|
|
|
|
def __iter__(self):
|
|
"""Yield vectors from the table.
|
|
|
|
YIELDS (numpy.ndarray): A vector.
|
|
"""
|
|
yield from self.data
|
|
|
|
def __len__(self):
|
|
"""Return the number of vectors that have been assigned.
|
|
|
|
RETURNS (int): The number of vectors in the data.
|
|
"""
|
|
return self.i
|
|
|
|
def __contains__(self, key):
|
|
"""Check whether a key has a vector entry in the table.
|
|
|
|
key (unicode / int): The key to check.
|
|
RETURNS (bool): Whether the key has a vector entry.
|
|
"""
|
|
if isinstance(key, basestring_):
|
|
key = self.strings[key]
|
|
return key in self.key2row
|
|
|
|
def add(self, key, *, vector=None, row=None):
|
|
"""Add a key to the table. Keys can be mapped to an existing vector
|
|
by setting `row`, or a new vector can be added.
|
|
|
|
key (unicode / int): The key to add.
|
|
vector (numpy.ndarray / None): A vector to add for the key.
|
|
row (int / None): The row-number of a vector to map the key to.
|
|
"""
|
|
if row is not None and vector is not None:
|
|
raise ValueError("Only one of 'row' and 'vector' may be set")
|
|
if isinstance(key, basestring_):
|
|
key = self.strings.add(key)
|
|
if key in self.key2row and vector is not None:
|
|
row = self.key2row[key]
|
|
elif key in self.key2row and row is not None:
|
|
self.key2row[key] = row
|
|
elif key not in self.key2row:
|
|
if row is not None:
|
|
self.key2row[key] = row
|
|
else:
|
|
self.key2row[key] = self.i
|
|
row = self.i
|
|
if row >= self.keys.shape[0]:
|
|
self.keys.resize((row*2,))
|
|
self.data.resize((row*2, self.data.shape[1]))
|
|
self.keys[self.i] = key
|
|
self.i += 1
|
|
if vector is not None:
|
|
self.data[row] = vector
|
|
return row
|
|
|
|
def items(self):
|
|
"""Iterate over `(string key, vector)` pairs, in order.
|
|
|
|
YIELDS (tuple): A key/vector pair.
|
|
"""
|
|
for i, key in enumerate(self.keys):
|
|
string = self.strings[key]
|
|
row = self.key2row[key]
|
|
yield string, self.data[row]
|
|
|
|
@property
|
|
def shape(self):
|
|
"""Get `(rows, dims)` tuples of number of rows and number of dimensions
|
|
in the vector table.
|
|
|
|
RETURNS (tuple): A `(rows, dims)` pair.
|
|
"""
|
|
return self.data.shape
|
|
|
|
def most_similar(self, key):
|
|
# TODO: implement
|
|
raise NotImplementedError
|
|
|
|
def from_glove(self, path):
|
|
"""Load GloVe vectors from a directory. Assumes binary format,
|
|
that the vocab is in a vocab.txt, and that vectors are named
|
|
vectors.{size}.[fd].bin, e.g. vectors.128.f.bin for 128d float32
|
|
vectors, vectors.300.d.bin for 300d float64 (double) vectors, etc.
|
|
By default GloVe outputs 64-bit vectors.
|
|
|
|
path (unicode / Path): The path to load the GloVe vectors from.
|
|
"""
|
|
path = util.ensure_path(path)
|
|
for name in path.iterdir():
|
|
if name.parts[-1].startswith('vectors'):
|
|
_, dims, dtype, _2 = name.parts[-1].split('.')
|
|
self.width = int(dims)
|
|
break
|
|
else:
|
|
raise IOError("Expected file named e.g. vectors.128.f.bin")
|
|
bin_loc = path / 'vectors.{dims}.{dtype}.bin'.format(dims=dims,
|
|
dtype=dtype)
|
|
with bin_loc.open('rb') as file_:
|
|
self.data = numpy.fromfile(file_, dtype='float64')
|
|
self.data = numpy.ascontiguousarray(self.data, dtype='float32')
|
|
n = 0
|
|
with (path / 'vocab.txt').open('r') as file_:
|
|
for line in file_:
|
|
self.add(line.strip())
|
|
n += 1
|
|
if (self.data.size % self.width) == 0:
|
|
self.data
|
|
|
|
def to_disk(self, path, **exclude):
|
|
"""Save the current state to a directory.
|
|
|
|
path (unicode / Path): A path to a directory, which will be created if
|
|
it doesn't exists. Either a string or a Path-like object.
|
|
"""
|
|
xp = get_array_module(self.data)
|
|
if xp is numpy:
|
|
save_array = lambda arr, file_: xp.save(file_, arr,
|
|
allow_pickle=False)
|
|
else:
|
|
save_array = lambda arr, file_: xp.save(file_, arr)
|
|
serializers = OrderedDict((
|
|
('vectors', lambda p: save_array(self.data, p.open('wb'))),
|
|
('keys', lambda p: xp.save(p.open('wb'), self.keys))
|
|
))
|
|
return util.to_disk(path, serializers, exclude)
|
|
|
|
def from_disk(self, path, **exclude):
|
|
"""Loads state from a directory. Modifies the object in place and
|
|
returns it.
|
|
|
|
path (unicode / Path): Directory path, string or Path-like object.
|
|
RETURNS (Vectors): The modified object.
|
|
"""
|
|
def load_keys(path):
|
|
if path.exists():
|
|
self.keys = numpy.load(path2str(path))
|
|
for i, key in enumerate(self.keys):
|
|
self.keys[i] = key
|
|
self.key2row[key] = i
|
|
|
|
def load_vectors(path):
|
|
xp = Model.ops.xp
|
|
if path.exists():
|
|
self.data = xp.load(path)
|
|
|
|
serializers = OrderedDict((
|
|
('keys', load_keys),
|
|
('vectors', load_vectors),
|
|
))
|
|
util.from_disk(path, serializers, exclude)
|
|
return self
|
|
|
|
def to_bytes(self, **exclude):
|
|
"""Serialize the current state to a binary string.
|
|
|
|
**exclude: Named attributes to prevent from being serialized.
|
|
RETURNS (bytes): The serialized form of the `Vectors` object.
|
|
"""
|
|
def serialize_weights():
|
|
if hasattr(self.data, 'to_bytes'):
|
|
return self.data.to_bytes()
|
|
else:
|
|
return msgpack.dumps(self.data)
|
|
serializers = OrderedDict((
|
|
('keys', lambda: msgpack.dumps(self.keys)),
|
|
('vectors', serialize_weights)
|
|
))
|
|
return util.to_bytes(serializers, exclude)
|
|
|
|
def from_bytes(self, data, **exclude):
|
|
"""Load state from a binary string.
|
|
|
|
data (bytes): The data to load from.
|
|
**exclude: Named attributes to prevent from being loaded.
|
|
RETURNS (Vectors): The `Vectors` object.
|
|
"""
|
|
def deserialize_weights(b):
|
|
if hasattr(self.data, 'from_bytes'):
|
|
self.data.from_bytes()
|
|
else:
|
|
self.data = msgpack.loads(b)
|
|
|
|
def load_keys(keys):
|
|
self.keys.resize((len(keys),))
|
|
for i, key in enumerate(keys):
|
|
self.keys[i] = key
|
|
self.key2row[key] = i
|
|
|
|
deserializers = OrderedDict((
|
|
('keys', lambda b: load_keys(msgpack.loads(b))),
|
|
('vectors', deserialize_weights)
|
|
))
|
|
util.from_bytes(data, deserializers, exclude)
|
|
return self
|