explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-07-10 16:22:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix ml/.

Browse Source
This commit is contained in:
Raphael Mitsch 2023-07-03 12:08:19 +02:00
parent 9de2257e7a
commit 8f59eeb772
2 changed files with 95 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
spacy/ml/parser_model.pxd
View File

@ -40,11 +40,16 @@ cdef ActivationsC alloc_activations(SizesC n) nogil
cdef void free_activations(const ActivationsC* A) nogil cdef void free_activations(const ActivationsC* A) nogil
cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states, cdef void predict_states(
const WeightsC* W, SizesC n) nogil CBlas cblas, ActivationsC* A, StateC** states, const WeightsC* W, SizesC n
) nogil
cdef int arg_max_if_valid(const weight_t* scores, const int* is_valid, int n) nogil cdef int arg_max_if_valid(const weight_t* scores, const int* is_valid, int n) nogil
cdef void cpu_log_loss(float* d_scores, cdef void cpu_log_loss(
const float* costs, const int* is_valid, const float* scores, int O) nogil float* d_scores,
const float* costs,
const int* is_valid,
const float* scores,
int O
) nogil

123
spacy/ml/parser_model.pyx
View File

@ -8,13 +8,13 @@ from thinc.backends.linalg cimport Vec, VecVec
import numpy import numpy
import numpy.random import numpy.random
from thinc.api import CupyOps, Model, NumpyOps, get_ops from thinc.api import CupyOps, Model, NumpyOps
from .. import util from .. import util
from ..errors import Errors from ..errors import Errors
from ..pipeline._parser_internals.stateclass cimport StateClass from ..pipeline._parser_internals.stateclass cimport StateClass
from ..typedefs cimport class_t, hash_t, weight_t from ..typedefs cimport weight_t
cdef WeightsC get_c_weights(model) except *: cdef WeightsC get_c_weights(model) except *:
@ -78,33 +78,48 @@ cdef void resize_activations(ActivationsC* A, SizesC n) nogil:
A.is_valid = <int*>calloc(n.states * n.classes, sizeof(A.is_valid[0])) A.is_valid = <int*>calloc(n.states * n.classes, sizeof(A.is_valid[0]))
A._max_size = n.states A._max_size = n.states
else: else:
A.token_ids = <int*>realloc(A.token_ids, A.token_ids = <int*>realloc(
n.states * n.feats * sizeof(A.token_ids[0])) A.token_ids, n.states * n.feats * sizeof(A.token_ids[0])
A.scores = <float*>realloc(A.scores, )
n.states * n.classes * sizeof(A.scores[0])) A.scores = <float*>realloc(
A.unmaxed = <float*>realloc(A.unmaxed, A.scores, n.states * n.classes * sizeof(A.scores[0])
n.states * n.hiddens * n.pieces * sizeof(A.unmaxed[0])) )
A.hiddens = <float*>realloc(A.hiddens, A.unmaxed = <float*>realloc(
n.states * n.hiddens * sizeof(A.hiddens[0])) A.unmaxed, n.states * n.hiddens * n.pieces * sizeof(A.unmaxed[0])
A.is_valid = <int*>realloc(A.is_valid, )
n.states * n.classes * sizeof(A.is_valid[0])) A.hiddens = <float*>realloc(
A.hiddens, n.states * n.hiddens * sizeof(A.hiddens[0])
)
A.is_valid = <int*>realloc(
A.is_valid, n.states * n.classes * sizeof(A.is_valid[0])
)
A._max_size = n.states A._max_size = n.states
A._curr_size = n.states A._curr_size = n.states
cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states, cdef void predict_states(
const WeightsC* W, SizesC n) nogil: CBlas cblas, ActivationsC* A, StateC** states, const WeightsC* W, SizesC n
cdef double one = 1.0 ) nogil:
resize_activations(A, n) resize_activations(A, n)
for i in range(n.states): for i in range(n.states):
states[i].set_context_tokens(&A.token_ids[i*n.feats], n.feats) states[i].set_context_tokens(&A.token_ids[i*n.feats], n.feats)
memset(A.unmaxed, 0, n.states * n.hiddens * n.pieces * sizeof(float)) memset(A.unmaxed, 0, n.states * n.hiddens * n.pieces * sizeof(float))
memset(A.hiddens, 0, n.states * n.hiddens * sizeof(float)) memset(A.hiddens, 0, n.states * n.hiddens * sizeof(float))
sum_state_features(cblas, A.unmaxed, sum_state_features(
W.feat_weights, A.token_ids, n.states, n.feats, n.hiddens * n.pieces) cblas,
A.unmaxed,
W.feat_weights,
A.token_ids,
n.states,
n.feats,
n.hiddens * n.pieces
)
for i in range(n.states): for i in range(n.states):
VecVec.add_i(&A.unmaxed[i*n.hiddens*n.pieces], VecVec.add_i(
W.feat_bias, 1., n.hiddens * n.pieces) &A.unmaxed[i*n.hiddens*n.pieces],
W.feat_bias, 1.,
n.hiddens * n.pieces
)
for j in range(n.hiddens): for j in range(n.hiddens):
index = i * n.hiddens * n.pieces + j * n.pieces index = i * n.hiddens * n.pieces + j * n.pieces
which = Vec.arg_max(&A.unmaxed[index], n.pieces) which = Vec.arg_max(&A.unmaxed[index], n.pieces)
@ -114,14 +129,15 @@ cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states,
memcpy(A.scores, A.hiddens, n.states * n.classes * sizeof(float)) memcpy(A.scores, A.hiddens, n.states * n.classes * sizeof(float))
else: else:
# Compute hidden-to-output # Compute hidden-to-output
sgemm(cblas)(False, True, n.states, n.classes, n.hiddens, sgemm(cblas)(
False, True, n.states, n.classes, n.hiddens,
1.0, <const float *>A.hiddens, n.hiddens, 1.0, <const float *>A.hiddens, n.hiddens,
<const float *>W.hidden_weights, n.hiddens, <const float *>W.hidden_weights, n.hiddens,
0.0, A.scores, n.classes) 0.0, A.scores, n.classes
)
# Add bias # Add bias
for i in range(n.states): for i in range(n.states):
VecVec.add_i(&A.scores[i*n.classes], VecVec.add_i(&A.scores[i*n.classes], W.hidden_bias, 1., n.classes)
W.hidden_bias, 1., n.classes)
# Set unseen classes to minimum value # Set unseen classes to minimum value
i = 0 i = 0
min_ = A.scores[0] min_ = A.scores[0]
@ -134,9 +150,16 @@ cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states,
A.scores[i*n.classes+j] = min_ A.scores[i*n.classes+j] = min_
cdef void sum_state_features(CBlas cblas, float* output, cdef void sum_state_features(
const float* cached, const int* token_ids, int B, int F, int O) nogil: CBlas cblas,
cdef int idx, b, f, i float* output,
const float* cached,
const int* token_ids,
int B,
int F,
intO
) nogil:
cdef int idx, b, f
cdef const float* feature cdef const float* feature
padding = cached padding = cached
cached += F * O cached += F * O
@ -153,9 +176,13 @@ cdef void sum_state_features(CBlas cblas, float* output,
token_ids += F token_ids += F
cdef void cpu_log_loss(float* d_scores, cdef void cpu_log_loss(
const float* costs, const int* is_valid, const float* scores, float* d_scores,
int O) nogil: const float* costs,
const int* is_valid,
const float* scores,
int O
) nogil:
"""Do multi-label log loss""" """Do multi-label log loss"""
cdef double max_, gmax, Z, gZ cdef double max_, gmax, Z, gZ
best = arg_max_if_gold(scores, costs, is_valid, O) best = arg_max_if_gold(scores, costs, is_valid, O)
@ -179,8 +206,9 @@ cdef void cpu_log_loss(float* d_scores,
d_scores[i] = exp(scores[i]-max_) / Z d_scores[i] = exp(scores[i]-max_) / Z
cdef int arg_max_if_gold(const weight_t* scores, const weight_t* costs, cdef int arg_max_if_gold(
const int* is_valid, int n) nogil: const weight_t* scores, const weight_t* costs, const int* is_valid, int n
) nogil:
# Find minimum cost # Find minimum cost
cdef float cost = 1 cdef float cost = 1
for i in range(n): for i in range(n):
@ -204,10 +232,17 @@ cdef int arg_max_if_valid(const weight_t* scores, const int* is_valid, int n) no
return best return best
class ParserStepModel(Model): class ParserStepModel(Model):
def __init__(self, docs, layers, *, has_upper, unseen_classes=None, train=True, def __init__(
dropout=0.1): self,
docs,
layers,
*,
has_upper,
unseen_classes=None,
train=True,
dropout=0.1
):
Model.__init__(self, name="parser_step_model", forward=step_forward) Model.__init__(self, name="parser_step_model", forward=step_forward)
self.attrs["has_upper"] = has_upper self.attrs["has_upper"] = has_upper
self.attrs["dropout_rate"] = dropout self.attrs["dropout_rate"] = dropout
@ -268,8 +303,10 @@ class ParserStepModel(Model):
return ids return ids
def backprop_step(self, token_ids, d_vector, get_d_tokvecs): def backprop_step(self, token_ids, d_vector, get_d_tokvecs):
if isinstance(self.state2vec.ops, CupyOps) \ if (
and not isinstance(token_ids, self.state2vec.ops.xp.ndarray): isinstance(self.state2vec.ops, CupyOps)
and not isinstance(token_ids, self.state2vec.ops.xp.ndarray)
):
# Move token_ids and d_vector to GPU, asynchronously # Move token_ids and d_vector to GPU, asynchronously
self.backprops.append(( self.backprops.append((
util.get_async(self.cuda_stream, token_ids), util.get_async(self.cuda_stream, token_ids),
@ -279,7 +316,6 @@ class ParserStepModel(Model):
else: else:
self.backprops.append((token_ids, d_vector, get_d_tokvecs)) self.backprops.append((token_ids, d_vector, get_d_tokvecs))
def finish_steps(self, golds): def finish_steps(self, golds):
# Add a padding vector to the d_tokvecs gradient, so that missing # Add a padding vector to the d_tokvecs gradient, so that missing
# values don't affect the real gradient. # values don't affect the real gradient.
@ -292,14 +328,15 @@ class ParserStepModel(Model):
ids = ids.flatten() ids = ids.flatten()
d_state_features = d_state_features.reshape( d_state_features = d_state_features.reshape(
(ids.size, d_state_features.shape[2])) (ids.size, d_state_features.shape[2]))
self.ops.scatter_add(d_tokvecs, ids, self.ops.scatter_add(d_tokvecs, ids, d_state_features)
d_state_features)
# Padded -- see update() # Padded -- see update()
self.bp_tokvecs(d_tokvecs[:-1]) self.bp_tokvecs(d_tokvecs[:-1])
return d_tokvecs return d_tokvecs
NUMPY_OPS = NumpyOps() NUMPY_OPS = NumpyOps()
def step_forward(model: ParserStepModel, states, is_train): def step_forward(model: ParserStepModel, states, is_train):
token_ids = model.get_token_ids(states) token_ids = model.get_token_ids(states)
vector, get_d_tokvecs = model.state2vec(token_ids, is_train) vector, get_d_tokvecs = model.state2vec(token_ids, is_train)
@ -312,7 +349,7 @@ def step_forward(model: ParserStepModel, states, is_train):
scores, get_d_vector = model.vec2scores(vector, is_train) scores, get_d_vector = model.vec2scores(vector, is_train)
else: else:
scores = NumpyOps().asarray(vector) scores = NumpyOps().asarray(vector)
get_d_vector = lambda d_scores: d_scores get_d_vector = lambda d_scores: d_scores # no-cython-lint: E731
# If the class is unseen, make sure its score is minimum # If the class is unseen, make sure its score is minimum
scores[:, model._class_mask == 0] = numpy.nanmin(scores) scores[:, model._class_mask == 0] = numpy.nanmin(scores)
@ -448,9 +485,11 @@ cdef class precompute_hiddens:
feat_weights = self.get_feat_weights() feat_weights = self.get_feat_weights()
cdef int[:, ::1] ids = token_ids cdef int[:, ::1] ids = token_ids
sum_state_features(cblas, <float*>state_vector.data, sum_state_features(
cblas, <float*>state_vector.data,
feat_weights, &ids[0, 0], feat_weights, &ids[0, 0],
token_ids.shape[0], self.nF, self.nO*self.nP) token_ids.shape[0], self.nF, self.nO*self.nP
)
state_vector += self.bias state_vector += self.bias
state_vector, bp_nonlinearity = self._nonlinearity(state_vector) state_vector, bp_nonlinearity = self._nonlinearity(state_vector)