explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-27 01:34:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove commented code from parser

Browse Source
This commit is contained in:
Matthew Honnibal 2017-05-25 14:55:09 -05:00
parent f403c2cd5f
commit 2cb7cc2db7
1 changed files with 8 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
spacy/syntax/nn_parser.pyx
View File

@ -429,18 +429,14 @@ cdef class Parser:
states, golds = self._init_gold_batch(docs, golds) states, golds = self._init_gold_batch(docs, golds)
max_length = min([len(doc) for doc in docs]) max_length = min([len(doc) for doc in docs])
#golds = [self.moves.preprocess_gold(g) for g in golds]
#states = self.moves.init_batch(docs)
state2vec, vec2scores = self.get_batch_model(len(states), tokvecs, cuda_stream, state2vec, vec2scores = self.get_batch_model(len(states), tokvecs, cuda_stream,
0.0) 0.0)
todo = [(s, g) for (s, g) in zip(states, golds) todo = [(s, g) for (s, g) in zip(states, golds)
if not s.is_final() and g is not None] if not s.is_final() and g is not None]
backprops = [] backprops = []
d_tokvecs = state2vec.ops.allocate(tokvecs.shape) d_tokvecs = state2vec.ops.allocate(tokvecs.shape)
cdef float loss = 0. cdef float loss = 0.
#while len(todo and len(todo) >= len(states):
while todo: while todo:
states, golds = zip(*todo) states, golds = zip(*todo)
@ -483,34 +479,33 @@ cdef class Parser:
long_doc[:N], and another representing long_doc[N:].""" long_doc[:N], and another representing long_doc[N:]."""
cdef StateClass state cdef StateClass state
lengths = [len(doc) for doc in docs] lengths = [len(doc) for doc in docs]
# Cap to min length
min_length = min(lengths) min_length = min(lengths)
offset = 0 offset = 0
states = [] states = []
extra_golds = [] extra_golds = []
cdef np.ndarray py_costs = numpy.zeros((self.moves.n_moves,), dtype='f') cdef Pool mem = Pool()
cdef np.ndarray py_is_valid = numpy.zeros((self.moves.n_moves,), dtype='i') costs = <float*>mem.alloc(self.moves.n_moves, sizeof(float))
costs = <float*>py_costs.data is_valid = <int*>mem.alloc(self.moves.n_moves, sizeof(int))
is_valid = <int*>py_is_valid.data
for doc, gold in zip(docs, golds): for doc, gold in zip(docs, golds):
gold = self.moves.preprocess_gold(gold) gold = self.moves.preprocess_gold(gold)
state = StateClass(doc, offset=offset) state = StateClass(doc, offset=offset)
self.moves.initialize_state(state.c) self.moves.initialize_state(state.c)
states.append(state) if not state.is_final():
extra_golds.append(gold) states.append(state)
extra_golds.append(gold)
start = min(min_length, len(doc)) start = min(min_length, len(doc))
while start < len(doc): while start < len(doc):
length = min(min_length, len(doc)-start) length = min(min_length, len(doc)-start)
state = StateClass(doc, offset=offset) state = StateClass(doc, offset=offset)
self.moves.initialize_state(state.c) self.moves.initialize_state(state.c)
while state.B(0) < start and not state.is_final(): while state.B(0) < start and not state.is_final():
py_is_valid.fill(0)
py_costs.fill(0)
self.moves.set_costs(is_valid, costs, state, gold) self.moves.set_costs(is_valid, costs, state, gold)
for i in range(self.moves.n_moves): for i in range(self.moves.n_moves):
if is_valid[i] and costs[i] <= 0: if is_valid[i] and costs[i] <= 0:
self.moves.c[i].do(state.c, self.moves.c[i].label) self.moves.c[i].do(state.c, self.moves.c[i].label)
break break
else:
raise ValueError("Could not find gold move")
start += length start += length
if not state.is_final(): if not state.is_final():
states.append(state) states.append(state)