Add destructors for states in TransitionSystem (#4686)

spacy/syntax/_beam_utils.pyx

@@ -69,7 +69,8 @@ cdef class ParserBeam(object):
         cdef StateC* st
         for state in states:
             beam = Beam(self.moves.n_moves, width, min_density=density)
-            beam.initialize(self.moves.init_beam_state, state.c.length,
+            beam.initialize(self.moves.init_beam_state,
+                            self.moves.del_beam_state, state.c.length,
                             state.c._sent)
             for i in range(beam.width):
                 st = <StateC*>beam.at(i)

spacy/syntax/arc_eager.pyx

@@ -324,10 +324,16 @@ cdef void* _init_state(Pool mem, int length, void* tokens) except NULL:
     return <void*>st
 
 
+cdef int _del_state(Pool mem, void* state, void* x) except -1:
+    cdef StateC* st = <StateC*>state
+    del st
+
+
 cdef class ArcEager(TransitionSystem):
     def __init__(self, *args, **kwargs):
         TransitionSystem.__init__(self, *args, **kwargs)
         self.init_beam_state = _init_state
+        self.del_beam_state = _del_state
 
     @classmethod
     def get_actions(cls, **kwargs):

spacy/syntax/transition_system.pxd

@@ -33,6 +33,8 @@ ctypedef int (*do_func_t)(StateC* state, attr_t label) nogil
 
 ctypedef void* (*init_state_t)(Pool mem, int length, void* tokens) except NULL
 
+ctypedef int (*del_state_t)(Pool mem, void* state, void* extra_args) except -1
+
 cdef class TransitionSystem:
     cdef Pool mem
     cdef StringStore strings
@@ -42,6 +44,7 @@ cdef class TransitionSystem:
     cdef public attr_t root_label
     cdef public freqs
     cdef init_state_t init_beam_state
+    cdef del_state_t del_beam_state
     cdef public object labels
 
     cdef int initialize_state(self, StateC* state) nogil

spacy/syntax/transition_system.pyx

@@ -30,6 +30,11 @@ cdef void* _init_state(Pool mem, int length, void* tokens) except NULL:
     return <void*>st
 
 
+cdef int _del_state(Pool mem, void* state, void* x) except -1:
+    cdef StateC* st = <StateC*>state
+    del st
+
+
 cdef class TransitionSystem:
     def __init__(self, StringStore string_table, labels_by_action=None, min_freq=None):
         self.mem = Pool()
@@ -44,6 +49,7 @@ cdef class TransitionSystem:
             self.initialize_actions(labels_by_action, min_freq=min_freq)
         self.root_label = self.strings.add('ROOT')
         self.init_beam_state = _init_state
+        self.del_beam_state = _del_state
 
     def __reduce__(self):
         return (self.__class__, (self.strings, self.labels), None, None)
@@ -72,7 +78,8 @@ cdef class TransitionSystem:
 
         for doc in docs:
             beam = Beam(self.n_moves, beam_width, min_density=beam_density)
-            beam.initialize(self.init_beam_state, doc.length, doc.c)
+            beam.initialize(self.init_beam_state, self.del_beam_state,
+                            doc.length, doc.c)
             for i in range(beam.width):
                 state = <StateC*>beam.at(i)
                 state.offset = offset