diff --git a/spacy/syntax/_beam_utils.pyx b/spacy/syntax/_beam_utils.pyx
index dc482f278..b1085c762 100644
--- a/spacy/syntax/_beam_utils.pyx
+++ b/spacy/syntax/_beam_utils.pyx
@@ -69,7 +69,8 @@ cdef class ParserBeam(object):
         cdef StateC* st
         for state in states:
             beam = Beam(self.moves.n_moves, width, min_density=density)
-            beam.initialize(self.moves.init_beam_state, state.c.length,
+            beam.initialize(self.moves.init_beam_state,
+                            self.moves.del_beam_state, state.c.length,
                             state.c._sent)
             for i in range(beam.width):
                 st = <StateC*>beam.at(i)
diff --git a/spacy/syntax/arc_eager.pyx b/spacy/syntax/arc_eager.pyx
index eb39124ce..efe8573c1 100644
--- a/spacy/syntax/arc_eager.pyx
+++ b/spacy/syntax/arc_eager.pyx
@@ -324,10 +324,16 @@ cdef void* _init_state(Pool mem, int length, void* tokens) except NULL:
     return <void*>st
 
 
+cdef int _del_state(Pool mem, void* state, void* x) except -1:
+    cdef StateC* st = <StateC*>state
+    del st
+
+
 cdef class ArcEager(TransitionSystem):
     def __init__(self, *args, **kwargs):
         TransitionSystem.__init__(self, *args, **kwargs)
         self.init_beam_state = _init_state
+        self.del_beam_state = _del_state
 
     @classmethod
     def get_actions(cls, **kwargs):
diff --git a/spacy/syntax/transition_system.pxd b/spacy/syntax/transition_system.pxd
index 45d9a787f..a5fe55918 100644
--- a/spacy/syntax/transition_system.pxd
+++ b/spacy/syntax/transition_system.pxd
@@ -33,6 +33,8 @@ ctypedef int (*do_func_t)(StateC* state, attr_t label) nogil
 
 ctypedef void* (*init_state_t)(Pool mem, int length, void* tokens) except NULL
 
+ctypedef int (*del_state_t)(Pool mem, void* state, void* extra_args) except -1
+
 cdef class TransitionSystem:
     cdef Pool mem
     cdef StringStore strings
@@ -42,6 +44,7 @@ cdef class TransitionSystem:
     cdef public attr_t root_label
     cdef public freqs
     cdef init_state_t init_beam_state
+    cdef del_state_t del_beam_state
     cdef public object labels
 
     cdef int initialize_state(self, StateC* state) nogil
diff --git a/spacy/syntax/transition_system.pyx b/spacy/syntax/transition_system.pyx
index 7876813e0..65097f114 100644
--- a/spacy/syntax/transition_system.pyx
+++ b/spacy/syntax/transition_system.pyx
@@ -30,6 +30,11 @@ cdef void* _init_state(Pool mem, int length, void* tokens) except NULL:
     return <void*>st
 
 
+cdef int _del_state(Pool mem, void* state, void* x) except -1:
+    cdef StateC* st = <StateC*>state
+    del st
+
+
 cdef class TransitionSystem:
     def __init__(self, StringStore string_table, labels_by_action=None, min_freq=None):
         self.mem = Pool()
@@ -44,6 +49,7 @@ cdef class TransitionSystem:
             self.initialize_actions(labels_by_action, min_freq=min_freq)
         self.root_label = self.strings.add('ROOT')
         self.init_beam_state = _init_state
+        self.del_beam_state = _del_state
 
     def __reduce__(self):
         return (self.__class__, (self.strings, self.labels), None, None)
@@ -72,7 +78,8 @@ cdef class TransitionSystem:
 
         for doc in docs:
             beam = Beam(self.n_moves, beam_width, min_density=beam_density)
-            beam.initialize(self.init_beam_state, doc.length, doc.c)
+            beam.initialize(self.init_beam_state, self.del_beam_state,
+                            doc.length, doc.c)
             for i in range(beam.width):
                 state = <StateC*>beam.at(i)
                 state.offset = offset