explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-06-02 04:03:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove freqs from transition_system

Browse Source
This commit is contained in:
Matthew Honnibal 2017-05-20 02:20:16 -05:00
parent 61fe55efba
commit 8b04b0af9f
3 changed files with 2 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spacy/syntax/arc_eager.pyx
View File

@ -358,9 +358,6 @@ cdef class ArcEager(TransitionSystem):
label = 'ROOT' label = 'ROOT'
gold.c.heads[i] = gold.heads[i] gold.c.heads[i] = gold.heads[i]
gold.c.labels[i] = self.strings[label] gold.c.labels[i] = self.strings[label]
# Count frequencies, for use in encoder
self.freqs[HEAD][gold.c.heads[i] - i] += 1
self.freqs[DEP][gold.c.labels[i]] += 1
cdef Transition lookup_transition(self, object name) except *: cdef Transition lookup_transition(self, object name) except *:
if '-' in name: if '-' in name:

13
spacy/syntax/ner.pyx
View File

@ -98,19 +98,6 @@ cdef class BiluoPushDown(TransitionSystem):
cdef int preprocess_gold(self, GoldParse gold) except -1: cdef int preprocess_gold(self, GoldParse gold) except -1:
for i in range(gold.length): for i in range(gold.length):
gold.c.ner[i] = self.lookup_transition(gold.ner[i]) gold.c.ner[i] = self.lookup_transition(gold.ner[i])
# Count frequencies, for use in encoder
if gold.c.ner[i].move in (BEGIN, UNIT):
self.freqs[ENT_IOB][3] += 1
self.freqs[ENT_TYPE][gold.c.ner[i].label] += 1
elif gold.c.ner[i].move in (IN, LAST):
self.freqs[ENT_IOB][2] += 1
self.freqs[ENT_TYPE][0] += 1
elif gold.c.ner[i].move == OUT:
self.freqs[ENT_IOB][1] += 1
self.freqs[ENT_TYPE][0] += 1
else:
self.freqs[ENT_IOB][1] += 1
self.freqs[ENT_TYPE][0] += 1
cdef Transition lookup_transition(self, object name) except *: cdef Transition lookup_transition(self, object name) except *:
if name == '-' or name == None: if name == '-' or name == None:

12
spacy/syntax/transition_system.pyx
View File

@ -26,7 +26,7 @@ cdef void* _init_state(Pool mem, int length, void* tokens) except NULL:
cdef class TransitionSystem: cdef class TransitionSystem:
def __init__(self, StringStore string_table, dict labels_by_action, _freqs=None): def __init__(self, StringStore string_table, dict labels_by_action):
self.mem = Pool() self.mem = Pool()
self.strings = string_table self.strings = string_table
self.n_moves = 0 self.n_moves = 0
@ -38,14 +38,6 @@ cdef class TransitionSystem:
for label_str in label_strs: for label_str in label_strs:
self.add_action(int(action), label_str) self.add_action(int(action), label_str)
self.root_label = self.strings['ROOT'] self.root_label = self.strings['ROOT']
self.freqs = {} if _freqs is None else _freqs
for attr in (TAG, HEAD, DEP, ENT_TYPE, ENT_IOB):
self.freqs[attr] = defaultdict(int)
self.freqs[attr][0] = 1
# Ensure we've seen heads. Need an official dependency length limit...
for i in range(10024):
self.freqs[HEAD][i] = 1
self.freqs[HEAD][-i] = 1
self.init_beam_state = _init_state self.init_beam_state = _init_state
def __reduce__(self): def __reduce__(self):
@ -55,7 +47,7 @@ cdef class TransitionSystem:
label_str = self.strings[trans.label] label_str = self.strings[trans.label]
labels_by_action.setdefault(trans.move, []).append(label_str) labels_by_action.setdefault(trans.move, []).append(label_str)
return (self.__class__, return (self.__class__,
(self.strings, labels_by_action, self.freqs), (self.strings, labels_by_action),
None, None) None, None)
def init_batch(self, docs): def init_batch(self, docs):