explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-12 04:38:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Work on character tagger

Browse Source
This commit is contained in:
Matthew Honnibal 2016-02-22 00:15:25 +01:00
parent 7a519ea5af
commit c3f334cef1
1 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
spacy/tagger.pyx
View File

@ -125,9 +125,9 @@ cdef class TaggerNeuralNet(NeuralNet):
cdef class CharacterTagger(NeuralNet): cdef class CharacterTagger(NeuralNet):
def __init__(self, n_classes, def __init__(self, n_classes,
depth=4, hidden_width=100, depth=5, hidden_width=20,
chars_width=5, chars_width=10,
words_width=20, shape_width=5, suffix_width=5, tags_width=5, words_width=20, shape_width=5, suffix_width=5, tags_width=20,
learn_rate=0.1): learn_rate=0.1):
input_length = 5 * chars_width * self.chars_per_word + 2 * tags_width input_length = 5 * chars_width * self.chars_per_word + 2 * tags_width
widths = [input_length] + [hidden_width] * depth + [n_classes] widths = [input_length] + [hidden_width] * depth + [n_classes]
@ -153,11 +153,15 @@ cdef class CharacterTagger(NeuralNet):
cdef int chars_per_word = self.chars_per_word cdef int chars_per_word = self.chars_per_word
cdef unicode string cdef unicode string
for string in (p2, p1, w, n1, n2): for string in (p2, p1, w, n1, n2):
for c in range(chars_per_word): for c in range(chars_per_word / 2):
eg.features[p].i = p eg.features[p].i = p
eg.features[p].key = ord(string[c]) eg.features[p].key = ord(string[c])
eg.features[p].value = 1.0 if string[c] != u'_' else 0.0 eg.features[p].value = 1.0 if string[c] != u'_' else 0.0
p += 1 p += 1
eg.features[p].i = p
eg.features[p].key = ord(string[-(c+1)])
eg.features[p].value = 1.0 if string[-(c+1)] != u'_' else 0.0
p += 1
eg.features[p].key = tokens[i-1].tag eg.features[p].key = tokens[i-1].tag
eg.features[p].value = 1.0 eg.features[p].value = 1.0
eg.features[p].i = p eg.features[p].i = p
@ -179,7 +183,7 @@ cdef class CharacterTagger(NeuralNet):
property chars_per_word: property chars_per_word:
def __get__(self): def __get__(self):
return 15 return 16
def _pad(word, nr_char): def _pad(word, nr_char):
@ -187,13 +191,12 @@ def _pad(word, nr_char):
pass pass
elif len(word) > nr_char: elif len(word) > nr_char:
split = nr_char / 2 split = nr_char / 2
word = word[:split+1] + word[-split:] word = word[:split] + word[-split:]
else: else:
word = word.ljust(nr_char, ' ') word = word.ljust(nr_char, '_')
assert len(word) == nr_char, repr(word) assert len(word) == nr_char, repr(word)
return word return word
cdef inline void _fill_from_token(atom_t* context, const TokenC* t) nogil: cdef inline void _fill_from_token(atom_t* context, const TokenC* t) nogil:
context[0] = t.lex.lower context[0] = t.lex.lower