explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-10-30 23:47:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Fix Issue #131: Force whitespace characters to attach syntactically to previous token, and ensure they cannot serve as stand-alone 'sentence' units.

Browse Source
This commit is contained in:
Matthew Honnibal 2015-10-10 15:53:30 +11:00
parent 8b39feefbe
commit 9dd2f25c74
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
spacy/syntax/arc_eager.pyx
View File

@ -380,10 +380,17 @@ cdef class ArcEager(TransitionSystem):
st.fast_forward()
cdef int finalize_state(self, StateClass st) nogil:
cdef int i
for i in range(st.length):
# Always attach spaces to the previous word
if Lexeme.c_check_flag(st._sent[i].lex, IS_SPACE):
st._sent[i].head = -1 if (i >= 1) else 1
if st._sent[i].sent_start and st._sent[i].head == -1:
st._sent[i].sent_start = False
# If we had this space token as the start of a sentence,
# move that sentence start forward one
if (i + 1) < st.length and not st._sent[i+1].sent_start:
st._sent[i+1].sent_start = True
elif st._sent[i].head == 0 and st._sent[i].dep == 0:
st._sent[i].dep = self.root_label
# If we're not using the Break transition, we segment via root-labelled