From 7baef5b7ff5bd2b76ef7a6c2d025ede483fab635 Mon Sep 17 00:00:00 2001
From: Matthew Honnibal <honnibal@gmail.com>
Date: Thu, 23 Oct 2014 04:01:17 +1100
Subject: [PATCH] * Fix padding on tokens

---
 spacy/tokens.pyx | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/spacy/tokens.pyx b/spacy/tokens.pyx
index 7b81f6cf1..d20e6c498 100644
--- a/spacy/tokens.pyx
+++ b/spacy/tokens.pyx
@@ -49,9 +49,8 @@ cdef class Tokens:
         self.lex = self._lex_ptr
         self.idx = self._idx_ptr
         self.pos = self._pos_ptr
-        for i in range(PADDING):
-            self.lex[i] = &EMPTY_LEXEME
-        for i in range(size, PADDING):
+        cdef int i
+        for i in range(size + (PADDING*2)):
             self.lex[i] = &EMPTY_LEXEME
         self.lex += PADDING
         self.idx += PADDING
@@ -85,7 +84,8 @@ cdef class Tokens:
         self.lex = self._lex_ptr + PADDING
         self.idx = self._idx_ptr + PADDING
         self.pos = self._pos_ptr + PADDING
-
+        for i in range(self.length, self.max_length + PADDING):
+            self.lex[i] = &EMPTY_LEXEME
 
     cdef int extend(self, int idx, LexemeC** lexemes, int n) except -1:
         cdef int i