explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-09-21 03:19:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Remove UniStr struct

Browse Source
This commit is contained in:
Matthew Honnibal 2015-07-22 13:39:17 +02:00
parent 2fc66e3723
commit 815bda201d
2 changed files with 2 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
spacy/serialize/packer.pyx
View File

@ -7,9 +7,6 @@ from libc.math cimport exp as c_exp
from libcpp.queue cimport priority_queue
from libcpp.pair cimport pair
from ..structs cimport UniStr
from ..strings cimport slice_unicode
from cymem.cymem cimport Address, Pool
from preshed.maps cimport PreshMap
from preshed.counter cimport PreshCounter
@ -175,14 +172,13 @@ cdef class Packer:
cdef Doc tokens = Doc(self.vocab)
cdef int start = 0
cdef bint is_spacy
cdef UniStr span
cdef int length = len(string)
cdef int i = 0
cdef bint is_end_token
for is_end_token in bits:
if is_end_token:
slice_unicode(&span, string, start, i+1)
lex = self.vocab.get(tokens.mem, &span)
span = string[start:i+1]
lex = self.vocab.get(tokens.mem, span)
is_spacy = (i+1) < length and string[i+1] == u' '
tokens.push_back(lex, is_spacy)
start = i + 1 + is_spacy

6
spacy/structs.pxd
View File

@ -78,9 +78,3 @@ cdef struct TokenC:
int ent_iob
int ent_type
cdef struct UniStr:
Py_UNICODE* chars
size_t n
hash_t key