mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 11:29:13 +03:00
25 lines
854 B
Cython
25 lines
854 B
Cython
from libcpp.vector cimport vector
|
|
from libc.stdint cimport uint64_t
|
|
|
|
from ext.sparsehash cimport dense_hash_map
|
|
|
|
# Circular import problems here
|
|
ctypedef size_t Lexeme_addr
|
|
ctypedef uint64_t StringHash
|
|
ctypedef dense_hash_map[StringHash, Lexeme_addr] Vocab
|
|
ctypedef int (*Splitter)(unicode word, size_t length)
|
|
|
|
|
|
from spacy.lexeme cimport Lexeme
|
|
|
|
cdef load_tokenization(Vocab& vocab, dict bacov, token_rules)
|
|
cdef vector[Lexeme_addr] tokenize(Vocab& vocab, dict bacov, Splitter splitter,
|
|
unicode string) except *
|
|
cdef Lexeme_addr lookup(Vocab& vocab, dict bacov, Splitter splitter, int start,
|
|
unicode string) except 0
|
|
cdef StringHash hash_string(unicode s, size_t length) except 0
|
|
cdef unicode unhash(dict bacov, StringHash hash_value)
|
|
|
|
|
|
cpdef vector[size_t] expand_chunk(size_t addr) except *
|