spaCy/spacy/lang.pxd

48 lines
1.1 KiB
Cython
Raw Normal View History

2014-08-03 00:51:52 +04:00
from libc.stdint cimport uint32_t
from libc.stdint cimport uint64_t
2014-08-25 18:42:22 +04:00
from spacy.word cimport Lexeme
2014-09-10 20:11:13 +04:00
from spacy.tokens cimport Tokens
from spacy.lexeme cimport LexemeC
from spacy._hashing cimport PointerHash
2014-09-10 20:11:13 +04:00
from libcpp.utility cimport pair
from libcpp.vector cimport vector
from libc.stdint cimport uint64_t, int64_t
2014-09-10 20:11:13 +04:00
cdef extern from "Python.h":
cdef bint Py_UNICODE_ISSPACE(Py_UNICODE ch)
cdef bint Py_UNICODE_ISALNUM(Py_UNICODE ch)
cdef struct String:
Py_UNICODE* chars
size_t n
uint64_t key
cdef class Lexicon:
2014-09-10 20:11:13 +04:00
cpdef readonly size_t size
cpdef Lexeme lookup(self, unicode string)
cdef size_t get(self, String* s)
2014-08-29 03:59:23 +04:00
cdef PointerHash _dict
2014-08-29 03:59:23 +04:00
cdef list _string_features
cdef list _flag_features
cdef class Language:
2014-08-29 03:59:23 +04:00
cdef unicode name
cdef PointerHash cache
cdef PointerHash specials
cpdef readonly Lexicon lexicon
2014-09-10 20:11:13 +04:00
cpdef readonly object tokens_class
2014-09-11 23:37:32 +04:00
cpdef Tokens tokenize(self, unicode text)
2014-08-29 03:59:23 +04:00
cpdef Lexeme lookup(self, unicode text)
2014-08-16 05:22:03 +04:00
2014-09-13 02:14:05 +04:00
cdef int _tokenize(self, Tokens tokens, String* string)
cdef int _split_one(self, Py_UNICODE* characters, size_t length)