spaCy/spacy/tokens/span.pyx

cimport numpy as np
from libcpp.memory cimport make_shared

import copy
import warnings

import numpy
from thinc.api import get_array_module

from ..attrs cimport *
from ..attrs cimport ORTH, attr_id_t
from ..lexeme cimport Lexeme
from ..structs cimport TokenC
from ..symbols cimport dep
from ..typedefs cimport attr_t
from .doc cimport _get_lca_matrix, get_token_attr, token_by_end, token_by_start
from .token cimport Token

from ..errors import Errors, Warnings
from ..util import normalize_slice
from .underscore import Underscore, get_ext_args


cdef class Span:
    """A slice from a Doc object.

    DOCS: https://spacy.io/api/span
    """
    @classmethod
    def set_extension(cls, name, **kwargs):
        """Define a custom attribute which becomes available as `Span._`.

        name (str): Name of the attribute to set.
        default: Optional default value of the attribute.
        getter (callable): Optional getter function.
        setter (callable): Optional setter function.
        method (callable): Optional method for method extension.
        force (bool): Force overwriting existing attribute.

        DOCS: https://spacy.io/api/span#set_extension
        USAGE: https://spacy.io/usage/processing-pipelines#custom-components-attributes
        """
        if cls.has_extension(name) and not kwargs.get("force", False):
            raise ValueError(Errors.E090.format(name=name, obj="Span"))
        Underscore.span_extensions[name] = get_ext_args(**kwargs)

    @classmethod
    def get_extension(cls, name):
        """Look up a previously registered extension by name.

        name (str): Name of the extension.
        RETURNS (tuple): A `(default, method, getter, setter)` tuple.

        DOCS: https://spacy.io/api/span#get_extension
        """
        return Underscore.span_extensions.get(name)

    @classmethod
    def has_extension(cls, name):
        """Check whether an extension has been registered.

        name (str): Name of the extension.
        RETURNS (bool): Whether the extension has been registered.

        DOCS: https://spacy.io/api/span#has_extension
        """
        return name in Underscore.span_extensions

    @classmethod
    def remove_extension(cls, name):
        """Remove a previously registered extension.

        name (str): Name of the extension.
        RETURNS (tuple): A `(default, method, getter, setter)` tuple of the
            removed extension.

        DOCS: https://spacy.io/api/span#remove_extension
        """
        if not cls.has_extension(name):
            raise ValueError(Errors.E046.format(name=name))
        return Underscore.span_extensions.pop(name)

    def __cinit__(self, Doc doc, int start, int end, label=0, vector=None,
                  vector_norm=None, kb_id=0, span_id=0):
        """Create a `Span` object from the slice `doc[start : end]`.

        doc (Doc): The parent document.
        start (int): The index of the first token of the span.
        end (int): The index of the first token after the span.
        label (Union[int, str]): A label to attach to the Span, e.g. for named
            entities.
        vector (ndarray[ndim=1, dtype='float32']): A meaning representation
            of the span.
        vector_norm (float): The L2 norm of the span's vector representation.
        kb_id (Union[int, str]): An identifier from a Knowledge Base to capture
            the meaning of a named entity.
        span_id (Union[int, str]): An identifier to associate with the span.

        DOCS: https://spacy.io/api/span#init
        """
        if not (0 <= start <= end <= len(doc)):
            raise IndexError(Errors.E035.format(start=start, end=end, length=len(doc)))
        self.doc = doc
        if isinstance(label, str):
            label = doc.vocab.strings.add(label)
        if isinstance(kb_id, str):
            kb_id = doc.vocab.strings.add(kb_id)
        if isinstance(span_id, str):
            span_id = doc.vocab.strings.add(span_id)
        if label not in doc.vocab.strings:
            raise ValueError(Errors.E084.format(label=label))

        start_char = doc[start].idx if start < doc.length else len(doc.text)
        if start == end:
            end_char = start_char
        else:
            end_char = doc[end - 1].idx + len(doc[end - 1])
        self.c = make_shared[SpanC](SpanC(
            label=label,
            kb_id=kb_id,
            id=span_id,
            start=start,
            end=end,
            start_char=start_char,
            end_char=end_char,
        ))
        self._vector = vector
        self._vector_norm = vector_norm

    def __richcmp__(self, Span other, int op):
        if other is None:
            if op == 0 or op == 1 or op == 2:
                return False
            else:
                return True

        self_tuple = self._cmp_tuple()
        other_tuple = other._cmp_tuple()
        # <
        if op == 0:
            return self_tuple < other_tuple
        # <=
        elif op == 1:
            return self_tuple <= other_tuple
        # ==
        elif op == 2:
            return self_tuple == other_tuple
        # !=
        elif op == 3:
            return self_tuple != other_tuple
        # >
        elif op == 4:
            return self_tuple > other_tuple
        # >=
        elif op == 5:
            return self_tuple >= other_tuple

    def __hash__(self):
        return hash(self._cmp_tuple())

    def _cmp_tuple(self):
        cdef SpanC* span_c = self.span_c()
        return (
            span_c.start_char,
            span_c.end_char,
            span_c.start,
            span_c.end,
            span_c.label,
            span_c.kb_id,
            span_c.id,
            self.doc,
        )

    def __len__(self):
        """Get the number of tokens in the span.

        RETURNS (int): The number of tokens in the span.

        DOCS: https://spacy.io/api/span#len
        """
        cdef SpanC* span_c = self.span_c()
        if span_c.end < span_c.start:
            return 0
        return span_c.end - span_c.start

    def __repr__(self):
        return self.text

    def __getitem__(self, object i):
        """Get a `Token` or a `Span` object

        i (int or tuple): The index of the token within the span, or slice of
            the span to get.
        RETURNS (Token or Span): The token at `span[i]`.

        DOCS: https://spacy.io/api/span#getitem
        """
        cdef SpanC* span_c = self.span_c()
        if isinstance(i, slice):
            start, end = normalize_slice(len(self), i.start, i.stop, i.step)
            return Span(self.doc, start + self.start, end + self.start)
        else:
            if i < 0:
                token_i = span_c.end + i
            else:
                token_i = span_c.start + i
            if span_c.start <= token_i < span_c.end:
                return self.doc[token_i]
            else:
                raise IndexError(Errors.E1002)

    def __iter__(self):
        """Iterate over `Token` objects.

        YIELDS (Token): A `Token` object.

        DOCS: https://spacy.io/api/span#iter
        """
        cdef SpanC* span_c = self.span_c()
        for i in range(span_c.start, span_c.end):
            yield self.doc[i]

    def __reduce__(self):
        raise NotImplementedError(Errors.E112)

    @property
    def _(self):
        """Custom extension attributes registered via `set_extension`."""
        cdef SpanC* span_c = self.span_c()
        return Underscore(Underscore.span_extensions, self,
                          start=span_c.start_char, end=span_c.end_char, label=self.label, kb_id=self.kb_id, span_id=self.id)

    def as_doc(self, *, bint copy_user_data=False, array_head=None, array=None):
        """Create a `Doc` object with a copy of the `Span`'s data.
        copy_user_data (bool): Whether or not to copy the original doc's user data.
        array_head (tuple): `Doc` array attrs, can be passed in to speed up computation.
        array (ndarray): `Doc` as array, can be passed in to speed up computation.
        RETURNS (Doc): The `Doc` copy of the span.

        DOCS: https://spacy.io/api/span#as_doc
        """
        words = [t.text for t in self]
        spaces = [bool(t.whitespace_) for t in self]
        cdef Doc doc = Doc(self.doc.vocab, words=words, spaces=spaces)
        if array_head is None:
            array_head = self.doc._get_array_attrs()
        if array is None:
            array = self.doc.to_array(array_head)
        array = array[self.start : self.end]
        self._fix_dep_copy(array_head, array)
        # Fix initial IOB so the entities are valid for doc.ents below.
        if len(array) > 0 and ENT_IOB in array_head:
            ent_iob_col = array_head.index(ENT_IOB)
            if array[0][ent_iob_col] == 1:
                array[0][ent_iob_col] = 3
        doc.from_array(array_head, array)
        # Set partial entities at the beginning or end of the span to have
        # missing entity annotation. Note: the initial partial entity could be
        # detected from the IOB annotation but the final partial entity can't,
        # so detect and remove both in the same way by checking self.ents.
        span_ents = {(ent.start, ent.end) for ent in self.ents}
        doc_ents = doc.ents
        if len(doc_ents) > 0:
            # Remove initial partial ent
            if (doc_ents[0].start + self.start, doc_ents[0].end + self.start) not in span_ents:
                doc.set_ents([], missing=[doc_ents[0]], default="unmodified")
            # Remove final partial ent
            if (doc_ents[-1].start + self.start, doc_ents[-1].end + self.start) not in span_ents:
                doc.set_ents([], missing=[doc_ents[-1]], default="unmodified")
        doc.noun_chunks_iterator = self.doc.noun_chunks_iterator
        doc.user_hooks = self.doc.user_hooks
        doc.user_span_hooks = self.doc.user_span_hooks
        doc.user_token_hooks = self.doc.user_token_hooks
        doc.vector = self.vector
        doc.vector_norm = self.vector_norm
        doc.tensor = self.doc.tensor[self.start : self.end]
        for key, value in self.doc.cats.items():
            if hasattr(key, "__len__") and len(key) == 3:
                cat_start, cat_end, cat_label = key
                if cat_start == self.start_char and cat_end == self.end_char:
                    doc.cats[cat_label] = value
        if copy_user_data:
            user_data = {}
            char_offset = self.start_char
            for key, value in self.doc.user_data.items():
                if isinstance(key, tuple) and len(key) == 4 and key[0] == "._.":
                    data_type = key[0]
                    name = key[1]
                    start = key[2]
                    end = key[3]
                    if start is not None or end is not None:
                        start -= char_offset
                        # Check if Span object
                        if end is not None:
                            end -= char_offset
                            _label = key[4]
                            _kb_id = key[5]
                            _span_id = key[6]
                            user_data[(data_type, name, start, end, _label, _kb_id, _span_id)] = copy.copy(value)
                        # Else Token object
                        else:
                            user_data[(data_type, name, start, end)] = copy.copy(value)
                else:
                    user_data[key] = copy.copy(value)
            doc.user_data = user_data
        return doc

    def _fix_dep_copy(self, attrs, array):
        """ Rewire dependency links to make sure their heads fall into the span
        while still keeping the correct number of sentences. """
        cdef int length = len(array)
        cdef attr_t value
        cdef int i, head_col, ancestor_i
        cdef SpanC* span_c = self.span_c()
        old_to_new_root = dict()
        if HEAD in attrs:
            head_col = attrs.index(HEAD)
            for i in range(length):
                # if the HEAD refers to a token outside this span, find a more appropriate ancestor
                token = self[i]
                ancestor_i = token.head.i - span_c.start   # span offset
                if ancestor_i not in range(length):
                    if DEP in attrs:
                        array[i, attrs.index(DEP)] = dep

                    # try finding an ancestor within this span
                    ancestors = token.ancestors
                    for ancestor in ancestors:
                        ancestor_i = ancestor.i - span_c.start
                        if ancestor_i in range(length):
                            array[i, head_col] = numpy.int32(ancestor_i - i).astype(numpy.uint64)

                # if there is no appropriate ancestor, define a new artificial root
                value = array[i, head_col]
                if (i+value) not in range(length):
                    new_root = old_to_new_root.get(ancestor_i, None)
                    if new_root is not None:
                        # take the same artificial root as a previous token from the same sentence
                        array[i, head_col] = numpy.int32(new_root - i).astype(numpy.uint64)
                    else:
                        # set this token as the new artificial root
                        array[i, head_col] = 0
                        old_to_new_root[ancestor_i] = i

        return array

    def get_lca_matrix(self):
        """Calculates a matrix of Lowest Common Ancestors (LCA) for a given
        `Span`, where LCA[i, j] is the index of the lowest common ancestor among
        the tokens span[i] and span[j]. If they have no common ancestor within
        the span, LCA[i, j] will be -1.

        RETURNS (np.array[ndim=2, dtype=numpy.int32]): LCA matrix with shape
            (n, n), where n = len(self).

        DOCS: https://spacy.io/api/span#get_lca_matrix
        """
        cdef SpanC* span_c = self.span_c()
        return numpy.asarray(_get_lca_matrix(self.doc, span_c.start, span_c.end))

    def similarity(self, other):
        """Make a semantic similarity estimate. The default estimate is cosine
        similarity using an average of word vectors.

        other (object): The object to compare with. By default, accepts `Doc`,
            `Span`, `Token` and `Lexeme` objects.
        RETURNS (float): A scalar similarity score. Higher is more similar.

        DOCS: https://spacy.io/api/span#similarity
        """
        if "similarity" in self.doc.user_span_hooks:
            return self.doc.user_span_hooks["similarity"](self, other)
        attr = getattr(self.doc.vocab.vectors, "attr", ORTH)
        cdef Token this_token
        cdef Token other_token
        cdef Lexeme other_lex
        if len(self) == 1 and isinstance(other, Token):
            this_token = self[0]
            other_token = other
            if Token.get_struct_attr(this_token.c, attr) == Token.get_struct_attr(other_token.c, attr):
                return 1.0
        elif len(self) == 1 and isinstance(other, Lexeme):
            this_token = self[0]
            other_lex = other
            if Token.get_struct_attr(this_token.c, attr) == Lexeme.get_struct_attr(other_lex.c, attr):
                return 1.0
        elif isinstance(other, (Doc, Span)) and len(self) == len(other):
            similar = True
            for i in range(len(self)):
                this_token = self[i]
                other_token = other[i]
                if Token.get_struct_attr(this_token.c, attr) != Token.get_struct_attr(other_token.c, attr):
                    similar = False
                    break
            if similar:
                return 1.0
        if self.vocab.vectors.n_keys == 0:
            warnings.warn(Warnings.W007.format(obj="Span"))
        if self.vector_norm == 0.0 or other.vector_norm == 0.0:
            if not self.has_vector or not other.has_vector:
                warnings.warn(Warnings.W008.format(obj="Span"))
            return 0.0
        vector = self.vector
        xp = get_array_module(vector)
        result = xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm)
        # ensure we get a scalar back (numpy does this automatically but cupy doesn't)
        return result.item()

    cpdef np.ndarray to_array(self, object py_attr_ids):
        """Given a list of M attribute IDs, export the tokens to a numpy
        `ndarray` of shape `(N, M)`, where `N` is the length of the document.
        The values will be 32-bit integers.

        attr_ids (list[int]): A list of attribute ID ints.
        RETURNS (numpy.ndarray[long, ndim=2]): A feature matrix, with one row
            per word, and one column per attribute indicated in the input
            `attr_ids`.
        """
        cdef int i, j
        cdef attr_id_t feature
        cdef np.ndarray[attr_t, ndim=2] output
        # Make an array from the attributes - otherwise our inner loop is Python
        # dict iteration
        cdef np.ndarray[attr_t, ndim=1] attr_ids = numpy.asarray(py_attr_ids, dtype=numpy.uint64)
        cdef int length = self.end - self.start
        output = numpy.ndarray(shape=(length, len(attr_ids)), dtype=numpy.uint64)
        for i in range(self.start, self.end):
            for j, feature in enumerate(attr_ids):
                output[i-self.start, j] = get_token_attr(&self.doc.c[i], feature)
        return output

    @property
    def vocab(self):
        """RETURNS (Vocab): The Span's Doc's vocab."""
        return self.doc.vocab

    @property
    def sent(self):
        """Obtain the sentence that contains this span. If the given span
        crosses sentence boundaries, return only the first sentence
        to which it belongs.

        RETURNS (Span): The sentence span that the span is a part of.
        """
        if "sent" in self.doc.user_span_hooks:
            return self.doc.user_span_hooks["sent"](self)
        elif "sents" in self.doc.user_hooks:
            for sentence in self.doc.user_hooks["sents"](self.doc):
                if sentence.start <= self.start < sentence.end:
                    return sentence
        # Use `sent_start` token attribute to find sentence boundaries
        cdef int n = 0
        if self.doc.has_annotation("SENT_START"):
            # Find start of the sentence
            start = self.start
            while self.doc.c[start].sent_start != 1 and start > 0:
                start += -1
            # Find end of the sentence - can be within the entity
            end = self.start + 1
            while end < self.doc.length and self.doc.c[end].sent_start != 1:
                end += 1
                n += 1
                if n >= self.doc.length:
                    break
            return self.doc[start:end]
        else:
            raise ValueError(Errors.E030)

    cdef SpanC* span_c(self):
        return self.c.get()

    @property
    def sents(self):
        """Obtain the sentences that contain this span. If the given span
        crosses sentence boundaries, return all sentences it is a part of.

        RETURNS (Tuple[Span]): All sentences that the span is a part of.

        DOCS: https://spacy.io/api/span#sents
        """
        cdef int start
        cdef int i

        if "sents" in self.doc.user_span_hooks:
            return tuple(self.doc.user_span_hooks["sents"](self))
        spans = []
        if "sents" in self.doc.user_hooks:
            for sentence in self.doc.user_hooks["sents"](self.doc):
                if sentence.end > self.start:
                    if sentence.start < self.end or sentence.start == self.start == self.end:
                        spans.append(sentence)
                    else:
                        break
        else:
            if not self.doc.has_annotation("SENT_START"):
                raise ValueError(Errors.E030)
            # Use `sent_start` token attribute to find sentence boundaries
            # Find start of the 1st sentence of the Span
            start = self.start
            while self.doc.c[start].sent_start != 1 and start > 0:
                start -= 1

            # Now, find all the sentences in the span
            for i in range(start + 1, self.doc.length):
                if self.doc.c[i].sent_start == 1:
                    spans.append(Span(self.doc, start, i))
                    start = i
                    if start >= self.end:
                        break
                elif i == self.doc.length - 1:
                    spans.append(Span(self.doc, start, self.doc.length))

            # Ensure that trailing parts of the Span instance are included in last element of .sents.
            if start == self.doc.length - 1:
                spans.append(Span(self.doc, start, self.doc.length))
        return tuple(spans)

    @property
    def ents(self):
        """The named entities that fall completely within the span. Returns
        a tuple of `Span` objects.

        RETURNS (Tuple[Span]): Entities in the span, one `Span` per entity.

        DOCS: https://spacy.io/api/span#ents
        """
        cdef Span ent
        cdef SpanC* span_c = self.span_c()
        cdef SpanC* ent_span_c
        ents = []
        for ent in self.doc.ents:
            ent_span_c = ent.span_c()
            if ent_span_c.start >= span_c.start:
                if ent_span_c.end <= span_c.end:
                    ents.append(ent)
                else:
                    break
        return tuple(ents)

    @property
    def has_vector(self):
        """A boolean value indicating whether a word vector is associated with
        the object.

        RETURNS (bool): Whether a word vector is associated with the object.

        DOCS: https://spacy.io/api/span#has_vector
        """
        if "has_vector" in self.doc.user_span_hooks:
            return self.doc.user_span_hooks["has_vector"](self)
        elif self.vocab.vectors.size > 0:
            return any(token.has_vector for token in self)
        else:
            return False

    @property
    def vector(self):
        """A real-valued meaning representation. Defaults to an average of the
        token vectors.

        RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array
            representing the span's semantics.

        DOCS: https://spacy.io/api/span#vector
        """
        if "vector" in self.doc.user_span_hooks:
            return self.doc.user_span_hooks["vector"](self)
        if self._vector is None:
            if not len(self):
                xp = get_array_module(self.vocab.vectors.data)
                self._vector = xp.zeros((self.vocab.vectors_length,), dtype="f")
            else:
                self._vector = sum(t.vector for t in self) / len(self)
        return self._vector

    @property
    def vector_norm(self):
        """The L2 norm of the span's vector representation.

        RETURNS (float): The L2 norm of the vector representation.

        DOCS: https://spacy.io/api/span#vector_norm
        """
        if "vector_norm" in self.doc.user_span_hooks:
            return self.doc.user_span_hooks["vector"](self)
        if self._vector_norm is None:
            vector = self.vector
            total = (vector*vector).sum()
            xp = get_array_module(vector)
            self._vector_norm = xp.sqrt(total) if total != 0. else 0.
        return self._vector_norm

    @property
    def tensor(self):
        """The span's slice of the doc's tensor.

        RETURNS (ndarray[ndim=2, dtype='float32']): A 2D numpy or cupy array
            representing the span's semantics.
        """
        if self.doc.tensor is None:
            return None
        return self.doc.tensor[self.start : self.end]

    @property
    def text(self):
        """RETURNS (str): The original verbatim text of the span."""
        text = self.text_with_ws
        if len(self) > 0 and self[-1].whitespace_:
            text = text[:-1]
        return text

    @property
    def text_with_ws(self):
        """The text content of the span with a trailing whitespace character if
        the last token has one.

        RETURNS (str): The text content of the span (with trailing
            whitespace).
        """
        return "".join([t.text_with_ws for t in self])

    @property
    def noun_chunks(self):
        """Iterate over the base noun phrases in the span. Yields base
        noun-phrase #[code Span] objects, if the language has a noun chunk iterator.
        Raises a NotImplementedError otherwise.

        A base noun phrase, or "NP chunk", is a noun
        phrase that does not permit other NPs to be nested within it – so no
        NP-level coordination, no prepositional phrases, and no relative
        clauses.

        RETURNS (Tuple[Span]): Noun chunks in the span.

        DOCS: https://spacy.io/api/span#noun_chunks
        """
        spans = []
        for span in self.doc.noun_chunks:
            if span.start >= self.start and span.end <= self.end:
                spans.append(span)
        return tuple(spans)

    @property
    def root(self):
        """The token with the shortest path to the root of the
        sentence (or the root itself). If multiple tokens are equally
        high in the tree, the first token is taken.

        RETURNS (Token): The root token.

        DOCS: https://spacy.io/api/span#root
        """
        if "root" in self.doc.user_span_hooks:
            return self.doc.user_span_hooks["root"](self)
        # This should probably be called 'head', and the other one called
        # 'gov'. But we went with 'head' elsewhere, and now we're stuck =/
        cdef int i
        cdef SpanC* span_c = self.span_c()
        # First, we scan through the Span, and check whether there's a word
        # with head==0, i.e. a sentence root. If so, we can return it. The
        # longer the span, the more likely it contains a sentence root, and
        # in this case we return in linear time.
        for i in range(span_c.start, span_c.end):
            if self.doc.c[i].head == 0:
                return self.doc[i]
        # If we don't have a sentence root, we do something that's not so
        # algorithmically clever, but I think should be quite fast,
        # especially for short spans.
        # For each word, we count the path length, and arg min this measure.
        # We could use better tree logic to save steps here...But I
        # think this should be okay.
        cdef int current_best = self.doc.length
        cdef int root = -1
        for i in range(span_c.start, span_c.end):
            if span_c.start <= (i+self.doc.c[i].head) < span_c.end:
                continue
            words_to_root = _count_words_to_root(&self.doc.c[i], self.doc.length)
            if words_to_root < current_best:
                current_best = words_to_root
                root = i
        if root == -1:
            return self.doc[span_c.start]
        else:
            return self.doc[root]

    def char_span(self, int start_idx, int end_idx, label=0, *, kb_id=0, vector=None, alignment_mode="strict", span_id=0):
        """Create a `Span` object from the slice `span.text[start : end]`.

        start_idx (int): The index of the first character of the span.
        end_idx (int): The index of the first character after the span.
        label (Union[int, str]): A label to attach to the Span, e.g. for
            named entities.
        kb_id (Union[int, str]):  An ID from a KB to capture the meaning of a named entity.
        vector (ndarray[ndim=1, dtype='float32']): A meaning representation of
            the span.
        alignment_mode (str): How character indices are aligned to token
            boundaries. Options: "strict" (character indices must be aligned
            with token boundaries), "contract" (span of all tokens completely
            within the character span), "expand" (span of all tokens at least
            partially covered by the character span). Defaults to "strict".
        span_id (Union[int, str]): An identifier to associate with the span.
        RETURNS (Span): The newly constructed object.
        """
        cdef SpanC* span_c = self.span_c()
        start_idx += span_c.start_char
        end_idx += span_c.start_char
        return self.doc.char_span(start_idx, end_idx, label=label, kb_id=kb_id, vector=vector, alignment_mode=alignment_mode, span_id=span_id)

    @property
    def conjuncts(self):
        """Tokens that are conjoined to the span's root.

        RETURNS (tuple): A tuple of Token objects.

        DOCS: https://spacy.io/api/span#lefts
        """
        return self.root.conjuncts

    @property
    def lefts(self):
        """Tokens that are to the left of the span, whose head is within the
        `Span`.

        YIELDS (Token):A left-child of a token of the span.

        DOCS: https://spacy.io/api/span#lefts
        """
        for token in reversed(self):  # Reverse, so we get tokens in order
            for left in token.lefts:
                if left.i < self.start:
                    yield left

    @property
    def rights(self):
        """Tokens that are to the right of the Span, whose head is within the
        `Span`.

        YIELDS (Token): A right-child of a token of the span.

        DOCS: https://spacy.io/api/span#rights
        """
        for token in self:
            for right in token.rights:
                if right.i >= self.end:
                    yield right

    @property
    def n_lefts(self):
        """The number of tokens that are to the left of the span, whose
        heads are within the span.

        RETURNS (int): The number of leftward immediate children of the
            span, in the syntactic dependency parse.

        DOCS: https://spacy.io/api/span#n_lefts
        """
        return len(list(self.lefts))

    @property
    def n_rights(self):
        """The number of tokens that are to the right of the span, whose
        heads are within the span.

        RETURNS (int): The number of rightward immediate children of the
            span, in the syntactic dependency parse.

        DOCS: https://spacy.io/api/span#n_rights
        """
        return len(list(self.rights))

    @property
    def subtree(self):
        """Tokens within the span and tokens which descend from them.

        YIELDS (Token): A token within the span, or a descendant from it.

        DOCS: https://spacy.io/api/span#subtree
        """
        for word in self.lefts:
            yield from word.subtree
        yield from self
        for word in self.rights:
            yield from word.subtree

    property start:
        def __get__(self):
            return self.span_c().start

        def __set__(self, int start):
            if start < 0 or start > self.doc.length:
                raise IndexError(Errors.E1032.format(var="start", obj="Doc", length=self.doc.length, value=start))
            cdef SpanC* span_c = self.span_c()
            if start > span_c.end:
                raise ValueError(Errors.E4007.format(var="start", value=start, op="<=", existing_var="end", existing_value=span_c.end))
            span_c.start = start
            span_c.start_char = self.doc.c[start].idx

    property end:
        def __get__(self):
            return self.span_c().end

        def __set__(self, int end):
            if end < 0 or end > self.doc.length:
                raise IndexError(Errors.E1032.format(var="end", obj="Doc", length=self.doc.length, value=end))
            cdef SpanC* span_c = self.span_c()
            if span_c.start > end:
                raise ValueError(Errors.E4007.format(var="end", value=end, op=">=", existing_var="start", existing_value=span_c.start))
            span_c.end = end
            if end > 0:
                span_c.end_char = self.doc.c[end-1].idx + self.doc.c[end-1].lex.length
            else:
                span_c.end_char = 0

    property start_char:
        def __get__(self):
            return self.span_c().start_char

        def __set__(self, int start_char):
            if start_char < 0 or start_char > len(self.doc.text):
                raise IndexError(Errors.E1032.format(var="start_char", obj="Doc text", length=len(self.doc.text), value=start_char))
            cdef int start = token_by_start(self.doc.c, self.doc.length, start_char)
            if start < 0:
                raise ValueError(Errors.E4008.format(value=start_char, pos="start"))
            cdef SpanC* span_c = self.span_c()
            if start_char > span_c.end_char:
                raise ValueError(Errors.E4007.format(var="start_char", value=start_char, op="<=", existing_var="end_char", existing_value=span_c.end_char))
            span_c.start_char = start_char
            span_c.start = start

    property end_char:
        def __get__(self):
            return self.span_c().end_char

        def __set__(self, int end_char):
            if end_char < 0 or end_char > len(self.doc.text):
                raise IndexError(Errors.E1032.format(var="end_char", obj="Doc text", length=len(self.doc.text), value=end_char))
            cdef int end = token_by_end(self.doc.c, self.doc.length, end_char)
            if end < 0:
                raise ValueError(Errors.E4008.format(value=end_char, pos="end"))
            cdef SpanC* span_c = self.span_c()
            if span_c.start_char > end_char:
                raise ValueError(Errors.E4007.format(var="end_char", value=end_char, op=">=", existing_var="start_char", existing_value=span_c.start_char))
            span_c.end_char = end_char
            span_c.end = end

    property label:
        def __get__(self):
            return self.span_c().label

        def __set__(self, attr_t label):
            if label != self.span_c().label :
                old_label = self.span_c().label
                self.span_c().label = label
                new = Underscore(Underscore.span_extensions, self, start=self.span_c().start_char, end=self.span_c().end_char, label=self.label, kb_id=self.kb_id, span_id=self.id)
                old = Underscore(Underscore.span_extensions, self, start=self.span_c().start_char, end=self.span_c().end_char, label=old_label, kb_id=self.kb_id, span_id=self.id)
                Underscore._replace_keys(old, new)

    property kb_id:
        def __get__(self):
            return self.span_c().kb_id

        def __set__(self, attr_t kb_id):
            if kb_id != self.span_c().kb_id :
                old_kb_id = self.span_c().kb_id
                self.span_c().kb_id = kb_id
                new = Underscore(Underscore.span_extensions, self, start=self.span_c().start_char, end=self.span_c().end_char, label=self.label, kb_id=self.kb_id, span_id=self.id)
                old = Underscore(Underscore.span_extensions, self, start=self.span_c().start_char, end=self.span_c().end_char, label=self.label, kb_id=old_kb_id, span_id=self.id)
                Underscore._replace_keys(old, new)

    property id:
        def __get__(self):
            return self.span_c().id

        def __set__(self, attr_t id):
            if id != self.span_c().id :
                old_id = self.span_c().id
                self.span_c().id = id
                new = Underscore(Underscore.span_extensions, self, start=self.span_c().start_char, end=self.span_c().end_char, label=self.label, kb_id=self.kb_id, span_id=self.id)
                old = Underscore(Underscore.span_extensions, self, start=self.span_c().start_char, end=self.span_c().end_char, label=self.label, kb_id=self.kb_id, span_id=old_id)
                Underscore._replace_keys(old, new)

    property ent_id:
        """Alias for the span's ID."""
        def __get__(self):
            return self.id

        def __set__(self, attr_t ent_id):
            self.id = ent_id

    @property
    def orth_(self):
        """Verbatim text content (identical to `Span.text`). Exists mostly for
        consistency with other attributes.

        RETURNS (str): The span's text."""
        return self.text

    @property
    def lemma_(self):
        """RETURNS (str): The span's lemma."""
        return "".join([t.lemma_ + t.whitespace_ for t in self]).strip()

    property label_:
        """The span's label."""
        def __get__(self):
            return self.doc.vocab.strings[self.label]

        def __set__(self, str label_):
            self.label = self.doc.vocab.strings.add(label_)

    property kb_id_:
        """The span's KB ID."""
        def __get__(self):
            return self.doc.vocab.strings[self.kb_id]

        def __set__(self, str kb_id_):
            self.kb_id = self.doc.vocab.strings.add(kb_id_)

    property id_:
        """The span's ID."""
        def __get__(self):
            return self.doc.vocab.strings[self.id]

        def __set__(self, str id_):
            self.id = self.doc.vocab.strings.add(id_)

    property ent_id_:
        """Alias for the span's ID."""
        def __get__(self):
            return self.id_

        def __set__(self, str ent_id_):
            self.id_ = ent_id_


cdef int _count_words_to_root(const TokenC* token, int sent_length) except -1:
    # Don't allow spaces to be the root, if there are
    # better candidates
    if Lexeme.c_check_flag(token.lex, IS_SPACE) and token.l_kids == 0 and token.r_kids == 0:
        return sent_length-1
    if Lexeme.c_check_flag(token.lex, IS_PUNCT) and token.l_kids == 0 and token.r_kids == 0:
        return sent_length-1
    cdef int n = 0
    while token.head != 0:
        token += token.head
        n += 1
        if n >= sent_length:
            raise RuntimeError(Errors.E039)
    return n