From f212303729cb0775bb00eebb6eef0a6c646f92da Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 21 Sep 2020 17:59:09 +0200 Subject: [PATCH] Add sent_starts to Doc.__init__ Add sent_starts to `Doc.__init__`. Officially specify `is_sent_start` values but also convert to and accept `sent_start` internally. --- spacy/tests/doc/test_doc_api.py | 20 ++++++++++++++ spacy/tokens/doc.pyx | 46 +++++++++++++++++++++++---------- website/docs/api/doc.md | 1 + 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/spacy/tests/doc/test_doc_api.py b/spacy/tests/doc/test_doc_api.py index c979931b1..0579642c4 100644 --- a/spacy/tests/doc/test_doc_api.py +++ b/spacy/tests/doc/test_doc_api.py @@ -9,6 +9,26 @@ from spacy.attrs import ENT_TYPE, ENT_IOB, SENT_START, HEAD, DEP, MORPH from ..util import get_doc +def test_doc_api_init(en_vocab): + # set sent_start by sent_starts + doc = Doc( + en_vocab, words=["a", "b", "c", "d"], sent_starts=[True, False, True, False] + ) + assert [t.is_sent_start for t in doc] == [True, False, True, False] + + # set sent_start by heads + doc = Doc( + en_vocab, words=["a", "b", "c", "d"], heads=[0, 0, 2, 2], deps=["dep"] * 4 + ) + assert [t.is_sent_start for t in doc] == [True, False, True, False] + + # heads override sent_starts + doc = Doc( + en_vocab, words=["a", "b", "c", "d"], sent_starts=[True] * 4, heads=[0, 0, 2, 2], deps=["dep"] * 4 + ) + assert [t.is_sent_start for t in doc] == [True, False, True, False] + + @pytest.mark.parametrize("text", [["one", "two", "three"]]) def test_doc_api_compare_by_string_position(en_vocab, text): doc = Doc(en_vocab, words=text) diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index 27efa6cef..c5f1f6801 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -171,6 +171,7 @@ cdef class Doc: lemmas=None, heads=None, deps=None, + sent_starts=None, ents=None, ): """Create a Doc object. @@ -183,13 +184,24 @@ cdef class Doc: words. True means that the word is followed by a space, False means it is not. If `None`, defaults to `[True]*len(words)` user_data (dict or None): Optional extra data to attach to the Doc. - tags (Optional[List[str]]): A list of unicode strings, of the same length as words, to assign as token.tag. Defaults to None. - pos (Optional[List[str]]): A list of unicode strings, of the same length as words, to assign as token.pos. Defaults to None. - morphs (Optional[List[str]]): A list of unicode strings, of the same length as words, to assign as token.morph. Defaults to None. - lemmas (Optional[List[str]]): A list of unicode strings, of the same length as words, to assign as token.lemma. Defaults to None. - heads (Optional[List[int]]): A list of values, of the same length as words, to assign as heads. Head indices are the position of the head in the doc. Defaults to None. - deps (Optional[List[str]]): A list of unicode strings, of the same length as words, to assign as token.dep. Defaults to None. - ents (Optional[List[Span]]): A list of spans to assign as doc.ents. Defaults to None. + tags (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.tag. Defaults to None. + pos (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.pos. Defaults to None. + morphs (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.morph. Defaults to None. + lemmas (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.lemma. Defaults to None. + heads (Optional[List[int]]): A list of values, of the same length as + words, to assign as heads. Head indices are the position of the + head in the doc. Defaults to None. + deps (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.dep. Defaults to None. + sent_starts (Optional[List[Union[bool, None]]]): A list of values, of + the same length as words, to assign as token.is_sent_start. Will be + overridden by heads if heads is provided. Defaults to None. + ents (Optional[List[Span]]): A list of spans to assign as doc.ents. + Defaults to None. DOCS: https://nightly.spacy.io/api/doc#init """ @@ -242,16 +254,24 @@ cdef class Doc: heads = [head - i for i, head in enumerate(heads)] if deps and not heads: heads = [0] * len(deps) + if sent_starts is not None: + for i in range(len(sent_starts)): + if sent_starts[i] is True: + sent_starts[i] = 1 + elif sent_starts[i] is False: + sent_starts[i] = -1 + elif sent_starts[i] is None or sent_starts[i] not in [-1, 0, 1]: + sent_starts[i] = 0 headings = [] values = [] - annotations = [pos, heads, deps, lemmas, tags, morphs] - possible_headings = [POS, HEAD, DEP, LEMMA, TAG, MORPH] + annotations = [pos, heads, deps, lemmas, tags, morphs, sent_starts] + possible_headings = [POS, HEAD, DEP, LEMMA, TAG, MORPH, SENT_START] for a, annot in enumerate(annotations): if annot is not None: if len(annot) != len(words): raise ValueError(Errors.E189) headings.append(possible_headings[a]) - if annot is not heads: + if annot is not heads and annot is not sent_starts: values.extend(annot) for value in values: self.vocab.strings.add(value) @@ -263,12 +283,12 @@ cdef class Doc: j = 0 for annot in annotations: if annot: - if annot is heads: + if annot is heads or annot is sent_starts: for i in range(len(words)): if attrs.ndim == 1: - attrs[i] = heads[i] + attrs[i] = annot[i] else: - attrs[i, j] = heads[i] + attrs[i, j] = annot[i] elif annot is morphs: for i in range(len(words)): morph_key = vocab.morphology.add(morphs[i]) diff --git a/website/docs/api/doc.md b/website/docs/api/doc.md index baf264b80..52f94a83d 100644 --- a/website/docs/api/doc.md +++ b/website/docs/api/doc.md @@ -43,6 +43,7 @@ Construct a `Doc` object. The most common way to get a `Doc` object is via the | lemmas | A list of strings, of the same length as words, to assign as `token.lemma` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | | heads | A list of values, of the same length as words, to assign as the head for each word. Head indices are the absolute position of the head in the doc. Defaults to `None`. ~~Optional[List[int]]~~ | | deps | A list of strings, of the same length as words, to assign as `token.dep` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| sent_starts | A list of values, of the same length as words, to assign as token.is_sent_start. Will be overridden by heads if heads is provided. Defaults to `None`. ~~Optional[List[Union[bool, None]]~~ | | ents | A list of spans to assign as doc.ents. Defaults to `None`. ~~Optional[List[Span]]~~ | ## Doc.\_\_getitem\_\_ {#getitem tag="method"}