From 75f7c15187ecad9be9d018810beaa546508944af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Wed, 12 Jan 2022 13:38:52 +0100 Subject: [PATCH 01/24] Span/SpanGroup: wrap SpanC in shared_ptr (#9869) * Span/SpanGroup: wrap SpanC in shared_ptr When a Span that was retrieved from a SpanGroup was modified, these changes were not reflected in the SpanGroup because the underlying SpanC struct was copied. This change applies the solution proposed by @nrodnova, to wrap SpanC in a shared_ptr. This makes a SpanGroup and Spans derived from it share the same SpanC. So, changes made through a Span are visible in the SpanGroup as well. Fixes #9556 * Test that a SpanGroup is modified through its Spans * SpanGroup.push_back: remove nogil Modifying std::vector is not thread-safe. * C++ >= 11 does not allow const T in vector * Add Span.span_c as a shorthand for Span.c.get Since this method is cdef'ed, it is only visible from Cython, so we avoid using raw pointers in Python Replace existing uses of span.c.get() to use this new method. * Fix formatting * Style fix: pointer types * SpanGroup.to_bytes: reduce number of shared_ptr::get calls * Mark SpanGroup modification test with issue Co-authored-by: Sofie Van Landeghem Co-authored-by: Sofie Van Landeghem --- spacy/pipeline/_parser_internals/ner.pyx | 28 +++--- spacy/tests/doc/test_span.py | 14 ++- spacy/tokens/span.pxd | 11 ++- spacy/tokens/span.pyx | 110 +++++++++++++---------- spacy/tokens/span_group.pxd | 5 +- spacy/tokens/span_group.pyx | 21 +++-- 6 files changed, 115 insertions(+), 74 deletions(-) diff --git a/spacy/pipeline/_parser_internals/ner.pyx b/spacy/pipeline/_parser_internals/ner.pyx index 3edeff19a..87410de0f 100644 --- a/spacy/pipeline/_parser_internals/ner.pyx +++ b/spacy/pipeline/_parser_internals/ner.pyx @@ -1,6 +1,8 @@ import os import random from libc.stdint cimport int32_t +from libcpp.memory cimport shared_ptr +from libcpp.vector cimport vector from cymem.cymem cimport Pool from collections import Counter @@ -42,9 +44,7 @@ MOVE_NAMES[OUT] = 'O' cdef struct GoldNERStateC: Transition* ner - SpanC* negs - int32_t length - int32_t nr_neg + vector[shared_ptr[SpanC]] negs cdef class BiluoGold: @@ -77,8 +77,6 @@ cdef GoldNERStateC create_gold_state( negs = [] assert example.x.length > 0 gs.ner = mem.alloc(example.x.length, sizeof(Transition)) - gs.negs = mem.alloc(len(negs), sizeof(SpanC)) - gs.nr_neg = len(negs) ner_ents, ner_tags = example.get_aligned_ents_and_ner() for i, ner_tag in enumerate(ner_tags): gs.ner[i] = moves.lookup_transition(ner_tag) @@ -92,8 +90,8 @@ cdef GoldNERStateC create_gold_state( # In order to handle negative samples, we need to maintain the full # (start, end, label) triple. If we break it down to the 'isnt B-LOC' # thing, we'll get blocked if there's an incorrect prefix. - for i, neg in enumerate(negs): - gs.negs[i] = neg.c + for neg in negs: + gs.negs.push_back(neg.c) return gs @@ -410,6 +408,8 @@ cdef class Begin: cdef int g_act = gold.ner[b0].move cdef attr_t g_tag = gold.ner[b0].label + cdef shared_ptr[SpanC] span + if g_act == MISSING: pass elif g_act == BEGIN: @@ -427,8 +427,8 @@ cdef class Begin: # be correct or not. However, we can at least tell whether we're # going to be opening an entity where there's only one possible # L. - for span in gold.negs[:gold.nr_neg]: - if span.label == label and span.start == b0: + for span in gold.negs: + if span.get().label == label and span.get().start == b0: cost += 1 break return cost @@ -573,8 +573,9 @@ cdef class Last: # If we have negative-example entities, integrate them into the objective, # by marking actions that close an entity that we know is incorrect # as costly. - for span in gold.negs[:gold.nr_neg]: - if span.label == label and (span.end-1) == b0 and span.start == ent_start: + cdef shared_ptr[SpanC] span + for span in gold.negs: + if span.get().label == label and (span.get().end-1) == b0 and span.get().start == ent_start: cost += 1 break return cost @@ -638,8 +639,9 @@ cdef class Unit: # This is fairly straight-forward for U- entities, as we have a single # action cdef int b0 = s.B(0) - for span in gold.negs[:gold.nr_neg]: - if span.label == label and span.start == b0 and span.end == (b0+1): + cdef shared_ptr[SpanC] span + for span in gold.negs: + if span.get().label == label and span.get().start == b0 and span.get().end == (b0+1): cost += 1 break return cost diff --git a/spacy/tests/doc/test_span.py b/spacy/tests/doc/test_span.py index 10aba5b94..0e7730b65 100644 --- a/spacy/tests/doc/test_span.py +++ b/spacy/tests/doc/test_span.py @@ -4,7 +4,7 @@ from numpy.testing import assert_array_equal from spacy.attrs import ORTH, LENGTH from spacy.lang.en import English -from spacy.tokens import Doc, Span, Token +from spacy.tokens import Doc, Span, SpanGroup, Token from spacy.vocab import Vocab from spacy.util import filter_spans from thinc.api import get_current_ops @@ -163,6 +163,18 @@ def test_char_span(doc, i_sent, i, j, text): assert span.text == text +@pytest.mark.issue(9556) +def test_modify_span_group(doc): + group = SpanGroup(doc, spans=doc.ents) + for span in group: + span.start = 0 + span.label = doc.vocab.strings["TEST"] + + # Span changes must be reflected in the span group + assert group[0].start == 0 + assert group[0].label == doc.vocab.strings["TEST"] + + def test_spans_sent_spans(doc): sents = list(doc.sents) assert sents[0].start == 0 diff --git a/spacy/tokens/span.pxd b/spacy/tokens/span.pxd index 78bee0a8c..85553068e 100644 --- a/spacy/tokens/span.pxd +++ b/spacy/tokens/span.pxd @@ -1,3 +1,4 @@ +from libcpp.memory cimport shared_ptr cimport numpy as np from .doc cimport Doc @@ -7,19 +8,21 @@ from ..structs cimport SpanC cdef class Span: cdef readonly Doc doc - cdef SpanC c + cdef shared_ptr[SpanC] c cdef public _vector cdef public _vector_norm @staticmethod - cdef inline Span cinit(Doc doc, SpanC span): + cdef inline Span cinit(Doc doc, const shared_ptr[SpanC] &span): cdef Span self = Span.__new__( Span, doc, - start=span.start, - end=span.end + start=span.get().start, + end=span.get().end ) self.c = span return self cpdef np.ndarray to_array(self, object features) + + cdef SpanC* span_c(self) diff --git a/spacy/tokens/span.pyx b/spacy/tokens/span.pyx index cd02cab36..c8fb0d1a2 100644 --- a/spacy/tokens/span.pyx +++ b/spacy/tokens/span.pyx @@ -1,5 +1,6 @@ cimport numpy as np from libc.math cimport sqrt +from libcpp.memory cimport make_shared import numpy from thinc.api import get_array_module @@ -109,14 +110,14 @@ cdef class Span: end_char = start_char else: end_char = doc[end - 1].idx + len(doc[end - 1]) - self.c = SpanC( + self.c = make_shared[SpanC](SpanC( label=label, kb_id=kb_id, start=start, end=end, start_char=start_char, end_char=end_char, - ) + )) self._vector = vector self._vector_norm = vector_norm @@ -126,41 +127,46 @@ cdef class Span: return False else: return True + + cdef SpanC* span_c = self.span_c() + cdef SpanC* other_span_c = other.span_c() + # < if op == 0: - return self.c.start_char < other.c.start_char + return span_c.start_char < other_span_c.start_char # <= elif op == 1: - return self.c.start_char <= other.c.start_char + return span_c.start_char <= other_span_c.start_char # == elif op == 2: # Do the cheap comparisons first return ( - (self.c.start_char == other.c.start_char) and \ - (self.c.end_char == other.c.end_char) and \ - (self.c.label == other.c.label) and \ - (self.c.kb_id == other.c.kb_id) and \ + (span_c.start_char == other_span_c.start_char) and \ + (span_c.end_char == other_span_c.end_char) and \ + (span_c.label == other_span_c.label) and \ + (span_c.kb_id == other_span_c.kb_id) and \ (self.doc == other.doc) ) # != elif op == 3: # Do the cheap comparisons first return not ( - (self.c.start_char == other.c.start_char) and \ - (self.c.end_char == other.c.end_char) and \ - (self.c.label == other.c.label) and \ - (self.c.kb_id == other.c.kb_id) and \ + (span_c.start_char == other_span_c.start_char) and \ + (span_c.end_char == other_span_c.end_char) and \ + (span_c.label == other_span_c.label) and \ + (span_c.kb_id == other_span_c.kb_id) and \ (self.doc == other.doc) ) # > elif op == 4: - return self.c.start_char > other.c.start_char + return span_c.start_char > other_span_c.start_char # >= elif op == 5: - return self.c.start_char >= other.c.start_char + return span_c.start_char >= other_span_c.start_char def __hash__(self): - return hash((self.doc, self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id)) + cdef SpanC* span_c = self.span_c() + return hash((self.doc, span_c.start_char, span_c.end_char, span_c.label, span_c.kb_id)) def __len__(self): """Get the number of tokens in the span. @@ -169,9 +175,10 @@ cdef class Span: DOCS: https://spacy.io/api/span#len """ - if self.c.end < self.c.start: + cdef SpanC* span_c = self.span_c() + if span_c.end < span_c.start: return 0 - return self.c.end - self.c.start + return span_c.end - span_c.start def __repr__(self): return self.text @@ -185,15 +192,16 @@ cdef class Span: DOCS: https://spacy.io/api/span#getitem """ + cdef SpanC* span_c = self.span_c() if isinstance(i, slice): start, end = normalize_slice(len(self), i.start, i.stop, i.step) return Span(self.doc, start + self.start, end + self.start) else: if i < 0: - token_i = self.c.end + i + token_i = span_c.end + i else: - token_i = self.c.start + i - if self.c.start <= token_i < self.c.end: + token_i = span_c.start + i + if span_c.start <= token_i < span_c.end: return self.doc[token_i] else: raise IndexError(Errors.E1002) @@ -205,7 +213,8 @@ cdef class Span: DOCS: https://spacy.io/api/span#iter """ - for i in range(self.c.start, self.c.end): + cdef SpanC* span_c = self.span_c() + for i in range(span_c.start, span_c.end): yield self.doc[i] def __reduce__(self): @@ -213,9 +222,10 @@ cdef class Span: @property def _(self): + cdef SpanC* span_c = self.span_c() """Custom extension attributes registered via `set_extension`.""" return Underscore(Underscore.span_extensions, self, - start=self.c.start_char, end=self.c.end_char) + start=span_c.start_char, end=span_c.end_char) def as_doc(self, *, bint copy_user_data=False, array_head=None, array=None): """Create a `Doc` object with a copy of the `Span`'s data. @@ -289,13 +299,14 @@ cdef class Span: cdef int length = len(array) cdef attr_t value cdef int i, head_col, ancestor_i + cdef SpanC* span_c = self.span_c() old_to_new_root = dict() if HEAD in attrs: head_col = attrs.index(HEAD) for i in range(length): # if the HEAD refers to a token outside this span, find a more appropriate ancestor token = self[i] - ancestor_i = token.head.i - self.c.start # span offset + ancestor_i = token.head.i - span_c.start # span offset if ancestor_i not in range(length): if DEP in attrs: array[i, attrs.index(DEP)] = dep @@ -303,7 +314,7 @@ cdef class Span: # try finding an ancestor within this span ancestors = token.ancestors for ancestor in ancestors: - ancestor_i = ancestor.i - self.c.start + ancestor_i = ancestor.i - span_c.start if ancestor_i in range(length): array[i, head_col] = ancestor_i - i @@ -332,7 +343,8 @@ cdef class Span: DOCS: https://spacy.io/api/span#get_lca_matrix """ - return numpy.asarray(_get_lca_matrix(self.doc, self.c.start, self.c.end)) + cdef SpanC* span_c = self.span_c() + return numpy.asarray(_get_lca_matrix(self.doc, span_c.start, span_c.end)) def similarity(self, other): """Make a semantic similarity estimate. The default estimate is cosine @@ -426,6 +438,9 @@ cdef class Span: else: raise ValueError(Errors.E030) + cdef SpanC* span_c(self): + return self.c.get() + @property def sents(self): """Obtain the sentences that contain this span. If the given span @@ -477,10 +492,13 @@ cdef class Span: DOCS: https://spacy.io/api/span#ents """ cdef Span ent + cdef SpanC* span_c = self.span_c() + cdef SpanC* ent_span_c ents = [] for ent in self.doc.ents: - if ent.c.start >= self.c.start: - if ent.c.end <= self.c.end: + ent_span_c = ent.span_c() + if ent_span_c.start >= span_c.start: + if ent_span_c.end <= span_c.end: ents.append(ent) else: break @@ -615,11 +633,12 @@ cdef class Span: # This should probably be called 'head', and the other one called # 'gov'. But we went with 'head' elsewhere, and now we're stuck =/ cdef int i + cdef SpanC* span_c = self.span_c() # First, we scan through the Span, and check whether there's a word # with head==0, i.e. a sentence root. If so, we can return it. The # longer the span, the more likely it contains a sentence root, and # in this case we return in linear time. - for i in range(self.c.start, self.c.end): + for i in range(span_c.start, span_c.end): if self.doc.c[i].head == 0: return self.doc[i] # If we don't have a sentence root, we do something that's not so @@ -630,15 +649,15 @@ cdef class Span: # think this should be okay. cdef int current_best = self.doc.length cdef int root = -1 - for i in range(self.c.start, self.c.end): - if self.c.start <= (i+self.doc.c[i].head) < self.c.end: + for i in range(span_c.start, span_c.end): + if span_c.start <= (i+self.doc.c[i].head) < span_c.end: continue words_to_root = _count_words_to_root(&self.doc.c[i], self.doc.length) if words_to_root < current_best: current_best = words_to_root root = i if root == -1: - return self.doc[self.c.start] + return self.doc[span_c.start] else: return self.doc[root] @@ -654,8 +673,9 @@ cdef class Span: the span. RETURNS (Span): The newly constructed object. """ - start_idx += self.c.start_char - end_idx += self.c.start_char + cdef SpanC* span_c = self.span_c() + start_idx += span_c.start_char + end_idx += span_c.start_char return self.doc.char_span(start_idx, end_idx, label=label, kb_id=kb_id, vector=vector) @property @@ -736,53 +756,53 @@ cdef class Span: property start: def __get__(self): - return self.c.start + return self.span_c().start def __set__(self, int start): if start < 0: raise IndexError("TODO") - self.c.start = start + self.span_c().start = start property end: def __get__(self): - return self.c.end + return self.span_c().end def __set__(self, int end): if end < 0: raise IndexError("TODO") - self.c.end = end + self.span_c().end = end property start_char: def __get__(self): - return self.c.start_char + return self.span_c().start_char def __set__(self, int start_char): if start_char < 0: raise IndexError("TODO") - self.c.start_char = start_char + self.span_c().start_char = start_char property end_char: def __get__(self): - return self.c.end_char + return self.span_c().end_char def __set__(self, int end_char): if end_char < 0: raise IndexError("TODO") - self.c.end_char = end_char + self.span_c().end_char = end_char property label: def __get__(self): - return self.c.label + return self.span_c().label def __set__(self, attr_t label): - self.c.label = label + self.span_c().label = label property kb_id: def __get__(self): - return self.c.kb_id + return self.span_c().kb_id def __set__(self, attr_t kb_id): - self.c.kb_id = kb_id + self.span_c().kb_id = kb_id property ent_id: """RETURNS (uint64): The entity ID.""" diff --git a/spacy/tokens/span_group.pxd b/spacy/tokens/span_group.pxd index 5074aa275..6b817578a 100644 --- a/spacy/tokens/span_group.pxd +++ b/spacy/tokens/span_group.pxd @@ -1,3 +1,4 @@ +from libcpp.memory cimport shared_ptr from libcpp.vector cimport vector from ..structs cimport SpanC @@ -5,6 +6,6 @@ cdef class SpanGroup: cdef public object _doc_ref cdef public str name cdef public dict attrs - cdef vector[SpanC] c + cdef vector[shared_ptr[SpanC]] c - cdef void push_back(self, SpanC span) nogil + cdef void push_back(self, const shared_ptr[SpanC] &span) diff --git a/spacy/tokens/span_group.pyx b/spacy/tokens/span_group.pyx index 6cfa75237..c310da785 100644 --- a/spacy/tokens/span_group.pyx +++ b/spacy/tokens/span_group.pyx @@ -5,6 +5,7 @@ import srsly from spacy.errors import Errors from .span cimport Span from libc.stdint cimport uint64_t, uint32_t, int32_t +from libcpp.memory cimport make_shared cdef class SpanGroup: @@ -135,9 +136,11 @@ cdef class SpanGroup: DOCS: https://spacy.io/api/spangroup#to_bytes """ + cdef SpanC* span_c output = {"name": self.name, "attrs": self.attrs, "spans": []} for i in range(self.c.size()): span = self.c[i] + span_c = span.get() # The struct.pack here is probably overkill, but it might help if # you're saving tonnes of spans, and it doesn't really add any # complexity. We do take care to specify little-endian byte order @@ -149,13 +152,13 @@ cdef class SpanGroup: # l: int32_t output["spans"].append(struct.pack( ">QQQllll", - span.id, - span.kb_id, - span.label, - span.start, - span.end, - span.start_char, - span.end_char + span_c.id, + span_c.kb_id, + span_c.label, + span_c.start, + span_c.end, + span_c.start_char, + span_c.end_char )) return srsly.msgpack_dumps(output) @@ -182,8 +185,8 @@ cdef class SpanGroup: span.end = items[4] span.start_char = items[5] span.end_char = items[6] - self.c.push_back(span) + self.c.push_back(make_shared[SpanC](span)) return self - cdef void push_back(self, SpanC span) nogil: + cdef void push_back(self, const shared_ptr[SpanC] &span): self.c.push_back(span) From 0e71bd973fa39e9b39f2ff4e023a42cde2064ff7 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 15 Apr 2022 15:34:58 +0200 Subject: [PATCH 02/24] Return doc offsets in Matcher on spans (#10576) The returned match offsets were only adjusted for `as_spans`, not generally. Because the `on_match` callbacks are always applied to the doc, the `Matcher` matches on spans should consistently use the doc offsets. --- spacy/matcher/matcher.pyx | 7 ++++--- spacy/tests/matcher/test_matcher_api.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/spacy/matcher/matcher.pyx b/spacy/matcher/matcher.pyx index 745d7cf43..863bd198c 100644 --- a/spacy/matcher/matcher.pyx +++ b/spacy/matcher/matcher.pyx @@ -252,6 +252,10 @@ cdef class Matcher: # non-overlapping ones this `match` can be either (start, end) or # (start, end, alignments) depending on `with_alignments=` option. for key, *match in matches: + # Adjust span matches to doc offsets + if isinstance(doclike, Span): + match[0] += doclike.start + match[1] += doclike.start span_filter = self._filter.get(key) if span_filter is not None: pairs = pairs_by_id.get(key, []) @@ -282,9 +286,6 @@ cdef class Matcher: if as_spans: final_results = [] for key, start, end, *_ in final_matches: - if isinstance(doclike, Span): - start += doclike.start - end += doclike.start final_results.append(Span(doc, start, end, label=key)) elif with_alignments: # convert alignments List[Dict[str, int]] --> List[int] diff --git a/spacy/tests/matcher/test_matcher_api.py b/spacy/tests/matcher/test_matcher_api.py index c02d65cdf..9d401382f 100644 --- a/spacy/tests/matcher/test_matcher_api.py +++ b/spacy/tests/matcher/test_matcher_api.py @@ -591,9 +591,16 @@ def test_matcher_span(matcher): doc = Doc(matcher.vocab, words=text.split()) span_js = doc[:3] span_java = doc[4:] - assert len(matcher(doc)) == 2 - assert len(matcher(span_js)) == 1 - assert len(matcher(span_java)) == 1 + doc_matches = matcher(doc) + span_js_matches = matcher(span_js) + span_java_matches = matcher(span_java) + assert len(doc_matches) == 2 + assert len(span_js_matches) == 1 + assert len(span_java_matches) == 1 + + # match offsets always refer to the doc + assert doc_matches[0] == span_js_matches[0] + assert doc_matches[1] == span_java_matches[0] def test_matcher_as_spans(matcher): From 7c6a97559d070f161131853dc03e600c6a0ca82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danie=CC=88l=20de=20Kok?= Date: Wed, 25 May 2022 14:06:45 +0200 Subject: [PATCH 03/24] Simplify GPU check This change removes `thinc.util.has_cupy` from the GPU presence check. Currently `gpu_is_available` already implies `has_cupy`. We also want to show this warning in the future when a machine has a non-CuPy GPU. --- spacy/cli/_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index df98e711f..bb7f2d352 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -12,7 +12,7 @@ from click.parser import split_arg_string from typer.main import get_command from contextlib import contextmanager from thinc.api import Config, ConfigValidationError, require_gpu -from thinc.util import has_cupy, gpu_is_available +from thinc.util import gpu_is_available from configparser import InterpolationError import os @@ -554,5 +554,5 @@ def setup_gpu(use_gpu: int, silent=None) -> None: require_gpu(use_gpu) else: local_msg.info("Using CPU") - if has_cupy and gpu_is_available(): + if gpu_is_available(): local_msg.info("To switch to GPU 0, use the option: --gpu-id 0") From 322c5a3ac4da412582e684e8ec2355b68efe7b27 Mon Sep 17 00:00:00 2001 From: Freddy Heppell Date: Sun, 29 May 2022 10:49:19 +0100 Subject: [PATCH 04/24] Fix misspelt keyword in StringStore example --- website/docs/api/stringstore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/api/stringstore.md b/website/docs/api/stringstore.md index d5f78dbab..cd414b1f0 100644 --- a/website/docs/api/stringstore.md +++ b/website/docs/api/stringstore.md @@ -161,7 +161,7 @@ Load state from a binary string. > #### Example > > ```python -> fron spacy.strings import StringStore +> from spacy.strings import StringStore > store_bytes = stringstore.to_bytes() > new_store = StringStore().from_bytes(store_bytes) > ``` From bf95f0a1dd41863b81adc3eda302b0389ad3f9e4 Mon Sep 17 00:00:00 2001 From: Peter Baumgartner <5107405+pmbaumgartner@users.noreply.github.com> Date: Mon, 30 May 2022 02:51:19 -0400 Subject: [PATCH 05/24] add doc cleaner to menu (#10862) --- website/docs/api/pipeline-functions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/website/docs/api/pipeline-functions.md b/website/docs/api/pipeline-functions.md index ff19d3e71..1b7017ca7 100644 --- a/website/docs/api/pipeline-functions.md +++ b/website/docs/api/pipeline-functions.md @@ -7,6 +7,7 @@ menu: - ['merge_entities', 'merge_entities'] - ['merge_subtokens', 'merge_subtokens'] - ['token_splitter', 'token_splitter'] + - ['doc_cleaner', 'doc_cleaner'] --- ## merge_noun_chunks {#merge_noun_chunks tag="function"} From 709d6d91148350a72b8ae2d8e0f077df61aa2c98 Mon Sep 17 00:00:00 2001 From: Max Tarlov <51882792+maxTarlov@users.noreply.github.com> Date: Mon, 30 May 2022 00:11:55 -0700 Subject: [PATCH 06/24] Update documentation for displacy style kwargs (#10841) * Update docs for displacy style kwargs Added "span" to the accepted values for the style kwarg in the displacy.serve and displacy.render top-level functions. These styles are new as of SpaCy 3.3, so I added the "new" tag for that option only * restored alpha ordering --- website/docs/api/top-level.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index 904a91ea9..889c6437c 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -239,7 +239,7 @@ browser. Will run a simple web server. | Name | Description | | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ | -| `style` | Visualization style, `"dep"` or `"ent"`. Defaults to `"dep"`. ~~str~~ | +| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | | `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | | `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | | `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | @@ -264,7 +264,7 @@ Render a dependency parse tree or named entity visualization. | Name | Description | | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span, dict]], Doc, Span, dict]~~ | -| `style` | Visualization style, `"dep"` or `"ent"`. Defaults to `"dep"`. ~~str~~ | +| `style` | Visualization style,`"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | | `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | | `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | | `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | From d4218366c53602967d5d345b345f4dd3e4413e71 Mon Sep 17 00:00:00 2001 From: richardpaulhudson Date: Mon, 30 May 2022 18:05:26 +0200 Subject: [PATCH 07/24] Update Holmes entry in universe.json --- website/meta/universe.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index e91e9ef44..2840c37d6 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -2799,13 +2799,13 @@ "id": "holmes", "title": "Holmes", "slogan": "Information extraction from English and German texts based on predicate logic", - "github": "msg-systems/holmes-extractor", - "url": "https://github.com/msg-systems/holmes-extractor", - "description": "Holmes is a Python 3 library that supports a number of use cases involving information extraction from English and German texts, including chatbot, structural extraction, topic matching and supervised document classification. There is a [website demonstrating intelligent search based on topic matching](https://holmes-demo.xt.msg.team).", + "github": "explosion/holmes-extractor", + "url": "https://github.com/explosion/holmes-extractor", + "description": "Holmes is a Python 3 library that supports a number of use cases involving information extraction from English and German texts, including chatbot, structural extraction, topic matching and supervised document classification. There is a [website demonstrating intelligent search based on topic matching](https://demo.holmes.prod.demos.explosion.services).", "pip": "holmes-extractor", - "category": ["conversational", "standalone"], + "category": ["pipeline", "standalone"], "tags": ["chatbots", "text-processing"], - "thumb": "https://raw.githubusercontent.com/msg-systems/holmes-extractor/master/docs/holmes_thumbnail.png", + "thumb": "https://raw.githubusercontent.com/explosion/holmes-extractor/master/docs/holmes_thumbnail.png", "code_example": [ "import holmes_extractor as holmes", "holmes_manager = holmes.Manager(model='en_core_web_lg')", From dca2e8c6442d67a398f4d432388f188986d25ca5 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 1 Jun 2022 07:41:28 +0900 Subject: [PATCH 08/24] Minor NEL type fixes (#10860) * Fix TODO about typing Fix was simple: just request an array2f. * Add type ignore Maxout has a more restrictive type than the residual layer expects (only Floats2d vs any Floats). * Various cleanup This moves a lot of lines around but doesn't change any functionality. Details: 1. use `continue` to reduce indentation 2. move sentence doc building inside conditional since it's otherwise unused 3. reduces some temporary assignments --- spacy/ml/models/entity_linker.py | 2 +- spacy/pipeline/entity_linker.py | 110 +++++++++++++++---------------- 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/spacy/ml/models/entity_linker.py b/spacy/ml/models/entity_linker.py index fba4b485f..287feecce 100644 --- a/spacy/ml/models/entity_linker.py +++ b/spacy/ml/models/entity_linker.py @@ -23,7 +23,7 @@ def build_nel_encoder( ((tok2vec >> list2ragged()) & build_span_maker()) >> extract_spans() >> reduce_mean() - >> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) + >> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore >> output_layer ) model.set_ref("output_layer", output_layer) diff --git a/spacy/pipeline/entity_linker.py b/spacy/pipeline/entity_linker.py index 12c3e382f..aa7985a9c 100644 --- a/spacy/pipeline/entity_linker.py +++ b/spacy/pipeline/entity_linker.py @@ -355,7 +355,7 @@ class EntityLinker(TrainablePipe): keep_ents.append(eidx) eidx += 1 - entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32") + entity_encodings = self.model.ops.asarray2f(entity_encodings, dtype="float32") selected_encodings = sentence_encodings[keep_ents] # if there are no matches, short circuit @@ -368,13 +368,12 @@ class EntityLinker(TrainablePipe): method="get_loss", msg="gold entities do not match up" ) raise RuntimeError(err) - # TODO: fix typing issue here - gradients = self.distance.get_grad(selected_encodings, entity_encodings) # type: ignore + gradients = self.distance.get_grad(selected_encodings, entity_encodings) # to match the input size, we need to give a zero gradient for items not in the kb out = self.model.ops.alloc2f(*sentence_encodings.shape) out[keep_ents] = gradients - loss = self.distance.get_loss(selected_encodings, entity_encodings) # type: ignore + loss = self.distance.get_loss(selected_encodings, entity_encodings) loss = loss / len(entity_encodings) return float(loss), out @@ -391,18 +390,21 @@ class EntityLinker(TrainablePipe): self.validate_kb() entity_count = 0 final_kb_ids: List[str] = [] + xp = self.model.ops.xp if not docs: return final_kb_ids if isinstance(docs, Doc): docs = [docs] for i, doc in enumerate(docs): + if len(doc) == 0: + continue sentences = [s for s in doc.sents] - if len(doc) > 0: - # Looping through each entity (TODO: rewrite) - for ent in doc.ents: - sent = ent.sent - sent_index = sentences.index(sent) - assert sent_index >= 0 + # Looping through each entity (TODO: rewrite) + for ent in doc.ents: + sent_index = sentences.index(ent.sent) + assert sent_index >= 0 + + if self.incl_context: # get n_neighbour sentences, clipped to the length of the document start_sentence = max(0, sent_index - self.n_sents) end_sentence = min(len(sentences) - 1, sent_index + self.n_sents) @@ -410,55 +412,53 @@ class EntityLinker(TrainablePipe): end_token = sentences[end_sentence].end sent_doc = doc[start_token:end_token].as_doc() # currently, the context is the same for each entity in a sentence (should be refined) - xp = self.model.ops.xp - if self.incl_context: - sentence_encoding = self.model.predict([sent_doc])[0] - sentence_encoding_t = sentence_encoding.T - sentence_norm = xp.linalg.norm(sentence_encoding_t) - entity_count += 1 - if ent.label_ in self.labels_discard: - # ignoring this entity - setting to NIL + sentence_encoding = self.model.predict([sent_doc])[0] + sentence_encoding_t = sentence_encoding.T + sentence_norm = xp.linalg.norm(sentence_encoding_t) + entity_count += 1 + if ent.label_ in self.labels_discard: + # ignoring this entity - setting to NIL + final_kb_ids.append(self.NIL) + else: + candidates = list(self.get_candidates(self.kb, ent)) + if not candidates: + # no prediction possible for this entity - setting to NIL final_kb_ids.append(self.NIL) + elif len(candidates) == 1: + # shortcut for efficiency reasons: take the 1 candidate + # TODO: thresholding + final_kb_ids.append(candidates[0].entity_) else: - candidates = list(self.get_candidates(self.kb, ent)) - if not candidates: - # no prediction possible for this entity - setting to NIL - final_kb_ids.append(self.NIL) - elif len(candidates) == 1: - # shortcut for efficiency reasons: take the 1 candidate - # TODO: thresholding - final_kb_ids.append(candidates[0].entity_) - else: - random.shuffle(candidates) - # set all prior probabilities to 0 if incl_prior=False - prior_probs = xp.asarray([c.prior_prob for c in candidates]) - if not self.incl_prior: - prior_probs = xp.asarray([0.0 for _ in candidates]) - scores = prior_probs - # add in similarity from the context - if self.incl_context: - entity_encodings = xp.asarray( - [c.entity_vector for c in candidates] - ) - entity_norm = xp.linalg.norm(entity_encodings, axis=1) - if len(entity_encodings) != len(prior_probs): - raise RuntimeError( - Errors.E147.format( - method="predict", - msg="vectors not of equal length", - ) + random.shuffle(candidates) + # set all prior probabilities to 0 if incl_prior=False + prior_probs = xp.asarray([c.prior_prob for c in candidates]) + if not self.incl_prior: + prior_probs = xp.asarray([0.0 for _ in candidates]) + scores = prior_probs + # add in similarity from the context + if self.incl_context: + entity_encodings = xp.asarray( + [c.entity_vector for c in candidates] + ) + entity_norm = xp.linalg.norm(entity_encodings, axis=1) + if len(entity_encodings) != len(prior_probs): + raise RuntimeError( + Errors.E147.format( + method="predict", + msg="vectors not of equal length", ) - # cosine similarity - sims = xp.dot(entity_encodings, sentence_encoding_t) / ( - sentence_norm * entity_norm ) - if sims.shape != prior_probs.shape: - raise ValueError(Errors.E161) - scores = prior_probs + sims - (prior_probs * sims) - # TODO: thresholding - best_index = scores.argmax().item() - best_candidate = candidates[best_index] - final_kb_ids.append(best_candidate.entity_) + # cosine similarity + sims = xp.dot(entity_encodings, sentence_encoding_t) / ( + sentence_norm * entity_norm + ) + if sims.shape != prior_probs.shape: + raise ValueError(Errors.E161) + scores = prior_probs + sims - (prior_probs * sims) + # TODO: thresholding + best_index = scores.argmax().item() + best_candidate = candidates[best_index] + final_kb_ids.append(best_candidate.entity_) if not (len(final_kb_ids) == entity_count): err = Errors.E147.format( method="predict", msg="result variables not of equal length" From f8b769e7bfb3717919fbd7566dca879b2a12c45f Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Wed, 1 Jun 2022 09:37:30 +0200 Subject: [PATCH 09/24] Add `test_slow_gpu` explosion-bot command (#10858) --- .github/workflows/explosionbot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/explosionbot.yml b/.github/workflows/explosionbot.yml index e29ce8fe8..d585ecd9c 100644 --- a/.github/workflows/explosionbot.yml +++ b/.github/workflows/explosionbot.yml @@ -23,5 +23,5 @@ jobs: env: INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }} INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }} - ENABLED_COMMANDS: "test_gpu,test_slow" + ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu" ALLOWED_TEAMS: "spaCy" From f7507c2327162a97a31cbe75732a0b512f4cd323 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Thu, 2 Jun 2022 00:10:16 +0200 Subject: [PATCH 10/24] fix typo + CI slow testing (#10835) * fix typo * one more typo --- spacy/pipeline/pipe.pyx | 2 +- spacy/training/example.pyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/pipeline/pipe.pyx b/spacy/pipeline/pipe.pyx index d24e4d574..4e3ae1cf0 100644 --- a/spacy/pipeline/pipe.pyx +++ b/spacy/pipeline/pipe.pyx @@ -31,7 +31,7 @@ cdef class Pipe: and returned. This usually happens under the hood when the nlp object is called on a text and all components are applied to the Doc. - docs (Doc): The Doc to process. + doc (Doc): The Doc to process. RETURNS (Doc): The processed Doc. DOCS: https://spacy.io/api/pipe#call diff --git a/spacy/training/example.pyx b/spacy/training/example.pyx index ab92f78c6..3035388a6 100644 --- a/spacy/training/example.pyx +++ b/spacy/training/example.pyx @@ -198,7 +198,7 @@ cdef class Example: def get_aligned_sent_starts(self): """Get list of SENT_START attributes aligned to the predicted tokenization. - If the reference has not sentence starts, return a list of None values. + If the reference does not have sentence starts, return a list of None values. """ if self.y.has_annotation("SENT_START"): align = self.alignment.y2x From a322d6d5f2f85c2da6cded4fcd6143d41b5a9e96 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 2 Jun 2022 13:12:53 +0200 Subject: [PATCH 11/24] Add SpanRuler component (#9880) * Add SpanRuler component Add a `SpanRuler` component similar to `EntityRuler` that saves a list of matched spans to `Doc.spans[spans_key]`. The matches from the token and phrase matchers are deduplicated and sorted before assignment but are not otherwise filtered. * Update spacy/pipeline/span_ruler.py Co-authored-by: Sofie Van Landeghem * Fix cast * Add self.key property * Use number of patterns as length * Remove patterns kwarg from init * Update spacy/tests/pipeline/test_span_ruler.py Co-authored-by: Sofie Van Landeghem * Add options for spans filter and setting to ents * Add `spans_filter` option as a registered function' * Make `spans_key` optional and if `None`, set to `doc.ents` instead of `doc.spans[spans_key]`. * Update and generalize tests * Add test for setting doc.ents, fix key property type * Fix typing * Allow independent doc.spans and doc.ents * If `spans_key` is set, set `doc.spans` with `spans_filter`. * If `annotate_ents` is set, set `doc.ents` with `ents_fitler`. * Use `util.filter_spans` by default as `ents_filter`. * Use a custom warning if the filter does not work for `doc.ents`. * Enable use of SpanC.id in Span * Support id in SpanRuler as Span.id * Update types * `id` can only be provided as string (already by `PatternType` definition) * Update all uses of Span.id/ent_id in Doc * Rename Span id kwarg to span_id * Update types and docs * Add ents filter to mimic EntityRuler overwrite_ents * Refactor `ents_filter` to take `entities, spans` args for more filtering options * Give registered filters more descriptive names * Allow registered `filter_spans` filter (`spacy.first_longest_spans_filter.v1`) to take any number of `Iterable[Span]` objects as args so it can be used for spans filter or ents filter * Implement future entity ruler as span ruler Implement a compatible `entity_ruler` as `future_entity_ruler` using `SpanRuler` as the underlying component: * Add `sort_key` and `sort_reverse` to allow the sorting behavior to be customized. (Necessary for the same sorting/filtering as in `EntityRuler`.) * Implement `overwrite_overlapping_ents_filter` and `preserve_existing_ents_filter` to support `EntityRuler.overwrite_ents` settings. * Add `remove_by_id` to support `EntityRuler.remove` functionality. * Refactor `entity_ruler` tests to parametrize all tests to test both `entity_ruler` and `future_entity_ruler` * Implement `SpanRuler.token_patterns` and `SpanRuler.phrase_patterns` properties. Additional changes: * Move all config settings to top-level attributes to avoid duplicating settings in the config vs. `span_ruler/cfg`. (Also avoids a lot of casting.) * Format * Fix filter make method name * Refactor to use same error for removing by label or ID * Also provide existing spans to spans filter * Support ids property * Remove token_patterns and phrase_patterns * Update docstrings * Add span ruler docs * Fix types * Apply suggestions from code review Co-authored-by: Sofie Van Landeghem * Move sorting into filters * Check for all tokens in seen tokens in entity ruler filters * Remove registered sort key * Set Token.ent_id in a backwards-compatible way in Doc.set_ents * Remove sort options from API docs * Update docstrings * Rename entity ruler filters * Fix and parameterize scoring * Add id to Span API docs * Fix typo in API docs * Include explicit labeled=True for scorer Co-authored-by: Sofie Van Landeghem --- spacy/errors.py | 6 +- spacy/pipeline/__init__.py | 2 + spacy/pipeline/entityruler.py | 9 +- spacy/pipeline/span_ruler.py | 569 ++++++++++++++++++++++ spacy/tests/doc/test_span.py | 12 + spacy/tests/pipeline/test_entity_ruler.py | 297 +++++++---- spacy/tests/pipeline/test_span_ruler.py | 465 ++++++++++++++++++ spacy/tokens/doc.pyx | 32 +- spacy/tokens/span.pyi | 3 +- spacy/tokens/span.pyx | 37 +- spacy/util.py | 9 + website/docs/api/entityruler.md | 2 +- website/docs/api/span.md | 7 +- website/docs/api/spanruler.md | 351 +++++++++++++ website/docs/usage/rule-based-matching.md | 103 ++++ website/meta/sidebars.json | 1 + 16 files changed, 1772 insertions(+), 133 deletions(-) create mode 100644 spacy/pipeline/span_ruler.py create mode 100644 spacy/tests/pipeline/test_span_ruler.py create mode 100644 website/docs/api/spanruler.md diff --git a/spacy/errors.py b/spacy/errors.py index c82ffe882..665b91659 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -532,6 +532,8 @@ class Errors(metaclass=ErrorsWithCodes): E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.") # New errors added in v3.x + E854 = ("Unable to set doc.ents. Check that the 'ents_filter' does not " + "permit overlapping spans.") E855 = ("Invalid {obj}: {obj} is not from the same doc.") E856 = ("Error accessing span at position {i}: out of bounds in span group " "of length {length}.") @@ -903,8 +905,8 @@ class Errors(metaclass=ErrorsWithCodes): E1022 = ("Words must be of type str or int, but input is of type '{wtype}'") E1023 = ("Couldn't read EntityRuler from the {path}. This file doesn't " "exist.") - E1024 = ("A pattern with ID \"{ent_id}\" is not present in EntityRuler " - "patterns.") + E1024 = ("A pattern with {attr_type} '{label}' is not present in " + "'{component}' patterns.") E1025 = ("Cannot intify the value '{value}' as an IOB string. The only " "supported values are: 'I', 'O', 'B' and ''") E1026 = ("Edit tree has an invalid format:\n{errors}") diff --git a/spacy/pipeline/__init__.py b/spacy/pipeline/__init__.py index 938ab08c6..26931606b 100644 --- a/spacy/pipeline/__init__.py +++ b/spacy/pipeline/__init__.py @@ -13,6 +13,7 @@ from .sentencizer import Sentencizer from .tagger import Tagger from .textcat import TextCategorizer from .spancat import SpanCategorizer +from .span_ruler import SpanRuler from .textcat_multilabel import MultiLabel_TextCategorizer from .tok2vec import Tok2Vec from .functions import merge_entities, merge_noun_chunks, merge_subtokens @@ -30,6 +31,7 @@ __all__ = [ "SentenceRecognizer", "Sentencizer", "SpanCategorizer", + "SpanRuler", "Tagger", "TextCategorizer", "Tok2Vec", diff --git a/spacy/pipeline/entityruler.py b/spacy/pipeline/entityruler.py index 4307af793..3cb1ca676 100644 --- a/spacy/pipeline/entityruler.py +++ b/spacy/pipeline/entityruler.py @@ -180,10 +180,7 @@ class EntityRuler(Pipe): if start not in seen_tokens and end - 1 not in seen_tokens: if match_id in self._ent_ids: label, ent_id = self._ent_ids[match_id] - span = Span(doc, start, end, label=label) - if ent_id: - for token in span: - token.ent_id_ = ent_id + span = Span(doc, start, end, label=label, span_id=ent_id) else: span = Span(doc, start, end, label=match_id) new_entities.append(span) @@ -357,7 +354,9 @@ class EntityRuler(Pipe): (label, eid) for (label, eid) in self._ent_ids.values() if eid == ent_id ] if not label_id_pairs: - raise ValueError(Errors.E1024.format(ent_id=ent_id)) + raise ValueError( + Errors.E1024.format(attr_type="ID", label=ent_id, component=self.name) + ) created_labels = [ self._create_label(label, eid) for (label, eid) in label_id_pairs ] diff --git a/spacy/pipeline/span_ruler.py b/spacy/pipeline/span_ruler.py new file mode 100644 index 000000000..807a4ffe5 --- /dev/null +++ b/spacy/pipeline/span_ruler.py @@ -0,0 +1,569 @@ +from typing import Optional, Union, List, Dict, Tuple, Iterable, Any, Callable +from typing import Sequence, Set, cast +import warnings +from functools import partial +from pathlib import Path +import srsly + +from .pipe import Pipe +from ..training import Example +from ..language import Language +from ..errors import Errors, Warnings +from ..util import ensure_path, SimpleFrozenList, registry +from ..tokens import Doc, Span +from ..scorer import Scorer +from ..matcher import Matcher, PhraseMatcher +from .. import util + +PatternType = Dict[str, Union[str, List[Dict[str, Any]]]] +DEFAULT_SPANS_KEY = "ruler" + + +@Language.factory( + "future_entity_ruler", + assigns=["doc.ents"], + default_config={ + "phrase_matcher_attr": None, + "validate": False, + "overwrite_ents": False, + "scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"}, + "ent_id_sep": "__unused__", + }, + default_score_weights={ + "ents_f": 1.0, + "ents_p": 0.0, + "ents_r": 0.0, + "ents_per_type": None, + }, +) +def make_entity_ruler( + nlp: Language, + name: str, + phrase_matcher_attr: Optional[Union[int, str]], + validate: bool, + overwrite_ents: bool, + scorer: Optional[Callable], + ent_id_sep: str, +): + if overwrite_ents: + ents_filter = prioritize_new_ents_filter + else: + ents_filter = prioritize_existing_ents_filter + return SpanRuler( + nlp, + name, + spans_key=None, + spans_filter=None, + annotate_ents=True, + ents_filter=ents_filter, + phrase_matcher_attr=phrase_matcher_attr, + validate=validate, + overwrite=False, + scorer=scorer, + ) + + +@Language.factory( + "span_ruler", + assigns=["doc.spans"], + default_config={ + "spans_key": DEFAULT_SPANS_KEY, + "spans_filter": None, + "annotate_ents": False, + "ents_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}, + "phrase_matcher_attr": None, + "validate": False, + "overwrite": True, + "scorer": { + "@scorers": "spacy.overlapping_labeled_spans_scorer.v1", + "spans_key": DEFAULT_SPANS_KEY, + }, + }, + default_score_weights={ + f"spans_{DEFAULT_SPANS_KEY}_f": 1.0, + f"spans_{DEFAULT_SPANS_KEY}_p": 0.0, + f"spans_{DEFAULT_SPANS_KEY}_r": 0.0, + f"spans_{DEFAULT_SPANS_KEY}_per_type": None, + }, +) +def make_span_ruler( + nlp: Language, + name: str, + spans_key: Optional[str], + spans_filter: Optional[Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]]], + annotate_ents: bool, + ents_filter: Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]], + phrase_matcher_attr: Optional[Union[int, str]], + validate: bool, + overwrite: bool, + scorer: Optional[Callable], +): + return SpanRuler( + nlp, + name, + spans_key=spans_key, + spans_filter=spans_filter, + annotate_ents=annotate_ents, + ents_filter=ents_filter, + phrase_matcher_attr=phrase_matcher_attr, + validate=validate, + overwrite=overwrite, + scorer=scorer, + ) + + +def prioritize_new_ents_filter( + entities: Iterable[Span], spans: Iterable[Span] +) -> List[Span]: + """Merge entities and spans into one list without overlaps by allowing + spans to overwrite any entities that they overlap with. Intended to + replicate the overwrite_ents=True behavior from the EntityRuler. + + entities (Iterable[Span]): The entities, already filtered for overlaps. + spans (Iterable[Span]): The spans to merge, may contain overlaps. + RETURNS (List[Span]): Filtered list of non-overlapping spans. + """ + get_sort_key = lambda span: (span.end - span.start, -span.start) + spans = sorted(spans, key=get_sort_key, reverse=True) + entities = list(entities) + new_entities = [] + seen_tokens: Set[int] = set() + for span in spans: + start = span.start + end = span.end + if all(token.i not in seen_tokens for token in span): + new_entities.append(span) + entities = [e for e in entities if not (e.start < end and e.end > start)] + seen_tokens.update(range(start, end)) + return entities + new_entities + + +@registry.misc("spacy.prioritize_new_ents_filter.v1") +def make_prioritize_new_ents_filter(): + return prioritize_new_ents_filter + + +def prioritize_existing_ents_filter( + entities: Iterable[Span], spans: Iterable[Span] +) -> List[Span]: + """Merge entities and spans into one list without overlaps by prioritizing + existing entities. Intended to replicate the overwrite_ents=False behavior + from the EntityRuler. + + entities (Iterable[Span]): The entities, already filtered for overlaps. + spans (Iterable[Span]): The spans to merge, may contain overlaps. + RETURNS (List[Span]): Filtered list of non-overlapping spans. + """ + get_sort_key = lambda span: (span.end - span.start, -span.start) + spans = sorted(spans, key=get_sort_key, reverse=True) + entities = list(entities) + new_entities = [] + seen_tokens: Set[int] = set() + seen_tokens.update(*(range(ent.start, ent.end) for ent in entities)) + for span in spans: + start = span.start + end = span.end + if all(token.i not in seen_tokens for token in span): + new_entities.append(span) + seen_tokens.update(range(start, end)) + return entities + new_entities + + +@registry.misc("spacy.prioritize_existing_ents_filter.v1") +def make_preverse_existing_ents_filter(): + return prioritize_existing_ents_filter + + +def overlapping_labeled_spans_score( + examples: Iterable[Example], *, spans_key=DEFAULT_SPANS_KEY, **kwargs +) -> Dict[str, Any]: + kwargs = dict(kwargs) + attr_prefix = f"spans_" + kwargs.setdefault("attr", f"{attr_prefix}{spans_key}") + kwargs.setdefault("allow_overlap", True) + kwargs.setdefault("labeled", True) + kwargs.setdefault( + "getter", lambda doc, key: doc.spans.get(key[len(attr_prefix) :], []) + ) + kwargs.setdefault("has_annotation", lambda doc: spans_key in doc.spans) + return Scorer.score_spans(examples, **kwargs) + + +@registry.scorers("spacy.overlapping_labeled_spans_scorer.v1") +def make_overlapping_labeled_spans_scorer(spans_key: str = DEFAULT_SPANS_KEY): + return partial(overlapping_labeled_spans_score, spans_key=spans_key) + + +class SpanRuler(Pipe): + """The SpanRuler lets you add spans to the `Doc.spans` using token-based + rules or exact phrase matches. + + DOCS: https://spacy.io/api/spanruler + USAGE: https://spacy.io/usage/rule-based-matching#spanruler + """ + + def __init__( + self, + nlp: Language, + name: str = "span_ruler", + *, + spans_key: Optional[str] = DEFAULT_SPANS_KEY, + spans_filter: Optional[ + Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]] + ] = None, + annotate_ents: bool = False, + ents_filter: Callable[ + [Iterable[Span], Iterable[Span]], Iterable[Span] + ] = util.filter_chain_spans, + phrase_matcher_attr: Optional[Union[int, str]] = None, + validate: bool = False, + overwrite: bool = False, + scorer: Optional[Callable] = partial( + overlapping_labeled_spans_score, spans_key=DEFAULT_SPANS_KEY + ), + ) -> None: + """Initialize the span ruler. If patterns are supplied here, they + need to be a list of dictionaries with a `"label"` and `"pattern"` + key. A pattern can either be a token pattern (list) or a phrase pattern + (string). For example: `{'label': 'ORG', 'pattern': 'Apple'}`. + + nlp (Language): The shared nlp object to pass the vocab to the matchers + and process phrase patterns. + name (str): Instance name of the current pipeline component. Typically + passed in automatically from the factory when the component is + added. Used to disable the current span ruler while creating + phrase patterns with the nlp object. + spans_key (Optional[str]): The spans key to save the spans under. If + `None`, no spans are saved. Defaults to "ruler". + spans_filter (Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]): + The optional method to filter spans before they are assigned to + doc.spans. Defaults to `None`. + annotate_ents (bool): Whether to save spans to doc.ents. Defaults to + `False`. + ents_filter (Callable[[Iterable[Span], Iterable[Span]], List[Span]]): + The method to filter spans before they are assigned to doc.ents. + Defaults to `util.filter_chain_spans`. + phrase_matcher_attr (Optional[Union[int, str]]): Token attribute to + match on, passed to the internal PhraseMatcher as `attr`. Defaults + to `None`. + validate (bool): Whether patterns should be validated, passed to + Matcher and PhraseMatcher as `validate`. + overwrite (bool): Whether to remove any existing spans under this spans + key if `spans_key` is set, and/or to remove any ents under `doc.ents` if + `annotate_ents` is set. Defaults to `True`. + scorer (Optional[Callable]): The scoring method. Defaults to + spacy.pipeline.span_ruler.overlapping_labeled_spans_score. + + DOCS: https://spacy.io/api/spanruler#init + """ + self.nlp = nlp + self.name = name + self.spans_key = spans_key + self.annotate_ents = annotate_ents + self.phrase_matcher_attr = phrase_matcher_attr + self.validate = validate + self.overwrite = overwrite + self.spans_filter = spans_filter + self.ents_filter = ents_filter + self.scorer = scorer + self._match_label_id_map: Dict[int, Dict[str, str]] = {} + self.clear() + + def __len__(self) -> int: + """The number of all labels added to the span ruler.""" + return len(self._patterns) + + def __contains__(self, label: str) -> bool: + """Whether a label is present in the patterns.""" + for label_id in self._match_label_id_map.values(): + if label_id["label"] == label: + return True + return False + + @property + def key(self) -> Optional[str]: + """Key of the doc.spans dict to save the spans under.""" + return self.spans_key + + def __call__(self, doc: Doc) -> Doc: + """Find matches in document and add them as entities. + + doc (Doc): The Doc object in the pipeline. + RETURNS (Doc): The Doc with added entities, if available. + + DOCS: https://spacy.io/api/spanruler#call + """ + error_handler = self.get_error_handler() + try: + matches = self.match(doc) + self.set_annotations(doc, matches) + return doc + except Exception as e: + return error_handler(self.name, self, [doc], e) + + def match(self, doc: Doc): + self._require_patterns() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="\\[W036") + matches = cast( + List[Tuple[int, int, int]], + list(self.matcher(doc)) + list(self.phrase_matcher(doc)), + ) + deduplicated_matches = set( + Span( + doc, + start, + end, + label=self._match_label_id_map[m_id]["label"], + span_id=self._match_label_id_map[m_id]["id"], + ) + for m_id, start, end in matches + if start != end + ) + return sorted(list(deduplicated_matches)) + + def set_annotations(self, doc, matches): + """Modify the document in place""" + # set doc.spans if spans_key is set + if self.key: + spans = [] + if self.key in doc.spans and not self.overwrite: + spans = doc.spans[self.key] + spans.extend( + self.spans_filter(spans, matches) if self.spans_filter else matches + ) + doc.spans[self.key] = spans + # set doc.ents if annotate_ents is set + if self.annotate_ents: + spans = [] + if not self.overwrite: + spans = list(doc.ents) + spans = self.ents_filter(spans, matches) + try: + doc.ents = sorted(spans) + except ValueError: + raise ValueError(Errors.E854) + + @property + def labels(self) -> Tuple[str, ...]: + """All labels present in the match patterns. + + RETURNS (set): The string labels. + + DOCS: https://spacy.io/api/spanruler#labels + """ + return tuple(sorted(set([cast(str, p["label"]) for p in self._patterns]))) + + @property + def ids(self) -> Tuple[str, ...]: + """All IDs present in the match patterns. + + RETURNS (set): The string IDs. + + DOCS: https://spacy.io/api/spanruler#ids + """ + return tuple( + sorted(set([cast(str, p.get("id")) for p in self._patterns]) - set([None])) + ) + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + patterns: Optional[Sequence[PatternType]] = None, + ): + """Initialize the pipe for training. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + patterns (Optional[Iterable[PatternType]]): The list of patterns. + + DOCS: https://spacy.io/api/spanruler#initialize + """ + self.clear() + if patterns: + self.add_patterns(patterns) # type: ignore[arg-type] + + @property + def patterns(self) -> List[PatternType]: + """Get all patterns that were added to the span ruler. + + RETURNS (list): The original patterns, one dictionary per pattern. + + DOCS: https://spacy.io/api/spanruler#patterns + """ + return self._patterns + + def add_patterns(self, patterns: List[PatternType]) -> None: + """Add patterns to the span ruler. A pattern can either be a token + pattern (list of dicts) or a phrase pattern (string). For example: + {'label': 'ORG', 'pattern': 'Apple'} + {'label': 'ORG', 'pattern': 'Apple', 'id': 'apple'} + {'label': 'GPE', 'pattern': [{'lower': 'san'}, {'lower': 'francisco'}]} + + patterns (list): The patterns to add. + + DOCS: https://spacy.io/api/spanruler#add_patterns + """ + + # disable the nlp components after this one in case they haven't been + # initialized / deserialized yet + try: + current_index = -1 + for i, (name, pipe) in enumerate(self.nlp.pipeline): + if self == pipe: + current_index = i + break + subsequent_pipes = [pipe for pipe in self.nlp.pipe_names[current_index:]] + except ValueError: + subsequent_pipes = [] + with self.nlp.select_pipes(disable=subsequent_pipes): + phrase_pattern_labels = [] + phrase_pattern_texts = [] + for entry in patterns: + p_label = cast(str, entry["label"]) + p_id = cast(str, entry.get("id", "")) + label = repr((p_label, p_id)) + self._match_label_id_map[self.nlp.vocab.strings.as_int(label)] = { + "label": p_label, + "id": p_id, + } + if isinstance(entry["pattern"], str): + phrase_pattern_labels.append(label) + phrase_pattern_texts.append(entry["pattern"]) + elif isinstance(entry["pattern"], list): + self.matcher.add(label, [entry["pattern"]]) + else: + raise ValueError(Errors.E097.format(pattern=entry["pattern"])) + self._patterns.append(entry) + for label, pattern in zip( + phrase_pattern_labels, + self.nlp.pipe(phrase_pattern_texts), + ): + self.phrase_matcher.add(label, [pattern]) + + def clear(self) -> None: + """Reset all patterns. + + RETURNS: None + DOCS: https://spacy.io/api/spanruler#clear + """ + self._patterns: List[PatternType] = [] + self.matcher: Matcher = Matcher(self.nlp.vocab, validate=self.validate) + self.phrase_matcher: PhraseMatcher = PhraseMatcher( + self.nlp.vocab, + attr=self.phrase_matcher_attr, + validate=self.validate, + ) + + def remove(self, label: str) -> None: + """Remove a pattern by its label. + + label (str): Label of the pattern to be removed. + RETURNS: None + DOCS: https://spacy.io/api/spanruler#remove + """ + if label not in self: + raise ValueError( + Errors.E1024.format(attr_type="label", label=label, component=self.name) + ) + self._patterns = [p for p in self._patterns if p["label"] != label] + for m_label in self._match_label_id_map: + if self._match_label_id_map[m_label]["label"] == label: + m_label_str = self.nlp.vocab.strings.as_string(m_label) + if m_label_str in self.phrase_matcher: + self.phrase_matcher.remove(m_label_str) + if m_label_str in self.matcher: + self.matcher.remove(m_label_str) + + def remove_by_id(self, pattern_id: str) -> None: + """Remove a pattern by its pattern ID. + + pattern_id (str): ID of the pattern to be removed. + RETURNS: None + DOCS: https://spacy.io/api/spanruler#remove_by_id + """ + orig_len = len(self) + self._patterns = [p for p in self._patterns if p.get("id") != pattern_id] + if orig_len == len(self): + raise ValueError( + Errors.E1024.format( + attr_type="ID", label=pattern_id, component=self.name + ) + ) + for m_label in self._match_label_id_map: + if self._match_label_id_map[m_label]["id"] == pattern_id: + m_label_str = self.nlp.vocab.strings.as_string(m_label) + if m_label_str in self.phrase_matcher: + self.phrase_matcher.remove(m_label_str) + if m_label_str in self.matcher: + self.matcher.remove(m_label_str) + + def _require_patterns(self) -> None: + """Raise a warning if this component has no patterns defined.""" + if len(self) == 0: + warnings.warn(Warnings.W036.format(name=self.name)) + + def from_bytes( + self, bytes_data: bytes, *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "SpanRuler": + """Load the span ruler from a bytestring. + + bytes_data (bytes): The bytestring to load. + RETURNS (SpanRuler): The loaded span ruler. + + DOCS: https://spacy.io/api/spanruler#from_bytes + """ + self.clear() + deserializers = { + "patterns": lambda b: self.add_patterns(srsly.json_loads(b)), + } + util.from_bytes(bytes_data, deserializers, exclude) + return self + + def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes: + """Serialize the span ruler to a bytestring. + + RETURNS (bytes): The serialized patterns. + + DOCS: https://spacy.io/api/spanruler#to_bytes + """ + serializers = { + "patterns": lambda: srsly.json_dumps(self.patterns), + } + return util.to_bytes(serializers, exclude) + + def from_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "SpanRuler": + """Load the span ruler from a directory. + + path (Union[str, Path]): A path to a directory. + RETURNS (SpanRuler): The loaded span ruler. + + DOCS: https://spacy.io/api/spanruler#from_disk + """ + self.clear() + path = ensure_path(path) + deserializers = { + "patterns": lambda p: self.add_patterns(srsly.read_jsonl(p)), + } + util.from_disk(path, deserializers, {}) + return self + + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """Save the span ruler patterns to a directory. + + path (Union[str, Path]): A path to a directory. + + DOCS: https://spacy.io/api/spanruler#to_disk + """ + path = ensure_path(path) + serializers = { + "patterns": lambda p: srsly.write_jsonl(p, self.patterns), + } + util.to_disk(path, serializers, {}) diff --git a/spacy/tests/doc/test_span.py b/spacy/tests/doc/test_span.py index c0496cabf..3676b35af 100644 --- a/spacy/tests/doc/test_span.py +++ b/spacy/tests/doc/test_span.py @@ -428,10 +428,19 @@ def test_span_string_label_kb_id(doc): assert span.kb_id == doc.vocab.strings["Q342"] +def test_span_string_label_id(doc): + span = Span(doc, 0, 1, label="hello", span_id="Q342") + assert span.label_ == "hello" + assert span.label == doc.vocab.strings["hello"] + assert span.id_ == "Q342" + assert span.id == doc.vocab.strings["Q342"] + + def test_span_attrs_writable(doc): span = Span(doc, 0, 1) span.label_ = "label" span.kb_id_ = "kb_id" + span.id_ = "id" def test_span_ents_property(doc): @@ -619,6 +628,9 @@ def test_span_comparison(doc): assert Span(doc, 0, 4, "LABEL", kb_id="KB_ID") <= Span(doc, 1, 3) assert Span(doc, 1, 3) > Span(doc, 0, 4, "LABEL", kb_id="KB_ID") assert Span(doc, 1, 3) >= Span(doc, 0, 4, "LABEL", kb_id="KB_ID") + + # Different id + assert Span(doc, 1, 3, span_id="AAA") < Span(doc, 1, 3, span_id="BBB") # fmt: on diff --git a/spacy/tests/pipeline/test_entity_ruler.py b/spacy/tests/pipeline/test_entity_ruler.py index f2031d0a9..bbd537cef 100644 --- a/spacy/tests/pipeline/test_entity_ruler.py +++ b/spacy/tests/pipeline/test_entity_ruler.py @@ -5,12 +5,15 @@ from spacy.tokens import Doc, Span from spacy.language import Language from spacy.lang.en import English from spacy.pipeline import EntityRuler, EntityRecognizer, merge_entities +from spacy.pipeline import SpanRuler from spacy.pipeline.ner import DEFAULT_NER_MODEL from spacy.errors import MatchPatternError from spacy.tests.util import make_tempdir from thinc.api import NumpyOps, get_current_ops +ENTITY_RULERS = ["entity_ruler", "future_entity_ruler"] + @pytest.fixture def nlp(): @@ -37,12 +40,14 @@ def add_ent_component(doc): @pytest.mark.issue(3345) -def test_issue3345(): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue3345(entity_ruler_factory): """Test case where preset entity crosses sentence boundary.""" nlp = English() doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"]) doc[4].is_sent_start = True - ruler = EntityRuler(nlp, patterns=[{"label": "GPE", "pattern": "New York"}]) + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns([{"label": "GPE", "pattern": "New York"}]) cfg = {"model": DEFAULT_NER_MODEL} model = registry.resolve(cfg, validate=True)["model"] ner = EntityRecognizer(doc.vocab, model) @@ -60,13 +65,18 @@ def test_issue3345(): @pytest.mark.issue(4849) -def test_issue4849(): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue4849(entity_ruler_factory): nlp = English() patterns = [ {"label": "PERSON", "pattern": "joe biden", "id": "joe-biden"}, {"label": "PERSON", "pattern": "bernie sanders", "id": "bernie-sanders"}, ] - ruler = nlp.add_pipe("entity_ruler", config={"phrase_matcher_attr": "LOWER"}) + ruler = nlp.add_pipe( + entity_ruler_factory, + name="entity_ruler", + config={"phrase_matcher_attr": "LOWER"}, + ) ruler.add_patterns(patterns) text = """ The left is starting to take aim at Democratic front-runner Joe Biden. @@ -86,10 +96,11 @@ def test_issue4849(): @pytest.mark.issue(5918) -def test_issue5918(): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue5918(entity_ruler_factory): # Test edge case when merging entities. nlp = English() - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ {"label": "ORG", "pattern": "Digicon Inc"}, {"label": "ORG", "pattern": "Rotan Mosle Inc's"}, @@ -114,9 +125,10 @@ def test_issue5918(): @pytest.mark.issue(8168) -def test_issue8168(): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue8168(entity_ruler_factory): nlp = English() - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ {"label": "ORG", "pattern": "Apple"}, { @@ -131,14 +143,17 @@ def test_issue8168(): }, ] ruler.add_patterns(patterns) - - assert ruler._ent_ids == {8043148519967183733: ("GPE", "san-francisco")} + doc = nlp("San Francisco San Fran") + assert all(t.ent_id_ == "san-francisco" for t in doc) @pytest.mark.issue(8216) -def test_entity_ruler_fix8216(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fix8216(nlp, patterns, entity_ruler_factory): """Test that patterns don't get added excessively.""" - ruler = nlp.add_pipe("entity_ruler", config={"validate": True}) + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"validate": True} + ) ruler.add_patterns(patterns) pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) assert pattern_count > 0 @@ -147,13 +162,16 @@ def test_entity_ruler_fix8216(nlp, patterns): assert after_count == pattern_count -def test_entity_ruler_init(nlp, patterns): - ruler = EntityRuler(nlp, patterns=patterns) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_init(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) assert len(ruler) == len(patterns) assert len(ruler.labels) == 4 assert "HELLO" in ruler assert "BYE" in ruler - ruler = nlp.add_pipe("entity_ruler") + nlp.remove_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) doc = nlp("hello world bye bye") assert len(doc.ents) == 2 @@ -161,20 +179,23 @@ def test_entity_ruler_init(nlp, patterns): assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_no_patterns_warns(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_no_patterns_warns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") assert len(ruler) == 0 assert len(ruler.labels) == 0 - nlp.add_pipe("entity_ruler") + nlp.remove_pipe("entity_ruler") + nlp.add_pipe(entity_ruler_factory, name="entity_ruler") assert nlp.pipe_names == ["entity_ruler"] with pytest.warns(UserWarning): doc = nlp("hello world bye bye") assert len(doc.ents) == 0 -def test_entity_ruler_init_patterns(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_init_patterns(nlp, patterns, entity_ruler_factory): # initialize with patterns - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") assert len(ruler.labels) == 0 ruler.initialize(lambda: [], patterns=patterns) assert len(ruler.labels) == 4 @@ -186,7 +207,7 @@ def test_entity_ruler_init_patterns(nlp, patterns): nlp.config["initialize"]["components"]["entity_ruler"] = { "patterns": {"@misc": "entity_ruler_patterns"} } - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") assert len(ruler.labels) == 0 nlp.initialize() assert len(ruler.labels) == 4 @@ -195,18 +216,20 @@ def test_entity_ruler_init_patterns(nlp, patterns): assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_init_clear(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_init_clear(nlp, patterns, entity_ruler_factory): """Test that initialization clears patterns.""" - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) assert len(ruler.labels) == 4 ruler.initialize(lambda: []) assert len(ruler.labels) == 0 -def test_entity_ruler_clear(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_clear(nlp, patterns, entity_ruler_factory): """Test that initialization clears patterns.""" - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) assert len(ruler.labels) == 4 doc = nlp("hello world") @@ -218,8 +241,9 @@ def test_entity_ruler_clear(nlp, patterns): assert len(doc.ents) == 0 -def test_entity_ruler_existing(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler") +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_existing(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) nlp.add_pipe("add_ent", before="entity_ruler") doc = nlp("OH HELLO WORLD bye bye") @@ -228,8 +252,11 @@ def test_entity_ruler_existing(nlp, patterns): assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_existing_overwrite(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True}) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_existing_overwrite(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True} + ) ruler.add_patterns(patterns) nlp.add_pipe("add_ent", before="entity_ruler") doc = nlp("OH HELLO WORLD bye bye") @@ -239,8 +266,11 @@ def test_entity_ruler_existing_overwrite(nlp, patterns): assert doc.ents[1].label_ == "BYE" -def test_entity_ruler_existing_complex(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True}) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_existing_complex(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True} + ) ruler.add_patterns(patterns) nlp.add_pipe("add_ent", before="entity_ruler") doc = nlp("foo foo bye bye") @@ -251,8 +281,11 @@ def test_entity_ruler_existing_complex(nlp, patterns): assert len(doc.ents[1]) == 2 -def test_entity_ruler_entity_id(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True}) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_entity_id(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True} + ) ruler.add_patterns(patterns) doc = nlp("Apple is a technology company") assert len(doc.ents) == 1 @@ -260,18 +293,21 @@ def test_entity_ruler_entity_id(nlp, patterns): assert doc.ents[0].ent_id_ == "a1" -def test_entity_ruler_cfg_ent_id_sep(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_cfg_ent_id_sep(nlp, patterns, entity_ruler_factory): config = {"overwrite_ents": True, "ent_id_sep": "**"} - ruler = nlp.add_pipe("entity_ruler", config=config) + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler", config=config) ruler.add_patterns(patterns) - assert "TECH_ORG**a1" in ruler.phrase_patterns doc = nlp("Apple is a technology company") + if isinstance(ruler, EntityRuler): + assert "TECH_ORG**a1" in ruler.phrase_patterns assert len(doc.ents) == 1 assert doc.ents[0].label_ == "TECH_ORG" assert doc.ents[0].ent_id_ == "a1" -def test_entity_ruler_serialize_bytes(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_bytes(nlp, patterns, entity_ruler_factory): ruler = EntityRuler(nlp, patterns=patterns) assert len(ruler) == len(patterns) assert len(ruler.labels) == 4 @@ -288,7 +324,10 @@ def test_entity_ruler_serialize_bytes(nlp, patterns): assert sorted(new_ruler.labels) == sorted(ruler.labels) -def test_entity_ruler_serialize_phrase_matcher_attr_bytes(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_phrase_matcher_attr_bytes( + nlp, patterns, entity_ruler_factory +): ruler = EntityRuler(nlp, phrase_matcher_attr="LOWER", patterns=patterns) assert len(ruler) == len(patterns) assert len(ruler.labels) == 4 @@ -303,8 +342,9 @@ def test_entity_ruler_serialize_phrase_matcher_attr_bytes(nlp, patterns): assert new_ruler.phrase_matcher_attr == "LOWER" -def test_entity_ruler_validate(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_validate(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") validated_ruler = EntityRuler(nlp, validate=True) valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]} @@ -322,32 +362,35 @@ def test_entity_ruler_validate(nlp): validated_ruler.add_patterns([invalid_pattern]) -def test_entity_ruler_properties(nlp, patterns): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_properties(nlp, patterns, entity_ruler_factory): ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) assert sorted(ruler.labels) == sorted(["HELLO", "BYE", "COMPLEX", "TECH_ORG"]) assert sorted(ruler.ent_ids) == ["a1", "a2"] -def test_entity_ruler_overlapping_spans(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_overlapping_spans(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ {"label": "FOOBAR", "pattern": "foo bar"}, {"label": "BARBAZ", "pattern": "bar baz"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("foo bar baz")) + doc = nlp("foo bar baz") assert len(doc.ents) == 1 assert doc.ents[0].label_ == "FOOBAR" @pytest.mark.parametrize("n_process", [1, 2]) -def test_entity_ruler_multiprocessing(nlp, n_process): +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_multiprocessing(nlp, n_process, entity_ruler_factory): if isinstance(get_current_ops, NumpyOps) or n_process < 2: texts = ["I enjoy eating Pizza Hut pizza."] patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut", "id": "1234"}] - ruler = nlp.add_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) for doc in nlp.pipe(texts, n_process=2): @@ -355,8 +398,9 @@ def test_entity_ruler_multiprocessing(nlp, n_process): assert ent.ent_id_ == "1234" -def test_entity_ruler_serialize_jsonl(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler") +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_jsonl(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) with make_tempdir() as d: ruler.to_disk(d / "test_ruler.jsonl") @@ -365,8 +409,9 @@ def test_entity_ruler_serialize_jsonl(nlp, patterns): ruler.from_disk(d / "non_existing.jsonl") # read from a bad jsonl file -def test_entity_ruler_serialize_dir(nlp, patterns): - ruler = nlp.add_pipe("entity_ruler") +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_dir(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") ruler.add_patterns(patterns) with make_tempdir() as d: ruler.to_disk(d / "test_ruler") @@ -375,52 +420,65 @@ def test_entity_ruler_serialize_dir(nlp, patterns): ruler.from_disk(d / "non_existing_dir") # read from a bad directory -def test_entity_ruler_remove_basic(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_basic(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "ORG", "pattern": "ACM"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("Duygu went to school")) + doc = nlp("Dina went to school") assert len(ruler.patterns) == 3 assert len(doc.ents) == 1 + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" in ruler.phrase_matcher assert doc.ents[0].label_ == "PERSON" - assert doc.ents[0].text == "Duygu" - assert "PERSON||duygu" in ruler.phrase_matcher - ruler.remove("duygu") - doc = ruler(nlp.make_doc("Duygu went to school")) + assert doc.ents[0].text == "Dina" + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + doc = nlp("Dina went to school") assert len(doc.ents) == 0 - assert "PERSON||duygu" not in ruler.phrase_matcher + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" not in ruler.phrase_matcher assert len(ruler.patterns) == 2 -def test_entity_ruler_remove_same_id_multiple_patterns(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_same_id_multiple_patterns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, - {"label": "ORG", "pattern": "DuyguCorp", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, + {"label": "ORG", "pattern": "DinaCorp", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("Duygu founded DuyguCorp and ACME.")) + doc = nlp("Dina founded DinaCorp and ACME.") assert len(ruler.patterns) == 3 - assert "PERSON||duygu" in ruler.phrase_matcher - assert "ORG||duygu" in ruler.phrase_matcher + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" in ruler.phrase_matcher + assert "ORG||dina" in ruler.phrase_matcher assert len(doc.ents) == 3 - ruler.remove("duygu") - doc = ruler(nlp.make_doc("Duygu founded DuyguCorp and ACME.")) + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + doc = nlp("Dina founded DinaCorp and ACME.") assert len(ruler.patterns) == 1 - assert "PERSON||duygu" not in ruler.phrase_matcher - assert "ORG||duygu" not in ruler.phrase_matcher + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" not in ruler.phrase_matcher + assert "ORG||dina" not in ruler.phrase_matcher assert len(doc.ents) == 1 -def test_entity_ruler_remove_nonexisting_pattern(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_nonexisting_pattern(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "ORG", "pattern": "ACM"}, ] @@ -428,82 +486,109 @@ def test_entity_ruler_remove_nonexisting_pattern(nlp): assert len(ruler.patterns) == 3 with pytest.raises(ValueError): ruler.remove("nepattern") - assert len(ruler.patterns) == 3 + if isinstance(ruler, SpanRuler): + with pytest.raises(ValueError): + ruler.remove_by_id("nepattern") -def test_entity_ruler_remove_several_patterns(nlp): - ruler = EntityRuler(nlp) + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_several_patterns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "ORG", "pattern": "ACM"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("Duygu founded her company ACME.")) + doc = nlp("Dina founded her company ACME.") assert len(ruler.patterns) == 3 assert len(doc.ents) == 2 assert doc.ents[0].label_ == "PERSON" - assert doc.ents[0].text == "Duygu" + assert doc.ents[0].text == "Dina" assert doc.ents[1].label_ == "ORG" assert doc.ents[1].text == "ACME" - ruler.remove("duygu") - doc = ruler(nlp.make_doc("Duygu founded her company ACME")) + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + doc = nlp("Dina founded her company ACME") assert len(ruler.patterns) == 2 assert len(doc.ents) == 1 assert doc.ents[0].label_ == "ORG" assert doc.ents[0].text == "ACME" - ruler.remove("acme") - doc = ruler(nlp.make_doc("Duygu founded her company ACME")) + if isinstance(ruler, EntityRuler): + ruler.remove("acme") + else: + ruler.remove_by_id("acme") + doc = nlp("Dina founded her company ACME") assert len(ruler.patterns) == 1 assert len(doc.ents) == 0 -def test_entity_ruler_remove_patterns_in_a_row(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_patterns_in_a_row(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "DATE", "pattern": "her birthday", "id": "bday"}, {"label": "ORG", "pattern": "ACM"}, ] ruler.add_patterns(patterns) - doc = ruler(nlp.make_doc("Duygu founded her company ACME on her birthday")) + doc = nlp("Dina founded her company ACME on her birthday") assert len(doc.ents) == 3 assert doc.ents[0].label_ == "PERSON" - assert doc.ents[0].text == "Duygu" + assert doc.ents[0].text == "Dina" assert doc.ents[1].label_ == "ORG" assert doc.ents[1].text == "ACME" assert doc.ents[2].label_ == "DATE" assert doc.ents[2].text == "her birthday" - ruler.remove("duygu") - ruler.remove("acme") - ruler.remove("bday") - doc = ruler(nlp.make_doc("Duygu went to school")) + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + ruler.remove("acme") + ruler.remove("bday") + else: + ruler.remove_by_id("dina") + ruler.remove_by_id("acme") + ruler.remove_by_id("bday") + doc = nlp("Dina went to school") assert len(doc.ents) == 0 -def test_entity_ruler_remove_all_patterns(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_all_patterns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [ - {"label": "PERSON", "pattern": "Duygu", "id": "duygu"}, + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, {"label": "ORG", "pattern": "ACME", "id": "acme"}, {"label": "DATE", "pattern": "her birthday", "id": "bday"}, ] ruler.add_patterns(patterns) assert len(ruler.patterns) == 3 - ruler.remove("duygu") + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") assert len(ruler.patterns) == 2 - ruler.remove("acme") + if isinstance(ruler, EntityRuler): + ruler.remove("acme") + else: + ruler.remove_by_id("acme") assert len(ruler.patterns) == 1 - ruler.remove("bday") + if isinstance(ruler, EntityRuler): + ruler.remove("bday") + else: + ruler.remove_by_id("bday") assert len(ruler.patterns) == 0 with pytest.warns(UserWarning): - doc = ruler(nlp.make_doc("Duygu founded her company ACME on her birthday")) + doc = nlp("Dina founded her company ACME on her birthday") assert len(doc.ents) == 0 -def test_entity_ruler_remove_and_add(nlp): - ruler = EntityRuler(nlp) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_and_add(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") patterns = [{"label": "DATE", "pattern": "last time"}] ruler.add_patterns(patterns) doc = ruler( @@ -524,7 +609,10 @@ def test_entity_ruler_remove_and_add(nlp): assert doc.ents[0].text == "last time" assert doc.ents[1].label_ == "DATE" assert doc.ents[1].text == "this time" - ruler.remove("ttime") + if isinstance(ruler, EntityRuler): + ruler.remove("ttime") + else: + ruler.remove_by_id("ttime") doc = ruler( nlp.make_doc("I saw him last time we met, this time he brought some flowers") ) @@ -547,7 +635,10 @@ def test_entity_ruler_remove_and_add(nlp): ) assert len(ruler.patterns) == 3 assert len(doc.ents) == 3 - ruler.remove("ttime") + if isinstance(ruler, EntityRuler): + ruler.remove("ttime") + else: + ruler.remove_by_id("ttime") doc = ruler( nlp.make_doc( "I saw him last time we met, this time he brought some flowers, another time some chocolate." diff --git a/spacy/tests/pipeline/test_span_ruler.py b/spacy/tests/pipeline/test_span_ruler.py new file mode 100644 index 000000000..794815359 --- /dev/null +++ b/spacy/tests/pipeline/test_span_ruler.py @@ -0,0 +1,465 @@ +import pytest + +import spacy +from spacy import registry +from spacy.errors import MatchPatternError +from spacy.tokens import Span +from spacy.training import Example +from spacy.tests.util import make_tempdir + +from thinc.api import NumpyOps, get_current_ops + + +@pytest.fixture +@registry.misc("span_ruler_patterns") +def patterns(): + return [ + {"label": "HELLO", "pattern": "hello world", "id": "hello1"}, + {"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]}, + {"label": "HELLO", "pattern": [{"ORTH": "HELLO"}], "id": "hello2"}, + {"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]}, + {"label": "TECH_ORG", "pattern": "Apple"}, + {"label": "TECH_ORG", "pattern": "Microsoft"}, + ] + + +@pytest.fixture +def overlapping_patterns(): + return [ + {"label": "FOOBAR", "pattern": "foo bar"}, + {"label": "BARBAZ", "pattern": "bar baz"}, + ] + + +@pytest.fixture +def person_org_patterns(): + return [ + {"label": "PERSON", "pattern": "Dina"}, + {"label": "ORG", "pattern": "ACME"}, + {"label": "ORG", "pattern": "ACM"}, + ] + + +@pytest.fixture +def person_org_date_patterns(person_org_patterns): + return person_org_patterns + [{"label": "DATE", "pattern": "June 14th"}] + + +def test_span_ruler_add_empty(patterns): + """Test that patterns don't get added excessively.""" + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"validate": True}) + ruler.add_patterns(patterns) + pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) + assert pattern_count > 0 + ruler.add_patterns([]) + after_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) + assert after_count == pattern_count + + +def test_span_ruler_init(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + assert "HELLO" in ruler + assert "BYE" in ruler + doc = nlp("hello world bye bye") + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][0].id_ == "hello1" + assert doc.spans["ruler"][1].label_ == "BYE" + assert doc.spans["ruler"][1].id_ == "" + + +def test_span_ruler_no_patterns_warns(): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + assert len(ruler) == 0 + assert len(ruler.labels) == 0 + assert nlp.pipe_names == ["span_ruler"] + with pytest.warns(UserWarning): + doc = nlp("hello world bye bye") + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_init_patterns(patterns): + # initialize with patterns + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + assert len(ruler.labels) == 0 + ruler.initialize(lambda: [], patterns=patterns) + assert len(ruler.labels) == 4 + doc = nlp("hello world bye bye") + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][1].label_ == "BYE" + nlp.remove_pipe("span_ruler") + # initialize with patterns from misc registry + nlp.config["initialize"]["components"]["span_ruler"] = { + "patterns": {"@misc": "span_ruler_patterns"} + } + ruler = nlp.add_pipe("span_ruler") + assert len(ruler.labels) == 0 + nlp.initialize() + assert len(ruler.labels) == 4 + doc = nlp("hello world bye bye") + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][1].label_ == "BYE" + + +def test_span_ruler_init_clear(patterns): + """Test that initialization clears patterns.""" + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler.labels) == 4 + ruler.initialize(lambda: []) + assert len(ruler.labels) == 0 + + +def test_span_ruler_clear(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler.labels) == 4 + doc = nlp("hello world") + assert len(doc.spans["ruler"]) == 1 + ruler.clear() + assert len(ruler.labels) == 0 + with pytest.warns(UserWarning): + doc = nlp("hello world") + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_existing(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"overwrite": False}) + ruler.add_patterns(patterns) + doc = nlp.make_doc("OH HELLO WORLD bye bye") + doc.spans["ruler"] = [doc[0:2]] + doc = nlp(doc) + assert len(doc.spans["ruler"]) == 3 + assert doc.spans["ruler"][0] == doc[0:2] + assert doc.spans["ruler"][1].label_ == "HELLO" + assert doc.spans["ruler"][1].id_ == "hello2" + assert doc.spans["ruler"][2].label_ == "BYE" + assert doc.spans["ruler"][2].id_ == "" + + +def test_span_ruler_existing_overwrite(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"overwrite": True}) + ruler.add_patterns(patterns) + doc = nlp.make_doc("OH HELLO WORLD bye bye") + doc.spans["ruler"] = [doc[0:2]] + doc = nlp(doc) + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][0].text == "HELLO" + assert doc.spans["ruler"][1].label_ == "BYE" + + +def test_span_ruler_serialize_bytes(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + ruler_bytes = ruler.to_bytes() + new_nlp = spacy.blank("xx") + new_ruler = new_nlp.add_pipe("span_ruler") + assert len(new_ruler) == 0 + assert len(new_ruler.labels) == 0 + new_ruler = new_ruler.from_bytes(ruler_bytes) + assert len(new_ruler) == len(patterns) + assert len(new_ruler.labels) == 4 + assert len(new_ruler.patterns) == len(ruler.patterns) + for pattern in ruler.patterns: + assert pattern in new_ruler.patterns + assert sorted(new_ruler.labels) == sorted(ruler.labels) + + +def test_span_ruler_validate(): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + validated_ruler = nlp.add_pipe( + "span_ruler", name="validated_span_ruler", config={"validate": True} + ) + + valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]} + invalid_pattern = {"label": "HELLO", "pattern": [{"ASDF": "HELLO"}]} + + # invalid pattern raises error without validate + with pytest.raises(ValueError): + ruler.add_patterns([invalid_pattern]) + + # valid pattern is added without errors with validate + validated_ruler.add_patterns([valid_pattern]) + + # invalid pattern raises error with validate + with pytest.raises(MatchPatternError): + validated_ruler.add_patterns([invalid_pattern]) + + +def test_span_ruler_properties(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"overwrite": True}) + ruler.add_patterns(patterns) + assert sorted(ruler.labels) == sorted(set([p["label"] for p in patterns])) + + +def test_span_ruler_overlapping_spans(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(overlapping_patterns) + doc = ruler(nlp.make_doc("foo bar baz")) + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "FOOBAR" + assert doc.spans["ruler"][1].label_ == "BARBAZ" + + +def test_span_ruler_scorer(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(overlapping_patterns) + text = "foo bar baz" + pred_doc = ruler(nlp.make_doc(text)) + assert len(pred_doc.spans["ruler"]) == 2 + assert pred_doc.spans["ruler"][0].label_ == "FOOBAR" + assert pred_doc.spans["ruler"][1].label_ == "BARBAZ" + + ref_doc = nlp.make_doc(text) + ref_doc.spans["ruler"] = [Span(ref_doc, 0, 2, label="FOOBAR")] + scores = nlp.evaluate([Example(pred_doc, ref_doc)]) + assert scores["spans_ruler_p"] == 0.5 + assert scores["spans_ruler_r"] == 1.0 + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_span_ruler_multiprocessing(n_process): + if isinstance(get_current_ops, NumpyOps) or n_process < 2: + texts = ["I enjoy eating Pizza Hut pizza."] + + patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut"}] + + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + + for doc in nlp.pipe(texts, n_process=2): + for ent in doc.spans["ruler"]: + assert ent.label_ == "FASTFOOD" + + +def test_span_ruler_serialize_dir(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + with make_tempdir() as d: + ruler.to_disk(d / "test_ruler") + ruler.from_disk(d / "test_ruler") # read from an existing directory + with pytest.raises(ValueError): + ruler.from_disk(d / "non_existing_dir") # read from a bad directory + + +def test_span_ruler_remove_basic(person_org_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_patterns) + doc = ruler(nlp.make_doc("Dina went to school")) + assert len(ruler.patterns) == 3 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "PERSON" + assert doc.spans["ruler"][0].text == "Dina" + ruler.remove("PERSON") + doc = ruler(nlp.make_doc("Dina went to school")) + assert len(doc.spans["ruler"]) == 0 + assert len(ruler.patterns) == 2 + + +def test_span_ruler_remove_nonexisting_pattern(person_org_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_patterns) + assert len(ruler.patterns) == 3 + with pytest.raises(ValueError): + ruler.remove("NE") + with pytest.raises(ValueError): + ruler.remove_by_id("NE") + + +def test_span_ruler_remove_several_patterns(person_org_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_patterns) + doc = ruler(nlp.make_doc("Dina founded the company ACME.")) + assert len(ruler.patterns) == 3 + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "PERSON" + assert doc.spans["ruler"][0].text == "Dina" + assert doc.spans["ruler"][1].label_ == "ORG" + assert doc.spans["ruler"][1].text == "ACME" + ruler.remove("PERSON") + doc = ruler(nlp.make_doc("Dina founded the company ACME")) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "ORG" + assert doc.spans["ruler"][0].text == "ACME" + ruler.remove("ORG") + with pytest.warns(UserWarning): + doc = ruler(nlp.make_doc("Dina founded the company ACME")) + assert len(ruler.patterns) == 0 + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_remove_patterns_in_a_row(person_org_date_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_date_patterns) + doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th")) + assert len(doc.spans["ruler"]) == 3 + assert doc.spans["ruler"][0].label_ == "PERSON" + assert doc.spans["ruler"][0].text == "Dina" + assert doc.spans["ruler"][1].label_ == "ORG" + assert doc.spans["ruler"][1].text == "ACME" + assert doc.spans["ruler"][2].label_ == "DATE" + assert doc.spans["ruler"][2].text == "June 14th" + ruler.remove("ORG") + ruler.remove("DATE") + doc = ruler(nlp.make_doc("Dina went to school")) + assert len(doc.spans["ruler"]) == 1 + + +def test_span_ruler_remove_all_patterns(person_org_date_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_date_patterns) + assert len(ruler.patterns) == 4 + ruler.remove("PERSON") + assert len(ruler.patterns) == 3 + ruler.remove("ORG") + assert len(ruler.patterns) == 1 + ruler.remove("DATE") + assert len(ruler.patterns) == 0 + with pytest.warns(UserWarning): + doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th")) + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_remove_and_add(): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + patterns1 = [{"label": "DATE1", "pattern": "last time"}] + ruler.add_patterns(patterns1) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 1 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "DATE1" + assert doc.spans["ruler"][0].text == "last time" + patterns2 = [{"label": "DATE2", "pattern": "this time"}] + ruler.add_patterns(patterns2) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "DATE1" + assert doc.spans["ruler"][0].text == "last time" + assert doc.spans["ruler"][1].label_ == "DATE2" + assert doc.spans["ruler"][1].text == "this time" + ruler.remove("DATE1") + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 1 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "DATE2" + assert doc.spans["ruler"][0].text == "this time" + ruler.add_patterns(patterns1) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 2 + patterns3 = [{"label": "DATE3", "pattern": "another time"}] + ruler.add_patterns(patterns3) + doc = ruler( + nlp.make_doc( + "I saw him last time we met, this time he brought some flowers, another time some chocolate." + ) + ) + assert len(ruler.patterns) == 3 + assert len(doc.spans["ruler"]) == 3 + ruler.remove("DATE3") + doc = ruler( + nlp.make_doc( + "I saw him last time we met, this time he brought some flowers, another time some chocolate." + ) + ) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 2 + + +def test_span_ruler_spans_filter(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe( + "span_ruler", + config={"spans_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}}, + ) + ruler.add_patterns(overlapping_patterns) + doc = ruler(nlp.make_doc("foo bar baz")) + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "FOOBAR" + + +def test_span_ruler_ents_default_filter(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"annotate_ents": True}) + ruler.add_patterns(overlapping_patterns) + doc = ruler(nlp.make_doc("foo bar baz")) + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "FOOBAR" + + +def test_span_ruler_ents_overwrite_filter(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe( + "span_ruler", + config={ + "annotate_ents": True, + "overwrite": False, + "ents_filter": {"@misc": "spacy.prioritize_new_ents_filter.v1"}, + }, + ) + ruler.add_patterns(overlapping_patterns) + # overlapping ents are clobbered, non-overlapping ents are preserved + doc = nlp.make_doc("foo bar baz a b c") + doc.ents = [Span(doc, 1, 3, label="BARBAZ"), Span(doc, 3, 6, label="ABC")] + doc = ruler(doc) + assert len(doc.ents) == 2 + assert doc.ents[0].label_ == "FOOBAR" + assert doc.ents[1].label_ == "ABC" + + +def test_span_ruler_ents_bad_filter(overlapping_patterns): + @registry.misc("test_pass_through_filter") + def make_pass_through_filter(): + def pass_through_filter(spans1, spans2): + return spans1 + spans2 + + return pass_through_filter + + nlp = spacy.blank("xx") + ruler = nlp.add_pipe( + "span_ruler", + config={ + "annotate_ents": True, + "ents_filter": {"@misc": "test_pass_through_filter"}, + }, + ) + ruler.add_patterns(overlapping_patterns) + with pytest.raises(ValueError): + ruler(nlp.make_doc("foo bar baz")) diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index d25247b13..c0b67fb7c 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -516,7 +516,7 @@ cdef class Doc: def doc(self): return self - def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, alignment_mode="strict"): + def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, alignment_mode="strict", span_id=0): """Create a `Span` object from the slice `doc.text[start_idx : end_idx]`. Returns None if no valid `Span` can be created. @@ -575,7 +575,7 @@ cdef class Doc: start += 1 # Currently we have the token index, we want the range-end index end += 1 - cdef Span span = Span(self, start, end, label=label, kb_id=kb_id, vector=vector) + cdef Span span = Span(self, start, end, label=label, kb_id=kb_id, span_id=span_id, vector=vector) return span def similarity(self, other): @@ -713,6 +713,7 @@ cdef class Doc: cdef int start = -1 cdef attr_t label = 0 cdef attr_t kb_id = 0 + cdef attr_t ent_id = 0 output = [] for i in range(self.length): token = &self.c[i] @@ -723,18 +724,20 @@ cdef class Doc: elif token.ent_iob == 2 or token.ent_iob == 0 or \ (token.ent_iob == 3 and token.ent_type == 0): if start != -1: - output.append(Span(self, start, i, label=label, kb_id=kb_id)) + output.append(Span(self, start, i, label=label, kb_id=kb_id, span_id=ent_id)) start = -1 label = 0 kb_id = 0 + ent_id = 0 elif token.ent_iob == 3: if start != -1: - output.append(Span(self, start, i, label=label, kb_id=kb_id)) + output.append(Span(self, start, i, label=label, kb_id=kb_id, span_id=ent_id)) start = i label = token.ent_type kb_id = token.ent_kb_id + ent_id = token.ent_id if start != -1: - output.append(Span(self, start, self.length, label=label, kb_id=kb_id)) + output.append(Span(self, start, self.length, label=label, kb_id=kb_id, span_id=ent_id)) # remove empty-label spans output = [o for o in output if o.label_ != ""] return tuple(output) @@ -743,14 +746,14 @@ cdef class Doc: # TODO: # 1. Test basic data-driven ORTH gazetteer # 2. Test more nuanced date and currency regex - cdef attr_t entity_type, kb_id + cdef attr_t entity_type, kb_id, ent_id cdef int ent_start, ent_end ent_spans = [] for ent_info in ents: - entity_type_, kb_id, ent_start, ent_end = get_entity_info(ent_info) + entity_type_, kb_id, ent_start, ent_end, ent_id = get_entity_info(ent_info) if isinstance(entity_type_, str): self.vocab.strings.add(entity_type_) - span = Span(self, ent_start, ent_end, label=entity_type_, kb_id=kb_id) + span = Span(self, ent_start, ent_end, label=entity_type_, kb_id=kb_id, span_id=ent_id) ent_spans.append(span) self.set_ents(ent_spans, default=SetEntsDefault.outside) @@ -801,6 +804,9 @@ cdef class Doc: self.c[i].ent_iob = 1 self.c[i].ent_type = span.label self.c[i].ent_kb_id = span.kb_id + # for backwards compatibility in v3, only set ent_id from + # span.id if it's set, otherwise don't override + self.c[i].ent_id = span.id if span.id else self.c[i].ent_id for span in blocked: for i in range(span.start, span.end): self.c[i].ent_iob = 3 @@ -1180,6 +1186,7 @@ cdef class Doc: span.end_char + char_offset, span.label, span.kb_id, + span.id, span.text, # included as a check )) char_offset += len(doc.text) @@ -1215,8 +1222,9 @@ cdef class Doc: span_tuple[1], label=span_tuple[2], kb_id=span_tuple[3], + span_id=span_tuple[4], ) - text = span_tuple[4] + text = span_tuple[5] if span is not None and span.text == text: concat_doc.spans[key].append(span) else: @@ -1772,16 +1780,18 @@ def fix_attributes(doc, attributes): def get_entity_info(ent_info): + ent_kb_id = 0 + ent_id = 0 if isinstance(ent_info, Span): ent_type = ent_info.label ent_kb_id = ent_info.kb_id start = ent_info.start end = ent_info.end + ent_id = ent_info.id elif len(ent_info) == 3: ent_type, start, end = ent_info - ent_kb_id = 0 elif len(ent_info) == 4: ent_type, ent_kb_id, start, end = ent_info else: ent_id, ent_kb_id, ent_type, start, end = ent_info - return ent_type, ent_kb_id, start, end + return ent_type, ent_kb_id, start, end, ent_id diff --git a/spacy/tokens/span.pyi b/spacy/tokens/span.pyi index 697051e81..4a4149652 100644 --- a/spacy/tokens/span.pyi +++ b/spacy/tokens/span.pyi @@ -48,7 +48,8 @@ class Span: label: Union[str, int] = ..., vector: Optional[Floats1d] = ..., vector_norm: Optional[float] = ..., - kb_id: Optional[int] = ..., + kb_id: Union[str, int] = ..., + span_id: Union[str, int] = ..., ) -> None: ... def __richcmp__(self, other: Span, op: int) -> bool: ... def __hash__(self) -> int: ... diff --git a/spacy/tokens/span.pyx b/spacy/tokens/span.pyx index 305d7caf4..ab888ae95 100644 --- a/spacy/tokens/span.pyx +++ b/spacy/tokens/span.pyx @@ -80,17 +80,20 @@ cdef class Span: return Underscore.span_extensions.pop(name) def __cinit__(self, Doc doc, int start, int end, label=0, vector=None, - vector_norm=None, kb_id=0): + vector_norm=None, kb_id=0, span_id=0): """Create a `Span` object from the slice `doc[start : end]`. doc (Doc): The parent document. start (int): The index of the first token of the span. end (int): The index of the first token after the span. - label (int or str): A label to attach to the Span, e.g. for named entities. + label (Union[int, str]): A label to attach to the Span, e.g. for named + entities. vector (ndarray[ndim=1, dtype='float32']): A meaning representation of the span. vector_norm (float): The L2 norm of the span's vector representation. - kb_id (uint64): An identifier from a Knowledge Base to capture the meaning of a named entity. + kb_id (Union[int, str]): An identifier from a Knowledge Base to capture + the meaning of a named entity. + span_id (Union[int, str]): An identifier to associate with the span. DOCS: https://spacy.io/api/span#init """ @@ -101,6 +104,8 @@ cdef class Span: label = doc.vocab.strings.add(label) if isinstance(kb_id, str): kb_id = doc.vocab.strings.add(kb_id) + if isinstance(span_id, str): + span_id = doc.vocab.strings.add(span_id) if label not in doc.vocab.strings: raise ValueError(Errors.E084.format(label=label)) @@ -112,6 +117,7 @@ cdef class Span: self.c = SpanC( label=label, kb_id=kb_id, + id=span_id, start=start, end=end, start_char=start_char, @@ -126,8 +132,8 @@ cdef class Span: return False else: return True - self_tuple = (self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.doc) - other_tuple = (other.c.start_char, other.c.end_char, other.c.label, other.c.kb_id, other.doc) + self_tuple = (self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.id, self.doc) + other_tuple = (other.c.start_char, other.c.end_char, other.c.label, other.c.kb_id, other.id, other.doc) # < if op == 0: return self_tuple < other_tuple @@ -148,7 +154,7 @@ cdef class Span: return self_tuple >= other_tuple def __hash__(self): - return hash((self.doc, self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id)) + return hash((self.doc, self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.c.id)) def __len__(self): """Get the number of tokens in the span. @@ -632,7 +638,7 @@ cdef class Span: else: return self.doc[root] - def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None): + def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, id=0): """Create a `Span` object from the slice `span.text[start : end]`. start (int): The index of the first character of the span. @@ -774,6 +780,13 @@ cdef class Span: def __set__(self, attr_t kb_id): self.c.kb_id = kb_id + property id: + def __get__(self): + return self.c.id + + def __set__(self, attr_t id): + self.c.id = id + property ent_id: """RETURNS (uint64): The entity ID.""" def __get__(self): @@ -812,13 +825,21 @@ cdef class Span: self.label = self.doc.vocab.strings.add(label_) property kb_id_: - """RETURNS (str): The named entity's KB ID.""" + """RETURNS (str): The span's KB ID.""" def __get__(self): return self.doc.vocab.strings[self.kb_id] def __set__(self, str kb_id_): self.kb_id = self.doc.vocab.strings.add(kb_id_) + property id_: + """RETURNS (str): The span's ID.""" + def __get__(self): + return self.doc.vocab.strings[self.id] + + def __set__(self, str id_): + self.id = self.doc.vocab.strings.add(id_) + cdef int _count_words_to_root(const TokenC* token, int sent_length) except -1: # Don't allow spaces to be the root, if there are diff --git a/spacy/util.py b/spacy/util.py index 5ca6e4032..0111c839e 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -1242,6 +1242,15 @@ def filter_spans(spans: Iterable["Span"]) -> List["Span"]: return result +def filter_chain_spans(*spans: Iterable["Span"]) -> List["Span"]: + return filter_spans(itertools.chain(*spans)) + + +@registry.misc("spacy.first_longest_spans_filter.v1") +def make_first_longest_spans_filter(): + return filter_chain_spans + + def to_bytes(getters: Dict[str, Callable[[], bytes]], exclude: Iterable[str]) -> bytes: return srsly.msgpack_dumps(to_dict(getters, exclude)) diff --git a/website/docs/api/entityruler.md b/website/docs/api/entityruler.md index 1ef283870..c2ba33f01 100644 --- a/website/docs/api/entityruler.md +++ b/website/docs/api/entityruler.md @@ -290,7 +290,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. > > ```python > ruler_bytes = ruler.to_bytes() -> ruler = nlp.add_pipe("enity_ruler") +> ruler = nlp.add_pipe("entity_ruler") > ruler.from_bytes(ruler_bytes) > ``` diff --git a/website/docs/api/span.md b/website/docs/api/span.md index d765a199c..89f608994 100644 --- a/website/docs/api/span.md +++ b/website/docs/api/span.md @@ -27,6 +27,7 @@ Create a `Span` object from the slice `doc[start : end]`. | `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | | `vector_norm` | The L2 norm of the document's vector representation. ~~float~~ | | `kb_id` | A knowledge base ID to attach to the span, e.g. for named entities. ~~Union[str, int]~~ | +| `span_id` | An ID to associate with the span. ~~Union[str, int]~~ | ## Span.\_\_getitem\_\_ {#getitem tag="method"} @@ -560,7 +561,9 @@ overlaps with will be returned. | `lemma_` | The span's lemma. Equivalent to `"".join(token.text_with_ws for token in span)`. ~~str~~ | | `kb_id` | The hash value of the knowledge base ID referred to by the span. ~~int~~ | | `kb_id_` | The knowledge base ID referred to by the span. ~~str~~ | -| `ent_id` | The hash value of the named entity the token is an instance of. ~~int~~ | -| `ent_id_` | The string ID of the named entity the token is an instance of. ~~str~~ | +| `ent_id` | The hash value of the named entity the root token is an instance of. ~~int~~ | +| `ent_id_` | The string ID of the named entity the root token is an instance of. ~~str~~ | +| `id` | The hash value of the span's ID. ~~int~~ | +| `id_` | The span's ID. ~~str~~ | | `sentiment` | A scalar value indicating the positivity or negativity of the span. ~~float~~ | | `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | diff --git a/website/docs/api/spanruler.md b/website/docs/api/spanruler.md new file mode 100644 index 000000000..b573f7c58 --- /dev/null +++ b/website/docs/api/spanruler.md @@ -0,0 +1,351 @@ +--- +title: SpanRuler +tag: class +source: spacy/pipeline/span_ruler.py +new: 3.3 +teaser: 'Pipeline component for rule-based span and named entity recognition' +api_string_name: span_ruler +api_trainable: false +--- + +The span ruler lets you add spans to [`Doc.spans`](/api/doc#spans) and/or +[`Doc.ents`](/api/doc#ents) using token-based rules or exact phrase matches. For +usage examples, see the docs on +[rule-based span matching](/usage/rule-based-matching#spanruler). + +## Assigned Attributes {#assigned-attributes} + +Matches will be saved to `Doc.spans[spans_key]` as a +[`SpanGroup`](/api/spangroup) and/or to `Doc.ents`, where the annotation is +saved in the `Token.ent_type` and `Token.ent_iob` fields. + +| Location | Value | +| ---------------------- | ----------------------------------------------------------------- | +| `Doc.spans[spans_key]` | The annotated spans. ~~SpanGroup~~ | +| `Doc.ents` | The annotated spans. ~~Tuple[Span]~~ | +| `Token.ent_iob` | An enum encoding of the IOB part of the named entity tag. ~~int~~ | +| `Token.ent_iob_` | The IOB part of the named entity tag. ~~str~~ | +| `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | +| `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | + +## Config and implementation {#config} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg`](/usage/training#config). + +> #### Example +> +> ```python +> config = { +> "spans_key": "my_spans", +> "validate": True, +> "overwrite": False, +> } +> nlp.add_pipe("span_ruler", config=config) +> ``` + +| Setting | Description | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/span_ruler.py +``` + +## SpanRuler.\_\_init\_\_ {#init tag="method"} + +Initialize the span ruler. If patterns are supplied here, they need to be a list +of dictionaries with a `"label"` and `"pattern"` key. A pattern can either be a +token pattern (list) or a phrase pattern (string). For example: +`{"label": "ORG", "pattern": "Apple"}`. + +> #### Example +> +> ```python +> # Construction via add_pipe +> ruler = nlp.add_pipe("span_ruler") +> +> # Construction from class +> from spacy.pipeline import SpanRuler +> ruler = SpanRuler(nlp, overwrite=True) +> ``` + +| Name | Description | +| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | +| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ | +| _keyword-only_ | | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | + +## SpanRuler.initialize {#initialize tag="method"} + +Initialize the component with data and used before training to load in rules +from a [pattern file](/usage/rule-based-matching/#spanruler-files). This method +is typically called by [`Language.initialize`](/api/language#initialize) and +lets you customize arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. Any existing patterns are removed on initialization. + +> #### Example +> +> ```python +> span_ruler = nlp.add_pipe("span_ruler") +> span_ruler.initialize(lambda: [], nlp=nlp, patterns=patterns) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.span_ruler] +> +> [initialize.components.span_ruler.patterns] +> @readers = "srsly.read_jsonl.v1" +> path = "corpus/span_ruler_patterns.jsonl +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Not used by the `SpanRuler`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ | + +## SpanRuler.\_\len\_\_ {#len tag="method"} + +The number of all patterns added to the span ruler. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> assert len(ruler) == 0 +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> assert len(ruler) == 1 +> ``` + +| Name | Description | +| ----------- | ------------------------------- | +| **RETURNS** | The number of patterns. ~~int~~ | + +## SpanRuler.\_\_contains\_\_ {#contains tag="method"} + +Whether a label is present in the patterns. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> assert "ORG" in ruler +> assert not "PERSON" in ruler +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| `label` | The label to check. ~~str~~ | +| **RETURNS** | Whether the span ruler contains the label. ~~bool~~ | + +## SpanRuler.\_\_call\_\_ {#call tag="method"} + +Find matches in the `Doc` and add them to `doc.spans[span_key]` and/or +`doc.ents`. Typically, this happens automatically after the component has been +added to the pipeline using [`nlp.add_pipe`](/api/language#add_pipe). If the +span ruler was initialized with `overwrite=True`, existing spans and entities +will be removed. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> +> doc = nlp("A text about Apple.") +> spans = [(span.text, span.label_) for span in doc.spans["ruler"]] +> assert spans == [("Apple", "ORG")] +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | +| **RETURNS** | The modified `Doc` with added spans/entities. ~~Doc~~ | + +## SpanRuler.add_patterns {#add_patterns tag="method"} + +Add patterns to the span ruler. A pattern can either be a token pattern (list of +dicts) or a phrase pattern (string). For more details, see the usage guide on +[rule-based matching](/usage/rule-based-matching). + +> #### Example +> +> ```python +> patterns = [ +> {"label": "ORG", "pattern": "Apple"}, +> {"label": "GPE", "pattern": [{"lower": "san"}, {"lower": "francisco"}]} +> ] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ``` + +| Name | Description | +| ---------- | ---------------------------------------------------------------- | +| `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ | + +## SpanRuler.remove {#remove tag="method"} + +Remove patterns by label from the span ruler. A `ValueError` is raised if the +label does not exist in any patterns. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ruler.remove("ORG") +> ``` + +| Name | Description | +| ------- | -------------------------------------- | +| `label` | The label of the pattern rule. ~~str~~ | + +## SpanRuler.remove_by_id {#remove_by_id tag="method"} + +Remove patterns by ID from the span ruler. A `ValueError` is raised if the ID +does not exist in any patterns. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ruler.remove_by_id("apple") +> ``` + +| Name | Description | +| ------------ | ----------------------------------- | +| `pattern_id` | The ID of the pattern rule. ~~str~~ | + +## SpanRuler.clear {#clear tag="method"} + +Remove all patterns the span ruler. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ruler.clear() +> ``` + +## SpanRuler.to_disk {#to_disk tag="method"} + +Save the span ruler patterns to a directory. The patterns will be saved as +newline-delimited JSON (JSONL). + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.to_disk("/path/to/span_ruler") +> ``` + +| Name | Description | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | + +## SpanRuler.from_disk {#from_disk tag="method"} + +Load the span ruler from a path. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.from_disk("/path/to/span_ruler") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | + +## SpanRuler.to_bytes {#to_bytes tag="method"} + +Serialize the span ruler to a bytestring. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler_bytes = ruler.to_bytes() +> ``` + +| Name | Description | +| ----------- | ---------------------------------- | +| **RETURNS** | The serialized patterns. ~~bytes~~ | + +## SpanRuler.from_bytes {#from_bytes tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> ruler_bytes = ruler.to_bytes() +> ruler = nlp.add_pipe("span_ruler") +> ruler.from_bytes(ruler_bytes) +> ``` + +| Name | Description | +| ------------ | ---------------------------------------------- | +| `bytes_data` | The bytestring to load. ~~bytes~~ | +| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | + +## SpanRuler.labels {#labels tag="property"} + +All labels present in the match patterns. + +| Name | Description | +| ----------- | -------------------------------------- | +| **RETURNS** | The string labels. ~~Tuple[str, ...]~~ | + +## SpanRuler.ids {#ids tag="property"} + +All IDs present in the `id` property of the match patterns. + +| Name | Description | +| ----------- | ----------------------------------- | +| **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ | + +## SpanRuler.patterns {#patterns tag="property"} + +All patterns that were added to the span ruler. + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------- | +| **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ | + +## Attributes {#attributes} + +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------- | +| `key` | The spans key that spans are saved under. ~~Optional[str]~~ | +| `matcher` | The underlying matcher used to process token patterns. ~~Matcher~~ | +| `phrase_matcher` | The underlying phrase matcher used to process phrase patterns. ~~PhraseMatcher~~ | diff --git a/website/docs/usage/rule-based-matching.md b/website/docs/usage/rule-based-matching.md index bf654c14f..13612ac35 100644 --- a/website/docs/usage/rule-based-matching.md +++ b/website/docs/usage/rule-based-matching.md @@ -6,6 +6,7 @@ menu: - ['Phrase Matcher', 'phrasematcher'] - ['Dependency Matcher', 'dependencymatcher'] - ['Entity Ruler', 'entityruler'] + - ['Span Ruler', 'spanruler'] - ['Models & Rules', 'models-rules'] --- @@ -1446,6 +1447,108 @@ with nlp.select_pipes(enable="tagger"): ruler.add_patterns(patterns) ``` +## Rule-based span matching {#spanruler new="3.3"} + +The [`SpanRuler`](/api/spanruler) is a generalized version of the entity ruler +that lets you add spans to `doc.spans` or `doc.ents` based on pattern +dictionaries, which makes it easy to combine rule-based and statistical pipeline +components. + +### Span patterns {#spanruler-patterns} + +The [pattern format](#entityruler-patterns) is the same as for the entity ruler: + +1. **Phrase patterns** for exact string matches (string). + + ```python + {"label": "ORG", "pattern": "Apple"} + ``` + +2. **Token patterns** with one dictionary describing one token (list). + + ```python + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]} + ``` + +### Using the span ruler {#spanruler-usage} + +The [`SpanRuler`](/api/spanruler) is a pipeline component that's typically added +via [`nlp.add_pipe`](/api/language#add_pipe). When the `nlp` object is called on +a text, it will find matches in the `doc` and add them as spans to +`doc.spans["ruler"]`, using the specified pattern label as the entity label. +Unlike in `doc.ents`, overlapping matches are allowed in `doc.spans`, so no +filtering is required, but optional filtering and sorting can be applied to the +spans before they're saved. + +```python +### {executable="true"} +import spacy + +nlp = spacy.blank("en") +ruler = nlp.add_pipe("span_ruler") +patterns = [{"label": "ORG", "pattern": "Apple"}, + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]}] +ruler.add_patterns(patterns) + +doc = nlp("Apple is opening its first big office in San Francisco.") +print([(span.text, span.label_) for span in doc.spans["ruler"]]) +``` + +The span ruler is designed to integrate with spaCy's existing pipeline +components and enhance the [SpanCategorizer](/api/spancat) and +[EntityRecognizer](/api/entityrecognizer). The `overwrite` setting determines +whether the existing annotation in `doc.spans` or `doc.ents` is preserved. +Because overlapping entities are not allowed for `doc.ents`, the entities are +always filtered, using [`util.filter_spans`](/api/top-level#util.filter_spans) +by default. See the [`SpanRuler` API docs](/api/spanruler) for more information +about how to customize the sorting and filtering of matched spans. + +```python +### {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +# only annotate doc.ents, not doc.spans +config = {"spans_key": None, "annotate_ents": True, "overwrite": False} +ruler = nlp.add_pipe("span_ruler", config=config) +patterns = [{"label": "ORG", "pattern": "MyCorp Inc."}] +ruler.add_patterns(patterns) + +doc = nlp("MyCorp Inc. is a company in the U.S.") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +### Using pattern files {#spanruler-files} + +You can save patterns in a JSONL file (newline-delimited JSON) to load with +[`SpanRuler.initialize`](/api/spanruler#initialize) or +[`SpanRuler.add_patterns`](/api/spanruler#add_patterns). + +```json +### patterns.jsonl +{"label": "ORG", "pattern": "Apple"} +{"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]} +``` + +```python +import srsly + +patterns = srsly.read_jsonl("patterns.jsonl") +ruler = nlp.add_pipe("span_ruler") +ruler.add_patterns(patterns) +``` + + + +Unlike the entity ruler, the span ruler cannot load patterns on initialization +with `SpanRuler(patterns=patterns)` or directly from a JSONL file path with +`SpanRuler.from_disk(jsonl_path)`. Patterns should be loaded from the JSONL file +separately and then added through +[`SpanRuler.initialize`](/api/spanruler#initialize]) or +[`SpanRuler.add_patterns`](/api/spanruler#add_patterns) as shown above. + + + ## Combining models and rules {#models-rules} You can combine statistical and rule-based components in a variety of ways. diff --git a/website/meta/sidebars.json b/website/meta/sidebars.json index cf3f1398e..c23f0a255 100644 --- a/website/meta/sidebars.json +++ b/website/meta/sidebars.json @@ -103,6 +103,7 @@ { "text": "SentenceRecognizer", "url": "/api/sentencerecognizer" }, { "text": "Sentencizer", "url": "/api/sentencizer" }, { "text": "SpanCategorizer", "url": "/api/spancategorizer" }, + { "text": "SpanRuler", "url": "/api/spanruler" }, { "text": "Tagger", "url": "/api/tagger" }, { "text": "TextCategorizer", "url": "/api/textcategorizer" }, { "text": "Tok2Vec", "url": "/api/tok2vec" }, From 8387ce4c01db48d92ac5638e18316c0f1fc8861e Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Thu, 2 Jun 2022 14:03:47 +0200 Subject: [PATCH 12/24] Add Doc.from_json() (#10688) * Implement Doc.from_json: rough draft. * Implement Doc.from_json: first draft with tests. * Implement Doc.from_json: added documentation on website for Doc.to_json(), Doc.from_json(). * Implement Doc.from_json: formatting changes. * Implement Doc.to_json(): reverting unrelated formatting changes. * Implement Doc.to_json(): fixing entity and span conversion. Moving fixture and doc <-> json conversion tests into single file. * Implement Doc.from_json(): replaced entity/span converters with doc.char_span() calls. * Implement Doc.from_json(): handling sentence boundaries in spans. * Implementing Doc.from_json(): added parser-free sentence boundaries transfer. * Implementing Doc.from_json(): added parser-free sentence boundaries transfer. * Implementing Doc.from_json(): incorporated various PR feedback. * Renaming fixture for document without dependencies. Co-authored-by: Adriane Boyd * Implementing Doc.from_json(): using two sent_starts instead of one. Co-authored-by: Adriane Boyd * Implementing Doc.from_json(): doc_without_dependency_parser() -> doc_without_deps. Co-authored-by: Adriane Boyd * Implementing Doc.from_json(): incorporating various PR feedback. Rebased on latest master. * Implementing Doc.from_json(): refactored Doc.from_json() to work with annotation IDs instead of their string representations. * Implement Doc.from_json(): reverting unwanted formatting/rebasing changes. * Implement Doc.from_json(): added check for char_span() calculation for entities. * Update spacy/tokens/doc.pyx Co-authored-by: Adriane Boyd * Implement Doc.from_json(): minor refactoring, additional check for token attribute consistency with corresponding test. * Implement Doc.from_json(): removed redundancy in annotation type key naming. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): Simplifying setting annotation values. Co-authored-by: Adriane Boyd * Implement doc.from_json(): renaming annot_types to token_attrs. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): adjustments for renaming of annot_types to token_attrs. * Implement Doc.from_json(): removing default categories. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): simplifying lexeme initialization. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): simplifying lexeme initialization. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): refactoring to only have keys for present annotations. * Implement Doc.from_json(): fix check for tokens' HEAD attributes. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): refactoring Doc.from_json(). * Implement Doc.from_json(): fixing span_group retrieval. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): fixing span retrieval. * Implement Doc.from_json(): added schema for Doc JSON format. Minor refactoring in Doc.from_json(). * Implement Doc.from_json(): added comment regarding Token and Span extension support. * Implement Doc.from_json(): renaming inconsistent_props to partial_attrs.. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): adjusting error message. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): extending E1038 message. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): added params to E1038 raises. * Implement Doc.from_json(): combined attribute collection with partial attributes check. * Implement Doc.from_json(): added optional schema validation. * Implement Doc.from_json(): fixed optional fields in schema, tests. * Implement Doc.from_json(): removed redundant None check for DEP. * Implement Doc.from_json(): added passing of schema validatoin message to E1037.. * Implement Doc.from_json(): removing redundant error E1040. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): changing message for E1037. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): adjusted website docs and docstring of Doc.from_json(). * Update spacy/tests/doc/test_json_doc_conversion.py * Implement Doc.from_json(): docstring update. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): docstring update. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): website docs update. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): docstring formatting. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): docstring formatting. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): fixing Doc reference in website docs. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): reformatted website/docs/api/doc.md. * Implement Doc.from_json(): bumped IDs of new errors to avoid merge conflicts. * Implement Doc.from_json(): fixing bug in tests. Co-authored-by: Adriane Boyd * Implement Doc.from_json(): fix setting of sentence starts for docs without DEP. * Implement Doc.from_json(): add check for valid char spans when manually setting sentence boundaries. Refactor sentence boundary setting slightly. Move error message for lack of support for partial token annotations to errors.py. * Implement Doc.from_json(): simplify token sentence start manipulation. Co-authored-by: Adriane Boyd * Combine related error messages * Update spacy/tests/doc/test_json_doc_conversion.py Co-authored-by: Adriane Boyd --- spacy/errors.py | 5 + spacy/schemas.py | 26 +++ spacy/tests/doc/test_json_doc_conversion.py | 191 ++++++++++++++++++++ spacy/tests/doc/test_to_json.py | 72 -------- spacy/tokens/doc.pyi | 3 + spacy/tokens/doc.pyx | 147 ++++++++++++++- website/docs/api/doc.md | 39 ++++ 7 files changed, 405 insertions(+), 78 deletions(-) create mode 100644 spacy/tests/doc/test_json_doc_conversion.py delete mode 100644 spacy/tests/doc/test_to_json.py diff --git a/spacy/errors.py b/spacy/errors.py index 665b91659..75e24789f 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -921,6 +921,11 @@ class Errors(metaclass=ErrorsWithCodes): E1035 = ("Token index {i} out of bounds ({length})") E1036 = ("Cannot index into NoneNode") E1037 = ("Invalid attribute value '{attr}'.") + E1038 = ("Invalid JSON input: {message}") + E1039 = ("The {obj} start or end annotations (start: {start}, end: {end}) " + "could not be aligned to token boundaries.") + E1040 = ("Doc.from_json requires all tokens to have the same attributes. " + "Some tokens do not contain annotation for: {partial_attrs}") # Deprecated model shortcuts, only used in errors and warnings diff --git a/spacy/schemas.py b/spacy/schemas.py index 7d87658f2..b284b82e5 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -485,3 +485,29 @@ class RecommendationSchema(BaseModel): word_vectors: Optional[str] = None transformer: Optional[RecommendationTrf] = None has_letters: bool = True + + +class DocJSONSchema(BaseModel): + """ + JSON/dict format for JSON representation of Doc objects. + """ + + cats: Optional[Dict[StrictStr, StrictFloat]] = Field( + None, title="Categories with corresponding probabilities" + ) + ents: Optional[List[Dict[StrictStr, Union[StrictInt, StrictStr]]]] = Field( + None, title="Information on entities" + ) + sents: Optional[List[Dict[StrictStr, StrictInt]]] = Field( + None, title="Indices of sentences' start and end indices" + ) + text: StrictStr = Field(..., title="Document text") + spans: Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] = Field( + None, title="Span information - end/start indices, label, KB ID" + ) + tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field( + ..., title="Token information - ID, start, annotations" + ) + _: Optional[Dict[StrictStr, Any]] = Field( + None, title="Any custom data stored in the document's _ attribute" + ) diff --git a/spacy/tests/doc/test_json_doc_conversion.py b/spacy/tests/doc/test_json_doc_conversion.py new file mode 100644 index 000000000..85e4def29 --- /dev/null +++ b/spacy/tests/doc/test_json_doc_conversion.py @@ -0,0 +1,191 @@ +import pytest +import spacy +from spacy import schemas +from spacy.tokens import Doc, Span + + +@pytest.fixture() +def doc(en_vocab): + words = ["c", "d", "e"] + pos = ["VERB", "NOUN", "NOUN"] + tags = ["VBP", "NN", "NN"] + heads = [0, 0, 1] + deps = ["ROOT", "dobj", "dobj"] + ents = ["O", "B-ORG", "O"] + morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"] + + return Doc( + en_vocab, + words=words, + pos=pos, + tags=tags, + heads=heads, + deps=deps, + ents=ents, + morphs=morphs, + ) + + +@pytest.fixture() +def doc_without_deps(en_vocab): + words = ["c", "d", "e"] + pos = ["VERB", "NOUN", "NOUN"] + tags = ["VBP", "NN", "NN"] + ents = ["O", "B-ORG", "O"] + morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"] + + return Doc( + en_vocab, + words=words, + pos=pos, + tags=tags, + ents=ents, + morphs=morphs, + sent_starts=[True, False, True], + ) + + +def test_doc_to_json(doc): + json_doc = doc.to_json() + assert json_doc["text"] == "c d e " + assert len(json_doc["tokens"]) == 3 + assert json_doc["tokens"][0]["pos"] == "VERB" + assert json_doc["tokens"][0]["tag"] == "VBP" + assert json_doc["tokens"][0]["dep"] == "ROOT" + assert len(json_doc["ents"]) == 1 + assert json_doc["ents"][0]["start"] == 2 # character offset! + assert json_doc["ents"][0]["end"] == 3 # character offset! + assert json_doc["ents"][0]["label"] == "ORG" + assert not schemas.validate(schemas.DocJSONSchema, json_doc) + + +def test_doc_to_json_underscore(doc): + Doc.set_extension("json_test1", default=False) + Doc.set_extension("json_test2", default=False) + doc._.json_test1 = "hello world" + doc._.json_test2 = [1, 2, 3] + json_doc = doc.to_json(underscore=["json_test1", "json_test2"]) + assert "_" in json_doc + assert json_doc["_"]["json_test1"] == "hello world" + assert json_doc["_"]["json_test2"] == [1, 2, 3] + assert not schemas.validate(schemas.DocJSONSchema, json_doc) + + +def test_doc_to_json_underscore_error_attr(doc): + """Test that Doc.to_json() raises an error if a custom attribute doesn't + exist in the ._ space.""" + with pytest.raises(ValueError): + doc.to_json(underscore=["json_test3"]) + + +def test_doc_to_json_underscore_error_serialize(doc): + """Test that Doc.to_json() raises an error if a custom attribute value + isn't JSON-serializable.""" + Doc.set_extension("json_test4", method=lambda doc: doc.text) + with pytest.raises(ValueError): + doc.to_json(underscore=["json_test4"]) + + +def test_doc_to_json_span(doc): + """Test that Doc.to_json() includes spans""" + doc.spans["test"] = [Span(doc, 0, 2, "test"), Span(doc, 0, 1, "test")] + json_doc = doc.to_json() + assert "spans" in json_doc + assert len(json_doc["spans"]) == 1 + assert len(json_doc["spans"]["test"]) == 2 + assert json_doc["spans"]["test"][0]["start"] == 0 + assert not schemas.validate(schemas.DocJSONSchema, json_doc) + + +def test_json_to_doc(doc): + new_doc = Doc(doc.vocab).from_json(doc.to_json(), validate=True) + new_tokens = [token for token in new_doc] + assert new_doc.text == doc.text == "c d e " + assert len(new_tokens) == len([token for token in doc]) == 3 + assert new_tokens[0].pos == doc[0].pos + assert new_tokens[0].tag == doc[0].tag + assert new_tokens[0].dep == doc[0].dep + assert new_tokens[0].head.idx == doc[0].head.idx + assert new_tokens[0].lemma == doc[0].lemma + assert len(new_doc.ents) == 1 + assert new_doc.ents[0].start == 1 + assert new_doc.ents[0].end == 2 + assert new_doc.ents[0].label_ == "ORG" + + +def test_json_to_doc_underscore(doc): + if not Doc.has_extension("json_test1"): + Doc.set_extension("json_test1", default=False) + if not Doc.has_extension("json_test2"): + Doc.set_extension("json_test2", default=False) + + doc._.json_test1 = "hello world" + doc._.json_test2 = [1, 2, 3] + json_doc = doc.to_json(underscore=["json_test1", "json_test2"]) + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)]) + assert new_doc._.json_test1 == "hello world" + assert new_doc._.json_test2 == [1, 2, 3] + + +def test_json_to_doc_spans(doc): + """Test that Doc.from_json() includes correct.spans.""" + doc.spans["test"] = [ + Span(doc, 0, 2, label="test"), + Span(doc, 0, 1, label="test", kb_id=7), + ] + json_doc = doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert len(new_doc.spans) == 1 + assert len(new_doc.spans["test"]) == 2 + for i in range(2): + assert new_doc.spans["test"][i].start == doc.spans["test"][i].start + assert new_doc.spans["test"][i].end == doc.spans["test"][i].end + assert new_doc.spans["test"][i].label == doc.spans["test"][i].label + assert new_doc.spans["test"][i].kb_id == doc.spans["test"][i].kb_id + + +def test_json_to_doc_sents(doc, doc_without_deps): + """Test that Doc.from_json() includes correct.sents.""" + for test_doc in (doc, doc_without_deps): + json_doc = test_doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert [sent.text for sent in test_doc.sents] == [ + sent.text for sent in new_doc.sents + ] + assert [token.is_sent_start for token in test_doc] == [ + token.is_sent_start for token in new_doc + ] + + +def test_json_to_doc_cats(doc): + """Test that Doc.from_json() includes correct .cats.""" + cats = {"A": 0.3, "B": 0.7} + doc.cats = cats + json_doc = doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert new_doc.cats == cats + + +def test_json_to_doc_spaces(): + """Test that Doc.from_json() preserves spaces correctly.""" + doc = spacy.blank("en")("This is just brilliant.") + json_doc = doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert doc.text == new_doc.text + + +def test_json_to_doc_attribute_consistency(doc): + """Test that Doc.from_json() raises an exception if tokens don't all have the same set of properties.""" + doc_json = doc.to_json() + doc_json["tokens"][1].pop("morph") + with pytest.raises(ValueError): + Doc(doc.vocab).from_json(doc_json) + + +def test_json_to_doc_validation_error(doc): + """Test that Doc.from_json() raises an exception when validating invalid input.""" + doc_json = doc.to_json() + doc_json.pop("tokens") + with pytest.raises(ValueError): + Doc(doc.vocab).from_json(doc_json, validate=True) diff --git a/spacy/tests/doc/test_to_json.py b/spacy/tests/doc/test_to_json.py deleted file mode 100644 index 202281654..000000000 --- a/spacy/tests/doc/test_to_json.py +++ /dev/null @@ -1,72 +0,0 @@ -import pytest -from spacy.tokens import Doc, Span - - -@pytest.fixture() -def doc(en_vocab): - words = ["c", "d", "e"] - pos = ["VERB", "NOUN", "NOUN"] - tags = ["VBP", "NN", "NN"] - heads = [0, 0, 0] - deps = ["ROOT", "dobj", "dobj"] - ents = ["O", "B-ORG", "O"] - morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"] - return Doc( - en_vocab, - words=words, - pos=pos, - tags=tags, - heads=heads, - deps=deps, - ents=ents, - morphs=morphs, - ) - - -def test_doc_to_json(doc): - json_doc = doc.to_json() - assert json_doc["text"] == "c d e " - assert len(json_doc["tokens"]) == 3 - assert json_doc["tokens"][0]["pos"] == "VERB" - assert json_doc["tokens"][0]["tag"] == "VBP" - assert json_doc["tokens"][0]["dep"] == "ROOT" - assert len(json_doc["ents"]) == 1 - assert json_doc["ents"][0]["start"] == 2 # character offset! - assert json_doc["ents"][0]["end"] == 3 # character offset! - assert json_doc["ents"][0]["label"] == "ORG" - - -def test_doc_to_json_underscore(doc): - Doc.set_extension("json_test1", default=False) - Doc.set_extension("json_test2", default=False) - doc._.json_test1 = "hello world" - doc._.json_test2 = [1, 2, 3] - json_doc = doc.to_json(underscore=["json_test1", "json_test2"]) - assert "_" in json_doc - assert json_doc["_"]["json_test1"] == "hello world" - assert json_doc["_"]["json_test2"] == [1, 2, 3] - - -def test_doc_to_json_underscore_error_attr(doc): - """Test that Doc.to_json() raises an error if a custom attribute doesn't - exist in the ._ space.""" - with pytest.raises(ValueError): - doc.to_json(underscore=["json_test3"]) - - -def test_doc_to_json_underscore_error_serialize(doc): - """Test that Doc.to_json() raises an error if a custom attribute value - isn't JSON-serializable.""" - Doc.set_extension("json_test4", method=lambda doc: doc.text) - with pytest.raises(ValueError): - doc.to_json(underscore=["json_test4"]) - - -def test_doc_to_json_span(doc): - """Test that Doc.to_json() includes spans""" - doc.spans["test"] = [Span(doc, 0, 2, "test"), Span(doc, 0, 1, "test")] - json_doc = doc.to_json() - assert "spans" in json_doc - assert len(json_doc["spans"]) == 1 - assert len(json_doc["spans"]["test"]) == 2 - assert json_doc["spans"]["test"][0]["start"] == 0 diff --git a/spacy/tokens/doc.pyi b/spacy/tokens/doc.pyi index 7e9340d58..a40fa74aa 100644 --- a/spacy/tokens/doc.pyi +++ b/spacy/tokens/doc.pyi @@ -170,6 +170,9 @@ class Doc: def extend_tensor(self, tensor: Floats2d) -> None: ... def retokenize(self) -> Retokenizer: ... def to_json(self, underscore: Optional[List[str]] = ...) -> Dict[str, Any]: ... + def from_json( + self, doc_json: Dict[str, Any] = ..., validate: bool = False + ) -> Doc: ... def to_utf8_array(self, nr_char: int = ...) -> Ints2d: ... @staticmethod def _get_array_attrs() -> Tuple[Any]: ... diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index c0b67fb7c..9bae9afd3 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -1,4 +1,6 @@ # cython: infer_types=True, bounds_check=False, profile=True +from typing import Set + cimport cython cimport numpy as np from libc.string cimport memcpy @@ -15,6 +17,7 @@ from thinc.api import get_array_module, get_current_ops from thinc.util import copy_array import warnings +import spacy.schemas from .span cimport Span from .token cimport MISSING_DEP from ._dict_proxies import SpanGroups @@ -34,7 +37,7 @@ from .. import parts_of_speech from .underscore import Underscore, get_ext_args from ._retokenize import Retokenizer from ._serialize import ALL_ATTRS as DOCBIN_ALL_ATTRS - +from ..util import get_words_and_spaces DEF PADDING = 5 @@ -1475,6 +1478,138 @@ cdef class Doc: remove_label_if_necessary(attributes[i]) retokenizer.merge(span, attributes[i]) + def from_json(self, doc_json, *, validate=False): + """Convert a JSON document generated by Doc.to_json() to a Doc. + + doc_json (Dict): JSON representation of doc object to load. + validate (bool): Whether to validate `doc_json` against the expected schema. + Defaults to False. + RETURNS (Doc): A doc instance corresponding to the specified JSON representation. + """ + + if validate: + schema_validation_message = spacy.schemas.validate(spacy.schemas.DocJSONSchema, doc_json) + if schema_validation_message: + raise ValueError(Errors.E1038.format(message=schema_validation_message)) + + ### Token-level properties ### + + words = [] + token_attrs_ids = (POS, HEAD, DEP, LEMMA, TAG, MORPH) + # Map annotation type IDs to their string equivalents. + token_attrs = {t: self.vocab.strings[t].lower() for t in token_attrs_ids} + token_annotations = {} + + # Gather token-level properties. + for token_json in doc_json["tokens"]: + words.append(doc_json["text"][token_json["start"]:token_json["end"]]) + for attr, attr_json in token_attrs.items(): + if attr_json in token_json: + if token_json["id"] == 0 and attr not in token_annotations: + token_annotations[attr] = [] + elif attr not in token_annotations: + raise ValueError(Errors.E1040.format(partial_attrs=attr)) + token_annotations[attr].append(token_json[attr_json]) + + # Initialize doc instance. + start = 0 + cdef const LexemeC* lex + cdef bint has_space + reconstructed_words, spaces = get_words_and_spaces(words, doc_json["text"]) + assert words == reconstructed_words + + for word, has_space in zip(words, spaces): + lex = self.vocab.get(self.mem, word) + self.push_back(lex, has_space) + + # Set remaining token-level attributes via Doc.from_array(). + if HEAD in token_annotations: + token_annotations[HEAD] = [ + head - i for i, head in enumerate(token_annotations[HEAD]) + ] + + if DEP in token_annotations and HEAD not in token_annotations: + token_annotations[HEAD] = [0] * len(token_annotations[DEP]) + if HEAD in token_annotations and DEP not in token_annotations: + raise ValueError(Errors.E1017) + if POS in token_annotations: + for pp in set(token_annotations[POS]): + if pp not in parts_of_speech.IDS: + raise ValueError(Errors.E1021.format(pp=pp)) + + # Collect token attributes, assert all tokens have exactly the same set of attributes. + attrs = [] + partial_attrs: Set[str] = set() + for attr in token_attrs.keys(): + if attr in token_annotations: + if len(token_annotations[attr]) != len(words): + partial_attrs.add(token_attrs[attr]) + attrs.append(attr) + if len(partial_attrs): + raise ValueError(Errors.E1040.format(partial_attrs=partial_attrs)) + + # If there are any other annotations, set them. + if attrs: + array = self.to_array(attrs) + if array.ndim == 1: + array = numpy.reshape(array, (array.size, 1)) + j = 0 + + for j, (attr, annot) in enumerate(token_annotations.items()): + if attr is HEAD: + for i in range(len(words)): + array[i, j] = annot[i] + elif attr is MORPH: + for i in range(len(words)): + array[i, j] = self.vocab.morphology.add(annot[i]) + else: + for i in range(len(words)): + array[i, j] = self.vocab.strings.add(annot[i]) + self.from_array(attrs, array) + + ### Span/document properties ### + + # Complement other document-level properties (cats, spans, ents). + self.cats = doc_json.get("cats", {}) + + # Set sentence boundaries, if dependency parser not available but sentences are specified in JSON. + if not self.has_annotation("DEP"): + for sent in doc_json.get("sents", {}): + char_span = self.char_span(sent["start"], sent["end"]) + if char_span is None: + raise ValueError(Errors.E1039.format(obj="sentence", start=sent["start"], end=sent["end"])) + char_span[0].is_sent_start = True + for token in char_span[1:]: + token.is_sent_start = False + + + for span_group in doc_json.get("spans", {}): + spans = [] + for span in doc_json["spans"][span_group]: + char_span = self.char_span(span["start"], span["end"], span["label"], span["kb_id"]) + if char_span is None: + raise ValueError(Errors.E1039.format(obj="span", start=span["start"], end=span["end"])) + spans.append(char_span) + self.spans[span_group] = spans + + if "ents" in doc_json: + ents = [] + for ent in doc_json["ents"]: + char_span = self.char_span(ent["start"], ent["end"], ent["label"]) + if char_span is None: + raise ValueError(Errors.E1039.format(obj="entity"), start=ent["start"], end=ent["end"]) + ents.append(char_span) + self.ents = ents + + # Add custom attributes. Note that only Doc extensions are currently considered, Token and Span extensions are + # not yet supported. + for attr in doc_json.get("_", {}): + if not Doc.has_extension(attr): + Doc.set_extension(attr) + self._.set(attr, doc_json["_"][attr]) + + return self + def to_json(self, underscore=None): """Convert a Doc to JSON. @@ -1485,12 +1620,10 @@ cdef class Doc: """ data = {"text": self.text} if self.has_annotation("ENT_IOB"): - data["ents"] = [{"start": ent.start_char, "end": ent.end_char, - "label": ent.label_} for ent in self.ents] + data["ents"] = [{"start": ent.start_char, "end": ent.end_char, "label": ent.label_} for ent in self.ents] if self.has_annotation("SENT_START"): sents = list(self.sents) - data["sents"] = [{"start": sent.start_char, "end": sent.end_char} - for sent in sents] + data["sents"] = [{"start": sent.start_char, "end": sent.end_char} for sent in sents] if self.cats: data["cats"] = self.cats data["tokens"] = [] @@ -1516,7 +1649,9 @@ cdef class Doc: for span_group in self.spans: data["spans"][span_group] = [] for span in self.spans[span_group]: - span_data = {"start": span.start_char, "end": span.end_char, "label": span.label_, "kb_id": span.kb_id_} + span_data = { + "start": span.start_char, "end": span.end_char, "label": span.label_, "kb_id": span.kb_id_ + } data["spans"][span_group].append(span_data) if underscore: diff --git a/website/docs/api/doc.md b/website/docs/api/doc.md index 0008cde31..f97f4ad83 100644 --- a/website/docs/api/doc.md +++ b/website/docs/api/doc.md @@ -481,6 +481,45 @@ Deserialize, i.e. import the document contents from a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Doc` object. ~~Doc~~ | +## Doc.to_json {#to_json tag="method"} + +Serializes a document to JSON. Note that this is format differs from the +deprecated [`JSON training format`](/api/data-formats#json-input). + +> #### Example +> +> ```python +> doc = nlp("All we have to decide is what to do with the time that is given us.") +> assert doc.to_json()["text"] == doc.text +> ``` + +| Name | Description | +| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `underscore` | Optional list of string names of custom `Doc` attributes. Attribute values need to be JSON-serializable. Values will be added to an `"_"` key in the data, e.g. `"_": {"foo": "bar"}`. ~~Optional[List[str]]~~ | +| **RETURNS** | The data in JSON format. ~~Dict[str, Any]~~ | + +## Doc.from_json {#from_json tag="method" new="3.3.1"} + +Deserializes a document from JSON, i.e. generates a document from the provided +JSON data as generated by [`Doc.to_json()`](/api/doc#to_json). + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> doc = nlp("All we have to decide is what to do with the time that is given us.") +> doc_json = doc.to_json() +> deserialized_doc = Doc(nlp.vocab).from_json(doc_json) +> assert deserialized_doc.text == doc.text == doc_json["text"] +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------- | +| `doc_json` | The Doc data in JSON format from [`Doc.to_json`](#to_json). ~~Dict[str, Any]~~ | +| _keyword-only_ | | +| `validate` | Whether to validate the JSON input against the expected schema for detailed debugging. Defaults to `False`. ~~bool~~ | +| **RETURNS** | A `Doc` corresponding to the provided JSON. ~~Doc~~ | + ## Doc.retokenize {#retokenize tag="contextmanager" new="2.1"} Context manager to handle retokenization of the `Doc`. Modifications to the From 7e13652d3630aedeb68b3e743079098ab805b06a Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 2 Jun 2022 15:53:03 +0200 Subject: [PATCH 13/24] Fix schemas import in Doc (#10898) --- spacy/tokens/doc.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index 9bae9afd3..93e7d0cae 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -17,7 +17,6 @@ from thinc.api import get_array_module, get_current_ops from thinc.util import copy_array import warnings -import spacy.schemas from .span cimport Span from .token cimport MISSING_DEP from ._dict_proxies import SpanGroups @@ -34,6 +33,7 @@ from ..errors import Errors, Warnings from ..morphology import Morphology from .. import util from .. import parts_of_speech +from .. import schemas from .underscore import Underscore, get_ext_args from ._retokenize import Retokenizer from ._serialize import ALL_ATTRS as DOCBIN_ALL_ATTRS @@ -1488,7 +1488,7 @@ cdef class Doc: """ if validate: - schema_validation_message = spacy.schemas.validate(spacy.schemas.DocJSONSchema, doc_json) + schema_validation_message = schemas.validate(schemas.DocJSONSchema, doc_json) if schema_validation_message: raise ValueError(Errors.E1038.format(message=schema_validation_message)) From 6c6b8da7cc8dfdacba975681955e26333d16c665 Mon Sep 17 00:00:00 2001 From: single-fingal <97484373+single-fingal@users.noreply.github.com> Date: Thu, 2 Jun 2022 06:56:27 -0700 Subject: [PATCH 14/24] Fix: De/Serialize `SpanGroups` including the SpanGroup keys (#10707) * fix: De/Serialize `SpanGroups` including the SpanGroup keys This prevents the loss of `SpanGroup`s that have the same .name as other `SpanGroup`s within the same `SpanGroups` object (upon de/serialization of the `SpanGroups`). Fixes #10685 * Maintain backwards compatibility for serialized `SpanGroups` (serialized as: a list of `SpanGroup`s, or b'') * Add tests for `SpanGroups` deserialization backwards-compatibility * Move a `SpanGroups` de/serialization test (test_issue10685) to tests/serialize/test_serialize_spangroups.py * Output a warning if deserializing a `SpanGroups` with duplicate .name-d `SpanGroup`s * Minor refactor * `SpanGroups.from_bytes` handles only `list` and `dict` types with `dict` as the expected default * For lists, keep first rather than last value encountered * Update error message * Rename and update tests * Update to preserve list serialization of SpanGroups To avoid breaking compatibility of serialized `Doc` and `DocBin` with earlier versions of spacy v3, revert back to a list-only serialization, but update the names just for serialization so that the SpanGroups keys override the SpanGroup names. * Preserve object identity and current key overwrite * Preserve SpanGroup object identity * Preserve last rather than first span group from SpanGroup list format without SpanGroups keys * Update inline comments * Fix types * Add type info for SpanGroup.copy * Deserialize `SpanGroup`s as copies when a single SpanGroup is the value for more than 1 `SpanGroups` key. This is because we serialize `SpanGroups` as dicts (to maintain backward- and forward-compatibility) and we can't assume `SpanGroup`s with the same bytes/serialization were the same (identical) object, pre-serialization. * Update spacy/tokens/_dict_proxies.py * Add more SpanGroups serialization tests Test that serialized SpanGroups maintain their Span order * small clarification on older spaCy version * Update spacy/tests/serialize/test_serialize_span_groups.py Co-authored-by: Adriane Boyd Co-authored-by: Sofie Van Landeghem --- spacy/errors.py | 5 + .../serialize/test_serialize_span_groups.py | 161 ++++++++++++++++++ spacy/tokens/_dict_proxies.py | 53 ++++-- spacy/tokens/span_group.pyi | 1 + 4 files changed, 210 insertions(+), 10 deletions(-) create mode 100644 spacy/tests/serialize/test_serialize_span_groups.py diff --git a/spacy/errors.py b/spacy/errors.py index 75e24789f..a58a593dc 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -204,6 +204,11 @@ class Warnings(metaclass=ErrorsWithCodes): "for the corpora used to train the language. Please check " "`nlp.meta[\"sources\"]` for any relevant links.") W119 = ("Overriding pipe name in `config` is not supported. Ignoring override '{name_in_config}'.") + W120 = ("Unable to load all spans in Doc.spans: more than one span group " + "with the name '{group_name}' was found in the saved spans data. " + "Only the last span group will be loaded under " + "Doc.spans['{group_name}']. Skipping span group with values: " + "{group_values}") class Errors(metaclass=ErrorsWithCodes): diff --git a/spacy/tests/serialize/test_serialize_span_groups.py b/spacy/tests/serialize/test_serialize_span_groups.py new file mode 100644 index 000000000..85313fcdc --- /dev/null +++ b/spacy/tests/serialize/test_serialize_span_groups.py @@ -0,0 +1,161 @@ +import pytest + +from spacy.tokens import Span, SpanGroup +from spacy.tokens._dict_proxies import SpanGroups + + +@pytest.mark.issue(10685) +def test_issue10685(en_tokenizer): + """Test `SpanGroups` de/serialization""" + # Start with a Doc with no SpanGroups + doc = en_tokenizer("Will it blend?") + + # Test empty `SpanGroups` de/serialization: + assert len(doc.spans) == 0 + doc.spans.from_bytes(doc.spans.to_bytes()) + assert len(doc.spans) == 0 + + # Test non-empty `SpanGroups` de/serialization: + doc.spans["test"] = SpanGroup(doc, name="test", spans=[doc[0:1]]) + doc.spans["test2"] = SpanGroup(doc, name="test", spans=[doc[1:2]]) + + def assert_spangroups(): + assert len(doc.spans) == 2 + assert doc.spans["test"].name == "test" + assert doc.spans["test2"].name == "test" + assert list(doc.spans["test"]) == [doc[0:1]] + assert list(doc.spans["test2"]) == [doc[1:2]] + + # Sanity check the currently-expected behavior + assert_spangroups() + + # Now test serialization/deserialization: + doc.spans.from_bytes(doc.spans.to_bytes()) + + assert_spangroups() + + +def test_span_groups_serialization_mismatches(en_tokenizer): + """Test the serialization of multiple mismatching `SpanGroups` keys and `SpanGroup.name`s""" + doc = en_tokenizer("How now, brown cow?") + # Some variety: + # 1 SpanGroup where its name matches its key + # 2 SpanGroups that have the same name--which is not a key + # 2 SpanGroups that have the same name--which is a key + # 1 SpanGroup that is a value for 2 different keys (where its name is a key) + # 1 SpanGroup that is a value for 2 different keys (where its name is not a key) + groups = doc.spans + groups["key1"] = SpanGroup(doc, name="key1", spans=[doc[0:1], doc[1:2]]) + groups["key2"] = SpanGroup(doc, name="too", spans=[doc[3:4], doc[4:5]]) + groups["key3"] = SpanGroup(doc, name="too", spans=[doc[1:2], doc[0:1]]) + groups["key4"] = SpanGroup(doc, name="key4", spans=[doc[0:1]]) + groups["key5"] = SpanGroup(doc, name="key4", spans=[doc[0:1]]) + sg6 = SpanGroup(doc, name="key6", spans=[doc[0:1]]) + groups["key6"] = sg6 + groups["key7"] = sg6 + sg8 = SpanGroup(doc, name="also", spans=[doc[1:2]]) + groups["key8"] = sg8 + groups["key9"] = sg8 + + regroups = SpanGroups(doc).from_bytes(groups.to_bytes()) + + # Assert regroups == groups + assert regroups.keys() == groups.keys() + for key, regroup in regroups.items(): + # Assert regroup == groups[key] + assert regroup.name == groups[key].name + assert list(regroup) == list(groups[key]) + + +@pytest.mark.parametrize( + "spans_bytes,doc_text,expected_spangroups,expected_warning", + # The bytestrings below were generated from an earlier version of spaCy + # that serialized `SpanGroups` as a list of SpanGroup bytes (via SpanGroups.to_bytes). + # Comments preceding the bytestrings indicate from what Doc they were created. + [ + # Empty SpanGroups: + (b"\x90", "", {}, False), + # doc = nlp("Will it blend?") + # doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]]) + ( + b"\x91\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04", + "Will it blend?", + {"test": {"name": "test", "spans": [(0, 1)]}}, + False, + ), + # doc = nlp("Will it blend?") + # doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]]) + # doc.spans['test2'] = SpanGroup(doc, name='test', spans=[doc[1:2]]) + ( + b"\x92\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x05\x00\x00\x00\x07", + "Will it blend?", + # We expect only 1 SpanGroup to be in doc.spans in this example + # because there are 2 `SpanGroup`s that have the same .name. See #10685. + {"test": {"name": "test", "spans": [(1, 2)]}}, + True, + ), + # doc = nlp('How now, brown cow?') + # doc.spans['key1'] = SpanGroup(doc, name='key1', spans=[doc[0:1], doc[1:2]]) + # doc.spans['key2'] = SpanGroup(doc, name='too', spans=[doc[3:4], doc[4:5]]) + # doc.spans['key3'] = SpanGroup(doc, name='too', spans=[doc[1:2], doc[0:1]]) + # doc.spans['key4'] = SpanGroup(doc, name='key4', spans=[doc[0:1]]) + # doc.spans['key5'] = SpanGroup(doc, name='key4', spans=[doc[0:1]]) + ( + b"\x95\xc4m\x83\xa4name\xa4key1\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00\t\x00\x00\x00\x0e\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x12\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03", + "How now, brown cow?", + { + "key1": {"name": "key1", "spans": [(0, 1), (1, 2)]}, + "too": {"name": "too", "spans": [(1, 2), (0, 1)]}, + "key4": {"name": "key4", "spans": [(0, 1)]}, + }, + True, + ), + ], +) +def test_deserialize_span_groups_compat( + en_tokenizer, spans_bytes, doc_text, expected_spangroups, expected_warning +): + """Test backwards-compatibility of `SpanGroups` deserialization. + This uses serializations (bytes) from a prior version of spaCy (before 3.3.1). + + spans_bytes (bytes): Serialized `SpanGroups` object. + doc_text (str): Doc text. + expected_spangroups (dict): + Dict mapping every expected (after deserialization) `SpanGroups` key + to a SpanGroup's "args", where a SpanGroup's args are given as a dict: + {"name": span_group.name, + "spans": [(span0.start, span0.end), ...]} + expected_warning (bool): Whether a warning is to be expected from .from_bytes() + --i.e. if more than 1 SpanGroup has the same .name within the `SpanGroups`. + """ + doc = en_tokenizer(doc_text) + + if expected_warning: + with pytest.warns(UserWarning): + doc.spans.from_bytes(spans_bytes) + else: + # TODO: explicitly check for lack of a warning + doc.spans.from_bytes(spans_bytes) + + assert doc.spans.keys() == expected_spangroups.keys() + for name, spangroup_args in expected_spangroups.items(): + assert doc.spans[name].name == spangroup_args["name"] + spans = [Span(doc, start, end) for start, end in spangroup_args["spans"]] + assert list(doc.spans[name]) == spans + + +def test_span_groups_serialization(en_tokenizer): + doc = en_tokenizer("0 1 2 3 4 5 6") + span_groups = SpanGroups(doc) + spans = [doc[0:2], doc[1:3]] + sg1 = SpanGroup(doc, spans=spans) + span_groups["key1"] = sg1 + span_groups["key2"] = sg1 + span_groups["key3"] = [] + reloaded_span_groups = SpanGroups(doc).from_bytes(span_groups.to_bytes()) + assert span_groups.keys() == reloaded_span_groups.keys() + for key, value in span_groups.items(): + assert all( + span == reloaded_span + for span, reloaded_span in zip(span_groups[key], reloaded_span_groups[key]) + ) diff --git a/spacy/tokens/_dict_proxies.py b/spacy/tokens/_dict_proxies.py index d9506769b..9630da261 100644 --- a/spacy/tokens/_dict_proxies.py +++ b/spacy/tokens/_dict_proxies.py @@ -1,10 +1,11 @@ -from typing import Iterable, Tuple, Union, Optional, TYPE_CHECKING +from typing import Dict, Iterable, List, Tuple, Union, Optional, TYPE_CHECKING +import warnings import weakref from collections import UserDict import srsly from .span_group import SpanGroup -from ..errors import Errors +from ..errors import Errors, Warnings if TYPE_CHECKING: @@ -16,7 +17,7 @@ if TYPE_CHECKING: # Why inherit from UserDict instead of dict here? # Well, the 'dict' class doesn't necessarily delegate everything nicely, # for performance reasons. The UserDict is slower but better behaved. -# See https://treyhunner.com/2019/04/why-you-shouldnt-inherit-from-list-and-dict-in-python/0ww +# See https://treyhunner.com/2019/04/why-you-shouldnt-inherit-from-list-and-dict-in-python/ class SpanGroups(UserDict): """A dict-like proxy held by the Doc, to control access to span groups.""" @@ -53,20 +54,52 @@ class SpanGroups(UserDict): return super().setdefault(key, default=default) def to_bytes(self) -> bytes: - # We don't need to serialize this as a dict, because the groups - # know their names. + # We serialize this as a dict in order to track the key(s) a SpanGroup + # is a value of (in a backward- and forward-compatible way), since + # a SpanGroup can have a key that doesn't match its `.name` (See #10685) if len(self) == 0: return self._EMPTY_BYTES - msg = [value.to_bytes() for value in self.values()] + msg: Dict[bytes, List[str]] = {} + for key, value in self.items(): + msg.setdefault(value.to_bytes(), []).append(key) return srsly.msgpack_dumps(msg) def from_bytes(self, bytes_data: bytes) -> "SpanGroups": - msg = [] if bytes_data == self._EMPTY_BYTES else srsly.msgpack_loads(bytes_data) + # backwards-compatibility: bytes_data may be one of: + # b'', a serialized empty list, a serialized list of SpanGroup bytes + # or a serialized dict of SpanGroup bytes -> keys + msg = ( + [] + if not bytes_data or bytes_data == self._EMPTY_BYTES + else srsly.msgpack_loads(bytes_data) + ) self.clear() doc = self._ensure_doc() - for value_bytes in msg: - group = SpanGroup(doc).from_bytes(value_bytes) - self[group.name] = group + if isinstance(msg, list): + # This is either the 1st version of `SpanGroups` serialization + # or there were no SpanGroups serialized + for value_bytes in msg: + group = SpanGroup(doc).from_bytes(value_bytes) + if group.name in self: + # Display a warning if `msg` contains `SpanGroup`s + # that have the same .name (attribute). + # Because, for `SpanGroups` serialized as lists, + # only 1 SpanGroup per .name is loaded. (See #10685) + warnings.warn( + Warnings.W120.format( + group_name=group.name, group_values=self[group.name] + ) + ) + self[group.name] = group + else: + for value_bytes, keys in msg.items(): + group = SpanGroup(doc).from_bytes(value_bytes) + # Deserialize `SpanGroup`s as copies because it's possible for two + # different `SpanGroup`s (pre-serialization) to have the same bytes + # (since they can have the same `.name`). + self[keys[0]] = group + for key in keys[1:]: + self[key] = group.copy() return self def _ensure_doc(self) -> "Doc": diff --git a/spacy/tokens/span_group.pyi b/spacy/tokens/span_group.pyi index 26efc3ba0..245eb4dbe 100644 --- a/spacy/tokens/span_group.pyi +++ b/spacy/tokens/span_group.pyi @@ -24,3 +24,4 @@ class SpanGroup: def __getitem__(self, i: int) -> Span: ... def to_bytes(self) -> bytes: ... def from_bytes(self, bytes_data: bytes) -> SpanGroup: ... + def copy(self) -> SpanGroup: ... From 430592b3ceb63e3025c807b8305f53f6e6238ad2 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 2 Jun 2022 17:22:34 +0200 Subject: [PATCH 15/24] Extend typing_extensions to <4.2.0 (#10899) --- requirements.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index fb874c550..b2929145e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ langcodes>=3.2.0,<4.0.0 # Official Python utilities setuptools packaging>=20.0 -typing_extensions>=3.7.4.1,<4.0.0.0; python_version < "3.8" +typing_extensions>=3.7.4.1,<4.2.0; python_version < "3.8" # Development dependencies pre-commit>=2.13.0 cython>=0.25,<3.0 diff --git a/setup.cfg b/setup.cfg index 7e252d62a..c6036a8b3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,7 @@ install_requires = # Official Python utilities setuptools packaging>=20.0 - typing_extensions>=3.7.4,<4.0.0.0; python_version < "3.8" + typing_extensions>=3.7.4,<4.2.0; python_version < "3.8" langcodes>=3.2.0,<4.0.0 [options.entry_points] From 41389ffe1e910a6ea85156a3516bfa345e317978 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Thu, 2 Jun 2022 20:06:49 +0200 Subject: [PATCH 16/24] Avoid pickling `Doc` inputs passed to `Language.pipe()` (#10864) * `Language.pipe()`: Serialize `Doc` objects to bytes when using multiprocessing to avoid pickling overhead * `Doc.to_dict()`: Serialize `_context` attribute (keeping in line with `(un)pickle_doc()` * Correct type annotations * Fix typo * `Doc`: Do not serialize `_context` * `Language.pipe`: Send context objects to child processes, Simplify `as_tuples` handling * Fix type annotation * `Language.pipe`: Simplify `as_tuple` multiprocessor handling * Cleanup code, fix typos * MyPy fixes * Move doc preparation function into `_multiprocessing_pipe` Whitespace changes * Remove superfluous comma * Rename `prepare_doc` to `prepare_input` * Update spacy/errors.py * Undo renaming for error Co-authored-by: Adriane Boyd --- spacy/errors.py | 1 + spacy/language.py | 55 ++++++++++++++++++++++-------- spacy/tests/doc/test_pickle_doc.py | 2 -- spacy/tokens/doc.pyx | 5 ++- 4 files changed, 43 insertions(+), 20 deletions(-) diff --git a/spacy/errors.py b/spacy/errors.py index a58a593dc..384a6a4d2 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -931,6 +931,7 @@ class Errors(metaclass=ErrorsWithCodes): "could not be aligned to token boundaries.") E1040 = ("Doc.from_json requires all tokens to have the same attributes. " "Some tokens do not contain annotation for: {partial_attrs}") + E1041 = ("Expected a string, Doc, or bytes as input, but got: {type}") # Deprecated model shortcuts, only used in errors and warnings diff --git a/spacy/language.py b/spacy/language.py index faca1f258..42847823f 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -1090,16 +1090,21 @@ class Language: ) return self.tokenizer(text) - def _ensure_doc(self, doc_like: Union[str, Doc]) -> Doc: - """Create a Doc if need be, or raise an error if the input is not a Doc or a string.""" + def _ensure_doc(self, doc_like: Union[str, Doc, bytes]) -> Doc: + """Create a Doc if need be, or raise an error if the input is not + a Doc, string, or a byte array (generated by Doc.to_bytes()).""" if isinstance(doc_like, Doc): return doc_like if isinstance(doc_like, str): return self.make_doc(doc_like) - raise ValueError(Errors.E866.format(type=type(doc_like))) + if isinstance(doc_like, bytes): + return Doc(self.vocab).from_bytes(doc_like) + raise ValueError(Errors.E1041.format(type=type(doc_like))) - def _ensure_doc_with_context(self, doc_like: Union[str, Doc], context: Any) -> Doc: - """Create a Doc if need be and add as_tuples context, or raise an error if the input is not a Doc or a string.""" + def _ensure_doc_with_context( + self, doc_like: Union[str, Doc, bytes], context: _AnyContext + ) -> Doc: + """Call _ensure_doc to generate a Doc and set its context object.""" doc = self._ensure_doc(doc_like) doc._context = context return doc @@ -1519,7 +1524,6 @@ class Language: DOCS: https://spacy.io/api/language#pipe """ - # Handle texts with context as tuples if as_tuples: texts = cast(Iterable[Tuple[Union[str, Doc], _AnyContext]], texts) docs_with_contexts = ( @@ -1597,8 +1601,21 @@ class Language: n_process: int, batch_size: int, ) -> Iterator[Doc]: + def prepare_input( + texts: Iterable[Union[str, Doc]] + ) -> Iterable[Tuple[Union[str, bytes], _AnyContext]]: + # Serialize Doc inputs to bytes to avoid incurring pickling + # overhead when they are passed to child processes. Also yield + # any context objects they might have separately (as they are not serialized). + for doc_like in texts: + if isinstance(doc_like, Doc): + yield (doc_like.to_bytes(), cast(_AnyContext, doc_like._context)) + else: + yield (doc_like, cast(_AnyContext, None)) + + serialized_texts_with_ctx = prepare_input(texts) # type: ignore # raw_texts is used later to stop iteration. - texts, raw_texts = itertools.tee(texts) + texts, raw_texts = itertools.tee(serialized_texts_with_ctx) # type: ignore # for sending texts to worker texts_q: List[mp.Queue] = [mp.Queue() for _ in range(n_process)] # for receiving byte-encoded docs from worker @@ -1618,7 +1635,13 @@ class Language: procs = [ mp.Process( target=_apply_pipes, - args=(self._ensure_doc, pipes, rch, sch, Underscore.get_state()), + args=( + self._ensure_doc_with_context, + pipes, + rch, + sch, + Underscore.get_state(), + ), ) for rch, sch in zip(texts_q, bytedocs_send_ch) ] @@ -1631,12 +1654,12 @@ class Language: recv.recv() for recv in cycle(bytedocs_recv_ch) ) try: - for i, (_, (byte_doc, byte_context, byte_error)) in enumerate( + for i, (_, (byte_doc, context, byte_error)) in enumerate( zip(raw_texts, byte_tuples), 1 ): if byte_doc is not None: doc = Doc(self.vocab).from_bytes(byte_doc) - doc._context = byte_context + doc._context = context yield doc elif byte_error is not None: error = srsly.msgpack_loads(byte_error) @@ -2163,7 +2186,7 @@ def _copy_examples(examples: Iterable[Example]) -> List[Example]: def _apply_pipes( - ensure_doc: Callable[[Union[str, Doc]], Doc], + ensure_doc: Callable[[Union[str, Doc, bytes], _AnyContext], Doc], pipes: Iterable[Callable[..., Iterator[Doc]]], receiver, sender, @@ -2184,17 +2207,19 @@ def _apply_pipes( Underscore.load_state(underscore_state) while True: try: - texts = receiver.get() - docs = (ensure_doc(text) for text in texts) + texts_with_ctx = receiver.get() + docs = ( + ensure_doc(doc_like, context) for doc_like, context in texts_with_ctx + ) for pipe in pipes: docs = pipe(docs) # type: ignore[arg-type, assignment] # Connection does not accept unpickable objects, so send list. byte_docs = [(doc.to_bytes(), doc._context, None) for doc in docs] - padding = [(None, None, None)] * (len(texts) - len(byte_docs)) + padding = [(None, None, None)] * (len(texts_with_ctx) - len(byte_docs)) sender.send(byte_docs + padding) # type: ignore[operator] except Exception: error_msg = [(None, None, srsly.msgpack_dumps(traceback.format_exc()))] - padding = [(None, None, None)] * (len(texts) - 1) + padding = [(None, None, None)] * (len(texts_with_ctx) - 1) sender.send(error_msg + padding) diff --git a/spacy/tests/doc/test_pickle_doc.py b/spacy/tests/doc/test_pickle_doc.py index 738a751a0..28cb66714 100644 --- a/spacy/tests/doc/test_pickle_doc.py +++ b/spacy/tests/doc/test_pickle_doc.py @@ -5,11 +5,9 @@ from spacy.compat import pickle def test_pickle_single_doc(): nlp = Language() doc = nlp("pickle roundtrip") - doc._context = 3 data = pickle.dumps(doc, 1) doc2 = pickle.loads(data) assert doc2.text == "pickle roundtrip" - assert doc2._context == 3 def test_list_of_docs_pickles_efficiently(): diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index 93e7d0cae..e38de02b4 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -1880,18 +1880,17 @@ cdef int [:,:] _get_lca_matrix(Doc doc, int start, int end): def pickle_doc(doc): bytes_data = doc.to_bytes(exclude=["vocab", "user_data", "user_hooks"]) hooks_and_data = (doc.user_data, doc.user_hooks, doc.user_span_hooks, - doc.user_token_hooks, doc._context) + doc.user_token_hooks) return (unpickle_doc, (doc.vocab, srsly.pickle_dumps(hooks_and_data), bytes_data)) def unpickle_doc(vocab, hooks_and_data, bytes_data): - user_data, doc_hooks, span_hooks, token_hooks, _context = srsly.pickle_loads(hooks_and_data) + user_data, doc_hooks, span_hooks, token_hooks = srsly.pickle_loads(hooks_and_data) doc = Doc(vocab, user_data=user_data).from_bytes(bytes_data, exclude=["user_data"]) doc.user_hooks.update(doc_hooks) doc.user_span_hooks.update(span_hooks) doc.user_token_hooks.update(token_hooks) - doc._context = _context return doc From 727ce6d1f59f688fa46fe8ae3c37e30cf106e291 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 3 Jun 2022 09:44:04 +0200 Subject: [PATCH 17/24] Remove English exceptions with mismatched features (#10873) Remove English contraction exceptions with mismatched features that lead to exceptions like "theses" and "thisre". --- spacy/lang/en/tokenizer_exceptions.py | 70 ++++++++++++++------------- spacy/tests/lang/en/test_tokenizer.py | 9 ++++ 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/spacy/lang/en/tokenizer_exceptions.py b/spacy/lang/en/tokenizer_exceptions.py index 2c20b8c27..7886e28cb 100644 --- a/spacy/lang/en/tokenizer_exceptions.py +++ b/spacy/lang/en/tokenizer_exceptions.py @@ -35,7 +35,7 @@ for pron in ["i"]: _exc[orth + "m"] = [ {ORTH: orth, NORM: pron}, - {ORTH: "m", "tenspect": 1, "number": 1}, + {ORTH: "m"}, ] _exc[orth + "'ma"] = [ @@ -139,26 +139,27 @@ for pron in ["he", "she", "it"]: # W-words, relative pronouns, prepositions etc. -for word in [ - "who", - "what", - "when", - "where", - "why", - "how", - "there", - "that", - "this", - "these", - "those", +for word, morph in [ + ("who", None), + ("what", None), + ("when", None), + ("where", None), + ("why", None), + ("how", None), + ("there", None), + ("that", "Number=Sing|Person=3"), + ("this", "Number=Sing|Person=3"), + ("these", "Number=Plur|Person=3"), + ("those", "Number=Plur|Person=3"), ]: for orth in [word, word.title()]: - _exc[orth + "'s"] = [ - {ORTH: orth, NORM: word}, - {ORTH: "'s", NORM: "'s"}, - ] + if morph != "Number=Plur|Person=3": + _exc[orth + "'s"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'s", NORM: "'s"}, + ] - _exc[orth + "s"] = [{ORTH: orth, NORM: word}, {ORTH: "s"}] + _exc[orth + "s"] = [{ORTH: orth, NORM: word}, {ORTH: "s"}] _exc[orth + "'ll"] = [ {ORTH: orth, NORM: word}, @@ -182,25 +183,26 @@ for word in [ {ORTH: "ve", NORM: "have"}, ] - _exc[orth + "'re"] = [ - {ORTH: orth, NORM: word}, - {ORTH: "'re", NORM: "are"}, - ] + if morph != "Number=Sing|Person=3": + _exc[orth + "'re"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'re", NORM: "are"}, + ] - _exc[orth + "re"] = [ - {ORTH: orth, NORM: word}, - {ORTH: "re", NORM: "are"}, - ] + _exc[orth + "re"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "re", NORM: "are"}, + ] - _exc[orth + "'ve"] = [ - {ORTH: orth, NORM: word}, - {ORTH: "'ve"}, - ] + _exc[orth + "'ve"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'ve"}, + ] - _exc[orth + "ve"] = [ - {ORTH: orth}, - {ORTH: "ve", NORM: "have"}, - ] + _exc[orth + "ve"] = [ + {ORTH: orth}, + {ORTH: "ve", NORM: "have"}, + ] _exc[orth + "'d"] = [ {ORTH: orth, NORM: word}, diff --git a/spacy/tests/lang/en/test_tokenizer.py b/spacy/tests/lang/en/test_tokenizer.py index e6d1d7d85..0133d00b0 100644 --- a/spacy/tests/lang/en/test_tokenizer.py +++ b/spacy/tests/lang/en/test_tokenizer.py @@ -167,3 +167,12 @@ def test_issue3521(en_tokenizer, word): tok = en_tokenizer(word)[1] # 'not' and 'would' should be stopwords, also in their abbreviated forms assert tok.is_stop + + +@pytest.mark.issue(10699) +@pytest.mark.parametrize("text", ["theses", "thisre"]) +def test_issue10699(en_tokenizer, text): + """Test that 'theses' and 'thisre' are excluded from the contractions + generated by the English tokenizer exceptions.""" + tokens = en_tokenizer(text) + assert len(tokens) == 1 From 24aafdffada136a6e195c93ca0df2380f0aefa40 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Jun 2022 11:01:55 +0200 Subject: [PATCH 18/24] Auto-format code with black (#10908) Co-authored-by: explosion-bot --- spacy/ml/models/entity_linker.py | 2 +- spacy/tests/pipeline/test_entity_ruler.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/spacy/ml/models/entity_linker.py b/spacy/ml/models/entity_linker.py index 287feecce..d847342a3 100644 --- a/spacy/ml/models/entity_linker.py +++ b/spacy/ml/models/entity_linker.py @@ -23,7 +23,7 @@ def build_nel_encoder( ((tok2vec >> list2ragged()) & build_span_maker()) >> extract_spans() >> reduce_mean() - >> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore + >> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore >> output_layer ) model.set_ref("output_layer", output_layer) diff --git a/spacy/tests/pipeline/test_entity_ruler.py b/spacy/tests/pipeline/test_entity_ruler.py index bbd537cef..6851e2a7c 100644 --- a/spacy/tests/pipeline/test_entity_ruler.py +++ b/spacy/tests/pipeline/test_entity_ruler.py @@ -491,7 +491,6 @@ def test_entity_ruler_remove_nonexisting_pattern(nlp, entity_ruler_factory): ruler.remove_by_id("nepattern") - @pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) def test_entity_ruler_remove_several_patterns(nlp, entity_ruler_factory): ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") From e7d2b2696662732f31efd768946988c47a298e9b Mon Sep 17 00:00:00 2001 From: vincent d warmerdam Date: Sun, 5 Jun 2022 11:57:58 +0200 Subject: [PATCH 19/24] Add spacy-report to universe (#10910) * Add spacy-report to universe * Remove extra comma Co-authored-by: Paul O'Leary McCann --- website/meta/universe.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index 2840c37d6..b7f340f52 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1,5 +1,25 @@ { "resources": [ + { + "id": "spacy-report", + "title": "spacy-report", + "slogan": "Generates interactive reports for spaCy models.", + "description": "The goal of spacy-report is to offer static reports for spaCy models that help users make better decisions on how the models can be used.", + "github": "koaning/spacy-report", + "pip": "spacy-report", + "thumb": "https://github.com/koaning/spacy-report/raw/main/icon.png", + "image": "https://raw.githubusercontent.com/koaning/spacy-report/main/gif.gif", + "code_example": [ + "python -m spacy report textcat training/model-best/ corpus/train.spacy corpus/dev.spacy" + ], + "category": ["visualizers", "research"], + "author": "Vincent D. Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning", + "website": "https://koaning.io" + } + }, { "id": "scrubadub_spacy", "title": "scrubadub_spacy", From c323789721d36dd9912fa20b9196f01d98581f0e Mon Sep 17 00:00:00 2001 From: Ilya Nikitin Date: Mon, 6 Jun 2022 08:32:36 +0300 Subject: [PATCH 20/24] `token.md`: Fix documentation of `Token.ancestors` (#10917) --- website/docs/api/token.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/api/token.md b/website/docs/api/token.md index 3c3d12d54..d43cd3ff1 100644 --- a/website/docs/api/token.md +++ b/website/docs/api/token.md @@ -221,7 +221,7 @@ dependency tree. ## Token.ancestors {#ancestors tag="property" model="parser"} -The rightmost token of this token's syntactic descendants. +A sequence of the token's syntactic ancestors (parents, grandparents, etc). > #### Example > From a4003532a3a616e972054fc10cd40b215bed53cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Wed, 8 Jun 2022 08:16:22 +0200 Subject: [PATCH 21/24] Update README: spaCy 3.3.1 is out now (#10927) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05c912ffa..bcdf0f844 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ production-ready [**training system**](https://spacy.io/usage/training) and easy model packaging, deployment and workflow management. spaCy is commercial open-source software, released under the MIT license. -💫 **Version 3.2 out now!** +💫 **Version 3.3.1 out now!** [Check out the release notes here.](https://github.com/explosion/spaCy/releases) [![Azure Pipelines](https://img.shields.io/azure-devops/build/explosion-ai/public/8/master.svg?logo=azure-pipelines&style=flat-square&label=build)](https://dev.azure.com/explosion-ai/public/_build?definitionId=8) From 763dcbf885d026be1412ea6b2219ceaad6125377 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Wed, 8 Jun 2022 14:45:04 +0200 Subject: [PATCH 22/24] Fix version in SpanRuler docs (#10925) * SpanRuler is new since 3.3.1 * update SpanRuler version since 3.3.1 --- website/docs/api/spanruler.md | 2 +- website/docs/usage/rule-based-matching.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/api/spanruler.md b/website/docs/api/spanruler.md index b573f7c58..a1c222714 100644 --- a/website/docs/api/spanruler.md +++ b/website/docs/api/spanruler.md @@ -2,7 +2,7 @@ title: SpanRuler tag: class source: spacy/pipeline/span_ruler.py -new: 3.3 +new: 3.3.1 teaser: 'Pipeline component for rule-based span and named entity recognition' api_string_name: span_ruler api_trainable: false diff --git a/website/docs/usage/rule-based-matching.md b/website/docs/usage/rule-based-matching.md index 13612ac35..e4ba4b2af 100644 --- a/website/docs/usage/rule-based-matching.md +++ b/website/docs/usage/rule-based-matching.md @@ -1447,7 +1447,7 @@ with nlp.select_pipes(enable="tagger"): ruler.add_patterns(patterns) ``` -## Rule-based span matching {#spanruler new="3.3"} +## Rule-based span matching {#spanruler new="3.3.1"} The [`SpanRuler`](/api/spanruler) is a generalized version of the entity ruler that lets you add spans to `doc.spans` or `doc.ents` based on pattern From d176afd32f99615360d1d23e622e187b7c5833f4 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 8 Jun 2022 23:24:14 +0900 Subject: [PATCH 23/24] Add note about multiple patterns (#10826) * Add note about multiple patterns * Move note to the top of method docs * Remove EntityRuler note --- website/docs/api/matcher.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/docs/api/matcher.md b/website/docs/api/matcher.md index 6c8cae211..9daa0658d 100644 --- a/website/docs/api/matcher.md +++ b/website/docs/api/matcher.md @@ -113,6 +113,10 @@ string where an integer is expected) or unexpected property names. Find all token sequences matching the supplied patterns on the `Doc` or `Span`. +Note that if a single label has multiple patterns associated with it, the +returned matches don't provide a way to tell which pattern was responsible for +the match. + > #### Example > > ```python @@ -131,7 +135,7 @@ Find all token sequences matching the supplied patterns on the `Doc` or `Span`. | _keyword-only_ | | | `as_spans` 3 | Instead of tuples, return a list of [`Span`](/api/span) objects of the matches, with the `match_id` assigned as the span label. Defaults to `False`. ~~bool~~ | | `allow_missing` 3 | Whether to skip checks for missing annotation for attributes included in patterns. Defaults to `False`. ~~bool~~ | -| `with_alignments` 3.0.6 | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ | +| `with_alignments` 3.0.6 | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token in the pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ | | **RETURNS** | A list of `(match_id, start, end)` tuples, describing the matches. A match tuple describes a span `doc[start:end`]. The `match_id` is the ID of the added match pattern. If `as_spans` is set to `True`, a list of `Span` objects is returned instead. ~~Union[List[Tuple[int, int, int]], List[Span]]~~ | ## Matcher.\_\_len\_\_ {#len tag="method" new="2"} From 1bb87f35bc5babbfc2969737100a2acffab74675 Mon Sep 17 00:00:00 2001 From: kadarakos Date: Wed, 8 Jun 2022 19:34:11 +0200 Subject: [PATCH 24/24] Detect cycle during projectivize (#10877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * detect cycle during projectivize * not complete test to detect cycle in projectivize * boolean to int type to propagate error * use unordered_set instead of set * moved error message to errors * removed cycle from test case * use find instead of count * cycle check: only perform one lookup * Return bool again from _has_head_as_ancestor Communicate presence of cycles through an output argument. * Switch to returning std::pair to encode presence of a cycle The has_cycle pointer is too easy to misuse. Ideally, we would have a sum type like Rust's `Result` here, but C++ is not there yet. * _is_non_proj_arc: clarify what we are returning * _has_head_as_ancestor: remove count We are now explicitly checking for cycles, so the algorithm must always terminate. Either we encounter the head, we find a root, or a cycle. * _is_nonproj_arc: simplify condition * Another refactor using C++ exceptions * Remove unused error code * Print graph with cycle on exception * Include .hh files in source package * Add FIXME comment * cycle detection test * find cycle when starting from problematic vertex Co-authored-by: Daniël de Kok --- MANIFEST.in | 2 +- spacy/pipeline/_parser_internals/nonproj.hh | 11 +++++ spacy/pipeline/_parser_internals/nonproj.pxd | 4 ++ spacy/pipeline/_parser_internals/nonproj.pyx | 48 ++++++++++++++++---- spacy/tests/parser/test_nonproj.py | 12 ++++- spacy/tests/training/test_training.py | 6 +-- 6 files changed, 67 insertions(+), 16 deletions(-) create mode 100644 spacy/pipeline/_parser_internals/nonproj.hh diff --git a/MANIFEST.in b/MANIFEST.in index b7826e456..8ded6f808 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml +recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml *.hh include LICENSE include README.md include pyproject.toml diff --git a/spacy/pipeline/_parser_internals/nonproj.hh b/spacy/pipeline/_parser_internals/nonproj.hh new file mode 100644 index 000000000..071fd57b4 --- /dev/null +++ b/spacy/pipeline/_parser_internals/nonproj.hh @@ -0,0 +1,11 @@ +#ifndef NONPROJ_HH +#define NONPROJ_HH + +#include +#include + +void raise_domain_error(std::string const &msg) { + throw std::domain_error(msg); +} + +#endif // NONPROJ_HH diff --git a/spacy/pipeline/_parser_internals/nonproj.pxd b/spacy/pipeline/_parser_internals/nonproj.pxd index e69de29bb..aabdf7ebe 100644 --- a/spacy/pipeline/_parser_internals/nonproj.pxd +++ b/spacy/pipeline/_parser_internals/nonproj.pxd @@ -0,0 +1,4 @@ +from libcpp.string cimport string + +cdef extern from "nonproj.hh": + cdef void raise_domain_error(const string& msg) nogil except + diff --git a/spacy/pipeline/_parser_internals/nonproj.pyx b/spacy/pipeline/_parser_internals/nonproj.pyx index 36163fcc3..d1b6e7066 100644 --- a/spacy/pipeline/_parser_internals/nonproj.pyx +++ b/spacy/pipeline/_parser_internals/nonproj.pyx @@ -4,10 +4,13 @@ for doing pseudo-projective parsing implementation uses the HEAD decoration scheme. """ from copy import copy +from cython.operator cimport preincrement as incr, dereference as deref from libc.limits cimport INT_MAX from libc.stdlib cimport abs from libcpp cimport bool +from libcpp.string cimport string, to_string from libcpp.vector cimport vector +from libcpp.unordered_set cimport unordered_set from ...tokens.doc cimport Doc, set_children_from_heads @@ -49,7 +52,7 @@ def is_nonproj_arc(tokenid, heads): return _is_nonproj_arc(tokenid, c_heads) -cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil: +cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil except *: # definition (e.g. Havelka 2007): an arc h -> d, h < d is non-projective # if there is a token k, h < k < d such that h is not # an ancestor of k. Same for h -> d, h > d @@ -58,32 +61,56 @@ cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil: return False elif head < 0: # unattached tokens cannot be non-projective return False - + cdef int start, end if head < tokenid: start, end = (head+1, tokenid) else: start, end = (tokenid+1, head) for k in range(start, end): - if _has_head_as_ancestor(k, head, heads): - continue - else: # head not in ancestors: d -> h is non-projective + if not _has_head_as_ancestor(k, head, heads): return True return False -cdef bool _has_head_as_ancestor(int tokenid, int head, const vector[int]& heads) nogil: +cdef bool _has_head_as_ancestor(int tokenid, int head, const vector[int]& heads) nogil except *: ancestor = tokenid - cnt = 0 - while cnt < heads.size(): + cdef unordered_set[int] seen_tokens + seen_tokens.insert(ancestor) + while True: + # Reached the head or a disconnected node if heads[ancestor] == head or heads[ancestor] < 0: return True + # Reached the root + if heads[ancestor] == ancestor: + return False ancestor = heads[ancestor] - cnt += 1 + result = seen_tokens.insert(ancestor) + # Found cycle + if not result.second: + raise_domain_error(heads_to_string(heads)) return False +cdef string heads_to_string(const vector[int]& heads) nogil: + cdef vector[int].const_iterator citer + cdef string cycle_str + + cycle_str.append("Found cycle in dependency graph: [") + + # FIXME: Rewrite using ostringstream when available in Cython. + citer = heads.const_begin() + while citer != heads.const_end(): + if citer != heads.const_begin(): + cycle_str.append(", ") + cycle_str.append(to_string(deref(citer))) + incr(citer) + cycle_str.append("]") + + return cycle_str + + def is_nonproj_tree(heads): cdef vector[int] c_heads = _heads_to_c(heads) # a tree is non-projective if at least one arc is non-projective @@ -176,11 +203,12 @@ def get_smallest_nonproj_arc_slow(heads): return _get_smallest_nonproj_arc(c_heads) -cdef int _get_smallest_nonproj_arc(const vector[int]& heads) nogil: +cdef int _get_smallest_nonproj_arc(const vector[int]& heads) nogil except -2: # return the smallest non-proj arc or None # where size is defined as the distance between dep and head # and ties are broken left to right cdef int smallest_size = INT_MAX + # -1 means its already projective. cdef int smallest_np_arc = -1 cdef int size cdef int tokenid diff --git a/spacy/tests/parser/test_nonproj.py b/spacy/tests/parser/test_nonproj.py index 60d000c44..b420c300f 100644 --- a/spacy/tests/parser/test_nonproj.py +++ b/spacy/tests/parser/test_nonproj.py @@ -49,7 +49,7 @@ def test_parser_contains_cycle(tree, cyclic_tree, partial_tree, multirooted_tree assert contains_cycle(multirooted_tree) is None -def test_parser_is_nonproj_arc(nonproj_tree, partial_tree, multirooted_tree): +def test_parser_is_nonproj_arc(cyclic_tree, nonproj_tree, partial_tree, multirooted_tree): assert is_nonproj_arc(0, nonproj_tree) is False assert is_nonproj_arc(1, nonproj_tree) is False assert is_nonproj_arc(2, nonproj_tree) is False @@ -62,15 +62,19 @@ def test_parser_is_nonproj_arc(nonproj_tree, partial_tree, multirooted_tree): assert is_nonproj_arc(7, partial_tree) is False assert is_nonproj_arc(17, multirooted_tree) is False assert is_nonproj_arc(16, multirooted_tree) is True + with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'): + is_nonproj_arc(6, cyclic_tree) def test_parser_is_nonproj_tree( - proj_tree, nonproj_tree, partial_tree, multirooted_tree + proj_tree, cyclic_tree, nonproj_tree, partial_tree, multirooted_tree ): assert is_nonproj_tree(proj_tree) is False assert is_nonproj_tree(nonproj_tree) is True assert is_nonproj_tree(partial_tree) is False assert is_nonproj_tree(multirooted_tree) is True + with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'): + is_nonproj_tree(cyclic_tree) def test_parser_pseudoprojectivity(en_vocab): @@ -84,8 +88,10 @@ def test_parser_pseudoprojectivity(en_vocab): tree = [1, 2, 2] nonproj_tree = [1, 2, 2, 4, 5, 2, 7, 4, 2] nonproj_tree2 = [9, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1] + cyclic_tree = [1, 2, 2, 4, 5, 3, 2] labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl", "punct"] labels2 = ["advmod", "root", "det", "nsubj", "advmod", "det", "dobj", "det", "nmod", "aux", "nmod", "advmod", "det", "amod", "punct"] + cyclic_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "punct"] # fmt: on assert nonproj.decompose("X||Y") == ("X", "Y") assert nonproj.decompose("X") == ("X", "") @@ -97,6 +103,8 @@ def test_parser_pseudoprojectivity(en_vocab): assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree2) == 10 # fmt: off proj_heads, deco_labels = nonproj.projectivize(nonproj_tree, labels) + with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'): + nonproj.projectivize(cyclic_tree, cyclic_labels) assert proj_heads == [1, 2, 2, 4, 5, 2, 7, 5, 2] assert deco_labels == ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl||dobj", "punct"] diff --git a/spacy/tests/training/test_training.py b/spacy/tests/training/test_training.py index 8e08a25fb..31bf7e07b 100644 --- a/spacy/tests/training/test_training.py +++ b/spacy/tests/training/test_training.py @@ -671,13 +671,13 @@ def test_gold_ner_missing_tags(en_tokenizer): def test_projectivize(en_tokenizer): doc = en_tokenizer("He pretty quickly walks away") - heads = [3, 2, 3, 0, 2] + heads = [3, 2, 3, 3, 2] deps = ["dep"] * len(heads) example = Example.from_dict(doc, {"heads": heads, "deps": deps}) proj_heads, proj_labels = example.get_aligned_parse(projectivize=True) nonproj_heads, nonproj_labels = example.get_aligned_parse(projectivize=False) - assert proj_heads == [3, 2, 3, 0, 3] - assert nonproj_heads == [3, 2, 3, 0, 2] + assert proj_heads == [3, 2, 3, 3, 3] + assert nonproj_heads == [3, 2, 3, 3, 2] def test_iob_to_biluo():