mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-25 17:36:30 +03:00
Merge remote-tracking branch 'upstream/master' into merge-master-v4-20220609
This commit is contained in:
commit
2f05c6824c
2
.github/workflows/explosionbot.yml
vendored
2
.github/workflows/explosionbot.yml
vendored
|
@ -23,5 +23,5 @@ jobs:
|
|||
env:
|
||||
INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
|
||||
INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }}
|
||||
ENABLED_COMMANDS: "test_gpu,test_slow"
|
||||
ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu"
|
||||
ALLOWED_TEAMS: "spaCy"
|
||||
|
|
1
.github/workflows/gputests.yml
vendored
1
.github/workflows/gputests.yml
vendored
|
@ -10,6 +10,7 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
branch: [master, v4]
|
||||
if: github.repository_owner == 'explosion'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Trigger buildkite build
|
||||
|
|
1
.github/workflows/slowtests.yml
vendored
1
.github/workflows/slowtests.yml
vendored
|
@ -10,6 +10,7 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
branch: [master, v4]
|
||||
if: github.repository_owner == 'explosion'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml
|
||||
recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml *.hh
|
||||
include LICENSE
|
||||
include README.md
|
||||
include pyproject.toml
|
||||
|
|
|
@ -16,7 +16,7 @@ production-ready [**training system**](https://spacy.io/usage/training) and easy
|
|||
model packaging, deployment and workflow management. spaCy is commercial
|
||||
open-source software, released under the MIT license.
|
||||
|
||||
💫 **Version 3.2 out now!**
|
||||
💫 **Version 3.3.1 out now!**
|
||||
[Check out the release notes here.](https://github.com/explosion/spaCy/releases)
|
||||
|
||||
[![Azure Pipelines](https://img.shields.io/azure-devops/build/explosion-ai/public/8/master.svg?logo=azure-pipelines&style=flat-square&label=build)](https://dev.azure.com/explosion-ai/public/_build?definitionId=8)
|
||||
|
|
|
@ -5,8 +5,7 @@ requires = [
|
|||
"cymem>=2.0.2,<2.1.0",
|
||||
"preshed>=3.0.2,<3.1.0",
|
||||
"murmurhash>=0.28.0,<1.1.0",
|
||||
"thinc>=8.0.14,<8.1.0",
|
||||
"blis>=0.4.0,<0.8.0",
|
||||
"thinc>=8.1.0.dev0,<8.2.0",
|
||||
"pathy",
|
||||
"numpy>=1.15.0",
|
||||
]
|
||||
|
|
|
@ -3,8 +3,7 @@ spacy-legacy>=3.0.9,<3.1.0
|
|||
spacy-loggers>=1.0.0,<2.0.0
|
||||
cymem>=2.0.2,<2.1.0
|
||||
preshed>=3.0.2,<3.1.0
|
||||
thinc>=8.0.14,<8.1.0
|
||||
blis>=0.4.0,<0.8.0
|
||||
thinc>=8.1.0.dev0,<8.2.0
|
||||
ml_datasets>=0.2.0,<0.3.0
|
||||
murmurhash>=0.28.0,<1.1.0
|
||||
wasabi>=0.9.1,<1.1.0
|
||||
|
@ -16,13 +15,13 @@ pathy>=0.3.5
|
|||
numpy>=1.15.0
|
||||
requests>=2.13.0,<3.0.0
|
||||
tqdm>=4.38.0,<5.0.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.9.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0
|
||||
jinja2
|
||||
langcodes>=3.2.0,<4.0.0
|
||||
# Official Python utilities
|
||||
setuptools
|
||||
packaging>=20.0
|
||||
typing_extensions>=3.7.4.1,<4.0.0.0; python_version < "3.8"
|
||||
typing_extensions>=3.7.4.1,<4.2.0; python_version < "3.8"
|
||||
# Development dependencies
|
||||
pre-commit>=2.13.0
|
||||
cython>=0.25,<3.0
|
||||
|
@ -31,7 +30,7 @@ pytest-timeout>=1.3.0,<2.0.0
|
|||
mock>=2.0.0,<3.0.0
|
||||
flake8>=3.8.0,<3.10.0
|
||||
hypothesis>=3.27.0,<7.0.0
|
||||
mypy==0.910
|
||||
mypy>=0.910,<=0.960
|
||||
types-dataclasses>=0.1.3; python_version < "3.7"
|
||||
types-mock>=0.1.1
|
||||
types-requests
|
||||
|
|
|
@ -38,7 +38,7 @@ setup_requires =
|
|||
cymem>=2.0.2,<2.1.0
|
||||
preshed>=3.0.2,<3.1.0
|
||||
murmurhash>=0.28.0,<1.1.0
|
||||
thinc>=8.0.14,<8.1.0
|
||||
thinc>=8.1.0.dev0,<8.2.0
|
||||
install_requires =
|
||||
# Our libraries
|
||||
spacy-legacy>=3.0.9,<3.1.0
|
||||
|
@ -46,8 +46,7 @@ install_requires =
|
|||
murmurhash>=0.28.0,<1.1.0
|
||||
cymem>=2.0.2,<2.1.0
|
||||
preshed>=3.0.2,<3.1.0
|
||||
thinc>=8.0.14,<8.1.0
|
||||
blis>=0.4.0,<0.8.0
|
||||
thinc>=8.1.0.dev0,<8.2.0
|
||||
wasabi>=0.9.1,<1.1.0
|
||||
srsly>=2.4.3,<3.0.0
|
||||
catalogue>=2.0.6,<2.1.0
|
||||
|
@ -57,12 +56,12 @@ install_requires =
|
|||
tqdm>=4.38.0,<5.0.0
|
||||
numpy>=1.15.0
|
||||
requests>=2.13.0,<3.0.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.9.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0
|
||||
jinja2
|
||||
# Official Python utilities
|
||||
setuptools
|
||||
packaging>=20.0
|
||||
typing_extensions>=3.7.4,<4.0.0.0; python_version < "3.8"
|
||||
typing_extensions>=3.7.4,<4.2.0; python_version < "3.8"
|
||||
langcodes>=3.2.0,<4.0.0
|
||||
|
||||
[options.entry_points]
|
||||
|
|
|
@ -12,7 +12,7 @@ from click.parser import split_arg_string
|
|||
from typer.main import get_command
|
||||
from contextlib import contextmanager
|
||||
from thinc.api import Config, ConfigValidationError, require_gpu
|
||||
from thinc.util import has_cupy, gpu_is_available
|
||||
from thinc.util import gpu_is_available
|
||||
from configparser import InterpolationError
|
||||
import os
|
||||
|
||||
|
@ -554,5 +554,5 @@ def setup_gpu(use_gpu: int, silent=None) -> None:
|
|||
require_gpu(use_gpu)
|
||||
else:
|
||||
local_msg.info("Using CPU")
|
||||
if has_cupy and gpu_is_available():
|
||||
if gpu_is_available():
|
||||
local_msg.info("To switch to GPU 0, use the option: --gpu-id 0")
|
||||
|
|
|
@ -6,6 +6,7 @@ import sys
|
|||
import srsly
|
||||
from wasabi import Printer, MESSAGES, msg
|
||||
import typer
|
||||
import math
|
||||
|
||||
from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides
|
||||
from ._util import import_code, debug_cli
|
||||
|
@ -30,6 +31,12 @@ DEP_LABEL_THRESHOLD = 20
|
|||
# Minimum number of expected examples to train a new pipeline
|
||||
BLANK_MODEL_MIN_THRESHOLD = 100
|
||||
BLANK_MODEL_THRESHOLD = 2000
|
||||
# Arbitrary threshold where SpanCat performs well
|
||||
SPAN_DISTINCT_THRESHOLD = 1
|
||||
# Arbitrary threshold where SpanCat performs well
|
||||
BOUNDARY_DISTINCT_THRESHOLD = 1
|
||||
# Arbitrary threshold for filtering span lengths during reporting (percentage)
|
||||
SPAN_LENGTH_THRESHOLD_PERCENTAGE = 90
|
||||
|
||||
|
||||
@debug_cli.command(
|
||||
|
@ -247,6 +254,69 @@ def debug_data(
|
|||
msg.warn(f"No examples for texts WITHOUT new label '{label}'")
|
||||
has_no_neg_warning = True
|
||||
|
||||
with msg.loading("Obtaining span characteristics..."):
|
||||
span_characteristics = _get_span_characteristics(
|
||||
train_dataset, gold_train_data, spans_key
|
||||
)
|
||||
|
||||
msg.info(f"Span characteristics for spans_key '{spans_key}'")
|
||||
msg.info("SD = Span Distinctiveness, BD = Boundary Distinctiveness")
|
||||
_print_span_characteristics(span_characteristics)
|
||||
|
||||
_span_freqs = _get_spans_length_freq_dist(
|
||||
gold_train_data["spans_length"][spans_key]
|
||||
)
|
||||
_filtered_span_freqs = _filter_spans_length_freq_dist(
|
||||
_span_freqs, threshold=SPAN_LENGTH_THRESHOLD_PERCENTAGE
|
||||
)
|
||||
|
||||
msg.info(
|
||||
f"Over {SPAN_LENGTH_THRESHOLD_PERCENTAGE}% of spans have lengths of 1 -- "
|
||||
f"{max(_filtered_span_freqs.keys())} "
|
||||
f"(min={span_characteristics['min_length']}, max={span_characteristics['max_length']}). "
|
||||
f"The most common span lengths are: {_format_freqs(_filtered_span_freqs)}. "
|
||||
"If you are using the n-gram suggester, note that omitting "
|
||||
"infrequent n-gram lengths can greatly improve speed and "
|
||||
"memory usage."
|
||||
)
|
||||
|
||||
msg.text(
|
||||
f"Full distribution of span lengths: {_format_freqs(_span_freqs)}",
|
||||
show=verbose,
|
||||
)
|
||||
|
||||
# Add report regarding span characteristics
|
||||
if span_characteristics["avg_sd"] < SPAN_DISTINCT_THRESHOLD:
|
||||
msg.warn("Spans may not be distinct from the rest of the corpus")
|
||||
else:
|
||||
msg.good("Spans are distinct from the rest of the corpus")
|
||||
|
||||
p_spans = span_characteristics["p_spans"].values()
|
||||
all_span_tokens: Counter = sum(p_spans, Counter())
|
||||
most_common_spans = [w for w, _ in all_span_tokens.most_common(10)]
|
||||
msg.text(
|
||||
"10 most common span tokens: {}".format(
|
||||
_format_labels(most_common_spans)
|
||||
),
|
||||
show=verbose,
|
||||
)
|
||||
|
||||
# Add report regarding span boundary characteristics
|
||||
if span_characteristics["avg_bd"] < BOUNDARY_DISTINCT_THRESHOLD:
|
||||
msg.warn("Boundary tokens are not distinct from the rest of the corpus")
|
||||
else:
|
||||
msg.good("Boundary tokens are distinct from the rest of the corpus")
|
||||
|
||||
p_bounds = span_characteristics["p_bounds"].values()
|
||||
all_span_bound_tokens: Counter = sum(p_bounds, Counter())
|
||||
most_common_bounds = [w for w, _ in all_span_bound_tokens.most_common(10)]
|
||||
msg.text(
|
||||
"10 most common span boundary tokens: {}".format(
|
||||
_format_labels(most_common_bounds)
|
||||
),
|
||||
show=verbose,
|
||||
)
|
||||
|
||||
if has_low_data_warning:
|
||||
msg.text(
|
||||
f"To train a new span type, your data should include at "
|
||||
|
@ -647,6 +717,9 @@ def _compile_gold(
|
|||
"words": Counter(),
|
||||
"roots": Counter(),
|
||||
"spancat": dict(),
|
||||
"spans_length": dict(),
|
||||
"spans_per_type": dict(),
|
||||
"sb_per_type": dict(),
|
||||
"ws_ents": 0,
|
||||
"boundary_cross_ents": 0,
|
||||
"n_words": 0,
|
||||
|
@ -692,14 +765,59 @@ def _compile_gold(
|
|||
elif label == "-":
|
||||
data["ner"]["-"] += 1
|
||||
if "spancat" in factory_names:
|
||||
for span_key in list(eg.reference.spans.keys()):
|
||||
if span_key not in data["spancat"]:
|
||||
data["spancat"][span_key] = Counter()
|
||||
for i, span in enumerate(eg.reference.spans[span_key]):
|
||||
for spans_key in list(eg.reference.spans.keys()):
|
||||
# Obtain the span frequency
|
||||
if spans_key not in data["spancat"]:
|
||||
data["spancat"][spans_key] = Counter()
|
||||
for i, span in enumerate(eg.reference.spans[spans_key]):
|
||||
if span.label_ is None:
|
||||
continue
|
||||
else:
|
||||
data["spancat"][span_key][span.label_] += 1
|
||||
data["spancat"][spans_key][span.label_] += 1
|
||||
|
||||
# Obtain the span length
|
||||
if spans_key not in data["spans_length"]:
|
||||
data["spans_length"][spans_key] = dict()
|
||||
for span in gold.spans[spans_key]:
|
||||
if span.label_ is None:
|
||||
continue
|
||||
if span.label_ not in data["spans_length"][spans_key]:
|
||||
data["spans_length"][spans_key][span.label_] = []
|
||||
data["spans_length"][spans_key][span.label_].append(len(span))
|
||||
|
||||
# Obtain spans per span type
|
||||
if spans_key not in data["spans_per_type"]:
|
||||
data["spans_per_type"][spans_key] = dict()
|
||||
for span in gold.spans[spans_key]:
|
||||
if span.label_ not in data["spans_per_type"][spans_key]:
|
||||
data["spans_per_type"][spans_key][span.label_] = []
|
||||
data["spans_per_type"][spans_key][span.label_].append(span)
|
||||
|
||||
# Obtain boundary tokens per span type
|
||||
window_size = 1
|
||||
if spans_key not in data["sb_per_type"]:
|
||||
data["sb_per_type"][spans_key] = dict()
|
||||
for span in gold.spans[spans_key]:
|
||||
if span.label_ not in data["sb_per_type"][spans_key]:
|
||||
# Creating a data structure that holds the start and
|
||||
# end tokens for each span type
|
||||
data["sb_per_type"][spans_key][span.label_] = {
|
||||
"start": [],
|
||||
"end": [],
|
||||
}
|
||||
for offset in range(window_size):
|
||||
sb_start_idx = span.start - (offset + 1)
|
||||
if sb_start_idx >= 0:
|
||||
data["sb_per_type"][spans_key][span.label_]["start"].append(
|
||||
gold[sb_start_idx : sb_start_idx + 1]
|
||||
)
|
||||
|
||||
sb_end_idx = span.end + (offset + 1)
|
||||
if sb_end_idx <= len(gold):
|
||||
data["sb_per_type"][spans_key][span.label_]["end"].append(
|
||||
gold[sb_end_idx - 1 : sb_end_idx]
|
||||
)
|
||||
|
||||
if "textcat" in factory_names or "textcat_multilabel" in factory_names:
|
||||
data["cats"].update(gold.cats)
|
||||
if any(val not in (0, 1) for val in gold.cats.values()):
|
||||
|
@ -770,6 +888,16 @@ def _format_labels(
|
|||
return ", ".join([f"'{l}'" for l in cast(Iterable[str], labels)])
|
||||
|
||||
|
||||
def _format_freqs(freqs: Dict[int, float], sort: bool = True) -> str:
|
||||
if sort:
|
||||
freqs = dict(sorted(freqs.items()))
|
||||
|
||||
_freqs = [(str(k), v) for k, v in freqs.items()]
|
||||
return ", ".join(
|
||||
[f"{l} ({c}%)" for l, c in cast(Iterable[Tuple[str, float]], _freqs)]
|
||||
)
|
||||
|
||||
|
||||
def _get_examples_without_label(
|
||||
data: Sequence[Example],
|
||||
label: str,
|
||||
|
@ -824,3 +952,158 @@ def _get_labels_from_spancat(nlp: Language) -> Dict[str, Set[str]]:
|
|||
labels[pipe.key] = set()
|
||||
labels[pipe.key].update(pipe.labels)
|
||||
return labels
|
||||
|
||||
|
||||
def _gmean(l: List) -> float:
|
||||
"""Compute geometric mean of a list"""
|
||||
return math.exp(math.fsum(math.log(i) for i in l) / len(l))
|
||||
|
||||
|
||||
def _wgt_average(metric: Dict[str, float], frequencies: Counter) -> float:
|
||||
total = sum(value * frequencies[span_type] for span_type, value in metric.items())
|
||||
return total / sum(frequencies.values())
|
||||
|
||||
|
||||
def _get_distribution(docs, normalize: bool = True) -> Counter:
|
||||
"""Get the frequency distribution given a set of Docs"""
|
||||
word_counts: Counter = Counter()
|
||||
for doc in docs:
|
||||
for token in doc:
|
||||
# Normalize the text
|
||||
t = token.text.lower().replace("``", '"').replace("''", '"')
|
||||
word_counts[t] += 1
|
||||
if normalize:
|
||||
total = sum(word_counts.values(), 0.0)
|
||||
word_counts = Counter({k: v / total for k, v in word_counts.items()})
|
||||
return word_counts
|
||||
|
||||
|
||||
def _get_kl_divergence(p: Counter, q: Counter) -> float:
|
||||
"""Compute the Kullback-Leibler divergence from two frequency distributions"""
|
||||
total = 0.0
|
||||
for word, p_word in p.items():
|
||||
total += p_word * math.log(p_word / q[word])
|
||||
return total
|
||||
|
||||
|
||||
def _format_span_row(span_data: List[Dict], labels: List[str]) -> List[Any]:
|
||||
"""Compile into one list for easier reporting"""
|
||||
d = {
|
||||
label: [label] + list(round(d[label], 2) for d in span_data) for label in labels
|
||||
}
|
||||
return list(d.values())
|
||||
|
||||
|
||||
def _get_span_characteristics(
|
||||
examples: List[Example], compiled_gold: Dict[str, Any], spans_key: str
|
||||
) -> Dict[str, Any]:
|
||||
"""Obtain all span characteristics"""
|
||||
data_labels = compiled_gold["spancat"][spans_key]
|
||||
# Get lengths
|
||||
span_length = {
|
||||
label: _gmean(l)
|
||||
for label, l in compiled_gold["spans_length"][spans_key].items()
|
||||
}
|
||||
min_lengths = [min(l) for l in compiled_gold["spans_length"][spans_key].values()]
|
||||
max_lengths = [max(l) for l in compiled_gold["spans_length"][spans_key].values()]
|
||||
|
||||
# Get relevant distributions: corpus, spans, span boundaries
|
||||
p_corpus = _get_distribution([eg.reference for eg in examples], normalize=True)
|
||||
p_spans = {
|
||||
label: _get_distribution(spans, normalize=True)
|
||||
for label, spans in compiled_gold["spans_per_type"][spans_key].items()
|
||||
}
|
||||
p_bounds = {
|
||||
label: _get_distribution(sb["start"] + sb["end"], normalize=True)
|
||||
for label, sb in compiled_gold["sb_per_type"][spans_key].items()
|
||||
}
|
||||
|
||||
# Compute for actual span characteristics
|
||||
span_distinctiveness = {
|
||||
label: _get_kl_divergence(freq_dist, p_corpus)
|
||||
for label, freq_dist in p_spans.items()
|
||||
}
|
||||
sb_distinctiveness = {
|
||||
label: _get_kl_divergence(freq_dist, p_corpus)
|
||||
for label, freq_dist in p_bounds.items()
|
||||
}
|
||||
|
||||
return {
|
||||
"sd": span_distinctiveness,
|
||||
"bd": sb_distinctiveness,
|
||||
"lengths": span_length,
|
||||
"min_length": min(min_lengths),
|
||||
"max_length": max(max_lengths),
|
||||
"avg_sd": _wgt_average(span_distinctiveness, data_labels),
|
||||
"avg_bd": _wgt_average(sb_distinctiveness, data_labels),
|
||||
"avg_length": _wgt_average(span_length, data_labels),
|
||||
"labels": list(data_labels.keys()),
|
||||
"p_spans": p_spans,
|
||||
"p_bounds": p_bounds,
|
||||
}
|
||||
|
||||
|
||||
def _print_span_characteristics(span_characteristics: Dict[str, Any]):
|
||||
"""Print all span characteristics into a table"""
|
||||
headers = ("Span Type", "Length", "SD", "BD")
|
||||
# Prepare table data with all span characteristics
|
||||
table_data = [
|
||||
span_characteristics["lengths"],
|
||||
span_characteristics["sd"],
|
||||
span_characteristics["bd"],
|
||||
]
|
||||
table = _format_span_row(
|
||||
span_data=table_data, labels=span_characteristics["labels"]
|
||||
)
|
||||
# Prepare table footer with weighted averages
|
||||
footer_data = [
|
||||
span_characteristics["avg_length"],
|
||||
span_characteristics["avg_sd"],
|
||||
span_characteristics["avg_bd"],
|
||||
]
|
||||
footer = ["Wgt. Average"] + [str(round(f, 2)) for f in footer_data]
|
||||
msg.table(table, footer=footer, header=headers, divider=True)
|
||||
|
||||
|
||||
def _get_spans_length_freq_dist(
|
||||
length_dict: Dict, threshold=SPAN_LENGTH_THRESHOLD_PERCENTAGE
|
||||
) -> Dict[int, float]:
|
||||
"""Get frequency distribution of spans length under a certain threshold"""
|
||||
all_span_lengths = []
|
||||
for _, lengths in length_dict.items():
|
||||
all_span_lengths.extend(lengths)
|
||||
|
||||
freq_dist: Counter = Counter()
|
||||
for i in all_span_lengths:
|
||||
if freq_dist.get(i):
|
||||
freq_dist[i] += 1
|
||||
else:
|
||||
freq_dist[i] = 1
|
||||
|
||||
# We will be working with percentages instead of raw counts
|
||||
freq_dist_percentage = {}
|
||||
for span_length, count in freq_dist.most_common():
|
||||
percentage = (count / len(all_span_lengths)) * 100.0
|
||||
percentage = round(percentage, 2)
|
||||
freq_dist_percentage[span_length] = percentage
|
||||
|
||||
return freq_dist_percentage
|
||||
|
||||
|
||||
def _filter_spans_length_freq_dist(
|
||||
freq_dist: Dict[int, float], threshold: int
|
||||
) -> Dict[int, float]:
|
||||
"""Filter frequency distribution with respect to a threshold
|
||||
|
||||
We're going to filter all the span lengths that fall
|
||||
around a percentage threshold when summed.
|
||||
"""
|
||||
total = 0.0
|
||||
filtered_freq_dist = {}
|
||||
for span_length, dist in freq_dist.items():
|
||||
if total >= threshold:
|
||||
break
|
||||
else:
|
||||
filtered_freq_dist[span_length] = dist
|
||||
total += dist
|
||||
return filtered_freq_dist
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import warnings
|
||||
from .compat import Literal
|
||||
|
||||
|
||||
class ErrorsWithCodes(type):
|
||||
|
@ -26,7 +27,10 @@ def setup_default_warnings():
|
|||
filter_warning("once", error_msg="[W114]")
|
||||
|
||||
|
||||
def filter_warning(action: str, error_msg: str):
|
||||
def filter_warning(
|
||||
action: Literal["default", "error", "ignore", "always", "module", "once"],
|
||||
error_msg: str,
|
||||
):
|
||||
"""Customize how spaCy should handle a certain warning.
|
||||
|
||||
error_msg (str): e.g. "W006", or a full error message
|
||||
|
@ -200,6 +204,11 @@ class Warnings(metaclass=ErrorsWithCodes):
|
|||
"for the corpora used to train the language. Please check "
|
||||
"`nlp.meta[\"sources\"]` for any relevant links.")
|
||||
W119 = ("Overriding pipe name in `config` is not supported. Ignoring override '{name_in_config}'.")
|
||||
W120 = ("Unable to load all spans in Doc.spans: more than one span group "
|
||||
"with the name '{group_name}' was found in the saved spans data. "
|
||||
"Only the last span group will be loaded under "
|
||||
"Doc.spans['{group_name}']. Skipping span group with values: "
|
||||
"{group_values}")
|
||||
|
||||
|
||||
class Errors(metaclass=ErrorsWithCodes):
|
||||
|
@ -445,10 +454,10 @@ class Errors(metaclass=ErrorsWithCodes):
|
|||
"same, but found '{nlp}' and '{vocab}' respectively.")
|
||||
E152 = ("The attribute {attr} is not supported for token patterns. "
|
||||
"Please use the option `validate=True` with the Matcher, PhraseMatcher, "
|
||||
"or EntityRuler for more details.")
|
||||
"EntityRuler or AttributeRuler for more details.")
|
||||
E153 = ("The value type {vtype} is not supported for token patterns. "
|
||||
"Please use the option validate=True with Matcher, PhraseMatcher, "
|
||||
"or EntityRuler for more details.")
|
||||
"EntityRuler or AttributeRuler for more details.")
|
||||
E154 = ("One of the attributes or values is not supported for token "
|
||||
"patterns. Please use the option `validate=True` with the Matcher, "
|
||||
"PhraseMatcher, or EntityRuler for more details.")
|
||||
|
@ -528,6 +537,8 @@ class Errors(metaclass=ErrorsWithCodes):
|
|||
E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.")
|
||||
|
||||
# New errors added in v3.x
|
||||
E854 = ("Unable to set doc.ents. Check that the 'ents_filter' does not "
|
||||
"permit overlapping spans.")
|
||||
E855 = ("Invalid {obj}: {obj} is not from the same doc.")
|
||||
E856 = ("Error accessing span at position {i}: out of bounds in span group "
|
||||
"of length {length}.")
|
||||
|
@ -899,8 +910,8 @@ class Errors(metaclass=ErrorsWithCodes):
|
|||
E1022 = ("Words must be of type str or int, but input is of type '{wtype}'")
|
||||
E1023 = ("Couldn't read EntityRuler from the {path}. This file doesn't "
|
||||
"exist.")
|
||||
E1024 = ("A pattern with ID \"{ent_id}\" is not present in EntityRuler "
|
||||
"patterns.")
|
||||
E1024 = ("A pattern with {attr_type} '{label}' is not present in "
|
||||
"'{component}' patterns.")
|
||||
E1025 = ("Cannot intify the value '{value}' as an IOB string. The only "
|
||||
"supported values are: 'I', 'O', 'B' and ''")
|
||||
E1026 = ("Edit tree has an invalid format:\n{errors}")
|
||||
|
@ -914,6 +925,13 @@ class Errors(metaclass=ErrorsWithCodes):
|
|||
E1034 = ("Node index {i} out of bounds ({length})")
|
||||
E1035 = ("Token index {i} out of bounds ({length})")
|
||||
E1036 = ("Cannot index into NoneNode")
|
||||
E1037 = ("Invalid attribute value '{attr}'.")
|
||||
E1038 = ("Invalid JSON input: {message}")
|
||||
E1039 = ("The {obj} start or end annotations (start: {start}, end: {end}) "
|
||||
"could not be aligned to token boundaries.")
|
||||
E1040 = ("Doc.from_json requires all tokens to have the same attributes. "
|
||||
"Some tokens do not contain annotation for: {partial_attrs}")
|
||||
E1041 = ("Expected a string, Doc, or bytes as input, but got: {type}")
|
||||
|
||||
|
||||
# Deprecated model shortcuts, only used in errors and warnings
|
||||
|
|
|
@ -273,6 +273,7 @@ GLOSSARY = {
|
|||
"relcl": "relative clause modifier",
|
||||
"reparandum": "overridden disfluency",
|
||||
"root": "root",
|
||||
"ROOT": "root",
|
||||
"vocative": "vocative",
|
||||
"xcomp": "open clausal complement",
|
||||
# Dependency labels (German)
|
||||
|
|
|
@ -35,7 +35,7 @@ for pron in ["i"]:
|
|||
|
||||
_exc[orth + "m"] = [
|
||||
{ORTH: orth, NORM: pron},
|
||||
{ORTH: "m", "tenspect": 1, "number": 1},
|
||||
{ORTH: "m"},
|
||||
]
|
||||
|
||||
_exc[orth + "'ma"] = [
|
||||
|
@ -139,26 +139,27 @@ for pron in ["he", "she", "it"]:
|
|||
|
||||
# W-words, relative pronouns, prepositions etc.
|
||||
|
||||
for word in [
|
||||
"who",
|
||||
"what",
|
||||
"when",
|
||||
"where",
|
||||
"why",
|
||||
"how",
|
||||
"there",
|
||||
"that",
|
||||
"this",
|
||||
"these",
|
||||
"those",
|
||||
for word, morph in [
|
||||
("who", None),
|
||||
("what", None),
|
||||
("when", None),
|
||||
("where", None),
|
||||
("why", None),
|
||||
("how", None),
|
||||
("there", None),
|
||||
("that", "Number=Sing|Person=3"),
|
||||
("this", "Number=Sing|Person=3"),
|
||||
("these", "Number=Plur|Person=3"),
|
||||
("those", "Number=Plur|Person=3"),
|
||||
]:
|
||||
for orth in [word, word.title()]:
|
||||
_exc[orth + "'s"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
{ORTH: "'s", NORM: "'s"},
|
||||
]
|
||||
if morph != "Number=Plur|Person=3":
|
||||
_exc[orth + "'s"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
{ORTH: "'s", NORM: "'s"},
|
||||
]
|
||||
|
||||
_exc[orth + "s"] = [{ORTH: orth, NORM: word}, {ORTH: "s"}]
|
||||
_exc[orth + "s"] = [{ORTH: orth, NORM: word}, {ORTH: "s"}]
|
||||
|
||||
_exc[orth + "'ll"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
|
@ -182,25 +183,26 @@ for word in [
|
|||
{ORTH: "ve", NORM: "have"},
|
||||
]
|
||||
|
||||
_exc[orth + "'re"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
{ORTH: "'re", NORM: "are"},
|
||||
]
|
||||
if morph != "Number=Sing|Person=3":
|
||||
_exc[orth + "'re"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
{ORTH: "'re", NORM: "are"},
|
||||
]
|
||||
|
||||
_exc[orth + "re"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
{ORTH: "re", NORM: "are"},
|
||||
]
|
||||
_exc[orth + "re"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
{ORTH: "re", NORM: "are"},
|
||||
]
|
||||
|
||||
_exc[orth + "'ve"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
{ORTH: "'ve"},
|
||||
]
|
||||
_exc[orth + "'ve"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
{ORTH: "'ve"},
|
||||
]
|
||||
|
||||
_exc[orth + "ve"] = [
|
||||
{ORTH: orth},
|
||||
{ORTH: "ve", NORM: "have"},
|
||||
]
|
||||
_exc[orth + "ve"] = [
|
||||
{ORTH: orth},
|
||||
{ORTH: "ve", NORM: "have"},
|
||||
]
|
||||
|
||||
_exc[orth + "'d"] = [
|
||||
{ORTH: orth, NORM: word},
|
||||
|
|
|
@ -1090,16 +1090,21 @@ class Language:
|
|||
)
|
||||
return self.tokenizer(text)
|
||||
|
||||
def _ensure_doc(self, doc_like: Union[str, Doc]) -> Doc:
|
||||
"""Create a Doc if need be, or raise an error if the input is not a Doc or a string."""
|
||||
def _ensure_doc(self, doc_like: Union[str, Doc, bytes]) -> Doc:
|
||||
"""Create a Doc if need be, or raise an error if the input is not
|
||||
a Doc, string, or a byte array (generated by Doc.to_bytes())."""
|
||||
if isinstance(doc_like, Doc):
|
||||
return doc_like
|
||||
if isinstance(doc_like, str):
|
||||
return self.make_doc(doc_like)
|
||||
raise ValueError(Errors.E866.format(type=type(doc_like)))
|
||||
if isinstance(doc_like, bytes):
|
||||
return Doc(self.vocab).from_bytes(doc_like)
|
||||
raise ValueError(Errors.E1041.format(type=type(doc_like)))
|
||||
|
||||
def _ensure_doc_with_context(self, doc_like: Union[str, Doc], context: Any) -> Doc:
|
||||
"""Create a Doc if need be and add as_tuples context, or raise an error if the input is not a Doc or a string."""
|
||||
def _ensure_doc_with_context(
|
||||
self, doc_like: Union[str, Doc, bytes], context: _AnyContext
|
||||
) -> Doc:
|
||||
"""Call _ensure_doc to generate a Doc and set its context object."""
|
||||
doc = self._ensure_doc(doc_like)
|
||||
doc._context = context
|
||||
return doc
|
||||
|
@ -1519,7 +1524,6 @@ class Language:
|
|||
|
||||
DOCS: https://spacy.io/api/language#pipe
|
||||
"""
|
||||
# Handle texts with context as tuples
|
||||
if as_tuples:
|
||||
texts = cast(Iterable[Tuple[Union[str, Doc], _AnyContext]], texts)
|
||||
docs_with_contexts = (
|
||||
|
@ -1597,8 +1601,21 @@ class Language:
|
|||
n_process: int,
|
||||
batch_size: int,
|
||||
) -> Iterator[Doc]:
|
||||
def prepare_input(
|
||||
texts: Iterable[Union[str, Doc]]
|
||||
) -> Iterable[Tuple[Union[str, bytes], _AnyContext]]:
|
||||
# Serialize Doc inputs to bytes to avoid incurring pickling
|
||||
# overhead when they are passed to child processes. Also yield
|
||||
# any context objects they might have separately (as they are not serialized).
|
||||
for doc_like in texts:
|
||||
if isinstance(doc_like, Doc):
|
||||
yield (doc_like.to_bytes(), cast(_AnyContext, doc_like._context))
|
||||
else:
|
||||
yield (doc_like, cast(_AnyContext, None))
|
||||
|
||||
serialized_texts_with_ctx = prepare_input(texts) # type: ignore
|
||||
# raw_texts is used later to stop iteration.
|
||||
texts, raw_texts = itertools.tee(texts)
|
||||
texts, raw_texts = itertools.tee(serialized_texts_with_ctx) # type: ignore
|
||||
# for sending texts to worker
|
||||
texts_q: List[mp.Queue] = [mp.Queue() for _ in range(n_process)]
|
||||
# for receiving byte-encoded docs from worker
|
||||
|
@ -1618,7 +1635,13 @@ class Language:
|
|||
procs = [
|
||||
mp.Process(
|
||||
target=_apply_pipes,
|
||||
args=(self._ensure_doc, pipes, rch, sch, Underscore.get_state()),
|
||||
args=(
|
||||
self._ensure_doc_with_context,
|
||||
pipes,
|
||||
rch,
|
||||
sch,
|
||||
Underscore.get_state(),
|
||||
),
|
||||
)
|
||||
for rch, sch in zip(texts_q, bytedocs_send_ch)
|
||||
]
|
||||
|
@ -1631,12 +1654,12 @@ class Language:
|
|||
recv.recv() for recv in cycle(bytedocs_recv_ch)
|
||||
)
|
||||
try:
|
||||
for i, (_, (byte_doc, byte_context, byte_error)) in enumerate(
|
||||
for i, (_, (byte_doc, context, byte_error)) in enumerate(
|
||||
zip(raw_texts, byte_tuples), 1
|
||||
):
|
||||
if byte_doc is not None:
|
||||
doc = Doc(self.vocab).from_bytes(byte_doc)
|
||||
doc._context = byte_context
|
||||
doc._context = context
|
||||
yield doc
|
||||
elif byte_error is not None:
|
||||
error = srsly.msgpack_loads(byte_error)
|
||||
|
@ -2163,7 +2186,7 @@ def _copy_examples(examples: Iterable[Example]) -> List[Example]:
|
|||
|
||||
|
||||
def _apply_pipes(
|
||||
ensure_doc: Callable[[Union[str, Doc]], Doc],
|
||||
ensure_doc: Callable[[Union[str, Doc, bytes], _AnyContext], Doc],
|
||||
pipes: Iterable[Callable[..., Iterator[Doc]]],
|
||||
receiver,
|
||||
sender,
|
||||
|
@ -2184,17 +2207,19 @@ def _apply_pipes(
|
|||
Underscore.load_state(underscore_state)
|
||||
while True:
|
||||
try:
|
||||
texts = receiver.get()
|
||||
docs = (ensure_doc(text) for text in texts)
|
||||
texts_with_ctx = receiver.get()
|
||||
docs = (
|
||||
ensure_doc(doc_like, context) for doc_like, context in texts_with_ctx
|
||||
)
|
||||
for pipe in pipes:
|
||||
docs = pipe(docs) # type: ignore[arg-type, assignment]
|
||||
# Connection does not accept unpickable objects, so send list.
|
||||
byte_docs = [(doc.to_bytes(), doc._context, None) for doc in docs]
|
||||
padding = [(None, None, None)] * (len(texts) - len(byte_docs))
|
||||
padding = [(None, None, None)] * (len(texts_with_ctx) - len(byte_docs))
|
||||
sender.send(byte_docs + padding) # type: ignore[operator]
|
||||
except Exception:
|
||||
error_msg = [(None, None, srsly.msgpack_dumps(traceback.format_exc()))]
|
||||
padding = [(None, None, None)] * (len(texts) - 1)
|
||||
padding = [(None, None, None)] * (len(texts_with_ctx) - 1)
|
||||
sender.send(error_msg + padding)
|
||||
|
||||
|
||||
|
|
|
@ -85,7 +85,7 @@ class Table(OrderedDict):
|
|||
value: The value to set.
|
||||
"""
|
||||
key = get_string_id(key)
|
||||
OrderedDict.__setitem__(self, key, value)
|
||||
OrderedDict.__setitem__(self, key, value) # type:ignore[assignment]
|
||||
self.bloom.add(key)
|
||||
|
||||
def set(self, key: Union[str, int], value: Any) -> None:
|
||||
|
@ -104,7 +104,7 @@ class Table(OrderedDict):
|
|||
RETURNS: The value.
|
||||
"""
|
||||
key = get_string_id(key)
|
||||
return OrderedDict.__getitem__(self, key)
|
||||
return OrderedDict.__getitem__(self, key) # type:ignore[index]
|
||||
|
||||
def get(self, key: Union[str, int], default: Optional[Any] = None) -> Any:
|
||||
"""Get the value for a given key. String keys will be hashed.
|
||||
|
@ -114,7 +114,7 @@ class Table(OrderedDict):
|
|||
RETURNS: The value.
|
||||
"""
|
||||
key = get_string_id(key)
|
||||
return OrderedDict.get(self, key, default)
|
||||
return OrderedDict.get(self, key, default) # type:ignore[arg-type]
|
||||
|
||||
def __contains__(self, key: Union[str, int]) -> bool: # type: ignore[override]
|
||||
"""Check whether a key is in the table. String keys will be hashed.
|
||||
|
|
|
@ -787,6 +787,7 @@ def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates):
|
|||
def _get_attr_values(spec, string_store):
|
||||
attr_values = []
|
||||
for attr, value in spec.items():
|
||||
input_attr = attr
|
||||
if isinstance(attr, str):
|
||||
attr = attr.upper()
|
||||
if attr == '_':
|
||||
|
@ -815,7 +816,7 @@ def _get_attr_values(spec, string_store):
|
|||
attr_values.append((attr, value))
|
||||
else:
|
||||
# should be caught in validation
|
||||
raise ValueError(Errors.E152.format(attr=attr))
|
||||
raise ValueError(Errors.E152.format(attr=input_attr))
|
||||
return attr_values
|
||||
|
||||
|
||||
|
|
|
@ -23,7 +23,7 @@ def build_nel_encoder(
|
|||
((tok2vec >> list2ragged()) & build_span_maker())
|
||||
>> extract_spans()
|
||||
>> reduce_mean()
|
||||
>> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore[arg-type]
|
||||
>> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore
|
||||
>> output_layer
|
||||
)
|
||||
model.set_ref("output_layer", output_layer)
|
||||
|
|
|
@ -72,7 +72,7 @@ def build_tb_parser_model(
|
|||
t2v_width = tok2vec.get_dim("nO") if tok2vec.has_dim("nO") else None
|
||||
tok2vec = chain(
|
||||
tok2vec,
|
||||
cast(Model[List["Floats2d"], Floats2d], list2array()),
|
||||
list2array(),
|
||||
Linear(hidden_width, t2v_width),
|
||||
)
|
||||
tok2vec.set_dim("nO", hidden_width)
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
from typing import Optional, List, cast
|
||||
from functools import partial
|
||||
from typing import Optional, List
|
||||
|
||||
from thinc.types import Floats2d
|
||||
from thinc.api import Model, reduce_mean, Linear, list2ragged, Logistic
|
||||
|
@ -59,7 +59,8 @@ def build_simple_cnn_text_classifier(
|
|||
resizable_layer=resizable_layer,
|
||||
)
|
||||
model.set_ref("tok2vec", tok2vec)
|
||||
model.set_dim("nO", nO) # type: ignore # TODO: remove type ignore once Thinc has been updated
|
||||
if nO is not None:
|
||||
model.set_dim("nO", cast(int, nO))
|
||||
model.attrs["multi_label"] = not exclusive_classes
|
||||
return model
|
||||
|
||||
|
@ -85,7 +86,7 @@ def build_bow_text_classifier(
|
|||
if not no_output_layer:
|
||||
fill_defaults["b"] = NEG_VALUE
|
||||
output_layer = softmax_activation() if exclusive_classes else Logistic()
|
||||
resizable_layer = resizable( # type: ignore[var-annotated]
|
||||
resizable_layer: Model[Floats2d, Floats2d] = resizable(
|
||||
sparse_linear,
|
||||
resize_layer=partial(resize_linear_weighted, fill_defaults=fill_defaults),
|
||||
)
|
||||
|
@ -93,7 +94,8 @@ def build_bow_text_classifier(
|
|||
model = with_cpu(model, model.ops)
|
||||
if output_layer:
|
||||
model = model >> with_cpu(output_layer, output_layer.ops)
|
||||
model.set_dim("nO", nO) # type: ignore[arg-type]
|
||||
if nO is not None:
|
||||
model.set_dim("nO", cast(int, nO))
|
||||
model.set_ref("output_layer", sparse_linear)
|
||||
model.attrs["multi_label"] = not exclusive_classes
|
||||
model.attrs["resize_output"] = partial(
|
||||
|
@ -129,8 +131,8 @@ def build_text_classifier_v2(
|
|||
output_layer = Linear(nO=nO, nI=nO_double) >> Logistic()
|
||||
model = (linear_model | cnn_model) >> output_layer
|
||||
model.set_ref("tok2vec", tok2vec)
|
||||
if model.has_dim("nO") is not False:
|
||||
model.set_dim("nO", nO) # type: ignore[arg-type]
|
||||
if model.has_dim("nO") is not False and nO is not None:
|
||||
model.set_dim("nO", cast(int, nO))
|
||||
model.set_ref("output_layer", linear_model.get_ref("output_layer"))
|
||||
model.set_ref("attention_layer", attention_layer)
|
||||
model.set_ref("maxout_layer", maxout_layer)
|
||||
|
@ -164,7 +166,7 @@ def build_text_classifier_lowdata(
|
|||
>> list2ragged()
|
||||
>> ParametricAttention(width)
|
||||
>> reduce_sum()
|
||||
>> residual(Relu(width, width)) ** 2 # type: ignore[arg-type]
|
||||
>> residual(Relu(width, width)) ** 2
|
||||
>> Linear(nO, width)
|
||||
)
|
||||
if dropout:
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
from typing import Optional, List, Union, cast
|
||||
from thinc.types import Floats2d, Ints2d, Ragged
|
||||
from thinc.types import Floats2d, Ints2d, Ragged, Ints1d
|
||||
from thinc.api import chain, clone, concatenate, with_array, with_padded
|
||||
from thinc.api import Model, noop, list2ragged, ragged2list, HashEmbed
|
||||
from thinc.api import expand_window, residual, Maxout, Mish, PyTorchLSTM
|
||||
|
@ -159,7 +159,7 @@ def MultiHashEmbed(
|
|||
embeddings = [make_hash_embed(i) for i in range(len(attrs))]
|
||||
concat_size = width * (len(embeddings) + include_static_vectors)
|
||||
max_out: Model[Ragged, Ragged] = with_array(
|
||||
Maxout(width, concat_size, nP=3, dropout=0.0, normalize=True) # type: ignore
|
||||
Maxout(width, concat_size, nP=3, dropout=0.0, normalize=True)
|
||||
)
|
||||
if include_static_vectors:
|
||||
feature_extractor: Model[List[Doc], Ragged] = chain(
|
||||
|
@ -173,7 +173,7 @@ def MultiHashEmbed(
|
|||
StaticVectors(width, dropout=0.0),
|
||||
),
|
||||
max_out,
|
||||
cast(Model[Ragged, List[Floats2d]], ragged2list()),
|
||||
ragged2list(),
|
||||
)
|
||||
else:
|
||||
model = chain(
|
||||
|
@ -181,7 +181,7 @@ def MultiHashEmbed(
|
|||
cast(Model[List[Ints2d], Ragged], list2ragged()),
|
||||
with_array(concatenate(*embeddings)),
|
||||
max_out,
|
||||
cast(Model[Ragged, List[Floats2d]], ragged2list()),
|
||||
ragged2list(),
|
||||
)
|
||||
return model
|
||||
|
||||
|
@ -232,12 +232,12 @@ def CharacterEmbed(
|
|||
feature_extractor: Model[List[Doc], Ragged] = chain(
|
||||
FeatureExtractor([feature]),
|
||||
cast(Model[List[Ints2d], Ragged], list2ragged()),
|
||||
with_array(HashEmbed(nO=width, nV=rows, column=0, seed=5)), # type: ignore
|
||||
with_array(HashEmbed(nO=width, nV=rows, column=0, seed=5)), # type: ignore[misc]
|
||||
)
|
||||
max_out: Model[Ragged, Ragged]
|
||||
if include_static_vectors:
|
||||
max_out = with_array(
|
||||
Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0) # type: ignore
|
||||
Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0)
|
||||
)
|
||||
model = chain(
|
||||
concatenate(
|
||||
|
@ -246,11 +246,11 @@ def CharacterEmbed(
|
|||
StaticVectors(width, dropout=0.0),
|
||||
),
|
||||
max_out,
|
||||
cast(Model[Ragged, List[Floats2d]], ragged2list()),
|
||||
ragged2list(),
|
||||
)
|
||||
else:
|
||||
max_out = with_array(
|
||||
Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0) # type: ignore
|
||||
Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0)
|
||||
)
|
||||
model = chain(
|
||||
concatenate(
|
||||
|
@ -258,7 +258,7 @@ def CharacterEmbed(
|
|||
feature_extractor,
|
||||
),
|
||||
max_out,
|
||||
cast(Model[Ragged, List[Floats2d]], ragged2list()),
|
||||
ragged2list(),
|
||||
)
|
||||
return model
|
||||
|
||||
|
@ -289,10 +289,10 @@ def MaxoutWindowEncoder(
|
|||
normalize=True,
|
||||
),
|
||||
)
|
||||
model = clone(residual(cnn), depth) # type: ignore[arg-type]
|
||||
model = clone(residual(cnn), depth)
|
||||
model.set_dim("nO", width)
|
||||
receptive_field = window_size * depth
|
||||
return with_array(model, pad=receptive_field) # type: ignore[arg-type]
|
||||
return with_array(model, pad=receptive_field)
|
||||
|
||||
|
||||
@registry.architectures("spacy.MishWindowEncoder.v2")
|
||||
|
@ -313,9 +313,9 @@ def MishWindowEncoder(
|
|||
expand_window(window_size=window_size),
|
||||
Mish(nO=width, nI=width * ((window_size * 2) + 1), dropout=0.0, normalize=True),
|
||||
)
|
||||
model = clone(residual(cnn), depth) # type: ignore[arg-type]
|
||||
model = clone(residual(cnn), depth)
|
||||
model.set_dim("nO", width)
|
||||
return with_array(model) # type: ignore[arg-type]
|
||||
return with_array(model)
|
||||
|
||||
|
||||
@registry.architectures("spacy.TorchBiLSTMEncoder.v1")
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
from libc.string cimport memset, memcpy
|
||||
from thinc.backends.cblas cimport CBlas
|
||||
from ..typedefs cimport weight_t, hash_t
|
||||
from ..pipeline._parser_internals._state cimport StateC
|
||||
|
||||
|
@ -38,7 +39,7 @@ cdef ActivationsC alloc_activations(SizesC n) nogil
|
|||
|
||||
cdef void free_activations(const ActivationsC* A) nogil
|
||||
|
||||
cdef void predict_states(ActivationsC* A, StateC** states,
|
||||
cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states,
|
||||
const WeightsC* W, SizesC n) nogil
|
||||
|
||||
cdef int arg_max_if_valid(const weight_t* scores, const int* is_valid, int n) nogil
|
||||
|
|
|
@ -4,11 +4,10 @@ from libc.math cimport exp
|
|||
from libc.string cimport memset, memcpy
|
||||
from libc.stdlib cimport calloc, free, realloc
|
||||
from thinc.backends.linalg cimport Vec, VecVec
|
||||
cimport blis.cy
|
||||
|
||||
import numpy
|
||||
import numpy.random
|
||||
from thinc.api import Model, CupyOps, NumpyOps
|
||||
from thinc.api import Model, CupyOps, NumpyOps, get_ops
|
||||
|
||||
from .. import util
|
||||
from ..errors import Errors
|
||||
|
@ -91,7 +90,7 @@ cdef void resize_activations(ActivationsC* A, SizesC n) nogil:
|
|||
A._curr_size = n.states
|
||||
|
||||
|
||||
cdef void predict_states(ActivationsC* A, StateC** states,
|
||||
cdef void predict_states(CBlas cblas, ActivationsC* A, StateC** states,
|
||||
const WeightsC* W, SizesC n) nogil:
|
||||
cdef double one = 1.0
|
||||
resize_activations(A, n)
|
||||
|
@ -99,7 +98,7 @@ cdef void predict_states(ActivationsC* A, StateC** states,
|
|||
states[i].set_context_tokens(&A.token_ids[i*n.feats], n.feats)
|
||||
memset(A.unmaxed, 0, n.states * n.hiddens * n.pieces * sizeof(float))
|
||||
memset(A.hiddens, 0, n.states * n.hiddens * sizeof(float))
|
||||
sum_state_features(A.unmaxed,
|
||||
sum_state_features(cblas, A.unmaxed,
|
||||
W.feat_weights, A.token_ids, n.states, n.feats, n.hiddens * n.pieces)
|
||||
for i in range(n.states):
|
||||
VecVec.add_i(&A.unmaxed[i*n.hiddens*n.pieces],
|
||||
|
@ -113,12 +112,10 @@ cdef void predict_states(ActivationsC* A, StateC** states,
|
|||
memcpy(A.scores, A.hiddens, n.states * n.classes * sizeof(float))
|
||||
else:
|
||||
# Compute hidden-to-output
|
||||
blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.TRANSPOSE,
|
||||
n.states, n.classes, n.hiddens, one,
|
||||
<float*>A.hiddens, n.hiddens, 1,
|
||||
<float*>W.hidden_weights, n.hiddens, 1,
|
||||
one,
|
||||
<float*>A.scores, n.classes, 1)
|
||||
cblas.sgemm()(False, True, n.states, n.classes, n.hiddens,
|
||||
1.0, <const float *>A.hiddens, n.hiddens,
|
||||
<const float *>W.hidden_weights, n.hiddens,
|
||||
0.0, A.scores, n.classes)
|
||||
# Add bias
|
||||
for i in range(n.states):
|
||||
VecVec.add_i(&A.scores[i*n.classes],
|
||||
|
@ -135,7 +132,7 @@ cdef void predict_states(ActivationsC* A, StateC** states,
|
|||
A.scores[i*n.classes+j] = min_
|
||||
|
||||
|
||||
cdef void sum_state_features(float* output,
|
||||
cdef void sum_state_features(CBlas cblas, float* output,
|
||||
const float* cached, const int* token_ids, int B, int F, int O) nogil:
|
||||
cdef int idx, b, f, i
|
||||
cdef const float* feature
|
||||
|
@ -150,9 +147,7 @@ cdef void sum_state_features(float* output,
|
|||
else:
|
||||
idx = token_ids[f] * id_stride + f*O
|
||||
feature = &cached[idx]
|
||||
blis.cy.axpyv(blis.cy.NO_CONJUGATE, O, one,
|
||||
<float*>feature, 1,
|
||||
&output[b*O], 1)
|
||||
cblas.saxpy()(O, one, <const float*>feature, 1, &output[b*O], 1)
|
||||
token_ids += F
|
||||
|
||||
|
||||
|
@ -443,9 +438,15 @@ cdef class precompute_hiddens:
|
|||
# - Output from backward on GPU
|
||||
bp_hiddens = self._bp_hiddens
|
||||
|
||||
cdef CBlas cblas
|
||||
if isinstance(self.ops, CupyOps):
|
||||
cblas = get_ops("cpu").cblas()
|
||||
else:
|
||||
cblas = self.ops.cblas()
|
||||
|
||||
feat_weights = self.get_feat_weights()
|
||||
cdef int[:, ::1] ids = token_ids
|
||||
sum_state_features(<float*>state_vector.data,
|
||||
sum_state_features(cblas, <float*>state_vector.data,
|
||||
feat_weights, &ids[0,0],
|
||||
token_ids.shape[0], self.nF, self.nO*self.nP)
|
||||
state_vector += self.bias
|
||||
|
|
|
@ -40,17 +40,15 @@ def forward(
|
|||
if not token_count:
|
||||
return _handle_empty(model.ops, model.get_dim("nO"))
|
||||
key_attr: int = model.attrs["key_attr"]
|
||||
keys: Ints1d = model.ops.flatten(
|
||||
cast(Sequence, [doc.to_array(key_attr) for doc in docs])
|
||||
)
|
||||
keys = model.ops.flatten([cast(Ints1d, doc.to_array(key_attr)) for doc in docs])
|
||||
vocab: Vocab = docs[0].vocab
|
||||
W = cast(Floats2d, model.ops.as_contig(model.get_param("W")))
|
||||
if vocab.vectors.mode == Mode.default:
|
||||
V = cast(Floats2d, model.ops.asarray(vocab.vectors.data))
|
||||
V = model.ops.asarray(vocab.vectors.data)
|
||||
rows = vocab.vectors.find(keys=keys)
|
||||
V = model.ops.as_contig(V[rows])
|
||||
elif vocab.vectors.mode == Mode.floret:
|
||||
V = cast(Floats2d, vocab.vectors.get_batch(keys))
|
||||
V = vocab.vectors.get_batch(keys)
|
||||
V = model.ops.as_contig(V)
|
||||
else:
|
||||
raise RuntimeError(Errors.E896)
|
||||
|
@ -62,9 +60,7 @@ def forward(
|
|||
# Convert negative indices to 0-vectors
|
||||
# TODO: more options for UNK tokens
|
||||
vectors_data[rows < 0] = 0
|
||||
output = Ragged(
|
||||
vectors_data, model.ops.asarray([len(doc) for doc in docs], dtype="i") # type: ignore
|
||||
)
|
||||
output = Ragged(vectors_data, model.ops.asarray1i([len(doc) for doc in docs]))
|
||||
mask = None
|
||||
if is_train:
|
||||
mask = _get_drop_mask(model.ops, W.shape[0], model.attrs.get("dropout_rate"))
|
||||
|
@ -77,7 +73,9 @@ def forward(
|
|||
model.inc_grad(
|
||||
"W",
|
||||
model.ops.gemm(
|
||||
cast(Floats2d, d_output.data), model.ops.as_contig(V), trans1=True
|
||||
cast(Floats2d, d_output.data),
|
||||
cast(Floats2d, model.ops.as_contig(V)),
|
||||
trans1=True,
|
||||
),
|
||||
)
|
||||
return []
|
||||
|
|
|
@ -13,6 +13,7 @@ from .sentencizer import Sentencizer
|
|||
from .tagger import Tagger
|
||||
from .textcat import TextCategorizer
|
||||
from .spancat import SpanCategorizer
|
||||
from .span_ruler import SpanRuler
|
||||
from .textcat_multilabel import MultiLabel_TextCategorizer
|
||||
from .tok2vec import Tok2Vec
|
||||
from .functions import merge_entities, merge_noun_chunks, merge_subtokens
|
||||
|
@ -30,6 +31,7 @@ __all__ = [
|
|||
"SentenceRecognizer",
|
||||
"Sentencizer",
|
||||
"SpanCategorizer",
|
||||
"SpanRuler",
|
||||
"Tagger",
|
||||
"TextCategorizer",
|
||||
"Tok2Vec",
|
||||
|
|
11
spacy/pipeline/_parser_internals/nonproj.hh
Normal file
11
spacy/pipeline/_parser_internals/nonproj.hh
Normal file
|
@ -0,0 +1,11 @@
|
|||
#ifndef NONPROJ_HH
|
||||
#define NONPROJ_HH
|
||||
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
void raise_domain_error(std::string const &msg) {
|
||||
throw std::domain_error(msg);
|
||||
}
|
||||
|
||||
#endif // NONPROJ_HH
|
|
@ -0,0 +1,4 @@
|
|||
from libcpp.string cimport string
|
||||
|
||||
cdef extern from "nonproj.hh":
|
||||
cdef void raise_domain_error(const string& msg) nogil except +
|
|
@ -4,10 +4,13 @@ for doing pseudo-projective parsing implementation uses the HEAD decoration
|
|||
scheme.
|
||||
"""
|
||||
from copy import copy
|
||||
from cython.operator cimport preincrement as incr, dereference as deref
|
||||
from libc.limits cimport INT_MAX
|
||||
from libc.stdlib cimport abs
|
||||
from libcpp cimport bool
|
||||
from libcpp.string cimport string, to_string
|
||||
from libcpp.vector cimport vector
|
||||
from libcpp.unordered_set cimport unordered_set
|
||||
|
||||
from ...tokens.doc cimport Doc, set_children_from_heads
|
||||
|
||||
|
@ -49,7 +52,7 @@ def is_nonproj_arc(tokenid, heads):
|
|||
return _is_nonproj_arc(tokenid, c_heads)
|
||||
|
||||
|
||||
cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil:
|
||||
cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil except *:
|
||||
# definition (e.g. Havelka 2007): an arc h -> d, h < d is non-projective
|
||||
# if there is a token k, h < k < d such that h is not
|
||||
# an ancestor of k. Same for h -> d, h > d
|
||||
|
@ -58,32 +61,56 @@ cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil:
|
|||
return False
|
||||
elif head < 0: # unattached tokens cannot be non-projective
|
||||
return False
|
||||
|
||||
|
||||
cdef int start, end
|
||||
if head < tokenid:
|
||||
start, end = (head+1, tokenid)
|
||||
else:
|
||||
start, end = (tokenid+1, head)
|
||||
for k in range(start, end):
|
||||
if _has_head_as_ancestor(k, head, heads):
|
||||
continue
|
||||
else: # head not in ancestors: d -> h is non-projective
|
||||
if not _has_head_as_ancestor(k, head, heads):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
cdef bool _has_head_as_ancestor(int tokenid, int head, const vector[int]& heads) nogil:
|
||||
cdef bool _has_head_as_ancestor(int tokenid, int head, const vector[int]& heads) nogil except *:
|
||||
ancestor = tokenid
|
||||
cnt = 0
|
||||
while cnt < heads.size():
|
||||
cdef unordered_set[int] seen_tokens
|
||||
seen_tokens.insert(ancestor)
|
||||
while True:
|
||||
# Reached the head or a disconnected node
|
||||
if heads[ancestor] == head or heads[ancestor] < 0:
|
||||
return True
|
||||
# Reached the root
|
||||
if heads[ancestor] == ancestor:
|
||||
return False
|
||||
ancestor = heads[ancestor]
|
||||
cnt += 1
|
||||
result = seen_tokens.insert(ancestor)
|
||||
# Found cycle
|
||||
if not result.second:
|
||||
raise_domain_error(heads_to_string(heads))
|
||||
|
||||
return False
|
||||
|
||||
|
||||
cdef string heads_to_string(const vector[int]& heads) nogil:
|
||||
cdef vector[int].const_iterator citer
|
||||
cdef string cycle_str
|
||||
|
||||
cycle_str.append("Found cycle in dependency graph: [")
|
||||
|
||||
# FIXME: Rewrite using ostringstream when available in Cython.
|
||||
citer = heads.const_begin()
|
||||
while citer != heads.const_end():
|
||||
if citer != heads.const_begin():
|
||||
cycle_str.append(", ")
|
||||
cycle_str.append(to_string(deref(citer)))
|
||||
incr(citer)
|
||||
cycle_str.append("]")
|
||||
|
||||
return cycle_str
|
||||
|
||||
|
||||
def is_nonproj_tree(heads):
|
||||
cdef vector[int] c_heads = _heads_to_c(heads)
|
||||
# a tree is non-projective if at least one arc is non-projective
|
||||
|
@ -176,11 +203,12 @@ def get_smallest_nonproj_arc_slow(heads):
|
|||
return _get_smallest_nonproj_arc(c_heads)
|
||||
|
||||
|
||||
cdef int _get_smallest_nonproj_arc(const vector[int]& heads) nogil:
|
||||
cdef int _get_smallest_nonproj_arc(const vector[int]& heads) nogil except -2:
|
||||
# return the smallest non-proj arc or None
|
||||
# where size is defined as the distance between dep and head
|
||||
# and ties are broken left to right
|
||||
cdef int smallest_size = INT_MAX
|
||||
# -1 means its already projective.
|
||||
cdef int smallest_np_arc = -1
|
||||
cdef int size
|
||||
cdef int tokenid
|
||||
|
|
|
@ -138,7 +138,7 @@ class EditTreeLemmatizer(TrainablePipe):
|
|||
|
||||
truths.append(eg_truths)
|
||||
|
||||
d_scores, loss = loss_func(scores, truths) # type: ignore
|
||||
d_scores, loss = loss_func(scores, truths)
|
||||
if self.model.ops.xp.isnan(loss):
|
||||
raise ValueError(Errors.E910.format(name=self.name))
|
||||
|
||||
|
|
|
@ -234,10 +234,11 @@ class EntityLinker(TrainablePipe):
|
|||
nO = self.kb.entity_vector_length
|
||||
doc_sample = []
|
||||
vector_sample = []
|
||||
for example in islice(get_examples(), 10):
|
||||
doc = example.x
|
||||
for eg in islice(get_examples(), 10):
|
||||
doc = eg.x
|
||||
if self.use_gold_ents:
|
||||
doc.ents = example.y.ents
|
||||
ents, _ = eg.get_aligned_ents_and_ner()
|
||||
doc.ents = ents
|
||||
doc_sample.append(doc)
|
||||
vector_sample.append(self.model.ops.alloc1f(nO))
|
||||
assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
|
||||
|
@ -312,7 +313,8 @@ class EntityLinker(TrainablePipe):
|
|||
|
||||
for doc, ex in zip(docs, examples):
|
||||
if self.use_gold_ents:
|
||||
doc.ents = ex.reference.ents
|
||||
ents, _ = ex.get_aligned_ents_and_ner()
|
||||
doc.ents = ents
|
||||
else:
|
||||
# only keep matching ents
|
||||
doc.ents = ex.get_matching_ents()
|
||||
|
@ -345,7 +347,7 @@ class EntityLinker(TrainablePipe):
|
|||
for eg in examples:
|
||||
kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True)
|
||||
|
||||
for ent in eg.reference.ents:
|
||||
for ent in eg.get_matching_ents():
|
||||
kb_id = kb_ids[ent.start]
|
||||
if kb_id:
|
||||
entity_encoding = self.kb.get_vector(kb_id)
|
||||
|
@ -353,22 +355,25 @@ class EntityLinker(TrainablePipe):
|
|||
keep_ents.append(eidx)
|
||||
|
||||
eidx += 1
|
||||
entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32")
|
||||
entity_encodings = self.model.ops.asarray2f(entity_encodings, dtype="float32")
|
||||
selected_encodings = sentence_encodings[keep_ents]
|
||||
|
||||
# If the entity encodings list is empty, then
|
||||
# if there are no matches, short circuit
|
||||
if not keep_ents:
|
||||
out = self.model.ops.alloc2f(*sentence_encodings.shape)
|
||||
return 0, out
|
||||
|
||||
if selected_encodings.shape != entity_encodings.shape:
|
||||
err = Errors.E147.format(
|
||||
method="get_loss", msg="gold entities do not match up"
|
||||
)
|
||||
raise RuntimeError(err)
|
||||
# TODO: fix typing issue here
|
||||
gradients = self.distance.get_grad(selected_encodings, entity_encodings) # type: ignore
|
||||
gradients = self.distance.get_grad(selected_encodings, entity_encodings)
|
||||
# to match the input size, we need to give a zero gradient for items not in the kb
|
||||
out = self.model.ops.alloc2f(*sentence_encodings.shape)
|
||||
out[keep_ents] = gradients
|
||||
|
||||
loss = self.distance.get_loss(selected_encodings, entity_encodings) # type: ignore
|
||||
loss = self.distance.get_loss(selected_encodings, entity_encodings)
|
||||
loss = loss / len(entity_encodings)
|
||||
return float(loss), out
|
||||
|
||||
|
@ -385,18 +390,21 @@ class EntityLinker(TrainablePipe):
|
|||
self.validate_kb()
|
||||
entity_count = 0
|
||||
final_kb_ids: List[str] = []
|
||||
xp = self.model.ops.xp
|
||||
if not docs:
|
||||
return final_kb_ids
|
||||
if isinstance(docs, Doc):
|
||||
docs = [docs]
|
||||
for i, doc in enumerate(docs):
|
||||
if len(doc) == 0:
|
||||
continue
|
||||
sentences = [s for s in doc.sents]
|
||||
if len(doc) > 0:
|
||||
# Looping through each entity (TODO: rewrite)
|
||||
for ent in doc.ents:
|
||||
sent = ent.sent
|
||||
sent_index = sentences.index(sent)
|
||||
assert sent_index >= 0
|
||||
# Looping through each entity (TODO: rewrite)
|
||||
for ent in doc.ents:
|
||||
sent_index = sentences.index(ent.sent)
|
||||
assert sent_index >= 0
|
||||
|
||||
if self.incl_context:
|
||||
# get n_neighbour sentences, clipped to the length of the document
|
||||
start_sentence = max(0, sent_index - self.n_sents)
|
||||
end_sentence = min(len(sentences) - 1, sent_index + self.n_sents)
|
||||
|
@ -404,55 +412,53 @@ class EntityLinker(TrainablePipe):
|
|||
end_token = sentences[end_sentence].end
|
||||
sent_doc = doc[start_token:end_token].as_doc()
|
||||
# currently, the context is the same for each entity in a sentence (should be refined)
|
||||
xp = self.model.ops.xp
|
||||
if self.incl_context:
|
||||
sentence_encoding = self.model.predict([sent_doc])[0]
|
||||
sentence_encoding_t = sentence_encoding.T
|
||||
sentence_norm = xp.linalg.norm(sentence_encoding_t)
|
||||
entity_count += 1
|
||||
if ent.label_ in self.labels_discard:
|
||||
# ignoring this entity - setting to NIL
|
||||
sentence_encoding = self.model.predict([sent_doc])[0]
|
||||
sentence_encoding_t = sentence_encoding.T
|
||||
sentence_norm = xp.linalg.norm(sentence_encoding_t)
|
||||
entity_count += 1
|
||||
if ent.label_ in self.labels_discard:
|
||||
# ignoring this entity - setting to NIL
|
||||
final_kb_ids.append(self.NIL)
|
||||
else:
|
||||
candidates = list(self.get_candidates(self.kb, ent))
|
||||
if not candidates:
|
||||
# no prediction possible for this entity - setting to NIL
|
||||
final_kb_ids.append(self.NIL)
|
||||
elif len(candidates) == 1:
|
||||
# shortcut for efficiency reasons: take the 1 candidate
|
||||
# TODO: thresholding
|
||||
final_kb_ids.append(candidates[0].entity_)
|
||||
else:
|
||||
candidates = list(self.get_candidates(self.kb, ent))
|
||||
if not candidates:
|
||||
# no prediction possible for this entity - setting to NIL
|
||||
final_kb_ids.append(self.NIL)
|
||||
elif len(candidates) == 1:
|
||||
# shortcut for efficiency reasons: take the 1 candidate
|
||||
# TODO: thresholding
|
||||
final_kb_ids.append(candidates[0].entity_)
|
||||
else:
|
||||
random.shuffle(candidates)
|
||||
# set all prior probabilities to 0 if incl_prior=False
|
||||
prior_probs = xp.asarray([c.prior_prob for c in candidates])
|
||||
if not self.incl_prior:
|
||||
prior_probs = xp.asarray([0.0 for _ in candidates])
|
||||
scores = prior_probs
|
||||
# add in similarity from the context
|
||||
if self.incl_context:
|
||||
entity_encodings = xp.asarray(
|
||||
[c.entity_vector for c in candidates]
|
||||
)
|
||||
entity_norm = xp.linalg.norm(entity_encodings, axis=1)
|
||||
if len(entity_encodings) != len(prior_probs):
|
||||
raise RuntimeError(
|
||||
Errors.E147.format(
|
||||
method="predict",
|
||||
msg="vectors not of equal length",
|
||||
)
|
||||
random.shuffle(candidates)
|
||||
# set all prior probabilities to 0 if incl_prior=False
|
||||
prior_probs = xp.asarray([c.prior_prob for c in candidates])
|
||||
if not self.incl_prior:
|
||||
prior_probs = xp.asarray([0.0 for _ in candidates])
|
||||
scores = prior_probs
|
||||
# add in similarity from the context
|
||||
if self.incl_context:
|
||||
entity_encodings = xp.asarray(
|
||||
[c.entity_vector for c in candidates]
|
||||
)
|
||||
entity_norm = xp.linalg.norm(entity_encodings, axis=1)
|
||||
if len(entity_encodings) != len(prior_probs):
|
||||
raise RuntimeError(
|
||||
Errors.E147.format(
|
||||
method="predict",
|
||||
msg="vectors not of equal length",
|
||||
)
|
||||
# cosine similarity
|
||||
sims = xp.dot(entity_encodings, sentence_encoding_t) / (
|
||||
sentence_norm * entity_norm
|
||||
)
|
||||
if sims.shape != prior_probs.shape:
|
||||
raise ValueError(Errors.E161)
|
||||
scores = prior_probs + sims - (prior_probs * sims)
|
||||
# TODO: thresholding
|
||||
best_index = scores.argmax().item()
|
||||
best_candidate = candidates[best_index]
|
||||
final_kb_ids.append(best_candidate.entity_)
|
||||
# cosine similarity
|
||||
sims = xp.dot(entity_encodings, sentence_encoding_t) / (
|
||||
sentence_norm * entity_norm
|
||||
)
|
||||
if sims.shape != prior_probs.shape:
|
||||
raise ValueError(Errors.E161)
|
||||
scores = prior_probs + sims - (prior_probs * sims)
|
||||
# TODO: thresholding
|
||||
best_index = scores.argmax().item()
|
||||
best_candidate = candidates[best_index]
|
||||
final_kb_ids.append(best_candidate.entity_)
|
||||
if not (len(final_kb_ids) == entity_count):
|
||||
err = Errors.E147.format(
|
||||
method="predict", msg="result variables not of equal length"
|
||||
|
|
|
@ -159,10 +159,8 @@ class EntityRuler(Pipe):
|
|||
self._require_patterns()
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", message="\\[W036")
|
||||
matches = cast(
|
||||
List[Tuple[int, int, int]],
|
||||
list(self.matcher(doc)) + list(self.phrase_matcher(doc)),
|
||||
)
|
||||
matches = list(self.matcher(doc)) + list(self.phrase_matcher(doc))
|
||||
|
||||
final_matches = set(
|
||||
[(m_id, start, end) for m_id, start, end in matches if start != end]
|
||||
)
|
||||
|
@ -182,10 +180,7 @@ class EntityRuler(Pipe):
|
|||
if start not in seen_tokens and end - 1 not in seen_tokens:
|
||||
if match_id in self._ent_ids:
|
||||
label, ent_id = self._ent_ids[match_id]
|
||||
span = Span(doc, start, end, label=label)
|
||||
if ent_id:
|
||||
for token in span:
|
||||
token.ent_id_ = ent_id
|
||||
span = Span(doc, start, end, label=label, span_id=ent_id)
|
||||
else:
|
||||
span = Span(doc, start, end, label=match_id)
|
||||
new_entities.append(span)
|
||||
|
@ -359,7 +354,9 @@ class EntityRuler(Pipe):
|
|||
(label, eid) for (label, eid) in self._ent_ids.values() if eid == ent_id
|
||||
]
|
||||
if not label_id_pairs:
|
||||
raise ValueError(Errors.E1024.format(ent_id=ent_id))
|
||||
raise ValueError(
|
||||
Errors.E1024.format(attr_type="ID", label=ent_id, component=self.name)
|
||||
)
|
||||
created_labels = [
|
||||
self._create_label(label, eid) for (label, eid) in label_id_pairs
|
||||
]
|
||||
|
|
|
@ -213,15 +213,14 @@ class EntityLinker_v1(TrainablePipe):
|
|||
if kb_id:
|
||||
entity_encoding = self.kb.get_vector(kb_id)
|
||||
entity_encodings.append(entity_encoding)
|
||||
entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32")
|
||||
entity_encodings = self.model.ops.asarray2f(entity_encodings)
|
||||
if sentence_encodings.shape != entity_encodings.shape:
|
||||
err = Errors.E147.format(
|
||||
method="get_loss", msg="gold entities do not match up"
|
||||
)
|
||||
raise RuntimeError(err)
|
||||
# TODO: fix typing issue here
|
||||
gradients = self.distance.get_grad(sentence_encodings, entity_encodings) # type: ignore
|
||||
loss = self.distance.get_loss(sentence_encodings, entity_encodings) # type: ignore
|
||||
gradients = self.distance.get_grad(sentence_encodings, entity_encodings)
|
||||
loss = self.distance.get_loss(sentence_encodings, entity_encodings)
|
||||
loss = loss / len(entity_encodings)
|
||||
return float(loss), gradients
|
||||
|
||||
|
|
|
@ -31,7 +31,7 @@ cdef class Pipe:
|
|||
and returned. This usually happens under the hood when the nlp object
|
||||
is called on a text and all components are applied to the Doc.
|
||||
|
||||
docs (Doc): The Doc to process.
|
||||
doc (Doc): The Doc to process.
|
||||
RETURNS (Doc): The processed Doc.
|
||||
|
||||
DOCS: https://spacy.io/api/pipe#call
|
||||
|
|
569
spacy/pipeline/span_ruler.py
Normal file
569
spacy/pipeline/span_ruler.py
Normal file
|
@ -0,0 +1,569 @@
|
|||
from typing import Optional, Union, List, Dict, Tuple, Iterable, Any, Callable
|
||||
from typing import Sequence, Set, cast
|
||||
import warnings
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
import srsly
|
||||
|
||||
from .pipe import Pipe
|
||||
from ..training import Example
|
||||
from ..language import Language
|
||||
from ..errors import Errors, Warnings
|
||||
from ..util import ensure_path, SimpleFrozenList, registry
|
||||
from ..tokens import Doc, Span
|
||||
from ..scorer import Scorer
|
||||
from ..matcher import Matcher, PhraseMatcher
|
||||
from .. import util
|
||||
|
||||
PatternType = Dict[str, Union[str, List[Dict[str, Any]]]]
|
||||
DEFAULT_SPANS_KEY = "ruler"
|
||||
|
||||
|
||||
@Language.factory(
|
||||
"future_entity_ruler",
|
||||
assigns=["doc.ents"],
|
||||
default_config={
|
||||
"phrase_matcher_attr": None,
|
||||
"validate": False,
|
||||
"overwrite_ents": False,
|
||||
"scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"},
|
||||
"ent_id_sep": "__unused__",
|
||||
},
|
||||
default_score_weights={
|
||||
"ents_f": 1.0,
|
||||
"ents_p": 0.0,
|
||||
"ents_r": 0.0,
|
||||
"ents_per_type": None,
|
||||
},
|
||||
)
|
||||
def make_entity_ruler(
|
||||
nlp: Language,
|
||||
name: str,
|
||||
phrase_matcher_attr: Optional[Union[int, str]],
|
||||
validate: bool,
|
||||
overwrite_ents: bool,
|
||||
scorer: Optional[Callable],
|
||||
ent_id_sep: str,
|
||||
):
|
||||
if overwrite_ents:
|
||||
ents_filter = prioritize_new_ents_filter
|
||||
else:
|
||||
ents_filter = prioritize_existing_ents_filter
|
||||
return SpanRuler(
|
||||
nlp,
|
||||
name,
|
||||
spans_key=None,
|
||||
spans_filter=None,
|
||||
annotate_ents=True,
|
||||
ents_filter=ents_filter,
|
||||
phrase_matcher_attr=phrase_matcher_attr,
|
||||
validate=validate,
|
||||
overwrite=False,
|
||||
scorer=scorer,
|
||||
)
|
||||
|
||||
|
||||
@Language.factory(
|
||||
"span_ruler",
|
||||
assigns=["doc.spans"],
|
||||
default_config={
|
||||
"spans_key": DEFAULT_SPANS_KEY,
|
||||
"spans_filter": None,
|
||||
"annotate_ents": False,
|
||||
"ents_filter": {"@misc": "spacy.first_longest_spans_filter.v1"},
|
||||
"phrase_matcher_attr": None,
|
||||
"validate": False,
|
||||
"overwrite": True,
|
||||
"scorer": {
|
||||
"@scorers": "spacy.overlapping_labeled_spans_scorer.v1",
|
||||
"spans_key": DEFAULT_SPANS_KEY,
|
||||
},
|
||||
},
|
||||
default_score_weights={
|
||||
f"spans_{DEFAULT_SPANS_KEY}_f": 1.0,
|
||||
f"spans_{DEFAULT_SPANS_KEY}_p": 0.0,
|
||||
f"spans_{DEFAULT_SPANS_KEY}_r": 0.0,
|
||||
f"spans_{DEFAULT_SPANS_KEY}_per_type": None,
|
||||
},
|
||||
)
|
||||
def make_span_ruler(
|
||||
nlp: Language,
|
||||
name: str,
|
||||
spans_key: Optional[str],
|
||||
spans_filter: Optional[Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]]],
|
||||
annotate_ents: bool,
|
||||
ents_filter: Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]],
|
||||
phrase_matcher_attr: Optional[Union[int, str]],
|
||||
validate: bool,
|
||||
overwrite: bool,
|
||||
scorer: Optional[Callable],
|
||||
):
|
||||
return SpanRuler(
|
||||
nlp,
|
||||
name,
|
||||
spans_key=spans_key,
|
||||
spans_filter=spans_filter,
|
||||
annotate_ents=annotate_ents,
|
||||
ents_filter=ents_filter,
|
||||
phrase_matcher_attr=phrase_matcher_attr,
|
||||
validate=validate,
|
||||
overwrite=overwrite,
|
||||
scorer=scorer,
|
||||
)
|
||||
|
||||
|
||||
def prioritize_new_ents_filter(
|
||||
entities: Iterable[Span], spans: Iterable[Span]
|
||||
) -> List[Span]:
|
||||
"""Merge entities and spans into one list without overlaps by allowing
|
||||
spans to overwrite any entities that they overlap with. Intended to
|
||||
replicate the overwrite_ents=True behavior from the EntityRuler.
|
||||
|
||||
entities (Iterable[Span]): The entities, already filtered for overlaps.
|
||||
spans (Iterable[Span]): The spans to merge, may contain overlaps.
|
||||
RETURNS (List[Span]): Filtered list of non-overlapping spans.
|
||||
"""
|
||||
get_sort_key = lambda span: (span.end - span.start, -span.start)
|
||||
spans = sorted(spans, key=get_sort_key, reverse=True)
|
||||
entities = list(entities)
|
||||
new_entities = []
|
||||
seen_tokens: Set[int] = set()
|
||||
for span in spans:
|
||||
start = span.start
|
||||
end = span.end
|
||||
if all(token.i not in seen_tokens for token in span):
|
||||
new_entities.append(span)
|
||||
entities = [e for e in entities if not (e.start < end and e.end > start)]
|
||||
seen_tokens.update(range(start, end))
|
||||
return entities + new_entities
|
||||
|
||||
|
||||
@registry.misc("spacy.prioritize_new_ents_filter.v1")
|
||||
def make_prioritize_new_ents_filter():
|
||||
return prioritize_new_ents_filter
|
||||
|
||||
|
||||
def prioritize_existing_ents_filter(
|
||||
entities: Iterable[Span], spans: Iterable[Span]
|
||||
) -> List[Span]:
|
||||
"""Merge entities and spans into one list without overlaps by prioritizing
|
||||
existing entities. Intended to replicate the overwrite_ents=False behavior
|
||||
from the EntityRuler.
|
||||
|
||||
entities (Iterable[Span]): The entities, already filtered for overlaps.
|
||||
spans (Iterable[Span]): The spans to merge, may contain overlaps.
|
||||
RETURNS (List[Span]): Filtered list of non-overlapping spans.
|
||||
"""
|
||||
get_sort_key = lambda span: (span.end - span.start, -span.start)
|
||||
spans = sorted(spans, key=get_sort_key, reverse=True)
|
||||
entities = list(entities)
|
||||
new_entities = []
|
||||
seen_tokens: Set[int] = set()
|
||||
seen_tokens.update(*(range(ent.start, ent.end) for ent in entities))
|
||||
for span in spans:
|
||||
start = span.start
|
||||
end = span.end
|
||||
if all(token.i not in seen_tokens for token in span):
|
||||
new_entities.append(span)
|
||||
seen_tokens.update(range(start, end))
|
||||
return entities + new_entities
|
||||
|
||||
|
||||
@registry.misc("spacy.prioritize_existing_ents_filter.v1")
|
||||
def make_preverse_existing_ents_filter():
|
||||
return prioritize_existing_ents_filter
|
||||
|
||||
|
||||
def overlapping_labeled_spans_score(
|
||||
examples: Iterable[Example], *, spans_key=DEFAULT_SPANS_KEY, **kwargs
|
||||
) -> Dict[str, Any]:
|
||||
kwargs = dict(kwargs)
|
||||
attr_prefix = f"spans_"
|
||||
kwargs.setdefault("attr", f"{attr_prefix}{spans_key}")
|
||||
kwargs.setdefault("allow_overlap", True)
|
||||
kwargs.setdefault("labeled", True)
|
||||
kwargs.setdefault(
|
||||
"getter", lambda doc, key: doc.spans.get(key[len(attr_prefix) :], [])
|
||||
)
|
||||
kwargs.setdefault("has_annotation", lambda doc: spans_key in doc.spans)
|
||||
return Scorer.score_spans(examples, **kwargs)
|
||||
|
||||
|
||||
@registry.scorers("spacy.overlapping_labeled_spans_scorer.v1")
|
||||
def make_overlapping_labeled_spans_scorer(spans_key: str = DEFAULT_SPANS_KEY):
|
||||
return partial(overlapping_labeled_spans_score, spans_key=spans_key)
|
||||
|
||||
|
||||
class SpanRuler(Pipe):
|
||||
"""The SpanRuler lets you add spans to the `Doc.spans` using token-based
|
||||
rules or exact phrase matches.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler
|
||||
USAGE: https://spacy.io/usage/rule-based-matching#spanruler
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
nlp: Language,
|
||||
name: str = "span_ruler",
|
||||
*,
|
||||
spans_key: Optional[str] = DEFAULT_SPANS_KEY,
|
||||
spans_filter: Optional[
|
||||
Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]]
|
||||
] = None,
|
||||
annotate_ents: bool = False,
|
||||
ents_filter: Callable[
|
||||
[Iterable[Span], Iterable[Span]], Iterable[Span]
|
||||
] = util.filter_chain_spans,
|
||||
phrase_matcher_attr: Optional[Union[int, str]] = None,
|
||||
validate: bool = False,
|
||||
overwrite: bool = False,
|
||||
scorer: Optional[Callable] = partial(
|
||||
overlapping_labeled_spans_score, spans_key=DEFAULT_SPANS_KEY
|
||||
),
|
||||
) -> None:
|
||||
"""Initialize the span ruler. If patterns are supplied here, they
|
||||
need to be a list of dictionaries with a `"label"` and `"pattern"`
|
||||
key. A pattern can either be a token pattern (list) or a phrase pattern
|
||||
(string). For example: `{'label': 'ORG', 'pattern': 'Apple'}`.
|
||||
|
||||
nlp (Language): The shared nlp object to pass the vocab to the matchers
|
||||
and process phrase patterns.
|
||||
name (str): Instance name of the current pipeline component. Typically
|
||||
passed in automatically from the factory when the component is
|
||||
added. Used to disable the current span ruler while creating
|
||||
phrase patterns with the nlp object.
|
||||
spans_key (Optional[str]): The spans key to save the spans under. If
|
||||
`None`, no spans are saved. Defaults to "ruler".
|
||||
spans_filter (Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]):
|
||||
The optional method to filter spans before they are assigned to
|
||||
doc.spans. Defaults to `None`.
|
||||
annotate_ents (bool): Whether to save spans to doc.ents. Defaults to
|
||||
`False`.
|
||||
ents_filter (Callable[[Iterable[Span], Iterable[Span]], List[Span]]):
|
||||
The method to filter spans before they are assigned to doc.ents.
|
||||
Defaults to `util.filter_chain_spans`.
|
||||
phrase_matcher_attr (Optional[Union[int, str]]): Token attribute to
|
||||
match on, passed to the internal PhraseMatcher as `attr`. Defaults
|
||||
to `None`.
|
||||
validate (bool): Whether patterns should be validated, passed to
|
||||
Matcher and PhraseMatcher as `validate`.
|
||||
overwrite (bool): Whether to remove any existing spans under this spans
|
||||
key if `spans_key` is set, and/or to remove any ents under `doc.ents` if
|
||||
`annotate_ents` is set. Defaults to `True`.
|
||||
scorer (Optional[Callable]): The scoring method. Defaults to
|
||||
spacy.pipeline.span_ruler.overlapping_labeled_spans_score.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#init
|
||||
"""
|
||||
self.nlp = nlp
|
||||
self.name = name
|
||||
self.spans_key = spans_key
|
||||
self.annotate_ents = annotate_ents
|
||||
self.phrase_matcher_attr = phrase_matcher_attr
|
||||
self.validate = validate
|
||||
self.overwrite = overwrite
|
||||
self.spans_filter = spans_filter
|
||||
self.ents_filter = ents_filter
|
||||
self.scorer = scorer
|
||||
self._match_label_id_map: Dict[int, Dict[str, str]] = {}
|
||||
self.clear()
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""The number of all labels added to the span ruler."""
|
||||
return len(self._patterns)
|
||||
|
||||
def __contains__(self, label: str) -> bool:
|
||||
"""Whether a label is present in the patterns."""
|
||||
for label_id in self._match_label_id_map.values():
|
||||
if label_id["label"] == label:
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def key(self) -> Optional[str]:
|
||||
"""Key of the doc.spans dict to save the spans under."""
|
||||
return self.spans_key
|
||||
|
||||
def __call__(self, doc: Doc) -> Doc:
|
||||
"""Find matches in document and add them as entities.
|
||||
|
||||
doc (Doc): The Doc object in the pipeline.
|
||||
RETURNS (Doc): The Doc with added entities, if available.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#call
|
||||
"""
|
||||
error_handler = self.get_error_handler()
|
||||
try:
|
||||
matches = self.match(doc)
|
||||
self.set_annotations(doc, matches)
|
||||
return doc
|
||||
except Exception as e:
|
||||
return error_handler(self.name, self, [doc], e)
|
||||
|
||||
def match(self, doc: Doc):
|
||||
self._require_patterns()
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", message="\\[W036")
|
||||
matches = cast(
|
||||
List[Tuple[int, int, int]],
|
||||
list(self.matcher(doc)) + list(self.phrase_matcher(doc)),
|
||||
)
|
||||
deduplicated_matches = set(
|
||||
Span(
|
||||
doc,
|
||||
start,
|
||||
end,
|
||||
label=self._match_label_id_map[m_id]["label"],
|
||||
span_id=self._match_label_id_map[m_id]["id"],
|
||||
)
|
||||
for m_id, start, end in matches
|
||||
if start != end
|
||||
)
|
||||
return sorted(list(deduplicated_matches))
|
||||
|
||||
def set_annotations(self, doc, matches):
|
||||
"""Modify the document in place"""
|
||||
# set doc.spans if spans_key is set
|
||||
if self.key:
|
||||
spans = []
|
||||
if self.key in doc.spans and not self.overwrite:
|
||||
spans = doc.spans[self.key]
|
||||
spans.extend(
|
||||
self.spans_filter(spans, matches) if self.spans_filter else matches
|
||||
)
|
||||
doc.spans[self.key] = spans
|
||||
# set doc.ents if annotate_ents is set
|
||||
if self.annotate_ents:
|
||||
spans = []
|
||||
if not self.overwrite:
|
||||
spans = list(doc.ents)
|
||||
spans = self.ents_filter(spans, matches)
|
||||
try:
|
||||
doc.ents = sorted(spans)
|
||||
except ValueError:
|
||||
raise ValueError(Errors.E854)
|
||||
|
||||
@property
|
||||
def labels(self) -> Tuple[str, ...]:
|
||||
"""All labels present in the match patterns.
|
||||
|
||||
RETURNS (set): The string labels.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#labels
|
||||
"""
|
||||
return tuple(sorted(set([cast(str, p["label"]) for p in self._patterns])))
|
||||
|
||||
@property
|
||||
def ids(self) -> Tuple[str, ...]:
|
||||
"""All IDs present in the match patterns.
|
||||
|
||||
RETURNS (set): The string IDs.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#ids
|
||||
"""
|
||||
return tuple(
|
||||
sorted(set([cast(str, p.get("id")) for p in self._patterns]) - set([None]))
|
||||
)
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
get_examples: Callable[[], Iterable[Example]],
|
||||
*,
|
||||
nlp: Optional[Language] = None,
|
||||
patterns: Optional[Sequence[PatternType]] = None,
|
||||
):
|
||||
"""Initialize the pipe for training.
|
||||
|
||||
get_examples (Callable[[], Iterable[Example]]): Function that
|
||||
returns a representative sample of gold-standard Example objects.
|
||||
nlp (Language): The current nlp object the component is part of.
|
||||
patterns (Optional[Iterable[PatternType]]): The list of patterns.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#initialize
|
||||
"""
|
||||
self.clear()
|
||||
if patterns:
|
||||
self.add_patterns(patterns) # type: ignore[arg-type]
|
||||
|
||||
@property
|
||||
def patterns(self) -> List[PatternType]:
|
||||
"""Get all patterns that were added to the span ruler.
|
||||
|
||||
RETURNS (list): The original patterns, one dictionary per pattern.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#patterns
|
||||
"""
|
||||
return self._patterns
|
||||
|
||||
def add_patterns(self, patterns: List[PatternType]) -> None:
|
||||
"""Add patterns to the span ruler. A pattern can either be a token
|
||||
pattern (list of dicts) or a phrase pattern (string). For example:
|
||||
{'label': 'ORG', 'pattern': 'Apple'}
|
||||
{'label': 'ORG', 'pattern': 'Apple', 'id': 'apple'}
|
||||
{'label': 'GPE', 'pattern': [{'lower': 'san'}, {'lower': 'francisco'}]}
|
||||
|
||||
patterns (list): The patterns to add.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#add_patterns
|
||||
"""
|
||||
|
||||
# disable the nlp components after this one in case they haven't been
|
||||
# initialized / deserialized yet
|
||||
try:
|
||||
current_index = -1
|
||||
for i, (name, pipe) in enumerate(self.nlp.pipeline):
|
||||
if self == pipe:
|
||||
current_index = i
|
||||
break
|
||||
subsequent_pipes = [pipe for pipe in self.nlp.pipe_names[current_index:]]
|
||||
except ValueError:
|
||||
subsequent_pipes = []
|
||||
with self.nlp.select_pipes(disable=subsequent_pipes):
|
||||
phrase_pattern_labels = []
|
||||
phrase_pattern_texts = []
|
||||
for entry in patterns:
|
||||
p_label = cast(str, entry["label"])
|
||||
p_id = cast(str, entry.get("id", ""))
|
||||
label = repr((p_label, p_id))
|
||||
self._match_label_id_map[self.nlp.vocab.strings.as_int(label)] = {
|
||||
"label": p_label,
|
||||
"id": p_id,
|
||||
}
|
||||
if isinstance(entry["pattern"], str):
|
||||
phrase_pattern_labels.append(label)
|
||||
phrase_pattern_texts.append(entry["pattern"])
|
||||
elif isinstance(entry["pattern"], list):
|
||||
self.matcher.add(label, [entry["pattern"]])
|
||||
else:
|
||||
raise ValueError(Errors.E097.format(pattern=entry["pattern"]))
|
||||
self._patterns.append(entry)
|
||||
for label, pattern in zip(
|
||||
phrase_pattern_labels,
|
||||
self.nlp.pipe(phrase_pattern_texts),
|
||||
):
|
||||
self.phrase_matcher.add(label, [pattern])
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Reset all patterns.
|
||||
|
||||
RETURNS: None
|
||||
DOCS: https://spacy.io/api/spanruler#clear
|
||||
"""
|
||||
self._patterns: List[PatternType] = []
|
||||
self.matcher: Matcher = Matcher(self.nlp.vocab, validate=self.validate)
|
||||
self.phrase_matcher: PhraseMatcher = PhraseMatcher(
|
||||
self.nlp.vocab,
|
||||
attr=self.phrase_matcher_attr,
|
||||
validate=self.validate,
|
||||
)
|
||||
|
||||
def remove(self, label: str) -> None:
|
||||
"""Remove a pattern by its label.
|
||||
|
||||
label (str): Label of the pattern to be removed.
|
||||
RETURNS: None
|
||||
DOCS: https://spacy.io/api/spanruler#remove
|
||||
"""
|
||||
if label not in self:
|
||||
raise ValueError(
|
||||
Errors.E1024.format(attr_type="label", label=label, component=self.name)
|
||||
)
|
||||
self._patterns = [p for p in self._patterns if p["label"] != label]
|
||||
for m_label in self._match_label_id_map:
|
||||
if self._match_label_id_map[m_label]["label"] == label:
|
||||
m_label_str = self.nlp.vocab.strings.as_string(m_label)
|
||||
if m_label_str in self.phrase_matcher:
|
||||
self.phrase_matcher.remove(m_label_str)
|
||||
if m_label_str in self.matcher:
|
||||
self.matcher.remove(m_label_str)
|
||||
|
||||
def remove_by_id(self, pattern_id: str) -> None:
|
||||
"""Remove a pattern by its pattern ID.
|
||||
|
||||
pattern_id (str): ID of the pattern to be removed.
|
||||
RETURNS: None
|
||||
DOCS: https://spacy.io/api/spanruler#remove_by_id
|
||||
"""
|
||||
orig_len = len(self)
|
||||
self._patterns = [p for p in self._patterns if p.get("id") != pattern_id]
|
||||
if orig_len == len(self):
|
||||
raise ValueError(
|
||||
Errors.E1024.format(
|
||||
attr_type="ID", label=pattern_id, component=self.name
|
||||
)
|
||||
)
|
||||
for m_label in self._match_label_id_map:
|
||||
if self._match_label_id_map[m_label]["id"] == pattern_id:
|
||||
m_label_str = self.nlp.vocab.strings.as_string(m_label)
|
||||
if m_label_str in self.phrase_matcher:
|
||||
self.phrase_matcher.remove(m_label_str)
|
||||
if m_label_str in self.matcher:
|
||||
self.matcher.remove(m_label_str)
|
||||
|
||||
def _require_patterns(self) -> None:
|
||||
"""Raise a warning if this component has no patterns defined."""
|
||||
if len(self) == 0:
|
||||
warnings.warn(Warnings.W036.format(name=self.name))
|
||||
|
||||
def from_bytes(
|
||||
self, bytes_data: bytes, *, exclude: Iterable[str] = SimpleFrozenList()
|
||||
) -> "SpanRuler":
|
||||
"""Load the span ruler from a bytestring.
|
||||
|
||||
bytes_data (bytes): The bytestring to load.
|
||||
RETURNS (SpanRuler): The loaded span ruler.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#from_bytes
|
||||
"""
|
||||
self.clear()
|
||||
deserializers = {
|
||||
"patterns": lambda b: self.add_patterns(srsly.json_loads(b)),
|
||||
}
|
||||
util.from_bytes(bytes_data, deserializers, exclude)
|
||||
return self
|
||||
|
||||
def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes:
|
||||
"""Serialize the span ruler to a bytestring.
|
||||
|
||||
RETURNS (bytes): The serialized patterns.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#to_bytes
|
||||
"""
|
||||
serializers = {
|
||||
"patterns": lambda: srsly.json_dumps(self.patterns),
|
||||
}
|
||||
return util.to_bytes(serializers, exclude)
|
||||
|
||||
def from_disk(
|
||||
self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
|
||||
) -> "SpanRuler":
|
||||
"""Load the span ruler from a directory.
|
||||
|
||||
path (Union[str, Path]): A path to a directory.
|
||||
RETURNS (SpanRuler): The loaded span ruler.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#from_disk
|
||||
"""
|
||||
self.clear()
|
||||
path = ensure_path(path)
|
||||
deserializers = {
|
||||
"patterns": lambda p: self.add_patterns(srsly.read_jsonl(p)),
|
||||
}
|
||||
util.from_disk(path, deserializers, {})
|
||||
return self
|
||||
|
||||
def to_disk(
|
||||
self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
|
||||
) -> None:
|
||||
"""Save the span ruler patterns to a directory.
|
||||
|
||||
path (Union[str, Path]): A path to a directory.
|
||||
|
||||
DOCS: https://spacy.io/api/spanruler#to_disk
|
||||
"""
|
||||
path = ensure_path(path)
|
||||
serializers = {
|
||||
"patterns": lambda p: srsly.write_jsonl(p, self.patterns),
|
||||
}
|
||||
util.to_disk(path, serializers, {})
|
|
@ -75,7 +75,7 @@ def build_ngram_suggester(sizes: List[int]) -> Suggester:
|
|||
if spans:
|
||||
assert spans[-1].ndim == 2, spans[-1].shape
|
||||
lengths.append(length)
|
||||
lengths_array = cast(Ints1d, ops.asarray(lengths, dtype="i"))
|
||||
lengths_array = ops.asarray1i(lengths)
|
||||
if len(spans) > 0:
|
||||
output = Ragged(ops.xp.vstack(spans), lengths_array)
|
||||
else:
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
from cymem.cymem cimport Pool
|
||||
from thinc.backends.cblas cimport CBlas
|
||||
|
||||
from ..vocab cimport Vocab
|
||||
from .trainable_pipe cimport TrainablePipe
|
||||
|
@ -12,7 +13,7 @@ cdef class Parser(TrainablePipe):
|
|||
cdef readonly TransitionSystem moves
|
||||
cdef public object _multitasks
|
||||
|
||||
cdef void _parseC(self, StateC** states,
|
||||
cdef void _parseC(self, CBlas cblas, StateC** states,
|
||||
WeightsC weights, SizesC sizes) nogil
|
||||
|
||||
cdef void c_transition_batch(self, StateC** states, const float* scores,
|
||||
|
|
|
@ -9,7 +9,7 @@ from libc.stdlib cimport calloc, free
|
|||
import random
|
||||
|
||||
import srsly
|
||||
from thinc.api import set_dropout_rate, CupyOps
|
||||
from thinc.api import get_ops, set_dropout_rate, CupyOps
|
||||
from thinc.extra.search cimport Beam
|
||||
import numpy.random
|
||||
import numpy
|
||||
|
@ -259,6 +259,12 @@ cdef class Parser(TrainablePipe):
|
|||
def greedy_parse(self, docs, drop=0.):
|
||||
cdef vector[StateC*] states
|
||||
cdef StateClass state
|
||||
ops = self.model.ops
|
||||
cdef CBlas cblas
|
||||
if isinstance(ops, CupyOps):
|
||||
cblas = get_ops("cpu").cblas()
|
||||
else:
|
||||
cblas = ops.cblas()
|
||||
self._ensure_labels_are_added(docs)
|
||||
set_dropout_rate(self.model, drop)
|
||||
batch = self.moves.init_batch(docs)
|
||||
|
@ -269,8 +275,7 @@ cdef class Parser(TrainablePipe):
|
|||
states.push_back(state.c)
|
||||
sizes = get_c_sizes(model, states.size())
|
||||
with nogil:
|
||||
self._parseC(&states[0],
|
||||
weights, sizes)
|
||||
self._parseC(cblas, &states[0], weights, sizes)
|
||||
model.clear_memory()
|
||||
del model
|
||||
return batch
|
||||
|
@ -297,14 +302,13 @@ cdef class Parser(TrainablePipe):
|
|||
del model
|
||||
return list(batch)
|
||||
|
||||
cdef void _parseC(self, StateC** states,
|
||||
cdef void _parseC(self, CBlas cblas, StateC** states,
|
||||
WeightsC weights, SizesC sizes) nogil:
|
||||
cdef int i, j
|
||||
cdef vector[StateC*] unfinished
|
||||
cdef ActivationsC activations = alloc_activations(sizes)
|
||||
while sizes.states >= 1:
|
||||
predict_states(&activations,
|
||||
states, &weights, sizes)
|
||||
predict_states(cblas, &activations, states, &weights, sizes)
|
||||
# Validate actions, argmax, take action.
|
||||
self.c_transition_batch(states,
|
||||
activations.scores, sizes.classes, sizes.states)
|
||||
|
|
|
@ -104,7 +104,7 @@ def get_arg_model(
|
|||
sig_args[param.name] = (annotation, default)
|
||||
is_strict = strict and not has_variable
|
||||
sig_args["__config__"] = ArgSchemaConfig if is_strict else ArgSchemaConfigExtra # type: ignore[assignment]
|
||||
return create_model(name, **sig_args) # type: ignore[arg-type, return-value]
|
||||
return create_model(name, **sig_args) # type: ignore[call-overload, arg-type, return-value]
|
||||
|
||||
|
||||
def validate_init_settings(
|
||||
|
@ -485,3 +485,29 @@ class RecommendationSchema(BaseModel):
|
|||
word_vectors: Optional[str] = None
|
||||
transformer: Optional[RecommendationTrf] = None
|
||||
has_letters: bool = True
|
||||
|
||||
|
||||
class DocJSONSchema(BaseModel):
|
||||
"""
|
||||
JSON/dict format for JSON representation of Doc objects.
|
||||
"""
|
||||
|
||||
cats: Optional[Dict[StrictStr, StrictFloat]] = Field(
|
||||
None, title="Categories with corresponding probabilities"
|
||||
)
|
||||
ents: Optional[List[Dict[StrictStr, Union[StrictInt, StrictStr]]]] = Field(
|
||||
None, title="Information on entities"
|
||||
)
|
||||
sents: Optional[List[Dict[StrictStr, StrictInt]]] = Field(
|
||||
None, title="Indices of sentences' start and end indices"
|
||||
)
|
||||
text: StrictStr = Field(..., title="Document text")
|
||||
spans: Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] = Field(
|
||||
None, title="Span information - end/start indices, label, KB ID"
|
||||
)
|
||||
tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field(
|
||||
..., title="Token information - ID, start, annotations"
|
||||
)
|
||||
_: Optional[Dict[StrictStr, Any]] = Field(
|
||||
None, title="Any custom data stored in the document's _ attribute"
|
||||
)
|
||||
|
|
191
spacy/tests/doc/test_json_doc_conversion.py
Normal file
191
spacy/tests/doc/test_json_doc_conversion.py
Normal file
|
@ -0,0 +1,191 @@
|
|||
import pytest
|
||||
import spacy
|
||||
from spacy import schemas
|
||||
from spacy.tokens import Doc, Span
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def doc(en_vocab):
|
||||
words = ["c", "d", "e"]
|
||||
pos = ["VERB", "NOUN", "NOUN"]
|
||||
tags = ["VBP", "NN", "NN"]
|
||||
heads = [0, 0, 1]
|
||||
deps = ["ROOT", "dobj", "dobj"]
|
||||
ents = ["O", "B-ORG", "O"]
|
||||
morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"]
|
||||
|
||||
return Doc(
|
||||
en_vocab,
|
||||
words=words,
|
||||
pos=pos,
|
||||
tags=tags,
|
||||
heads=heads,
|
||||
deps=deps,
|
||||
ents=ents,
|
||||
morphs=morphs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def doc_without_deps(en_vocab):
|
||||
words = ["c", "d", "e"]
|
||||
pos = ["VERB", "NOUN", "NOUN"]
|
||||
tags = ["VBP", "NN", "NN"]
|
||||
ents = ["O", "B-ORG", "O"]
|
||||
morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"]
|
||||
|
||||
return Doc(
|
||||
en_vocab,
|
||||
words=words,
|
||||
pos=pos,
|
||||
tags=tags,
|
||||
ents=ents,
|
||||
morphs=morphs,
|
||||
sent_starts=[True, False, True],
|
||||
)
|
||||
|
||||
|
||||
def test_doc_to_json(doc):
|
||||
json_doc = doc.to_json()
|
||||
assert json_doc["text"] == "c d e "
|
||||
assert len(json_doc["tokens"]) == 3
|
||||
assert json_doc["tokens"][0]["pos"] == "VERB"
|
||||
assert json_doc["tokens"][0]["tag"] == "VBP"
|
||||
assert json_doc["tokens"][0]["dep"] == "ROOT"
|
||||
assert len(json_doc["ents"]) == 1
|
||||
assert json_doc["ents"][0]["start"] == 2 # character offset!
|
||||
assert json_doc["ents"][0]["end"] == 3 # character offset!
|
||||
assert json_doc["ents"][0]["label"] == "ORG"
|
||||
assert not schemas.validate(schemas.DocJSONSchema, json_doc)
|
||||
|
||||
|
||||
def test_doc_to_json_underscore(doc):
|
||||
Doc.set_extension("json_test1", default=False)
|
||||
Doc.set_extension("json_test2", default=False)
|
||||
doc._.json_test1 = "hello world"
|
||||
doc._.json_test2 = [1, 2, 3]
|
||||
json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
|
||||
assert "_" in json_doc
|
||||
assert json_doc["_"]["json_test1"] == "hello world"
|
||||
assert json_doc["_"]["json_test2"] == [1, 2, 3]
|
||||
assert not schemas.validate(schemas.DocJSONSchema, json_doc)
|
||||
|
||||
|
||||
def test_doc_to_json_underscore_error_attr(doc):
|
||||
"""Test that Doc.to_json() raises an error if a custom attribute doesn't
|
||||
exist in the ._ space."""
|
||||
with pytest.raises(ValueError):
|
||||
doc.to_json(underscore=["json_test3"])
|
||||
|
||||
|
||||
def test_doc_to_json_underscore_error_serialize(doc):
|
||||
"""Test that Doc.to_json() raises an error if a custom attribute value
|
||||
isn't JSON-serializable."""
|
||||
Doc.set_extension("json_test4", method=lambda doc: doc.text)
|
||||
with pytest.raises(ValueError):
|
||||
doc.to_json(underscore=["json_test4"])
|
||||
|
||||
|
||||
def test_doc_to_json_span(doc):
|
||||
"""Test that Doc.to_json() includes spans"""
|
||||
doc.spans["test"] = [Span(doc, 0, 2, "test"), Span(doc, 0, 1, "test")]
|
||||
json_doc = doc.to_json()
|
||||
assert "spans" in json_doc
|
||||
assert len(json_doc["spans"]) == 1
|
||||
assert len(json_doc["spans"]["test"]) == 2
|
||||
assert json_doc["spans"]["test"][0]["start"] == 0
|
||||
assert not schemas.validate(schemas.DocJSONSchema, json_doc)
|
||||
|
||||
|
||||
def test_json_to_doc(doc):
|
||||
new_doc = Doc(doc.vocab).from_json(doc.to_json(), validate=True)
|
||||
new_tokens = [token for token in new_doc]
|
||||
assert new_doc.text == doc.text == "c d e "
|
||||
assert len(new_tokens) == len([token for token in doc]) == 3
|
||||
assert new_tokens[0].pos == doc[0].pos
|
||||
assert new_tokens[0].tag == doc[0].tag
|
||||
assert new_tokens[0].dep == doc[0].dep
|
||||
assert new_tokens[0].head.idx == doc[0].head.idx
|
||||
assert new_tokens[0].lemma == doc[0].lemma
|
||||
assert len(new_doc.ents) == 1
|
||||
assert new_doc.ents[0].start == 1
|
||||
assert new_doc.ents[0].end == 2
|
||||
assert new_doc.ents[0].label_ == "ORG"
|
||||
|
||||
|
||||
def test_json_to_doc_underscore(doc):
|
||||
if not Doc.has_extension("json_test1"):
|
||||
Doc.set_extension("json_test1", default=False)
|
||||
if not Doc.has_extension("json_test2"):
|
||||
Doc.set_extension("json_test2", default=False)
|
||||
|
||||
doc._.json_test1 = "hello world"
|
||||
doc._.json_test2 = [1, 2, 3]
|
||||
json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
|
||||
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||
assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)])
|
||||
assert new_doc._.json_test1 == "hello world"
|
||||
assert new_doc._.json_test2 == [1, 2, 3]
|
||||
|
||||
|
||||
def test_json_to_doc_spans(doc):
|
||||
"""Test that Doc.from_json() includes correct.spans."""
|
||||
doc.spans["test"] = [
|
||||
Span(doc, 0, 2, label="test"),
|
||||
Span(doc, 0, 1, label="test", kb_id=7),
|
||||
]
|
||||
json_doc = doc.to_json()
|
||||
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||
assert len(new_doc.spans) == 1
|
||||
assert len(new_doc.spans["test"]) == 2
|
||||
for i in range(2):
|
||||
assert new_doc.spans["test"][i].start == doc.spans["test"][i].start
|
||||
assert new_doc.spans["test"][i].end == doc.spans["test"][i].end
|
||||
assert new_doc.spans["test"][i].label == doc.spans["test"][i].label
|
||||
assert new_doc.spans["test"][i].kb_id == doc.spans["test"][i].kb_id
|
||||
|
||||
|
||||
def test_json_to_doc_sents(doc, doc_without_deps):
|
||||
"""Test that Doc.from_json() includes correct.sents."""
|
||||
for test_doc in (doc, doc_without_deps):
|
||||
json_doc = test_doc.to_json()
|
||||
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||
assert [sent.text for sent in test_doc.sents] == [
|
||||
sent.text for sent in new_doc.sents
|
||||
]
|
||||
assert [token.is_sent_start for token in test_doc] == [
|
||||
token.is_sent_start for token in new_doc
|
||||
]
|
||||
|
||||
|
||||
def test_json_to_doc_cats(doc):
|
||||
"""Test that Doc.from_json() includes correct .cats."""
|
||||
cats = {"A": 0.3, "B": 0.7}
|
||||
doc.cats = cats
|
||||
json_doc = doc.to_json()
|
||||
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||
assert new_doc.cats == cats
|
||||
|
||||
|
||||
def test_json_to_doc_spaces():
|
||||
"""Test that Doc.from_json() preserves spaces correctly."""
|
||||
doc = spacy.blank("en")("This is just brilliant.")
|
||||
json_doc = doc.to_json()
|
||||
new_doc = Doc(doc.vocab).from_json(json_doc, validate=True)
|
||||
assert doc.text == new_doc.text
|
||||
|
||||
|
||||
def test_json_to_doc_attribute_consistency(doc):
|
||||
"""Test that Doc.from_json() raises an exception if tokens don't all have the same set of properties."""
|
||||
doc_json = doc.to_json()
|
||||
doc_json["tokens"][1].pop("morph")
|
||||
with pytest.raises(ValueError):
|
||||
Doc(doc.vocab).from_json(doc_json)
|
||||
|
||||
|
||||
def test_json_to_doc_validation_error(doc):
|
||||
"""Test that Doc.from_json() raises an exception when validating invalid input."""
|
||||
doc_json = doc.to_json()
|
||||
doc_json.pop("tokens")
|
||||
with pytest.raises(ValueError):
|
||||
Doc(doc.vocab).from_json(doc_json, validate=True)
|
|
@ -5,11 +5,9 @@ from spacy.compat import pickle
|
|||
def test_pickle_single_doc():
|
||||
nlp = Language()
|
||||
doc = nlp("pickle roundtrip")
|
||||
doc._context = 3
|
||||
data = pickle.dumps(doc, 1)
|
||||
doc2 = pickle.loads(data)
|
||||
assert doc2.text == "pickle roundtrip"
|
||||
assert doc2._context == 3
|
||||
|
||||
|
||||
def test_list_of_docs_pickles_efficiently():
|
||||
|
|
|
@ -440,10 +440,19 @@ def test_span_string_label_kb_id(doc):
|
|||
assert span.kb_id == doc.vocab.strings["Q342"]
|
||||
|
||||
|
||||
def test_span_string_label_id(doc):
|
||||
span = Span(doc, 0, 1, label="hello", span_id="Q342")
|
||||
assert span.label_ == "hello"
|
||||
assert span.label == doc.vocab.strings["hello"]
|
||||
assert span.id_ == "Q342"
|
||||
assert span.id == doc.vocab.strings["Q342"]
|
||||
|
||||
|
||||
def test_span_attrs_writable(doc):
|
||||
span = Span(doc, 0, 1)
|
||||
span.label_ = "label"
|
||||
span.kb_id_ = "kb_id"
|
||||
span.id_ = "id"
|
||||
|
||||
|
||||
def test_span_ents_property(doc):
|
||||
|
@ -631,6 +640,9 @@ def test_span_comparison(doc):
|
|||
assert Span(doc, 0, 4, "LABEL", kb_id="KB_ID") <= Span(doc, 1, 3)
|
||||
assert Span(doc, 1, 3) > Span(doc, 0, 4, "LABEL", kb_id="KB_ID")
|
||||
assert Span(doc, 1, 3) >= Span(doc, 0, 4, "LABEL", kb_id="KB_ID")
|
||||
|
||||
# Different id
|
||||
assert Span(doc, 1, 3, span_id="AAA") < Span(doc, 1, 3, span_id="BBB")
|
||||
# fmt: on
|
||||
|
||||
|
||||
|
|
|
@ -1,72 +0,0 @@
|
|||
import pytest
|
||||
from spacy.tokens import Doc, Span
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def doc(en_vocab):
|
||||
words = ["c", "d", "e"]
|
||||
pos = ["VERB", "NOUN", "NOUN"]
|
||||
tags = ["VBP", "NN", "NN"]
|
||||
heads = [0, 0, 0]
|
||||
deps = ["ROOT", "dobj", "dobj"]
|
||||
ents = ["O", "B-ORG", "O"]
|
||||
morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"]
|
||||
return Doc(
|
||||
en_vocab,
|
||||
words=words,
|
||||
pos=pos,
|
||||
tags=tags,
|
||||
heads=heads,
|
||||
deps=deps,
|
||||
ents=ents,
|
||||
morphs=morphs,
|
||||
)
|
||||
|
||||
|
||||
def test_doc_to_json(doc):
|
||||
json_doc = doc.to_json()
|
||||
assert json_doc["text"] == "c d e "
|
||||
assert len(json_doc["tokens"]) == 3
|
||||
assert json_doc["tokens"][0]["pos"] == "VERB"
|
||||
assert json_doc["tokens"][0]["tag"] == "VBP"
|
||||
assert json_doc["tokens"][0]["dep"] == "ROOT"
|
||||
assert len(json_doc["ents"]) == 1
|
||||
assert json_doc["ents"][0]["start"] == 2 # character offset!
|
||||
assert json_doc["ents"][0]["end"] == 3 # character offset!
|
||||
assert json_doc["ents"][0]["label"] == "ORG"
|
||||
|
||||
|
||||
def test_doc_to_json_underscore(doc):
|
||||
Doc.set_extension("json_test1", default=False)
|
||||
Doc.set_extension("json_test2", default=False)
|
||||
doc._.json_test1 = "hello world"
|
||||
doc._.json_test2 = [1, 2, 3]
|
||||
json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
|
||||
assert "_" in json_doc
|
||||
assert json_doc["_"]["json_test1"] == "hello world"
|
||||
assert json_doc["_"]["json_test2"] == [1, 2, 3]
|
||||
|
||||
|
||||
def test_doc_to_json_underscore_error_attr(doc):
|
||||
"""Test that Doc.to_json() raises an error if a custom attribute doesn't
|
||||
exist in the ._ space."""
|
||||
with pytest.raises(ValueError):
|
||||
doc.to_json(underscore=["json_test3"])
|
||||
|
||||
|
||||
def test_doc_to_json_underscore_error_serialize(doc):
|
||||
"""Test that Doc.to_json() raises an error if a custom attribute value
|
||||
isn't JSON-serializable."""
|
||||
Doc.set_extension("json_test4", method=lambda doc: doc.text)
|
||||
with pytest.raises(ValueError):
|
||||
doc.to_json(underscore=["json_test4"])
|
||||
|
||||
|
||||
def test_doc_to_json_span(doc):
|
||||
"""Test that Doc.to_json() includes spans"""
|
||||
doc.spans["test"] = [Span(doc, 0, 2, "test"), Span(doc, 0, 1, "test")]
|
||||
json_doc = doc.to_json()
|
||||
assert "spans" in json_doc
|
||||
assert len(json_doc["spans"]) == 1
|
||||
assert len(json_doc["spans"]["test"]) == 2
|
||||
assert json_doc["spans"]["test"][0]["start"] == 0
|
|
@ -167,3 +167,12 @@ def test_issue3521(en_tokenizer, word):
|
|||
tok = en_tokenizer(word)[1]
|
||||
# 'not' and 'would' should be stopwords, also in their abbreviated forms
|
||||
assert tok.is_stop
|
||||
|
||||
|
||||
@pytest.mark.issue(10699)
|
||||
@pytest.mark.parametrize("text", ["theses", "thisre"])
|
||||
def test_issue10699(en_tokenizer, text):
|
||||
"""Test that 'theses' and 'thisre' are excluded from the contractions
|
||||
generated by the English tokenizer exceptions."""
|
||||
tokens = en_tokenizer(text)
|
||||
assert len(tokens) == 1
|
||||
|
|
|
@ -49,7 +49,7 @@ def test_parser_contains_cycle(tree, cyclic_tree, partial_tree, multirooted_tree
|
|||
assert contains_cycle(multirooted_tree) is None
|
||||
|
||||
|
||||
def test_parser_is_nonproj_arc(nonproj_tree, partial_tree, multirooted_tree):
|
||||
def test_parser_is_nonproj_arc(cyclic_tree, nonproj_tree, partial_tree, multirooted_tree):
|
||||
assert is_nonproj_arc(0, nonproj_tree) is False
|
||||
assert is_nonproj_arc(1, nonproj_tree) is False
|
||||
assert is_nonproj_arc(2, nonproj_tree) is False
|
||||
|
@ -62,15 +62,19 @@ def test_parser_is_nonproj_arc(nonproj_tree, partial_tree, multirooted_tree):
|
|||
assert is_nonproj_arc(7, partial_tree) is False
|
||||
assert is_nonproj_arc(17, multirooted_tree) is False
|
||||
assert is_nonproj_arc(16, multirooted_tree) is True
|
||||
with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'):
|
||||
is_nonproj_arc(6, cyclic_tree)
|
||||
|
||||
|
||||
def test_parser_is_nonproj_tree(
|
||||
proj_tree, nonproj_tree, partial_tree, multirooted_tree
|
||||
proj_tree, cyclic_tree, nonproj_tree, partial_tree, multirooted_tree
|
||||
):
|
||||
assert is_nonproj_tree(proj_tree) is False
|
||||
assert is_nonproj_tree(nonproj_tree) is True
|
||||
assert is_nonproj_tree(partial_tree) is False
|
||||
assert is_nonproj_tree(multirooted_tree) is True
|
||||
with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'):
|
||||
is_nonproj_tree(cyclic_tree)
|
||||
|
||||
|
||||
def test_parser_pseudoprojectivity(en_vocab):
|
||||
|
@ -84,8 +88,10 @@ def test_parser_pseudoprojectivity(en_vocab):
|
|||
tree = [1, 2, 2]
|
||||
nonproj_tree = [1, 2, 2, 4, 5, 2, 7, 4, 2]
|
||||
nonproj_tree2 = [9, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1]
|
||||
cyclic_tree = [1, 2, 2, 4, 5, 3, 2]
|
||||
labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl", "punct"]
|
||||
labels2 = ["advmod", "root", "det", "nsubj", "advmod", "det", "dobj", "det", "nmod", "aux", "nmod", "advmod", "det", "amod", "punct"]
|
||||
cyclic_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "punct"]
|
||||
# fmt: on
|
||||
assert nonproj.decompose("X||Y") == ("X", "Y")
|
||||
assert nonproj.decompose("X") == ("X", "")
|
||||
|
@ -97,6 +103,8 @@ def test_parser_pseudoprojectivity(en_vocab):
|
|||
assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree2) == 10
|
||||
# fmt: off
|
||||
proj_heads, deco_labels = nonproj.projectivize(nonproj_tree, labels)
|
||||
with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'):
|
||||
nonproj.projectivize(cyclic_tree, cyclic_labels)
|
||||
assert proj_heads == [1, 2, 2, 4, 5, 2, 7, 5, 2]
|
||||
assert deco_labels == ["det", "nsubj", "root", "det", "dobj", "aux",
|
||||
"nsubj", "acl||dobj", "punct"]
|
||||
|
|
|
@ -14,7 +14,7 @@ from spacy.pipeline.legacy import EntityLinker_v1
|
|||
from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL
|
||||
from spacy.scorer import Scorer
|
||||
from spacy.tests.util import make_tempdir
|
||||
from spacy.tokens import Span
|
||||
from spacy.tokens import Span, Doc
|
||||
from spacy.training import Example
|
||||
from spacy.util import ensure_path
|
||||
from spacy.vocab import Vocab
|
||||
|
@ -1075,3 +1075,43 @@ def test_no_gold_ents(patterns):
|
|||
|
||||
# this will run the pipeline on the examples and shouldn't crash
|
||||
results = nlp.evaluate(train_examples)
|
||||
|
||||
|
||||
@pytest.mark.issue(9575)
|
||||
def test_tokenization_mismatch():
|
||||
nlp = English()
|
||||
# include a matching entity so that update isn't skipped
|
||||
doc1 = Doc(
|
||||
nlp.vocab,
|
||||
words=["Kirby", "123456"],
|
||||
spaces=[True, False],
|
||||
ents=["B-CHARACTER", "B-CARDINAL"],
|
||||
)
|
||||
doc2 = Doc(
|
||||
nlp.vocab,
|
||||
words=["Kirby", "123", "456"],
|
||||
spaces=[True, False, False],
|
||||
ents=["B-CHARACTER", "B-CARDINAL", "B-CARDINAL"],
|
||||
)
|
||||
|
||||
eg = Example(doc1, doc2)
|
||||
train_examples = [eg]
|
||||
vector_length = 3
|
||||
|
||||
def create_kb(vocab):
|
||||
# create placeholder KB
|
||||
mykb = KnowledgeBase(vocab, entity_vector_length=vector_length)
|
||||
mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3])
|
||||
mykb.add_alias("Kirby", ["Q613241"], [0.9])
|
||||
return mykb
|
||||
|
||||
entity_linker = nlp.add_pipe("entity_linker", last=True)
|
||||
entity_linker.set_kb(create_kb)
|
||||
|
||||
optimizer = nlp.initialize(get_examples=lambda: train_examples)
|
||||
for i in range(2):
|
||||
losses = {}
|
||||
nlp.update(train_examples, sgd=optimizer, losses=losses)
|
||||
|
||||
nlp.add_pipe("sentencizer", first=True)
|
||||
results = nlp.evaluate(train_examples)
|
||||
|
|
|
@ -5,12 +5,15 @@ from spacy.tokens import Doc, Span
|
|||
from spacy.language import Language
|
||||
from spacy.lang.en import English
|
||||
from spacy.pipeline import EntityRuler, EntityRecognizer, merge_entities
|
||||
from spacy.pipeline import SpanRuler
|
||||
from spacy.pipeline.ner import DEFAULT_NER_MODEL
|
||||
from spacy.errors import MatchPatternError
|
||||
from spacy.tests.util import make_tempdir
|
||||
|
||||
from thinc.api import NumpyOps, get_current_ops
|
||||
|
||||
ENTITY_RULERS = ["entity_ruler", "future_entity_ruler"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def nlp():
|
||||
|
@ -37,12 +40,14 @@ def add_ent_component(doc):
|
|||
|
||||
|
||||
@pytest.mark.issue(3345)
|
||||
def test_issue3345():
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_issue3345(entity_ruler_factory):
|
||||
"""Test case where preset entity crosses sentence boundary."""
|
||||
nlp = English()
|
||||
doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"])
|
||||
doc[4].is_sent_start = True
|
||||
ruler = EntityRuler(nlp, patterns=[{"label": "GPE", "pattern": "New York"}])
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
ruler.add_patterns([{"label": "GPE", "pattern": "New York"}])
|
||||
cfg = {"model": DEFAULT_NER_MODEL}
|
||||
model = registry.resolve(cfg, validate=True)["model"]
|
||||
ner = EntityRecognizer(doc.vocab, model)
|
||||
|
@ -60,13 +65,18 @@ def test_issue3345():
|
|||
|
||||
|
||||
@pytest.mark.issue(4849)
|
||||
def test_issue4849():
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_issue4849(entity_ruler_factory):
|
||||
nlp = English()
|
||||
patterns = [
|
||||
{"label": "PERSON", "pattern": "joe biden", "id": "joe-biden"},
|
||||
{"label": "PERSON", "pattern": "bernie sanders", "id": "bernie-sanders"},
|
||||
]
|
||||
ruler = nlp.add_pipe("entity_ruler", config={"phrase_matcher_attr": "LOWER"})
|
||||
ruler = nlp.add_pipe(
|
||||
entity_ruler_factory,
|
||||
name="entity_ruler",
|
||||
config={"phrase_matcher_attr": "LOWER"},
|
||||
)
|
||||
ruler.add_patterns(patterns)
|
||||
text = """
|
||||
The left is starting to take aim at Democratic front-runner Joe Biden.
|
||||
|
@ -86,10 +96,11 @@ def test_issue4849():
|
|||
|
||||
|
||||
@pytest.mark.issue(5918)
|
||||
def test_issue5918():
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_issue5918(entity_ruler_factory):
|
||||
# Test edge case when merging entities.
|
||||
nlp = English()
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [
|
||||
{"label": "ORG", "pattern": "Digicon Inc"},
|
||||
{"label": "ORG", "pattern": "Rotan Mosle Inc's"},
|
||||
|
@ -114,9 +125,10 @@ def test_issue5918():
|
|||
|
||||
|
||||
@pytest.mark.issue(8168)
|
||||
def test_issue8168():
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_issue8168(entity_ruler_factory):
|
||||
nlp = English()
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [
|
||||
{"label": "ORG", "pattern": "Apple"},
|
||||
{
|
||||
|
@ -131,14 +143,17 @@ def test_issue8168():
|
|||
},
|
||||
]
|
||||
ruler.add_patterns(patterns)
|
||||
|
||||
assert ruler._ent_ids == {8043148519967183733: ("GPE", "san-francisco")}
|
||||
doc = nlp("San Francisco San Fran")
|
||||
assert all(t.ent_id_ == "san-francisco" for t in doc)
|
||||
|
||||
|
||||
@pytest.mark.issue(8216)
|
||||
def test_entity_ruler_fix8216(nlp, patterns):
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_fix8216(nlp, patterns, entity_ruler_factory):
|
||||
"""Test that patterns don't get added excessively."""
|
||||
ruler = nlp.add_pipe("entity_ruler", config={"validate": True})
|
||||
ruler = nlp.add_pipe(
|
||||
entity_ruler_factory, name="entity_ruler", config={"validate": True}
|
||||
)
|
||||
ruler.add_patterns(patterns)
|
||||
pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values())
|
||||
assert pattern_count > 0
|
||||
|
@ -147,13 +162,16 @@ def test_entity_ruler_fix8216(nlp, patterns):
|
|||
assert after_count == pattern_count
|
||||
|
||||
|
||||
def test_entity_ruler_init(nlp, patterns):
|
||||
ruler = EntityRuler(nlp, patterns=patterns)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_init(nlp, patterns, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
assert len(ruler) == len(patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
assert "HELLO" in ruler
|
||||
assert "BYE" in ruler
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
nlp.remove_pipe("entity_ruler")
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
doc = nlp("hello world bye bye")
|
||||
assert len(doc.ents) == 2
|
||||
|
@ -161,20 +179,23 @@ def test_entity_ruler_init(nlp, patterns):
|
|||
assert doc.ents[1].label_ == "BYE"
|
||||
|
||||
|
||||
def test_entity_ruler_no_patterns_warns(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_no_patterns_warns(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
assert len(ruler) == 0
|
||||
assert len(ruler.labels) == 0
|
||||
nlp.add_pipe("entity_ruler")
|
||||
nlp.remove_pipe("entity_ruler")
|
||||
nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
assert nlp.pipe_names == ["entity_ruler"]
|
||||
with pytest.warns(UserWarning):
|
||||
doc = nlp("hello world bye bye")
|
||||
assert len(doc.ents) == 0
|
||||
|
||||
|
||||
def test_entity_ruler_init_patterns(nlp, patterns):
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_init_patterns(nlp, patterns, entity_ruler_factory):
|
||||
# initialize with patterns
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
assert len(ruler.labels) == 0
|
||||
ruler.initialize(lambda: [], patterns=patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
|
@ -186,7 +207,7 @@ def test_entity_ruler_init_patterns(nlp, patterns):
|
|||
nlp.config["initialize"]["components"]["entity_ruler"] = {
|
||||
"patterns": {"@misc": "entity_ruler_patterns"}
|
||||
}
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
assert len(ruler.labels) == 0
|
||||
nlp.initialize()
|
||||
assert len(ruler.labels) == 4
|
||||
|
@ -195,18 +216,20 @@ def test_entity_ruler_init_patterns(nlp, patterns):
|
|||
assert doc.ents[1].label_ == "BYE"
|
||||
|
||||
|
||||
def test_entity_ruler_init_clear(nlp, patterns):
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_init_clear(nlp, patterns, entity_ruler_factory):
|
||||
"""Test that initialization clears patterns."""
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
ruler.initialize(lambda: [])
|
||||
assert len(ruler.labels) == 0
|
||||
|
||||
|
||||
def test_entity_ruler_clear(nlp, patterns):
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_clear(nlp, patterns, entity_ruler_factory):
|
||||
"""Test that initialization clears patterns."""
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
doc = nlp("hello world")
|
||||
|
@ -218,8 +241,9 @@ def test_entity_ruler_clear(nlp, patterns):
|
|||
assert len(doc.ents) == 0
|
||||
|
||||
|
||||
def test_entity_ruler_existing(nlp, patterns):
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_existing(nlp, patterns, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
nlp.add_pipe("add_ent", before="entity_ruler")
|
||||
doc = nlp("OH HELLO WORLD bye bye")
|
||||
|
@ -228,8 +252,11 @@ def test_entity_ruler_existing(nlp, patterns):
|
|||
assert doc.ents[1].label_ == "BYE"
|
||||
|
||||
|
||||
def test_entity_ruler_existing_overwrite(nlp, patterns):
|
||||
ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True})
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_existing_overwrite(nlp, patterns, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(
|
||||
entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True}
|
||||
)
|
||||
ruler.add_patterns(patterns)
|
||||
nlp.add_pipe("add_ent", before="entity_ruler")
|
||||
doc = nlp("OH HELLO WORLD bye bye")
|
||||
|
@ -239,8 +266,11 @@ def test_entity_ruler_existing_overwrite(nlp, patterns):
|
|||
assert doc.ents[1].label_ == "BYE"
|
||||
|
||||
|
||||
def test_entity_ruler_existing_complex(nlp, patterns):
|
||||
ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True})
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_existing_complex(nlp, patterns, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(
|
||||
entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True}
|
||||
)
|
||||
ruler.add_patterns(patterns)
|
||||
nlp.add_pipe("add_ent", before="entity_ruler")
|
||||
doc = nlp("foo foo bye bye")
|
||||
|
@ -251,8 +281,11 @@ def test_entity_ruler_existing_complex(nlp, patterns):
|
|||
assert len(doc.ents[1]) == 2
|
||||
|
||||
|
||||
def test_entity_ruler_entity_id(nlp, patterns):
|
||||
ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True})
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_entity_id(nlp, patterns, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(
|
||||
entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True}
|
||||
)
|
||||
ruler.add_patterns(patterns)
|
||||
doc = nlp("Apple is a technology company")
|
||||
assert len(doc.ents) == 1
|
||||
|
@ -260,18 +293,21 @@ def test_entity_ruler_entity_id(nlp, patterns):
|
|||
assert doc.ents[0].ent_id_ == "a1"
|
||||
|
||||
|
||||
def test_entity_ruler_cfg_ent_id_sep(nlp, patterns):
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_cfg_ent_id_sep(nlp, patterns, entity_ruler_factory):
|
||||
config = {"overwrite_ents": True, "ent_id_sep": "**"}
|
||||
ruler = nlp.add_pipe("entity_ruler", config=config)
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler", config=config)
|
||||
ruler.add_patterns(patterns)
|
||||
assert "TECH_ORG**a1" in ruler.phrase_patterns
|
||||
doc = nlp("Apple is a technology company")
|
||||
if isinstance(ruler, EntityRuler):
|
||||
assert "TECH_ORG**a1" in ruler.phrase_patterns
|
||||
assert len(doc.ents) == 1
|
||||
assert doc.ents[0].label_ == "TECH_ORG"
|
||||
assert doc.ents[0].ent_id_ == "a1"
|
||||
|
||||
|
||||
def test_entity_ruler_serialize_bytes(nlp, patterns):
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_serialize_bytes(nlp, patterns, entity_ruler_factory):
|
||||
ruler = EntityRuler(nlp, patterns=patterns)
|
||||
assert len(ruler) == len(patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
|
@ -288,7 +324,10 @@ def test_entity_ruler_serialize_bytes(nlp, patterns):
|
|||
assert sorted(new_ruler.labels) == sorted(ruler.labels)
|
||||
|
||||
|
||||
def test_entity_ruler_serialize_phrase_matcher_attr_bytes(nlp, patterns):
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_serialize_phrase_matcher_attr_bytes(
|
||||
nlp, patterns, entity_ruler_factory
|
||||
):
|
||||
ruler = EntityRuler(nlp, phrase_matcher_attr="LOWER", patterns=patterns)
|
||||
assert len(ruler) == len(patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
|
@ -303,8 +342,9 @@ def test_entity_ruler_serialize_phrase_matcher_attr_bytes(nlp, patterns):
|
|||
assert new_ruler.phrase_matcher_attr == "LOWER"
|
||||
|
||||
|
||||
def test_entity_ruler_validate(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_validate(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
validated_ruler = EntityRuler(nlp, validate=True)
|
||||
|
||||
valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]}
|
||||
|
@ -322,32 +362,35 @@ def test_entity_ruler_validate(nlp):
|
|||
validated_ruler.add_patterns([invalid_pattern])
|
||||
|
||||
|
||||
def test_entity_ruler_properties(nlp, patterns):
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_properties(nlp, patterns, entity_ruler_factory):
|
||||
ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True)
|
||||
assert sorted(ruler.labels) == sorted(["HELLO", "BYE", "COMPLEX", "TECH_ORG"])
|
||||
assert sorted(ruler.ent_ids) == ["a1", "a2"]
|
||||
|
||||
|
||||
def test_entity_ruler_overlapping_spans(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_overlapping_spans(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [
|
||||
{"label": "FOOBAR", "pattern": "foo bar"},
|
||||
{"label": "BARBAZ", "pattern": "bar baz"},
|
||||
]
|
||||
ruler.add_patterns(patterns)
|
||||
doc = ruler(nlp.make_doc("foo bar baz"))
|
||||
doc = nlp("foo bar baz")
|
||||
assert len(doc.ents) == 1
|
||||
assert doc.ents[0].label_ == "FOOBAR"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_process", [1, 2])
|
||||
def test_entity_ruler_multiprocessing(nlp, n_process):
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_multiprocessing(nlp, n_process, entity_ruler_factory):
|
||||
if isinstance(get_current_ops, NumpyOps) or n_process < 2:
|
||||
texts = ["I enjoy eating Pizza Hut pizza."]
|
||||
|
||||
patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut", "id": "1234"}]
|
||||
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
|
||||
for doc in nlp.pipe(texts, n_process=2):
|
||||
|
@ -355,8 +398,9 @@ def test_entity_ruler_multiprocessing(nlp, n_process):
|
|||
assert ent.ent_id_ == "1234"
|
||||
|
||||
|
||||
def test_entity_ruler_serialize_jsonl(nlp, patterns):
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_serialize_jsonl(nlp, patterns, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
with make_tempdir() as d:
|
||||
ruler.to_disk(d / "test_ruler.jsonl")
|
||||
|
@ -365,8 +409,9 @@ def test_entity_ruler_serialize_jsonl(nlp, patterns):
|
|||
ruler.from_disk(d / "non_existing.jsonl") # read from a bad jsonl file
|
||||
|
||||
|
||||
def test_entity_ruler_serialize_dir(nlp, patterns):
|
||||
ruler = nlp.add_pipe("entity_ruler")
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_serialize_dir(nlp, patterns, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
with make_tempdir() as d:
|
||||
ruler.to_disk(d / "test_ruler")
|
||||
|
@ -375,52 +420,65 @@ def test_entity_ruler_serialize_dir(nlp, patterns):
|
|||
ruler.from_disk(d / "non_existing_dir") # read from a bad directory
|
||||
|
||||
|
||||
def test_entity_ruler_remove_basic(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_remove_basic(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [
|
||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
||||
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||
{"label": "ORG", "pattern": "ACM"},
|
||||
]
|
||||
ruler.add_patterns(patterns)
|
||||
doc = ruler(nlp.make_doc("Duygu went to school"))
|
||||
doc = nlp("Dina went to school")
|
||||
assert len(ruler.patterns) == 3
|
||||
assert len(doc.ents) == 1
|
||||
if isinstance(ruler, EntityRuler):
|
||||
assert "PERSON||dina" in ruler.phrase_matcher
|
||||
assert doc.ents[0].label_ == "PERSON"
|
||||
assert doc.ents[0].text == "Duygu"
|
||||
assert "PERSON||duygu" in ruler.phrase_matcher
|
||||
ruler.remove("duygu")
|
||||
doc = ruler(nlp.make_doc("Duygu went to school"))
|
||||
assert doc.ents[0].text == "Dina"
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("dina")
|
||||
else:
|
||||
ruler.remove_by_id("dina")
|
||||
doc = nlp("Dina went to school")
|
||||
assert len(doc.ents) == 0
|
||||
assert "PERSON||duygu" not in ruler.phrase_matcher
|
||||
if isinstance(ruler, EntityRuler):
|
||||
assert "PERSON||dina" not in ruler.phrase_matcher
|
||||
assert len(ruler.patterns) == 2
|
||||
|
||||
|
||||
def test_entity_ruler_remove_same_id_multiple_patterns(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_remove_same_id_multiple_patterns(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [
|
||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
||||
{"label": "ORG", "pattern": "DuyguCorp", "id": "duygu"},
|
||||
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||
{"label": "ORG", "pattern": "DinaCorp", "id": "dina"},
|
||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||
]
|
||||
ruler.add_patterns(patterns)
|
||||
doc = ruler(nlp.make_doc("Duygu founded DuyguCorp and ACME."))
|
||||
doc = nlp("Dina founded DinaCorp and ACME.")
|
||||
assert len(ruler.patterns) == 3
|
||||
assert "PERSON||duygu" in ruler.phrase_matcher
|
||||
assert "ORG||duygu" in ruler.phrase_matcher
|
||||
if isinstance(ruler, EntityRuler):
|
||||
assert "PERSON||dina" in ruler.phrase_matcher
|
||||
assert "ORG||dina" in ruler.phrase_matcher
|
||||
assert len(doc.ents) == 3
|
||||
ruler.remove("duygu")
|
||||
doc = ruler(nlp.make_doc("Duygu founded DuyguCorp and ACME."))
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("dina")
|
||||
else:
|
||||
ruler.remove_by_id("dina")
|
||||
doc = nlp("Dina founded DinaCorp and ACME.")
|
||||
assert len(ruler.patterns) == 1
|
||||
assert "PERSON||duygu" not in ruler.phrase_matcher
|
||||
assert "ORG||duygu" not in ruler.phrase_matcher
|
||||
if isinstance(ruler, EntityRuler):
|
||||
assert "PERSON||dina" not in ruler.phrase_matcher
|
||||
assert "ORG||dina" not in ruler.phrase_matcher
|
||||
assert len(doc.ents) == 1
|
||||
|
||||
|
||||
def test_entity_ruler_remove_nonexisting_pattern(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_remove_nonexisting_pattern(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [
|
||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
||||
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||
{"label": "ORG", "pattern": "ACM"},
|
||||
]
|
||||
|
@ -428,82 +486,108 @@ def test_entity_ruler_remove_nonexisting_pattern(nlp):
|
|||
assert len(ruler.patterns) == 3
|
||||
with pytest.raises(ValueError):
|
||||
ruler.remove("nepattern")
|
||||
assert len(ruler.patterns) == 3
|
||||
if isinstance(ruler, SpanRuler):
|
||||
with pytest.raises(ValueError):
|
||||
ruler.remove_by_id("nepattern")
|
||||
|
||||
|
||||
def test_entity_ruler_remove_several_patterns(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_remove_several_patterns(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [
|
||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
||||
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||
{"label": "ORG", "pattern": "ACM"},
|
||||
]
|
||||
ruler.add_patterns(patterns)
|
||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME."))
|
||||
doc = nlp("Dina founded her company ACME.")
|
||||
assert len(ruler.patterns) == 3
|
||||
assert len(doc.ents) == 2
|
||||
assert doc.ents[0].label_ == "PERSON"
|
||||
assert doc.ents[0].text == "Duygu"
|
||||
assert doc.ents[0].text == "Dina"
|
||||
assert doc.ents[1].label_ == "ORG"
|
||||
assert doc.ents[1].text == "ACME"
|
||||
ruler.remove("duygu")
|
||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME"))
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("dina")
|
||||
else:
|
||||
ruler.remove_by_id("dina")
|
||||
doc = nlp("Dina founded her company ACME")
|
||||
assert len(ruler.patterns) == 2
|
||||
assert len(doc.ents) == 1
|
||||
assert doc.ents[0].label_ == "ORG"
|
||||
assert doc.ents[0].text == "ACME"
|
||||
ruler.remove("acme")
|
||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME"))
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("acme")
|
||||
else:
|
||||
ruler.remove_by_id("acme")
|
||||
doc = nlp("Dina founded her company ACME")
|
||||
assert len(ruler.patterns) == 1
|
||||
assert len(doc.ents) == 0
|
||||
|
||||
|
||||
def test_entity_ruler_remove_patterns_in_a_row(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_remove_patterns_in_a_row(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [
|
||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
||||
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||
{"label": "DATE", "pattern": "her birthday", "id": "bday"},
|
||||
{"label": "ORG", "pattern": "ACM"},
|
||||
]
|
||||
ruler.add_patterns(patterns)
|
||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME on her birthday"))
|
||||
doc = nlp("Dina founded her company ACME on her birthday")
|
||||
assert len(doc.ents) == 3
|
||||
assert doc.ents[0].label_ == "PERSON"
|
||||
assert doc.ents[0].text == "Duygu"
|
||||
assert doc.ents[0].text == "Dina"
|
||||
assert doc.ents[1].label_ == "ORG"
|
||||
assert doc.ents[1].text == "ACME"
|
||||
assert doc.ents[2].label_ == "DATE"
|
||||
assert doc.ents[2].text == "her birthday"
|
||||
ruler.remove("duygu")
|
||||
ruler.remove("acme")
|
||||
ruler.remove("bday")
|
||||
doc = ruler(nlp.make_doc("Duygu went to school"))
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("dina")
|
||||
ruler.remove("acme")
|
||||
ruler.remove("bday")
|
||||
else:
|
||||
ruler.remove_by_id("dina")
|
||||
ruler.remove_by_id("acme")
|
||||
ruler.remove_by_id("bday")
|
||||
doc = nlp("Dina went to school")
|
||||
assert len(doc.ents) == 0
|
||||
|
||||
|
||||
def test_entity_ruler_remove_all_patterns(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_remove_all_patterns(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [
|
||||
{"label": "PERSON", "pattern": "Duygu", "id": "duygu"},
|
||||
{"label": "PERSON", "pattern": "Dina", "id": "dina"},
|
||||
{"label": "ORG", "pattern": "ACME", "id": "acme"},
|
||||
{"label": "DATE", "pattern": "her birthday", "id": "bday"},
|
||||
]
|
||||
ruler.add_patterns(patterns)
|
||||
assert len(ruler.patterns) == 3
|
||||
ruler.remove("duygu")
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("dina")
|
||||
else:
|
||||
ruler.remove_by_id("dina")
|
||||
assert len(ruler.patterns) == 2
|
||||
ruler.remove("acme")
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("acme")
|
||||
else:
|
||||
ruler.remove_by_id("acme")
|
||||
assert len(ruler.patterns) == 1
|
||||
ruler.remove("bday")
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("bday")
|
||||
else:
|
||||
ruler.remove_by_id("bday")
|
||||
assert len(ruler.patterns) == 0
|
||||
with pytest.warns(UserWarning):
|
||||
doc = ruler(nlp.make_doc("Duygu founded her company ACME on her birthday"))
|
||||
doc = nlp("Dina founded her company ACME on her birthday")
|
||||
assert len(doc.ents) == 0
|
||||
|
||||
|
||||
def test_entity_ruler_remove_and_add(nlp):
|
||||
ruler = EntityRuler(nlp)
|
||||
@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS)
|
||||
def test_entity_ruler_remove_and_add(nlp, entity_ruler_factory):
|
||||
ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler")
|
||||
patterns = [{"label": "DATE", "pattern": "last time"}]
|
||||
ruler.add_patterns(patterns)
|
||||
doc = ruler(
|
||||
|
@ -524,7 +608,10 @@ def test_entity_ruler_remove_and_add(nlp):
|
|||
assert doc.ents[0].text == "last time"
|
||||
assert doc.ents[1].label_ == "DATE"
|
||||
assert doc.ents[1].text == "this time"
|
||||
ruler.remove("ttime")
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("ttime")
|
||||
else:
|
||||
ruler.remove_by_id("ttime")
|
||||
doc = ruler(
|
||||
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||
)
|
||||
|
@ -547,7 +634,10 @@ def test_entity_ruler_remove_and_add(nlp):
|
|||
)
|
||||
assert len(ruler.patterns) == 3
|
||||
assert len(doc.ents) == 3
|
||||
ruler.remove("ttime")
|
||||
if isinstance(ruler, EntityRuler):
|
||||
ruler.remove("ttime")
|
||||
else:
|
||||
ruler.remove_by_id("ttime")
|
||||
doc = ruler(
|
||||
nlp.make_doc(
|
||||
"I saw him last time we met, this time he brought some flowers, another time some chocolate."
|
||||
|
|
465
spacy/tests/pipeline/test_span_ruler.py
Normal file
465
spacy/tests/pipeline/test_span_ruler.py
Normal file
|
@ -0,0 +1,465 @@
|
|||
import pytest
|
||||
|
||||
import spacy
|
||||
from spacy import registry
|
||||
from spacy.errors import MatchPatternError
|
||||
from spacy.tokens import Span
|
||||
from spacy.training import Example
|
||||
from spacy.tests.util import make_tempdir
|
||||
|
||||
from thinc.api import NumpyOps, get_current_ops
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@registry.misc("span_ruler_patterns")
|
||||
def patterns():
|
||||
return [
|
||||
{"label": "HELLO", "pattern": "hello world", "id": "hello1"},
|
||||
{"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]},
|
||||
{"label": "HELLO", "pattern": [{"ORTH": "HELLO"}], "id": "hello2"},
|
||||
{"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]},
|
||||
{"label": "TECH_ORG", "pattern": "Apple"},
|
||||
{"label": "TECH_ORG", "pattern": "Microsoft"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def overlapping_patterns():
|
||||
return [
|
||||
{"label": "FOOBAR", "pattern": "foo bar"},
|
||||
{"label": "BARBAZ", "pattern": "bar baz"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def person_org_patterns():
|
||||
return [
|
||||
{"label": "PERSON", "pattern": "Dina"},
|
||||
{"label": "ORG", "pattern": "ACME"},
|
||||
{"label": "ORG", "pattern": "ACM"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def person_org_date_patterns(person_org_patterns):
|
||||
return person_org_patterns + [{"label": "DATE", "pattern": "June 14th"}]
|
||||
|
||||
|
||||
def test_span_ruler_add_empty(patterns):
|
||||
"""Test that patterns don't get added excessively."""
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler", config={"validate": True})
|
||||
ruler.add_patterns(patterns)
|
||||
pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values())
|
||||
assert pattern_count > 0
|
||||
ruler.add_patterns([])
|
||||
after_count = sum(len(mm) for mm in ruler.matcher._patterns.values())
|
||||
assert after_count == pattern_count
|
||||
|
||||
|
||||
def test_span_ruler_init(patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
assert len(ruler) == len(patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
assert "HELLO" in ruler
|
||||
assert "BYE" in ruler
|
||||
doc = nlp("hello world bye bye")
|
||||
assert len(doc.spans["ruler"]) == 2
|
||||
assert doc.spans["ruler"][0].label_ == "HELLO"
|
||||
assert doc.spans["ruler"][0].id_ == "hello1"
|
||||
assert doc.spans["ruler"][1].label_ == "BYE"
|
||||
assert doc.spans["ruler"][1].id_ == ""
|
||||
|
||||
|
||||
def test_span_ruler_no_patterns_warns():
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
assert len(ruler) == 0
|
||||
assert len(ruler.labels) == 0
|
||||
assert nlp.pipe_names == ["span_ruler"]
|
||||
with pytest.warns(UserWarning):
|
||||
doc = nlp("hello world bye bye")
|
||||
assert len(doc.spans["ruler"]) == 0
|
||||
|
||||
|
||||
def test_span_ruler_init_patterns(patterns):
|
||||
# initialize with patterns
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
assert len(ruler.labels) == 0
|
||||
ruler.initialize(lambda: [], patterns=patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
doc = nlp("hello world bye bye")
|
||||
assert doc.spans["ruler"][0].label_ == "HELLO"
|
||||
assert doc.spans["ruler"][1].label_ == "BYE"
|
||||
nlp.remove_pipe("span_ruler")
|
||||
# initialize with patterns from misc registry
|
||||
nlp.config["initialize"]["components"]["span_ruler"] = {
|
||||
"patterns": {"@misc": "span_ruler_patterns"}
|
||||
}
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
assert len(ruler.labels) == 0
|
||||
nlp.initialize()
|
||||
assert len(ruler.labels) == 4
|
||||
doc = nlp("hello world bye bye")
|
||||
assert doc.spans["ruler"][0].label_ == "HELLO"
|
||||
assert doc.spans["ruler"][1].label_ == "BYE"
|
||||
|
||||
|
||||
def test_span_ruler_init_clear(patterns):
|
||||
"""Test that initialization clears patterns."""
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
ruler.initialize(lambda: [])
|
||||
assert len(ruler.labels) == 0
|
||||
|
||||
|
||||
def test_span_ruler_clear(patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
doc = nlp("hello world")
|
||||
assert len(doc.spans["ruler"]) == 1
|
||||
ruler.clear()
|
||||
assert len(ruler.labels) == 0
|
||||
with pytest.warns(UserWarning):
|
||||
doc = nlp("hello world")
|
||||
assert len(doc.spans["ruler"]) == 0
|
||||
|
||||
|
||||
def test_span_ruler_existing(patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler", config={"overwrite": False})
|
||||
ruler.add_patterns(patterns)
|
||||
doc = nlp.make_doc("OH HELLO WORLD bye bye")
|
||||
doc.spans["ruler"] = [doc[0:2]]
|
||||
doc = nlp(doc)
|
||||
assert len(doc.spans["ruler"]) == 3
|
||||
assert doc.spans["ruler"][0] == doc[0:2]
|
||||
assert doc.spans["ruler"][1].label_ == "HELLO"
|
||||
assert doc.spans["ruler"][1].id_ == "hello2"
|
||||
assert doc.spans["ruler"][2].label_ == "BYE"
|
||||
assert doc.spans["ruler"][2].id_ == ""
|
||||
|
||||
|
||||
def test_span_ruler_existing_overwrite(patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler", config={"overwrite": True})
|
||||
ruler.add_patterns(patterns)
|
||||
doc = nlp.make_doc("OH HELLO WORLD bye bye")
|
||||
doc.spans["ruler"] = [doc[0:2]]
|
||||
doc = nlp(doc)
|
||||
assert len(doc.spans["ruler"]) == 2
|
||||
assert doc.spans["ruler"][0].label_ == "HELLO"
|
||||
assert doc.spans["ruler"][0].text == "HELLO"
|
||||
assert doc.spans["ruler"][1].label_ == "BYE"
|
||||
|
||||
|
||||
def test_span_ruler_serialize_bytes(patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
assert len(ruler) == len(patterns)
|
||||
assert len(ruler.labels) == 4
|
||||
ruler_bytes = ruler.to_bytes()
|
||||
new_nlp = spacy.blank("xx")
|
||||
new_ruler = new_nlp.add_pipe("span_ruler")
|
||||
assert len(new_ruler) == 0
|
||||
assert len(new_ruler.labels) == 0
|
||||
new_ruler = new_ruler.from_bytes(ruler_bytes)
|
||||
assert len(new_ruler) == len(patterns)
|
||||
assert len(new_ruler.labels) == 4
|
||||
assert len(new_ruler.patterns) == len(ruler.patterns)
|
||||
for pattern in ruler.patterns:
|
||||
assert pattern in new_ruler.patterns
|
||||
assert sorted(new_ruler.labels) == sorted(ruler.labels)
|
||||
|
||||
|
||||
def test_span_ruler_validate():
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
validated_ruler = nlp.add_pipe(
|
||||
"span_ruler", name="validated_span_ruler", config={"validate": True}
|
||||
)
|
||||
|
||||
valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]}
|
||||
invalid_pattern = {"label": "HELLO", "pattern": [{"ASDF": "HELLO"}]}
|
||||
|
||||
# invalid pattern raises error without validate
|
||||
with pytest.raises(ValueError):
|
||||
ruler.add_patterns([invalid_pattern])
|
||||
|
||||
# valid pattern is added without errors with validate
|
||||
validated_ruler.add_patterns([valid_pattern])
|
||||
|
||||
# invalid pattern raises error with validate
|
||||
with pytest.raises(MatchPatternError):
|
||||
validated_ruler.add_patterns([invalid_pattern])
|
||||
|
||||
|
||||
def test_span_ruler_properties(patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler", config={"overwrite": True})
|
||||
ruler.add_patterns(patterns)
|
||||
assert sorted(ruler.labels) == sorted(set([p["label"] for p in patterns]))
|
||||
|
||||
|
||||
def test_span_ruler_overlapping_spans(overlapping_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(overlapping_patterns)
|
||||
doc = ruler(nlp.make_doc("foo bar baz"))
|
||||
assert len(doc.spans["ruler"]) == 2
|
||||
assert doc.spans["ruler"][0].label_ == "FOOBAR"
|
||||
assert doc.spans["ruler"][1].label_ == "BARBAZ"
|
||||
|
||||
|
||||
def test_span_ruler_scorer(overlapping_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(overlapping_patterns)
|
||||
text = "foo bar baz"
|
||||
pred_doc = ruler(nlp.make_doc(text))
|
||||
assert len(pred_doc.spans["ruler"]) == 2
|
||||
assert pred_doc.spans["ruler"][0].label_ == "FOOBAR"
|
||||
assert pred_doc.spans["ruler"][1].label_ == "BARBAZ"
|
||||
|
||||
ref_doc = nlp.make_doc(text)
|
||||
ref_doc.spans["ruler"] = [Span(ref_doc, 0, 2, label="FOOBAR")]
|
||||
scores = nlp.evaluate([Example(pred_doc, ref_doc)])
|
||||
assert scores["spans_ruler_p"] == 0.5
|
||||
assert scores["spans_ruler_r"] == 1.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_process", [1, 2])
|
||||
def test_span_ruler_multiprocessing(n_process):
|
||||
if isinstance(get_current_ops, NumpyOps) or n_process < 2:
|
||||
texts = ["I enjoy eating Pizza Hut pizza."]
|
||||
|
||||
patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut"}]
|
||||
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
|
||||
for doc in nlp.pipe(texts, n_process=2):
|
||||
for ent in doc.spans["ruler"]:
|
||||
assert ent.label_ == "FASTFOOD"
|
||||
|
||||
|
||||
def test_span_ruler_serialize_dir(patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
with make_tempdir() as d:
|
||||
ruler.to_disk(d / "test_ruler")
|
||||
ruler.from_disk(d / "test_ruler") # read from an existing directory
|
||||
with pytest.raises(ValueError):
|
||||
ruler.from_disk(d / "non_existing_dir") # read from a bad directory
|
||||
|
||||
|
||||
def test_span_ruler_remove_basic(person_org_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(person_org_patterns)
|
||||
doc = ruler(nlp.make_doc("Dina went to school"))
|
||||
assert len(ruler.patterns) == 3
|
||||
assert len(doc.spans["ruler"]) == 1
|
||||
assert doc.spans["ruler"][0].label_ == "PERSON"
|
||||
assert doc.spans["ruler"][0].text == "Dina"
|
||||
ruler.remove("PERSON")
|
||||
doc = ruler(nlp.make_doc("Dina went to school"))
|
||||
assert len(doc.spans["ruler"]) == 0
|
||||
assert len(ruler.patterns) == 2
|
||||
|
||||
|
||||
def test_span_ruler_remove_nonexisting_pattern(person_org_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(person_org_patterns)
|
||||
assert len(ruler.patterns) == 3
|
||||
with pytest.raises(ValueError):
|
||||
ruler.remove("NE")
|
||||
with pytest.raises(ValueError):
|
||||
ruler.remove_by_id("NE")
|
||||
|
||||
|
||||
def test_span_ruler_remove_several_patterns(person_org_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(person_org_patterns)
|
||||
doc = ruler(nlp.make_doc("Dina founded the company ACME."))
|
||||
assert len(ruler.patterns) == 3
|
||||
assert len(doc.spans["ruler"]) == 2
|
||||
assert doc.spans["ruler"][0].label_ == "PERSON"
|
||||
assert doc.spans["ruler"][0].text == "Dina"
|
||||
assert doc.spans["ruler"][1].label_ == "ORG"
|
||||
assert doc.spans["ruler"][1].text == "ACME"
|
||||
ruler.remove("PERSON")
|
||||
doc = ruler(nlp.make_doc("Dina founded the company ACME"))
|
||||
assert len(ruler.patterns) == 2
|
||||
assert len(doc.spans["ruler"]) == 1
|
||||
assert doc.spans["ruler"][0].label_ == "ORG"
|
||||
assert doc.spans["ruler"][0].text == "ACME"
|
||||
ruler.remove("ORG")
|
||||
with pytest.warns(UserWarning):
|
||||
doc = ruler(nlp.make_doc("Dina founded the company ACME"))
|
||||
assert len(ruler.patterns) == 0
|
||||
assert len(doc.spans["ruler"]) == 0
|
||||
|
||||
|
||||
def test_span_ruler_remove_patterns_in_a_row(person_org_date_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(person_org_date_patterns)
|
||||
doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th"))
|
||||
assert len(doc.spans["ruler"]) == 3
|
||||
assert doc.spans["ruler"][0].label_ == "PERSON"
|
||||
assert doc.spans["ruler"][0].text == "Dina"
|
||||
assert doc.spans["ruler"][1].label_ == "ORG"
|
||||
assert doc.spans["ruler"][1].text == "ACME"
|
||||
assert doc.spans["ruler"][2].label_ == "DATE"
|
||||
assert doc.spans["ruler"][2].text == "June 14th"
|
||||
ruler.remove("ORG")
|
||||
ruler.remove("DATE")
|
||||
doc = ruler(nlp.make_doc("Dina went to school"))
|
||||
assert len(doc.spans["ruler"]) == 1
|
||||
|
||||
|
||||
def test_span_ruler_remove_all_patterns(person_org_date_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(person_org_date_patterns)
|
||||
assert len(ruler.patterns) == 4
|
||||
ruler.remove("PERSON")
|
||||
assert len(ruler.patterns) == 3
|
||||
ruler.remove("ORG")
|
||||
assert len(ruler.patterns) == 1
|
||||
ruler.remove("DATE")
|
||||
assert len(ruler.patterns) == 0
|
||||
with pytest.warns(UserWarning):
|
||||
doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th"))
|
||||
assert len(doc.spans["ruler"]) == 0
|
||||
|
||||
|
||||
def test_span_ruler_remove_and_add():
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
patterns1 = [{"label": "DATE1", "pattern": "last time"}]
|
||||
ruler.add_patterns(patterns1)
|
||||
doc = ruler(
|
||||
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||
)
|
||||
assert len(ruler.patterns) == 1
|
||||
assert len(doc.spans["ruler"]) == 1
|
||||
assert doc.spans["ruler"][0].label_ == "DATE1"
|
||||
assert doc.spans["ruler"][0].text == "last time"
|
||||
patterns2 = [{"label": "DATE2", "pattern": "this time"}]
|
||||
ruler.add_patterns(patterns2)
|
||||
doc = ruler(
|
||||
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||
)
|
||||
assert len(ruler.patterns) == 2
|
||||
assert len(doc.spans["ruler"]) == 2
|
||||
assert doc.spans["ruler"][0].label_ == "DATE1"
|
||||
assert doc.spans["ruler"][0].text == "last time"
|
||||
assert doc.spans["ruler"][1].label_ == "DATE2"
|
||||
assert doc.spans["ruler"][1].text == "this time"
|
||||
ruler.remove("DATE1")
|
||||
doc = ruler(
|
||||
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||
)
|
||||
assert len(ruler.patterns) == 1
|
||||
assert len(doc.spans["ruler"]) == 1
|
||||
assert doc.spans["ruler"][0].label_ == "DATE2"
|
||||
assert doc.spans["ruler"][0].text == "this time"
|
||||
ruler.add_patterns(patterns1)
|
||||
doc = ruler(
|
||||
nlp.make_doc("I saw him last time we met, this time he brought some flowers")
|
||||
)
|
||||
assert len(ruler.patterns) == 2
|
||||
assert len(doc.spans["ruler"]) == 2
|
||||
patterns3 = [{"label": "DATE3", "pattern": "another time"}]
|
||||
ruler.add_patterns(patterns3)
|
||||
doc = ruler(
|
||||
nlp.make_doc(
|
||||
"I saw him last time we met, this time he brought some flowers, another time some chocolate."
|
||||
)
|
||||
)
|
||||
assert len(ruler.patterns) == 3
|
||||
assert len(doc.spans["ruler"]) == 3
|
||||
ruler.remove("DATE3")
|
||||
doc = ruler(
|
||||
nlp.make_doc(
|
||||
"I saw him last time we met, this time he brought some flowers, another time some chocolate."
|
||||
)
|
||||
)
|
||||
assert len(ruler.patterns) == 2
|
||||
assert len(doc.spans["ruler"]) == 2
|
||||
|
||||
|
||||
def test_span_ruler_spans_filter(overlapping_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe(
|
||||
"span_ruler",
|
||||
config={"spans_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}},
|
||||
)
|
||||
ruler.add_patterns(overlapping_patterns)
|
||||
doc = ruler(nlp.make_doc("foo bar baz"))
|
||||
assert len(doc.spans["ruler"]) == 1
|
||||
assert doc.spans["ruler"][0].label_ == "FOOBAR"
|
||||
|
||||
|
||||
def test_span_ruler_ents_default_filter(overlapping_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe("span_ruler", config={"annotate_ents": True})
|
||||
ruler.add_patterns(overlapping_patterns)
|
||||
doc = ruler(nlp.make_doc("foo bar baz"))
|
||||
assert len(doc.ents) == 1
|
||||
assert doc.ents[0].label_ == "FOOBAR"
|
||||
|
||||
|
||||
def test_span_ruler_ents_overwrite_filter(overlapping_patterns):
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe(
|
||||
"span_ruler",
|
||||
config={
|
||||
"annotate_ents": True,
|
||||
"overwrite": False,
|
||||
"ents_filter": {"@misc": "spacy.prioritize_new_ents_filter.v1"},
|
||||
},
|
||||
)
|
||||
ruler.add_patterns(overlapping_patterns)
|
||||
# overlapping ents are clobbered, non-overlapping ents are preserved
|
||||
doc = nlp.make_doc("foo bar baz a b c")
|
||||
doc.ents = [Span(doc, 1, 3, label="BARBAZ"), Span(doc, 3, 6, label="ABC")]
|
||||
doc = ruler(doc)
|
||||
assert len(doc.ents) == 2
|
||||
assert doc.ents[0].label_ == "FOOBAR"
|
||||
assert doc.ents[1].label_ == "ABC"
|
||||
|
||||
|
||||
def test_span_ruler_ents_bad_filter(overlapping_patterns):
|
||||
@registry.misc("test_pass_through_filter")
|
||||
def make_pass_through_filter():
|
||||
def pass_through_filter(spans1, spans2):
|
||||
return spans1 + spans2
|
||||
|
||||
return pass_through_filter
|
||||
|
||||
nlp = spacy.blank("xx")
|
||||
ruler = nlp.add_pipe(
|
||||
"span_ruler",
|
||||
config={
|
||||
"annotate_ents": True,
|
||||
"ents_filter": {"@misc": "test_pass_through_filter"},
|
||||
},
|
||||
)
|
||||
ruler.add_patterns(overlapping_patterns)
|
||||
with pytest.raises(ValueError):
|
||||
ruler(nlp.make_doc("foo bar baz"))
|
161
spacy/tests/serialize/test_serialize_span_groups.py
Normal file
161
spacy/tests/serialize/test_serialize_span_groups.py
Normal file
|
@ -0,0 +1,161 @@
|
|||
import pytest
|
||||
|
||||
from spacy.tokens import Span, SpanGroup
|
||||
from spacy.tokens._dict_proxies import SpanGroups
|
||||
|
||||
|
||||
@pytest.mark.issue(10685)
|
||||
def test_issue10685(en_tokenizer):
|
||||
"""Test `SpanGroups` de/serialization"""
|
||||
# Start with a Doc with no SpanGroups
|
||||
doc = en_tokenizer("Will it blend?")
|
||||
|
||||
# Test empty `SpanGroups` de/serialization:
|
||||
assert len(doc.spans) == 0
|
||||
doc.spans.from_bytes(doc.spans.to_bytes())
|
||||
assert len(doc.spans) == 0
|
||||
|
||||
# Test non-empty `SpanGroups` de/serialization:
|
||||
doc.spans["test"] = SpanGroup(doc, name="test", spans=[doc[0:1]])
|
||||
doc.spans["test2"] = SpanGroup(doc, name="test", spans=[doc[1:2]])
|
||||
|
||||
def assert_spangroups():
|
||||
assert len(doc.spans) == 2
|
||||
assert doc.spans["test"].name == "test"
|
||||
assert doc.spans["test2"].name == "test"
|
||||
assert list(doc.spans["test"]) == [doc[0:1]]
|
||||
assert list(doc.spans["test2"]) == [doc[1:2]]
|
||||
|
||||
# Sanity check the currently-expected behavior
|
||||
assert_spangroups()
|
||||
|
||||
# Now test serialization/deserialization:
|
||||
doc.spans.from_bytes(doc.spans.to_bytes())
|
||||
|
||||
assert_spangroups()
|
||||
|
||||
|
||||
def test_span_groups_serialization_mismatches(en_tokenizer):
|
||||
"""Test the serialization of multiple mismatching `SpanGroups` keys and `SpanGroup.name`s"""
|
||||
doc = en_tokenizer("How now, brown cow?")
|
||||
# Some variety:
|
||||
# 1 SpanGroup where its name matches its key
|
||||
# 2 SpanGroups that have the same name--which is not a key
|
||||
# 2 SpanGroups that have the same name--which is a key
|
||||
# 1 SpanGroup that is a value for 2 different keys (where its name is a key)
|
||||
# 1 SpanGroup that is a value for 2 different keys (where its name is not a key)
|
||||
groups = doc.spans
|
||||
groups["key1"] = SpanGroup(doc, name="key1", spans=[doc[0:1], doc[1:2]])
|
||||
groups["key2"] = SpanGroup(doc, name="too", spans=[doc[3:4], doc[4:5]])
|
||||
groups["key3"] = SpanGroup(doc, name="too", spans=[doc[1:2], doc[0:1]])
|
||||
groups["key4"] = SpanGroup(doc, name="key4", spans=[doc[0:1]])
|
||||
groups["key5"] = SpanGroup(doc, name="key4", spans=[doc[0:1]])
|
||||
sg6 = SpanGroup(doc, name="key6", spans=[doc[0:1]])
|
||||
groups["key6"] = sg6
|
||||
groups["key7"] = sg6
|
||||
sg8 = SpanGroup(doc, name="also", spans=[doc[1:2]])
|
||||
groups["key8"] = sg8
|
||||
groups["key9"] = sg8
|
||||
|
||||
regroups = SpanGroups(doc).from_bytes(groups.to_bytes())
|
||||
|
||||
# Assert regroups == groups
|
||||
assert regroups.keys() == groups.keys()
|
||||
for key, regroup in regroups.items():
|
||||
# Assert regroup == groups[key]
|
||||
assert regroup.name == groups[key].name
|
||||
assert list(regroup) == list(groups[key])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"spans_bytes,doc_text,expected_spangroups,expected_warning",
|
||||
# The bytestrings below were generated from an earlier version of spaCy
|
||||
# that serialized `SpanGroups` as a list of SpanGroup bytes (via SpanGroups.to_bytes).
|
||||
# Comments preceding the bytestrings indicate from what Doc they were created.
|
||||
[
|
||||
# Empty SpanGroups:
|
||||
(b"\x90", "", {}, False),
|
||||
# doc = nlp("Will it blend?")
|
||||
# doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]])
|
||||
(
|
||||
b"\x91\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04",
|
||||
"Will it blend?",
|
||||
{"test": {"name": "test", "spans": [(0, 1)]}},
|
||||
False,
|
||||
),
|
||||
# doc = nlp("Will it blend?")
|
||||
# doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]])
|
||||
# doc.spans['test2'] = SpanGroup(doc, name='test', spans=[doc[1:2]])
|
||||
(
|
||||
b"\x92\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x05\x00\x00\x00\x07",
|
||||
"Will it blend?",
|
||||
# We expect only 1 SpanGroup to be in doc.spans in this example
|
||||
# because there are 2 `SpanGroup`s that have the same .name. See #10685.
|
||||
{"test": {"name": "test", "spans": [(1, 2)]}},
|
||||
True,
|
||||
),
|
||||
# doc = nlp('How now, brown cow?')
|
||||
# doc.spans['key1'] = SpanGroup(doc, name='key1', spans=[doc[0:1], doc[1:2]])
|
||||
# doc.spans['key2'] = SpanGroup(doc, name='too', spans=[doc[3:4], doc[4:5]])
|
||||
# doc.spans['key3'] = SpanGroup(doc, name='too', spans=[doc[1:2], doc[0:1]])
|
||||
# doc.spans['key4'] = SpanGroup(doc, name='key4', spans=[doc[0:1]])
|
||||
# doc.spans['key5'] = SpanGroup(doc, name='key4', spans=[doc[0:1]])
|
||||
(
|
||||
b"\x95\xc4m\x83\xa4name\xa4key1\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00\t\x00\x00\x00\x0e\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x12\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03",
|
||||
"How now, brown cow?",
|
||||
{
|
||||
"key1": {"name": "key1", "spans": [(0, 1), (1, 2)]},
|
||||
"too": {"name": "too", "spans": [(1, 2), (0, 1)]},
|
||||
"key4": {"name": "key4", "spans": [(0, 1)]},
|
||||
},
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_deserialize_span_groups_compat(
|
||||
en_tokenizer, spans_bytes, doc_text, expected_spangroups, expected_warning
|
||||
):
|
||||
"""Test backwards-compatibility of `SpanGroups` deserialization.
|
||||
This uses serializations (bytes) from a prior version of spaCy (before 3.3.1).
|
||||
|
||||
spans_bytes (bytes): Serialized `SpanGroups` object.
|
||||
doc_text (str): Doc text.
|
||||
expected_spangroups (dict):
|
||||
Dict mapping every expected (after deserialization) `SpanGroups` key
|
||||
to a SpanGroup's "args", where a SpanGroup's args are given as a dict:
|
||||
{"name": span_group.name,
|
||||
"spans": [(span0.start, span0.end), ...]}
|
||||
expected_warning (bool): Whether a warning is to be expected from .from_bytes()
|
||||
--i.e. if more than 1 SpanGroup has the same .name within the `SpanGroups`.
|
||||
"""
|
||||
doc = en_tokenizer(doc_text)
|
||||
|
||||
if expected_warning:
|
||||
with pytest.warns(UserWarning):
|
||||
doc.spans.from_bytes(spans_bytes)
|
||||
else:
|
||||
# TODO: explicitly check for lack of a warning
|
||||
doc.spans.from_bytes(spans_bytes)
|
||||
|
||||
assert doc.spans.keys() == expected_spangroups.keys()
|
||||
for name, spangroup_args in expected_spangroups.items():
|
||||
assert doc.spans[name].name == spangroup_args["name"]
|
||||
spans = [Span(doc, start, end) for start, end in spangroup_args["spans"]]
|
||||
assert list(doc.spans[name]) == spans
|
||||
|
||||
|
||||
def test_span_groups_serialization(en_tokenizer):
|
||||
doc = en_tokenizer("0 1 2 3 4 5 6")
|
||||
span_groups = SpanGroups(doc)
|
||||
spans = [doc[0:2], doc[1:3]]
|
||||
sg1 = SpanGroup(doc, spans=spans)
|
||||
span_groups["key1"] = sg1
|
||||
span_groups["key2"] = sg1
|
||||
span_groups["key3"] = []
|
||||
reloaded_span_groups = SpanGroups(doc).from_bytes(span_groups.to_bytes())
|
||||
assert span_groups.keys() == reloaded_span_groups.keys()
|
||||
for key, value in span_groups.items():
|
||||
assert all(
|
||||
span == reloaded_span
|
||||
for span, reloaded_span in zip(span_groups[key], reloaded_span_groups[key])
|
||||
)
|
|
@ -1,4 +1,7 @@
|
|||
import os
|
||||
import math
|
||||
from random import sample
|
||||
from typing import Counter
|
||||
|
||||
import pytest
|
||||
import srsly
|
||||
|
@ -14,6 +17,10 @@ from spacy.cli._util import substitute_project_variables
|
|||
from spacy.cli._util import validate_project_commands
|
||||
from spacy.cli.debug_data import _compile_gold, _get_labels_from_model
|
||||
from spacy.cli.debug_data import _get_labels_from_spancat
|
||||
from spacy.cli.debug_data import _get_distribution, _get_kl_divergence
|
||||
from spacy.cli.debug_data import _get_span_characteristics
|
||||
from spacy.cli.debug_data import _print_span_characteristics
|
||||
from spacy.cli.debug_data import _get_spans_length_freq_dist
|
||||
from spacy.cli.download import get_compatibility, get_version
|
||||
from spacy.cli.init_config import RECOMMENDATIONS, init_config, fill_config
|
||||
from spacy.cli.package import get_third_party_dependencies
|
||||
|
@ -24,6 +31,7 @@ from spacy.lang.nl import Dutch
|
|||
from spacy.language import Language
|
||||
from spacy.schemas import ProjectConfigSchema, RecommendationSchema, validate
|
||||
from spacy.tokens import Doc
|
||||
from spacy.tokens.span import Span
|
||||
from spacy.training import Example, docs_to_json, offsets_to_biluo_tags
|
||||
from spacy.training.converters import conll_ner_to_docs, conllu_to_docs
|
||||
from spacy.training.converters import iob_to_docs
|
||||
|
@ -740,3 +748,110 @@ def test_debug_data_compile_gold():
|
|||
eg = Example(pred, ref)
|
||||
data = _compile_gold([eg], ["ner"], nlp, True)
|
||||
assert data["boundary_cross_ents"] == 1
|
||||
|
||||
|
||||
def test_debug_data_compile_gold_for_spans():
|
||||
nlp = English()
|
||||
spans_key = "sc"
|
||||
|
||||
pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||
pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")]
|
||||
ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||
ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")]
|
||||
eg = Example(pred, ref)
|
||||
|
||||
data = _compile_gold([eg], ["spancat"], nlp, True)
|
||||
|
||||
assert data["spancat"][spans_key] == Counter({"ORG": 1, "GPE": 1})
|
||||
assert data["spans_length"][spans_key] == {"ORG": [3], "GPE": [1]}
|
||||
assert data["spans_per_type"][spans_key] == {
|
||||
"ORG": [Span(ref, 3, 6, "ORG")],
|
||||
"GPE": [Span(ref, 5, 6, "GPE")],
|
||||
}
|
||||
assert data["sb_per_type"][spans_key] == {
|
||||
"ORG": {"start": [ref[2:3]], "end": [ref[6:7]]},
|
||||
"GPE": {"start": [ref[4:5]], "end": [ref[6:7]]},
|
||||
}
|
||||
|
||||
|
||||
def test_frequency_distribution_is_correct():
|
||||
nlp = English()
|
||||
docs = [
|
||||
Doc(nlp.vocab, words=["Bank", "of", "China"]),
|
||||
Doc(nlp.vocab, words=["China"]),
|
||||
]
|
||||
|
||||
expected = Counter({"china": 0.5, "bank": 0.25, "of": 0.25})
|
||||
freq_distribution = _get_distribution(docs, normalize=True)
|
||||
assert freq_distribution == expected
|
||||
|
||||
|
||||
def test_kl_divergence_computation_is_correct():
|
||||
p = Counter({"a": 0.5, "b": 0.25})
|
||||
q = Counter({"a": 0.25, "b": 0.50, "c": 0.15, "d": 0.10})
|
||||
result = _get_kl_divergence(p, q)
|
||||
expected = 0.1733
|
||||
assert math.isclose(result, expected, rel_tol=1e-3)
|
||||
|
||||
|
||||
def test_get_span_characteristics_return_value():
|
||||
nlp = English()
|
||||
spans_key = "sc"
|
||||
|
||||
pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||
pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")]
|
||||
ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||
ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")]
|
||||
eg = Example(pred, ref)
|
||||
|
||||
examples = [eg]
|
||||
data = _compile_gold(examples, ["spancat"], nlp, True)
|
||||
span_characteristics = _get_span_characteristics(
|
||||
examples=examples, compiled_gold=data, spans_key=spans_key
|
||||
)
|
||||
|
||||
assert {"sd", "bd", "lengths"}.issubset(span_characteristics.keys())
|
||||
assert span_characteristics["min_length"] == 1
|
||||
assert span_characteristics["max_length"] == 3
|
||||
|
||||
|
||||
def test_ensure_print_span_characteristics_wont_fail():
|
||||
"""Test if interface between two methods aren't destroyed if refactored"""
|
||||
nlp = English()
|
||||
spans_key = "sc"
|
||||
|
||||
pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||
pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")]
|
||||
ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
|
||||
ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")]
|
||||
eg = Example(pred, ref)
|
||||
|
||||
examples = [eg]
|
||||
data = _compile_gold(examples, ["spancat"], nlp, True)
|
||||
span_characteristics = _get_span_characteristics(
|
||||
examples=examples, compiled_gold=data, spans_key=spans_key
|
||||
)
|
||||
_print_span_characteristics(span_characteristics)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("threshold", [70, 80, 85, 90, 95])
|
||||
def test_span_length_freq_dist_threshold_must_be_correct(threshold):
|
||||
sample_span_lengths = {
|
||||
"span_type_1": [1, 4, 4, 5],
|
||||
"span_type_2": [5, 3, 3, 2],
|
||||
"span_type_3": [3, 1, 3, 3],
|
||||
}
|
||||
span_freqs = _get_spans_length_freq_dist(sample_span_lengths, threshold)
|
||||
assert sum(span_freqs.values()) >= threshold
|
||||
|
||||
|
||||
def test_span_length_freq_dist_output_must_be_correct():
|
||||
sample_span_lengths = {
|
||||
"span_type_1": [1, 4, 4, 5],
|
||||
"span_type_2": [5, 3, 3, 2],
|
||||
"span_type_3": [3, 1, 3, 3],
|
||||
}
|
||||
threshold = 90
|
||||
span_freqs = _get_spans_length_freq_dist(sample_span_lengths, threshold)
|
||||
assert sum(span_freqs.values()) >= threshold
|
||||
assert list(span_freqs.keys()) == [3, 1, 4, 5, 2]
|
||||
|
|
|
@ -671,13 +671,13 @@ def test_gold_ner_missing_tags(en_tokenizer):
|
|||
|
||||
def test_projectivize(en_tokenizer):
|
||||
doc = en_tokenizer("He pretty quickly walks away")
|
||||
heads = [3, 2, 3, 0, 2]
|
||||
heads = [3, 2, 3, 3, 2]
|
||||
deps = ["dep"] * len(heads)
|
||||
example = Example.from_dict(doc, {"heads": heads, "deps": deps})
|
||||
proj_heads, proj_labels = example.get_aligned_parse(projectivize=True)
|
||||
nonproj_heads, nonproj_labels = example.get_aligned_parse(projectivize=False)
|
||||
assert proj_heads == [3, 2, 3, 0, 3]
|
||||
assert nonproj_heads == [3, 2, 3, 0, 2]
|
||||
assert proj_heads == [3, 2, 3, 3, 3]
|
||||
assert nonproj_heads == [3, 2, 3, 3, 2]
|
||||
|
||||
|
||||
def test_iob_to_biluo():
|
||||
|
|
|
@ -1,10 +1,11 @@
|
|||
from typing import Iterable, Tuple, Union, Optional, TYPE_CHECKING
|
||||
from typing import Dict, Iterable, List, Tuple, Union, Optional, TYPE_CHECKING
|
||||
import warnings
|
||||
import weakref
|
||||
from collections import UserDict
|
||||
import srsly
|
||||
|
||||
from .span_group import SpanGroup
|
||||
from ..errors import Errors
|
||||
from ..errors import Errors, Warnings
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
@ -16,7 +17,7 @@ if TYPE_CHECKING:
|
|||
# Why inherit from UserDict instead of dict here?
|
||||
# Well, the 'dict' class doesn't necessarily delegate everything nicely,
|
||||
# for performance reasons. The UserDict is slower but better behaved.
|
||||
# See https://treyhunner.com/2019/04/why-you-shouldnt-inherit-from-list-and-dict-in-python/0ww
|
||||
# See https://treyhunner.com/2019/04/why-you-shouldnt-inherit-from-list-and-dict-in-python/
|
||||
class SpanGroups(UserDict):
|
||||
"""A dict-like proxy held by the Doc, to control access to span groups."""
|
||||
|
||||
|
@ -53,20 +54,52 @@ class SpanGroups(UserDict):
|
|||
return super().setdefault(key, default=default)
|
||||
|
||||
def to_bytes(self) -> bytes:
|
||||
# We don't need to serialize this as a dict, because the groups
|
||||
# know their names.
|
||||
# We serialize this as a dict in order to track the key(s) a SpanGroup
|
||||
# is a value of (in a backward- and forward-compatible way), since
|
||||
# a SpanGroup can have a key that doesn't match its `.name` (See #10685)
|
||||
if len(self) == 0:
|
||||
return self._EMPTY_BYTES
|
||||
msg = [value.to_bytes() for value in self.values()]
|
||||
msg: Dict[bytes, List[str]] = {}
|
||||
for key, value in self.items():
|
||||
msg.setdefault(value.to_bytes(), []).append(key)
|
||||
return srsly.msgpack_dumps(msg)
|
||||
|
||||
def from_bytes(self, bytes_data: bytes) -> "SpanGroups":
|
||||
msg = [] if bytes_data == self._EMPTY_BYTES else srsly.msgpack_loads(bytes_data)
|
||||
# backwards-compatibility: bytes_data may be one of:
|
||||
# b'', a serialized empty list, a serialized list of SpanGroup bytes
|
||||
# or a serialized dict of SpanGroup bytes -> keys
|
||||
msg = (
|
||||
[]
|
||||
if not bytes_data or bytes_data == self._EMPTY_BYTES
|
||||
else srsly.msgpack_loads(bytes_data)
|
||||
)
|
||||
self.clear()
|
||||
doc = self._ensure_doc()
|
||||
for value_bytes in msg:
|
||||
group = SpanGroup(doc).from_bytes(value_bytes)
|
||||
self[group.name] = group
|
||||
if isinstance(msg, list):
|
||||
# This is either the 1st version of `SpanGroups` serialization
|
||||
# or there were no SpanGroups serialized
|
||||
for value_bytes in msg:
|
||||
group = SpanGroup(doc).from_bytes(value_bytes)
|
||||
if group.name in self:
|
||||
# Display a warning if `msg` contains `SpanGroup`s
|
||||
# that have the same .name (attribute).
|
||||
# Because, for `SpanGroups` serialized as lists,
|
||||
# only 1 SpanGroup per .name is loaded. (See #10685)
|
||||
warnings.warn(
|
||||
Warnings.W120.format(
|
||||
group_name=group.name, group_values=self[group.name]
|
||||
)
|
||||
)
|
||||
self[group.name] = group
|
||||
else:
|
||||
for value_bytes, keys in msg.items():
|
||||
group = SpanGroup(doc).from_bytes(value_bytes)
|
||||
# Deserialize `SpanGroup`s as copies because it's possible for two
|
||||
# different `SpanGroup`s (pre-serialization) to have the same bytes
|
||||
# (since they can have the same `.name`).
|
||||
self[keys[0]] = group
|
||||
for key in keys[1:]:
|
||||
self[key] = group.copy()
|
||||
return self
|
||||
|
||||
def _ensure_doc(self) -> "Doc":
|
||||
|
|
|
@ -170,6 +170,9 @@ class Doc:
|
|||
def extend_tensor(self, tensor: Floats2d) -> None: ...
|
||||
def retokenize(self) -> Retokenizer: ...
|
||||
def to_json(self, underscore: Optional[List[str]] = ...) -> Dict[str, Any]: ...
|
||||
def from_json(
|
||||
self, doc_json: Dict[str, Any] = ..., validate: bool = False
|
||||
) -> Doc: ...
|
||||
def to_utf8_array(self, nr_char: int = ...) -> Ints2d: ...
|
||||
@staticmethod
|
||||
def _get_array_attrs() -> Tuple[Any]: ...
|
||||
|
|
|
@ -1,4 +1,6 @@
|
|||
# cython: infer_types=True, bounds_check=False, profile=True
|
||||
from typing import Set
|
||||
|
||||
cimport cython
|
||||
cimport numpy as np
|
||||
from libc.string cimport memcpy
|
||||
|
@ -31,10 +33,11 @@ from ..errors import Errors, Warnings
|
|||
from ..morphology import Morphology
|
||||
from .. import util
|
||||
from .. import parts_of_speech
|
||||
from .. import schemas
|
||||
from .underscore import Underscore, get_ext_args
|
||||
from ._retokenize import Retokenizer
|
||||
from ._serialize import ALL_ATTRS as DOCBIN_ALL_ATTRS
|
||||
|
||||
from ..util import get_words_and_spaces
|
||||
|
||||
DEF PADDING = 5
|
||||
|
||||
|
@ -414,6 +417,7 @@ cdef class Doc:
|
|||
"""
|
||||
|
||||
# empty docs are always annotated
|
||||
input_attr = attr
|
||||
if self.length == 0:
|
||||
return True
|
||||
cdef int i
|
||||
|
@ -423,6 +427,10 @@ cdef class Doc:
|
|||
elif attr == "IS_SENT_END" or attr == self.vocab.strings["IS_SENT_END"]:
|
||||
attr = SENT_START
|
||||
attr = intify_attr(attr)
|
||||
if attr is None:
|
||||
raise ValueError(
|
||||
Errors.E1037.format(attr=input_attr)
|
||||
)
|
||||
# adjust attributes
|
||||
if attr == HEAD:
|
||||
# HEAD does not have an unset state, so rely on DEP
|
||||
|
@ -511,7 +519,7 @@ cdef class Doc:
|
|||
def doc(self):
|
||||
return self
|
||||
|
||||
def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, alignment_mode="strict"):
|
||||
def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, alignment_mode="strict", span_id=0):
|
||||
"""Create a `Span` object from the slice
|
||||
`doc.text[start_idx : end_idx]`. Returns None if no valid `Span` can be
|
||||
created.
|
||||
|
@ -570,7 +578,7 @@ cdef class Doc:
|
|||
start += 1
|
||||
# Currently we have the token index, we want the range-end index
|
||||
end += 1
|
||||
cdef Span span = Span(self, start, end, label=label, kb_id=kb_id, vector=vector)
|
||||
cdef Span span = Span(self, start, end, label=label, kb_id=kb_id, span_id=span_id, vector=vector)
|
||||
return span
|
||||
|
||||
def similarity(self, other):
|
||||
|
@ -708,6 +716,7 @@ cdef class Doc:
|
|||
cdef int start = -1
|
||||
cdef attr_t label = 0
|
||||
cdef attr_t kb_id = 0
|
||||
cdef attr_t ent_id = 0
|
||||
output = []
|
||||
for i in range(self.length):
|
||||
token = &self.c[i]
|
||||
|
@ -718,18 +727,20 @@ cdef class Doc:
|
|||
elif token.ent_iob == 2 or token.ent_iob == 0 or \
|
||||
(token.ent_iob == 3 and token.ent_type == 0):
|
||||
if start != -1:
|
||||
output.append(Span(self, start, i, label=label, kb_id=kb_id))
|
||||
output.append(Span(self, start, i, label=label, kb_id=kb_id, span_id=ent_id))
|
||||
start = -1
|
||||
label = 0
|
||||
kb_id = 0
|
||||
ent_id = 0
|
||||
elif token.ent_iob == 3:
|
||||
if start != -1:
|
||||
output.append(Span(self, start, i, label=label, kb_id=kb_id))
|
||||
output.append(Span(self, start, i, label=label, kb_id=kb_id, span_id=ent_id))
|
||||
start = i
|
||||
label = token.ent_type
|
||||
kb_id = token.ent_kb_id
|
||||
ent_id = token.ent_id
|
||||
if start != -1:
|
||||
output.append(Span(self, start, self.length, label=label, kb_id=kb_id))
|
||||
output.append(Span(self, start, self.length, label=label, kb_id=kb_id, span_id=ent_id))
|
||||
# remove empty-label spans
|
||||
output = [o for o in output if o.label_ != ""]
|
||||
return tuple(output)
|
||||
|
@ -738,14 +749,14 @@ cdef class Doc:
|
|||
# TODO:
|
||||
# 1. Test basic data-driven ORTH gazetteer
|
||||
# 2. Test more nuanced date and currency regex
|
||||
cdef attr_t entity_type, kb_id
|
||||
cdef attr_t entity_type, kb_id, ent_id
|
||||
cdef int ent_start, ent_end
|
||||
ent_spans = []
|
||||
for ent_info in ents:
|
||||
entity_type_, kb_id, ent_start, ent_end = get_entity_info(ent_info)
|
||||
entity_type_, kb_id, ent_start, ent_end, ent_id = get_entity_info(ent_info)
|
||||
if isinstance(entity_type_, str):
|
||||
self.vocab.strings.add(entity_type_)
|
||||
span = Span(self, ent_start, ent_end, label=entity_type_, kb_id=kb_id)
|
||||
span = Span(self, ent_start, ent_end, label=entity_type_, kb_id=kb_id, span_id=ent_id)
|
||||
ent_spans.append(span)
|
||||
self.set_ents(ent_spans, default=SetEntsDefault.outside)
|
||||
|
||||
|
@ -796,6 +807,9 @@ cdef class Doc:
|
|||
self.c[i].ent_iob = 1
|
||||
self.c[i].ent_type = span.label
|
||||
self.c[i].ent_kb_id = span.kb_id
|
||||
# for backwards compatibility in v3, only set ent_id from
|
||||
# span.id if it's set, otherwise don't override
|
||||
self.c[i].ent_id = span.id if span.id else self.c[i].ent_id
|
||||
for span in blocked:
|
||||
for i in range(span.start, span.end):
|
||||
self.c[i].ent_iob = 3
|
||||
|
@ -1175,6 +1189,7 @@ cdef class Doc:
|
|||
span.end_char + char_offset,
|
||||
span.label,
|
||||
span.kb_id,
|
||||
span.id,
|
||||
span.text, # included as a check
|
||||
))
|
||||
char_offset += len(doc.text)
|
||||
|
@ -1210,8 +1225,9 @@ cdef class Doc:
|
|||
span_tuple[1],
|
||||
label=span_tuple[2],
|
||||
kb_id=span_tuple[3],
|
||||
span_id=span_tuple[4],
|
||||
)
|
||||
text = span_tuple[4]
|
||||
text = span_tuple[5]
|
||||
if span is not None and span.text == text:
|
||||
concat_doc.spans[key].append(span)
|
||||
else:
|
||||
|
@ -1462,6 +1478,138 @@ cdef class Doc:
|
|||
remove_label_if_necessary(attributes[i])
|
||||
retokenizer.merge(span, attributes[i])
|
||||
|
||||
def from_json(self, doc_json, *, validate=False):
|
||||
"""Convert a JSON document generated by Doc.to_json() to a Doc.
|
||||
|
||||
doc_json (Dict): JSON representation of doc object to load.
|
||||
validate (bool): Whether to validate `doc_json` against the expected schema.
|
||||
Defaults to False.
|
||||
RETURNS (Doc): A doc instance corresponding to the specified JSON representation.
|
||||
"""
|
||||
|
||||
if validate:
|
||||
schema_validation_message = schemas.validate(schemas.DocJSONSchema, doc_json)
|
||||
if schema_validation_message:
|
||||
raise ValueError(Errors.E1038.format(message=schema_validation_message))
|
||||
|
||||
### Token-level properties ###
|
||||
|
||||
words = []
|
||||
token_attrs_ids = (POS, HEAD, DEP, LEMMA, TAG, MORPH)
|
||||
# Map annotation type IDs to their string equivalents.
|
||||
token_attrs = {t: self.vocab.strings[t].lower() for t in token_attrs_ids}
|
||||
token_annotations = {}
|
||||
|
||||
# Gather token-level properties.
|
||||
for token_json in doc_json["tokens"]:
|
||||
words.append(doc_json["text"][token_json["start"]:token_json["end"]])
|
||||
for attr, attr_json in token_attrs.items():
|
||||
if attr_json in token_json:
|
||||
if token_json["id"] == 0 and attr not in token_annotations:
|
||||
token_annotations[attr] = []
|
||||
elif attr not in token_annotations:
|
||||
raise ValueError(Errors.E1040.format(partial_attrs=attr))
|
||||
token_annotations[attr].append(token_json[attr_json])
|
||||
|
||||
# Initialize doc instance.
|
||||
start = 0
|
||||
cdef const LexemeC* lex
|
||||
cdef bint has_space
|
||||
reconstructed_words, spaces = get_words_and_spaces(words, doc_json["text"])
|
||||
assert words == reconstructed_words
|
||||
|
||||
for word, has_space in zip(words, spaces):
|
||||
lex = self.vocab.get(self.mem, word)
|
||||
self.push_back(lex, has_space)
|
||||
|
||||
# Set remaining token-level attributes via Doc.from_array().
|
||||
if HEAD in token_annotations:
|
||||
token_annotations[HEAD] = [
|
||||
head - i for i, head in enumerate(token_annotations[HEAD])
|
||||
]
|
||||
|
||||
if DEP in token_annotations and HEAD not in token_annotations:
|
||||
token_annotations[HEAD] = [0] * len(token_annotations[DEP])
|
||||
if HEAD in token_annotations and DEP not in token_annotations:
|
||||
raise ValueError(Errors.E1017)
|
||||
if POS in token_annotations:
|
||||
for pp in set(token_annotations[POS]):
|
||||
if pp not in parts_of_speech.IDS:
|
||||
raise ValueError(Errors.E1021.format(pp=pp))
|
||||
|
||||
# Collect token attributes, assert all tokens have exactly the same set of attributes.
|
||||
attrs = []
|
||||
partial_attrs: Set[str] = set()
|
||||
for attr in token_attrs.keys():
|
||||
if attr in token_annotations:
|
||||
if len(token_annotations[attr]) != len(words):
|
||||
partial_attrs.add(token_attrs[attr])
|
||||
attrs.append(attr)
|
||||
if len(partial_attrs):
|
||||
raise ValueError(Errors.E1040.format(partial_attrs=partial_attrs))
|
||||
|
||||
# If there are any other annotations, set them.
|
||||
if attrs:
|
||||
array = self.to_array(attrs)
|
||||
if array.ndim == 1:
|
||||
array = numpy.reshape(array, (array.size, 1))
|
||||
j = 0
|
||||
|
||||
for j, (attr, annot) in enumerate(token_annotations.items()):
|
||||
if attr is HEAD:
|
||||
for i in range(len(words)):
|
||||
array[i, j] = annot[i]
|
||||
elif attr is MORPH:
|
||||
for i in range(len(words)):
|
||||
array[i, j] = self.vocab.morphology.add(annot[i])
|
||||
else:
|
||||
for i in range(len(words)):
|
||||
array[i, j] = self.vocab.strings.add(annot[i])
|
||||
self.from_array(attrs, array)
|
||||
|
||||
### Span/document properties ###
|
||||
|
||||
# Complement other document-level properties (cats, spans, ents).
|
||||
self.cats = doc_json.get("cats", {})
|
||||
|
||||
# Set sentence boundaries, if dependency parser not available but sentences are specified in JSON.
|
||||
if not self.has_annotation("DEP"):
|
||||
for sent in doc_json.get("sents", {}):
|
||||
char_span = self.char_span(sent["start"], sent["end"])
|
||||
if char_span is None:
|
||||
raise ValueError(Errors.E1039.format(obj="sentence", start=sent["start"], end=sent["end"]))
|
||||
char_span[0].is_sent_start = True
|
||||
for token in char_span[1:]:
|
||||
token.is_sent_start = False
|
||||
|
||||
|
||||
for span_group in doc_json.get("spans", {}):
|
||||
spans = []
|
||||
for span in doc_json["spans"][span_group]:
|
||||
char_span = self.char_span(span["start"], span["end"], span["label"], span["kb_id"])
|
||||
if char_span is None:
|
||||
raise ValueError(Errors.E1039.format(obj="span", start=span["start"], end=span["end"]))
|
||||
spans.append(char_span)
|
||||
self.spans[span_group] = spans
|
||||
|
||||
if "ents" in doc_json:
|
||||
ents = []
|
||||
for ent in doc_json["ents"]:
|
||||
char_span = self.char_span(ent["start"], ent["end"], ent["label"])
|
||||
if char_span is None:
|
||||
raise ValueError(Errors.E1039.format(obj="entity"), start=ent["start"], end=ent["end"])
|
||||
ents.append(char_span)
|
||||
self.ents = ents
|
||||
|
||||
# Add custom attributes. Note that only Doc extensions are currently considered, Token and Span extensions are
|
||||
# not yet supported.
|
||||
for attr in doc_json.get("_", {}):
|
||||
if not Doc.has_extension(attr):
|
||||
Doc.set_extension(attr)
|
||||
self._.set(attr, doc_json["_"][attr])
|
||||
|
||||
return self
|
||||
|
||||
def to_json(self, underscore=None):
|
||||
"""Convert a Doc to JSON.
|
||||
|
||||
|
@ -1472,12 +1620,10 @@ cdef class Doc:
|
|||
"""
|
||||
data = {"text": self.text}
|
||||
if self.has_annotation("ENT_IOB"):
|
||||
data["ents"] = [{"start": ent.start_char, "end": ent.end_char,
|
||||
"label": ent.label_} for ent in self.ents]
|
||||
data["ents"] = [{"start": ent.start_char, "end": ent.end_char, "label": ent.label_} for ent in self.ents]
|
||||
if self.has_annotation("SENT_START"):
|
||||
sents = list(self.sents)
|
||||
data["sents"] = [{"start": sent.start_char, "end": sent.end_char}
|
||||
for sent in sents]
|
||||
data["sents"] = [{"start": sent.start_char, "end": sent.end_char} for sent in sents]
|
||||
if self.cats:
|
||||
data["cats"] = self.cats
|
||||
data["tokens"] = []
|
||||
|
@ -1503,7 +1649,9 @@ cdef class Doc:
|
|||
for span_group in self.spans:
|
||||
data["spans"][span_group] = []
|
||||
for span in self.spans[span_group]:
|
||||
span_data = {"start": span.start_char, "end": span.end_char, "label": span.label_, "kb_id": span.kb_id_}
|
||||
span_data = {
|
||||
"start": span.start_char, "end": span.end_char, "label": span.label_, "kb_id": span.kb_id_
|
||||
}
|
||||
data["spans"][span_group].append(span_data)
|
||||
|
||||
if underscore:
|
||||
|
@ -1732,18 +1880,17 @@ cdef int [:,:] _get_lca_matrix(Doc doc, int start, int end):
|
|||
def pickle_doc(doc):
|
||||
bytes_data = doc.to_bytes(exclude=["vocab", "user_data", "user_hooks"])
|
||||
hooks_and_data = (doc.user_data, doc.user_hooks, doc.user_span_hooks,
|
||||
doc.user_token_hooks, doc._context)
|
||||
doc.user_token_hooks)
|
||||
return (unpickle_doc, (doc.vocab, srsly.pickle_dumps(hooks_and_data), bytes_data))
|
||||
|
||||
|
||||
def unpickle_doc(vocab, hooks_and_data, bytes_data):
|
||||
user_data, doc_hooks, span_hooks, token_hooks, _context = srsly.pickle_loads(hooks_and_data)
|
||||
user_data, doc_hooks, span_hooks, token_hooks = srsly.pickle_loads(hooks_and_data)
|
||||
|
||||
doc = Doc(vocab, user_data=user_data).from_bytes(bytes_data, exclude=["user_data"])
|
||||
doc.user_hooks.update(doc_hooks)
|
||||
doc.user_span_hooks.update(span_hooks)
|
||||
doc.user_token_hooks.update(token_hooks)
|
||||
doc._context = _context
|
||||
return doc
|
||||
|
||||
|
||||
|
@ -1767,16 +1914,18 @@ def fix_attributes(doc, attributes):
|
|||
|
||||
|
||||
def get_entity_info(ent_info):
|
||||
ent_kb_id = 0
|
||||
ent_id = 0
|
||||
if isinstance(ent_info, Span):
|
||||
ent_type = ent_info.label
|
||||
ent_kb_id = ent_info.kb_id
|
||||
start = ent_info.start
|
||||
end = ent_info.end
|
||||
ent_id = ent_info.id
|
||||
elif len(ent_info) == 3:
|
||||
ent_type, start, end = ent_info
|
||||
ent_kb_id = 0
|
||||
elif len(ent_info) == 4:
|
||||
ent_type, ent_kb_id, start, end = ent_info
|
||||
else:
|
||||
ent_id, ent_kb_id, ent_type, start, end = ent_info
|
||||
return ent_type, ent_kb_id, start, end
|
||||
return ent_type, ent_kb_id, start, end, ent_id
|
||||
|
|
|
@ -48,7 +48,8 @@ class Span:
|
|||
label: Union[str, int] = ...,
|
||||
vector: Optional[Floats1d] = ...,
|
||||
vector_norm: Optional[float] = ...,
|
||||
kb_id: Optional[int] = ...,
|
||||
kb_id: Union[str, int] = ...,
|
||||
span_id: Union[str, int] = ...,
|
||||
) -> None: ...
|
||||
def __richcmp__(self, other: Span, op: int) -> bool: ...
|
||||
def __hash__(self) -> int: ...
|
||||
|
|
|
@ -81,17 +81,20 @@ cdef class Span:
|
|||
return Underscore.span_extensions.pop(name)
|
||||
|
||||
def __cinit__(self, Doc doc, int start, int end, label=0, vector=None,
|
||||
vector_norm=None, kb_id=0):
|
||||
vector_norm=None, kb_id=0, span_id=0):
|
||||
"""Create a `Span` object from the slice `doc[start : end]`.
|
||||
|
||||
doc (Doc): The parent document.
|
||||
start (int): The index of the first token of the span.
|
||||
end (int): The index of the first token after the span.
|
||||
label (int or str): A label to attach to the Span, e.g. for named entities.
|
||||
label (Union[int, str]): A label to attach to the Span, e.g. for named
|
||||
entities.
|
||||
vector (ndarray[ndim=1, dtype='float32']): A meaning representation
|
||||
of the span.
|
||||
vector_norm (float): The L2 norm of the span's vector representation.
|
||||
kb_id (uint64): An identifier from a Knowledge Base to capture the meaning of a named entity.
|
||||
kb_id (Union[int, str]): An identifier from a Knowledge Base to capture
|
||||
the meaning of a named entity.
|
||||
span_id (Union[int, str]): An identifier to associate with the span.
|
||||
|
||||
DOCS: https://spacy.io/api/span#init
|
||||
"""
|
||||
|
@ -102,6 +105,8 @@ cdef class Span:
|
|||
label = doc.vocab.strings.add(label)
|
||||
if isinstance(kb_id, str):
|
||||
kb_id = doc.vocab.strings.add(kb_id)
|
||||
if isinstance(span_id, str):
|
||||
span_id = doc.vocab.strings.add(span_id)
|
||||
if label not in doc.vocab.strings:
|
||||
raise ValueError(Errors.E084.format(label=label))
|
||||
|
||||
|
@ -113,6 +118,7 @@ cdef class Span:
|
|||
self.c = make_shared[SpanC](SpanC(
|
||||
label=label,
|
||||
kb_id=kb_id,
|
||||
id=span_id,
|
||||
start=start,
|
||||
end=end,
|
||||
start_char=start_char,
|
||||
|
@ -130,8 +136,8 @@ cdef class Span:
|
|||
|
||||
cdef SpanC* span_c = self.span_c()
|
||||
cdef SpanC* other_span_c = other.span_c()
|
||||
self_tuple = (span_c.start_char, span_c.end_char, span_c.label, span_c.kb_id, self.doc)
|
||||
other_tuple = (other_span_c.start_char, other_span_c.end_char, other_span_c.label, other_span_c.kb_id, other.doc)
|
||||
self_tuple = (span_c.start_char, span_c.end_char, span_c.label, span_c.kb_id, self.id, self.doc)
|
||||
other_tuple = (other_span_c.start_char, other_span_c.end_char, other_span_c.label, other_span_c.kb_id, other.id, other.doc)
|
||||
# <
|
||||
if op == 0:
|
||||
return self_tuple < other_tuple
|
||||
|
@ -153,7 +159,7 @@ cdef class Span:
|
|||
|
||||
def __hash__(self):
|
||||
cdef SpanC* span_c = self.span_c()
|
||||
return hash((self.doc, span_c.start_char, span_c.end_char, span_c.label, span_c.kb_id))
|
||||
return hash((self.doc, span_c.start_char, span_c.end_char, span_c.label, span_c.kb_id, span_c.id))
|
||||
|
||||
def __len__(self):
|
||||
"""Get the number of tokens in the span.
|
||||
|
@ -650,7 +656,7 @@ cdef class Span:
|
|||
else:
|
||||
return self.doc[root]
|
||||
|
||||
def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None):
|
||||
def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, id=0):
|
||||
"""Create a `Span` object from the slice `span.text[start : end]`.
|
||||
|
||||
start (int): The index of the first character of the span.
|
||||
|
@ -793,6 +799,15 @@ cdef class Span:
|
|||
def __set__(self, attr_t kb_id):
|
||||
self.span_c().kb_id = kb_id
|
||||
|
||||
property id:
|
||||
def __get__(self):
|
||||
cdef SpanC* span_c = self.span_c()
|
||||
return span_c.id
|
||||
|
||||
def __set__(self, attr_t id):
|
||||
cdef SpanC* span_c = self.span_c()
|
||||
span_c.id = id
|
||||
|
||||
property ent_id:
|
||||
"""RETURNS (uint64): The entity ID."""
|
||||
def __get__(self):
|
||||
|
@ -831,13 +846,21 @@ cdef class Span:
|
|||
self.label = self.doc.vocab.strings.add(label_)
|
||||
|
||||
property kb_id_:
|
||||
"""RETURNS (str): The named entity's KB ID."""
|
||||
"""RETURNS (str): The span's KB ID."""
|
||||
def __get__(self):
|
||||
return self.doc.vocab.strings[self.kb_id]
|
||||
|
||||
def __set__(self, str kb_id_):
|
||||
self.kb_id = self.doc.vocab.strings.add(kb_id_)
|
||||
|
||||
property id_:
|
||||
"""RETURNS (str): The span's ID."""
|
||||
def __get__(self):
|
||||
return self.doc.vocab.strings[self.id]
|
||||
|
||||
def __set__(self, str id_):
|
||||
self.id = self.doc.vocab.strings.add(id_)
|
||||
|
||||
|
||||
cdef int _count_words_to_root(const TokenC* token, int sent_length) except -1:
|
||||
# Don't allow spaces to be the root, if there are
|
||||
|
|
|
@ -24,3 +24,4 @@ class SpanGroup:
|
|||
def __getitem__(self, i: int) -> Span: ...
|
||||
def to_bytes(self) -> bytes: ...
|
||||
def from_bytes(self, bytes_data: bytes) -> SpanGroup: ...
|
||||
def copy(self) -> SpanGroup: ...
|
||||
|
|
|
@ -198,7 +198,7 @@ cdef class Example:
|
|||
|
||||
def get_aligned_sent_starts(self):
|
||||
"""Get list of SENT_START attributes aligned to the predicted tokenization.
|
||||
If the reference has not sentence starts, return a list of None values.
|
||||
If the reference does not have sentence starts, return a list of None values.
|
||||
"""
|
||||
if self.y.has_annotation("SENT_START"):
|
||||
align = self.alignment.y2x
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
from typing import List, Mapping, NoReturn, Union, Dict, Any, Set
|
||||
from typing import List, Mapping, NoReturn, Union, Dict, Any, Set, cast
|
||||
from typing import Optional, Iterable, Callable, Tuple, Type
|
||||
from typing import Iterator, Type, Pattern, Generator, TYPE_CHECKING
|
||||
from types import ModuleType
|
||||
|
@ -294,7 +294,7 @@ def find_matching_language(lang: str) -> Optional[str]:
|
|||
|
||||
# Find out which language modules we have
|
||||
possible_languages = []
|
||||
for modinfo in pkgutil.iter_modules(spacy.lang.__path__): # type: ignore
|
||||
for modinfo in pkgutil.iter_modules(spacy.lang.__path__): # type: ignore[attr-defined]
|
||||
code = modinfo.name
|
||||
if code == "xx":
|
||||
# Temporarily make 'xx' into a valid language code
|
||||
|
@ -391,7 +391,8 @@ def get_module_path(module: ModuleType) -> Path:
|
|||
"""
|
||||
if not hasattr(module, "__module__"):
|
||||
raise ValueError(Errors.E169.format(module=repr(module)))
|
||||
return Path(sys.modules[module.__module__].__file__).parent
|
||||
file_path = Path(cast(os.PathLike, sys.modules[module.__module__].__file__))
|
||||
return file_path.parent
|
||||
|
||||
|
||||
def load_model(
|
||||
|
@ -878,7 +879,7 @@ def get_package_path(name: str) -> Path:
|
|||
# Here we're importing the module just to find it. This is worryingly
|
||||
# indirect, but it's otherwise very difficult to find the package.
|
||||
pkg = importlib.import_module(name)
|
||||
return Path(pkg.__file__).parent
|
||||
return Path(cast(Union[str, os.PathLike], pkg.__file__)).parent
|
||||
|
||||
|
||||
def replace_model_node(model: Model, target: Model, replacement: Model) -> None:
|
||||
|
@ -1241,6 +1242,15 @@ def filter_spans(spans: Iterable["Span"]) -> List["Span"]:
|
|||
return result
|
||||
|
||||
|
||||
def filter_chain_spans(*spans: Iterable["Span"]) -> List["Span"]:
|
||||
return filter_spans(itertools.chain(*spans))
|
||||
|
||||
|
||||
@registry.misc("spacy.first_longest_spans_filter.v1")
|
||||
def make_first_longest_spans_filter():
|
||||
return filter_chain_spans
|
||||
|
||||
|
||||
def to_bytes(getters: Dict[str, Callable[[], bytes]], exclude: Iterable[str]) -> bytes:
|
||||
return srsly.msgpack_dumps(to_dict(getters, exclude))
|
||||
|
||||
|
@ -1675,7 +1685,7 @@ def packages_distributions() -> Dict[str, List[str]]:
|
|||
it's not available in the builtin importlib.metadata.
|
||||
"""
|
||||
pkg_to_dist = defaultdict(list)
|
||||
for dist in importlib_metadata.distributions(): # type: ignore[attr-defined]
|
||||
for dist in importlib_metadata.distributions():
|
||||
for pkg in (dist.read_text("top_level.txt") or "").split():
|
||||
pkg_to_dist[pkg].append(dist.metadata["Name"])
|
||||
return dict(pkg_to_dist)
|
||||
|
|
|
@ -466,6 +466,18 @@ takes the same arguments as `train` and reads settings off the
|
|||
|
||||
</Infobox>
|
||||
|
||||
<Infobox title="Notes on span characteristics" emoji="💡">
|
||||
|
||||
If your pipeline contains a `spancat` component, then this command will also
|
||||
report span characteristics such as the average span length and the span (or
|
||||
span boundary) distinctiveness. The distinctiveness measure shows how different
|
||||
the tokens are with respect to the rest of the corpus using the KL-divergence of
|
||||
the token distributions. To learn more, you can check out Papay et al.'s work on
|
||||
[*Dissecting Span Identification Tasks with Performance Prediction* (EMNLP
|
||||
2020)](https://aclanthology.org/2020.emnlp-main.396/).
|
||||
|
||||
</Infobox>
|
||||
|
||||
```cli
|
||||
$ python -m spacy debug data [config_path] [--code] [--ignore-warnings] [--verbose] [--no-format] [overrides]
|
||||
```
|
||||
|
@ -1323,7 +1335,7 @@ $ python -m spacy project run [subcommand] [project_dir] [--force] [--dry]
|
|||
| `subcommand` | Name of the command or workflow to run. ~~str (positional)~~ |
|
||||
| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ |
|
||||
| `--force`, `-F` | Force re-running steps, even if nothing changed. ~~bool (flag)~~ |
|
||||
| `--dry`, `-D` | Perform a dry run and don't execute scripts. ~~bool (flag)~~ |
|
||||
| `--dry`, `-D` | Perform a dry run and don't execute scripts. ~~bool (flag)~~ |
|
||||
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
|
||||
| **EXECUTES** | The command defined in the `project.yml`. |
|
||||
|
||||
|
@ -1441,12 +1453,12 @@ For more examples, see the templates in our
|
|||
|
||||
</Accordion>
|
||||
|
||||
| Name | Description |
|
||||
| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ |
|
||||
| `--output`, `-o` | Path to output file or `-` for stdout (default). If a file is specified and it already exists and contains auto-generated docs, only the auto-generated docs section is replaced. ~~Path (positional)~~ |
|
||||
| `--no-emoji`, `-NE` | Don't use emoji in the titles. ~~bool (flag)~~ |
|
||||
| **CREATES** | The Markdown-formatted project documentation. |
|
||||
| Name | Description |
|
||||
| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ |
|
||||
| `--output`, `-o` | Path to output file or `-` for stdout (default). If a file is specified and it already exists and contains auto-generated docs, only the auto-generated docs section is replaced. ~~Path (positional)~~ |
|
||||
| `--no-emoji`, `-NE` | Don't use emoji in the titles. ~~bool (flag)~~ |
|
||||
| **CREATES** | The Markdown-formatted project documentation. |
|
||||
|
||||
### project dvc {#project-dvc tag="command"}
|
||||
|
||||
|
@ -1485,7 +1497,7 @@ $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose]
|
|||
| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ |
|
||||
| `workflow` | Name of workflow defined in `project.yml`. Defaults to first workflow if not set. ~~Optional[str] \(option)~~ |
|
||||
| `--force`, `-F` | Force-updating config file. ~~bool (flag)~~ |
|
||||
| `--verbose`, `-V` | Print more output generated by DVC. ~~bool (flag)~~ |
|
||||
| `--verbose`, `-V` | Print more output generated by DVC. ~~bool (flag)~~ |
|
||||
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
|
||||
| **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. |
|
||||
|
||||
|
@ -1576,5 +1588,5 @@ $ python -m spacy huggingface-hub push [whl_path] [--org] [--msg] [--local-repo]
|
|||
| `--org`, `-o` | Optional name of organization to which the pipeline should be uploaded. ~~str (option)~~ |
|
||||
| `--msg`, `-m` | Commit message to use for update. Defaults to `"Update spaCy pipeline"`. ~~str (option)~~ |
|
||||
| `--local-repo`, `-l` | Local path to the model repository (will be created if it doesn't exist). Defaults to `hub` in the current working directory. ~~Path (option)~~ |
|
||||
| `--verbose`, `-V` | Output additional info for debugging, e.g. the full generated hub metadata. ~~bool (flag)~~ |
|
||||
| `--verbose`, `-V` | Output additional info for debugging, e.g. the full generated hub metadata. ~~bool (flag)~~ |
|
||||
| **UPLOADS** | The pipeline to the hub. |
|
||||
|
|
|
@ -37,13 +37,13 @@ streaming.
|
|||
> augmenter = null
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Path~~ |
|
||||
| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ |
|
||||
| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ |
|
||||
| Name | Description |
|
||||
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Path~~ |
|
||||
| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ |
|
||||
| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ |
|
||||
|
||||
```python
|
||||
%%GITHUB_SPACY/spacy/training/corpus.py
|
||||
|
@ -71,15 +71,15 @@ train/test skew.
|
|||
> corpus = Corpus("./data", limit=10)
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `path` | The directory or filename to read from. ~~Union[str, Path]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. Defaults to `False`. ~~bool~~ |
|
||||
| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `augmenter` | Optional data augmentation callback. ~~Callable[[Language, Example], Iterable[Example]]~~ |
|
||||
| `shuffle` | Whether to shuffle the examples. Defaults to `False`. ~~bool~~ |
|
||||
| Name | Description |
|
||||
| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `path` | The directory or filename to read from. ~~Union[str, Path]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. Defaults to `False`. ~~bool~~ |
|
||||
| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `augmenter` | Optional data augmentation callback. ~~Callable[[Language, Example], Iterable[Example]]~~ |
|
||||
| `shuffle` | Whether to shuffle the examples. Defaults to `False`. ~~bool~~ |
|
||||
|
||||
## Corpus.\_\_call\_\_ {#call tag="method"}
|
||||
|
||||
|
|
|
@ -481,6 +481,45 @@ Deserialize, i.e. import the document contents from a binary string.
|
|||
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
|
||||
| **RETURNS** | The `Doc` object. ~~Doc~~ |
|
||||
|
||||
## Doc.to_json {#to_json tag="method"}
|
||||
|
||||
Serializes a document to JSON. Note that this is format differs from the
|
||||
deprecated [`JSON training format`](/api/data-formats#json-input).
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> doc = nlp("All we have to decide is what to do with the time that is given us.")
|
||||
> assert doc.to_json()["text"] == doc.text
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `underscore` | Optional list of string names of custom `Doc` attributes. Attribute values need to be JSON-serializable. Values will be added to an `"_"` key in the data, e.g. `"_": {"foo": "bar"}`. ~~Optional[List[str]]~~ |
|
||||
| **RETURNS** | The data in JSON format. ~~Dict[str, Any]~~ |
|
||||
|
||||
## Doc.from_json {#from_json tag="method" new="3.3.1"}
|
||||
|
||||
Deserializes a document from JSON, i.e. generates a document from the provided
|
||||
JSON data as generated by [`Doc.to_json()`](/api/doc#to_json).
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> from spacy.tokens import Doc
|
||||
> doc = nlp("All we have to decide is what to do with the time that is given us.")
|
||||
> doc_json = doc.to_json()
|
||||
> deserialized_doc = Doc(nlp.vocab).from_json(doc_json)
|
||||
> assert deserialized_doc.text == doc.text == doc_json["text"]
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| -------------- | -------------------------------------------------------------------------------------------------------------------- |
|
||||
| `doc_json` | The Doc data in JSON format from [`Doc.to_json`](#to_json). ~~Dict[str, Any]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `validate` | Whether to validate the JSON input against the expected schema for detailed debugging. Defaults to `False`. ~~bool~~ |
|
||||
| **RETURNS** | A `Doc` corresponding to the provided JSON. ~~Doc~~ |
|
||||
|
||||
## Doc.retokenize {#retokenize tag="contextmanager" new="2.1"}
|
||||
|
||||
Context manager to handle retokenization of the `Doc`. Modifications to the
|
||||
|
|
|
@ -290,7 +290,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
|
|||
>
|
||||
> ```python
|
||||
> ruler_bytes = ruler.to_bytes()
|
||||
> ruler = nlp.add_pipe("enity_ruler")
|
||||
> ruler = nlp.add_pipe("entity_ruler")
|
||||
> ruler.from_bytes(ruler_bytes)
|
||||
> ```
|
||||
|
||||
|
|
|
@ -1123,7 +1123,7 @@ instance and factory instance.
|
|||
| `factory` | The name of the registered component factory. ~~str~~ |
|
||||
| `default_config` | The default config, describing the default values of the factory arguments. ~~Dict[str, Any]~~ |
|
||||
| `assigns` | `Doc` or `Token` attributes assigned by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ |
|
||||
| `requires` | `Doc` or `Token` attributes required by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ |
|
||||
| `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~ |
|
||||
| `requires` | `Doc` or `Token` attributes required by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ |
|
||||
| `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~ |
|
||||
| `default_score_weights` | The scores to report during training, and their default weight towards the final score used to select the best model. Weights should sum to `1.0` per component and will be combined and normalized for the whole pipeline. If a weight is set to `None`, the score will not be logged or weighted. ~~Dict[str, Optional[float]]~~ |
|
||||
| `scores` | All scores set by the components if it's trainable, e.g. `["ents_f", "ents_r", "ents_p"]`. Based on the `default_score_weights` and used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ |
|
||||
|
|
|
@ -30,26 +30,26 @@ pattern keys correspond to a number of
|
|||
[`Token` attributes](/api/token#attributes). The supported attributes for
|
||||
rule-based matching are:
|
||||
|
||||
| Attribute | Description |
|
||||
| ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `ORTH` | The exact verbatim text of a token. ~~str~~ |
|
||||
| `TEXT` <Tag variant="new">2.1</Tag> | The exact verbatim text of a token. ~~str~~ |
|
||||
| `NORM` | The normalized form of the token text. ~~str~~ |
|
||||
| `LOWER` | The lowercase form of the token text. ~~str~~ |
|
||||
| `LENGTH` | The length of the token text. ~~int~~ |
|
||||
| `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ |
|
||||
| `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ |
|
||||
| `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ |
|
||||
| `IS_SENT_START` | Token is start of sentence. ~~bool~~ |
|
||||
| `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ |
|
||||
| `SPACY` | Token has a trailing space. ~~bool~~ |
|
||||
| `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. ~~str~~ |
|
||||
| `ENT_TYPE` | The token's entity label. ~~str~~ |
|
||||
| `ENT_IOB` | The IOB part of the token's entity tag. ~~str~~ |
|
||||
| `ENT_ID` | The token's entity ID (`ent_id`). ~~str~~ |
|
||||
| `ENT_KB_ID` | The token's entity knowledge base ID (`ent_kb_id`). ~~str~~ |
|
||||
| `_` <Tag variant="new">2.1</Tag> | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ |
|
||||
| `OP` | Operator or quantifier to determine how often to match a token pattern. ~~str~~ |
|
||||
| Attribute | Description |
|
||||
| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `ORTH` | The exact verbatim text of a token. ~~str~~ |
|
||||
| `TEXT` <Tag variant="new">2.1</Tag> | The exact verbatim text of a token. ~~str~~ |
|
||||
| `NORM` | The normalized form of the token text. ~~str~~ |
|
||||
| `LOWER` | The lowercase form of the token text. ~~str~~ |
|
||||
| `LENGTH` | The length of the token text. ~~int~~ |
|
||||
| `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ |
|
||||
| `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ |
|
||||
| `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ |
|
||||
| `IS_SENT_START` | Token is start of sentence. ~~bool~~ |
|
||||
| `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ |
|
||||
| `SPACY` | Token has a trailing space. ~~bool~~ |
|
||||
| `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. ~~str~~ |
|
||||
| `ENT_TYPE` | The token's entity label. ~~str~~ |
|
||||
| `ENT_IOB` | The IOB part of the token's entity tag. ~~str~~ |
|
||||
| `ENT_ID` | The token's entity ID (`ent_id`). ~~str~~ |
|
||||
| `ENT_KB_ID` | The token's entity knowledge base ID (`ent_kb_id`). ~~str~~ |
|
||||
| `_` <Tag variant="new">2.1</Tag> | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ |
|
||||
| `OP` | Operator or quantifier to determine how often to match a token pattern. ~~str~~ |
|
||||
|
||||
Operators and quantifiers define **how often** a token pattern should be
|
||||
matched:
|
||||
|
@ -113,6 +113,10 @@ string where an integer is expected) or unexpected property names.
|
|||
|
||||
Find all token sequences matching the supplied patterns on the `Doc` or `Span`.
|
||||
|
||||
Note that if a single label has multiple patterns associated with it, the
|
||||
returned matches don't provide a way to tell which pattern was responsible for
|
||||
the match.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
|
@ -131,7 +135,7 @@ Find all token sequences matching the supplied patterns on the `Doc` or `Span`.
|
|||
| _keyword-only_ | |
|
||||
| `as_spans` <Tag variant="new">3</Tag> | Instead of tuples, return a list of [`Span`](/api/span) objects of the matches, with the `match_id` assigned as the span label. Defaults to `False`. ~~bool~~ |
|
||||
| `allow_missing` <Tag variant="new">3</Tag> | Whether to skip checks for missing annotation for attributes included in patterns. Defaults to `False`. ~~bool~~ |
|
||||
| `with_alignments` <Tag variant="new">3.0.6</Tag> | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ |
|
||||
| `with_alignments` <Tag variant="new">3.0.6</Tag> | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token in the pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ |
|
||||
| **RETURNS** | A list of `(match_id, start, end)` tuples, describing the matches. A match tuple describes a span `doc[start:end`]. The `match_id` is the ID of the added match pattern. If `as_spans` is set to `True`, a list of `Span` objects is returned instead. ~~Union[List[Tuple[int, int, int]], List[Span]]~~ |
|
||||
|
||||
## Matcher.\_\_len\_\_ {#len tag="method" new="2"}
|
||||
|
|
|
@ -7,6 +7,7 @@ menu:
|
|||
- ['merge_entities', 'merge_entities']
|
||||
- ['merge_subtokens', 'merge_subtokens']
|
||||
- ['token_splitter', 'token_splitter']
|
||||
- ['doc_cleaner', 'doc_cleaner']
|
||||
---
|
||||
|
||||
## merge_noun_chunks {#merge_noun_chunks tag="function"}
|
||||
|
|
|
@ -27,6 +27,7 @@ Create a `Span` object from the slice `doc[start : end]`.
|
|||
| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
|
||||
| `vector_norm` | The L2 norm of the document's vector representation. ~~float~~ |
|
||||
| `kb_id` | A knowledge base ID to attach to the span, e.g. for named entities. ~~Union[str, int]~~ |
|
||||
| `span_id` | An ID to associate with the span. ~~Union[str, int]~~ |
|
||||
|
||||
## Span.\_\_getitem\_\_ {#getitem tag="method"}
|
||||
|
||||
|
@ -560,7 +561,9 @@ overlaps with will be returned.
|
|||
| `lemma_` | The span's lemma. Equivalent to `"".join(token.text_with_ws for token in span)`. ~~str~~ |
|
||||
| `kb_id` | The hash value of the knowledge base ID referred to by the span. ~~int~~ |
|
||||
| `kb_id_` | The knowledge base ID referred to by the span. ~~str~~ |
|
||||
| `ent_id` | The hash value of the named entity the token is an instance of. ~~int~~ |
|
||||
| `ent_id_` | The string ID of the named entity the token is an instance of. ~~str~~ |
|
||||
| `ent_id` | The hash value of the named entity the root token is an instance of. ~~int~~ |
|
||||
| `ent_id_` | The string ID of the named entity the root token is an instance of. ~~str~~ |
|
||||
| `id` | The hash value of the span's ID. ~~int~~ |
|
||||
| `id_` | The span's ID. ~~str~~ |
|
||||
| `sentiment` | A scalar value indicating the positivity or negativity of the span. ~~float~~ |
|
||||
| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ |
|
||||
|
|
351
website/docs/api/spanruler.md
Normal file
351
website/docs/api/spanruler.md
Normal file
|
@ -0,0 +1,351 @@
|
|||
---
|
||||
title: SpanRuler
|
||||
tag: class
|
||||
source: spacy/pipeline/span_ruler.py
|
||||
new: 3.3.1
|
||||
teaser: 'Pipeline component for rule-based span and named entity recognition'
|
||||
api_string_name: span_ruler
|
||||
api_trainable: false
|
||||
---
|
||||
|
||||
The span ruler lets you add spans to [`Doc.spans`](/api/doc#spans) and/or
|
||||
[`Doc.ents`](/api/doc#ents) using token-based rules or exact phrase matches. For
|
||||
usage examples, see the docs on
|
||||
[rule-based span matching](/usage/rule-based-matching#spanruler).
|
||||
|
||||
## Assigned Attributes {#assigned-attributes}
|
||||
|
||||
Matches will be saved to `Doc.spans[spans_key]` as a
|
||||
[`SpanGroup`](/api/spangroup) and/or to `Doc.ents`, where the annotation is
|
||||
saved in the `Token.ent_type` and `Token.ent_iob` fields.
|
||||
|
||||
| Location | Value |
|
||||
| ---------------------- | ----------------------------------------------------------------- |
|
||||
| `Doc.spans[spans_key]` | The annotated spans. ~~SpanGroup~~ |
|
||||
| `Doc.ents` | The annotated spans. ~~Tuple[Span]~~ |
|
||||
| `Token.ent_iob` | An enum encoding of the IOB part of the named entity tag. ~~int~~ |
|
||||
| `Token.ent_iob_` | The IOB part of the named entity tag. ~~str~~ |
|
||||
| `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ |
|
||||
| `Token.ent_type_` | The label part of the named entity tag. ~~str~~ |
|
||||
|
||||
## Config and implementation {#config}
|
||||
|
||||
The default config is defined by the pipeline component factory and describes
|
||||
how the component should be configured. You can override its settings via the
|
||||
`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your
|
||||
[`config.cfg`](/usage/training#config).
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> config = {
|
||||
> "spans_key": "my_spans",
|
||||
> "validate": True,
|
||||
> "overwrite": False,
|
||||
> }
|
||||
> nlp.add_pipe("span_ruler", config=config)
|
||||
> ```
|
||||
|
||||
| Setting | Description |
|
||||
| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ |
|
||||
| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ |
|
||||
| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ |
|
||||
| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ |
|
||||
| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ |
|
||||
| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ |
|
||||
| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ |
|
||||
| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ |
|
||||
|
||||
```python
|
||||
%%GITHUB_SPACY/spacy/pipeline/span_ruler.py
|
||||
```
|
||||
|
||||
## SpanRuler.\_\_init\_\_ {#init tag="method"}
|
||||
|
||||
Initialize the span ruler. If patterns are supplied here, they need to be a list
|
||||
of dictionaries with a `"label"` and `"pattern"` key. A pattern can either be a
|
||||
token pattern (list) or a phrase pattern (string). For example:
|
||||
`{"label": "ORG", "pattern": "Apple"}`.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> # Construction via add_pipe
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
>
|
||||
> # Construction from class
|
||||
> from spacy.pipeline import SpanRuler
|
||||
> ruler = SpanRuler(nlp, overwrite=True)
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ |
|
||||
| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ |
|
||||
| _keyword-only_ | |
|
||||
| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ |
|
||||
| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ |
|
||||
| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ |
|
||||
| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ |
|
||||
| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ |
|
||||
| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ |
|
||||
| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ |
|
||||
| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ |
|
||||
|
||||
## SpanRuler.initialize {#initialize tag="method"}
|
||||
|
||||
Initialize the component with data and used before training to load in rules
|
||||
from a [pattern file](/usage/rule-based-matching/#spanruler-files). This method
|
||||
is typically called by [`Language.initialize`](/api/language#initialize) and
|
||||
lets you customize arguments it receives via the
|
||||
[`[initialize.components]`](/api/data-formats#config-initialize) block in the
|
||||
config. Any existing patterns are removed on initialization.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> span_ruler = nlp.add_pipe("span_ruler")
|
||||
> span_ruler.initialize(lambda: [], nlp=nlp, patterns=patterns)
|
||||
> ```
|
||||
>
|
||||
> ```ini
|
||||
> ### config.cfg
|
||||
> [initialize.components.span_ruler]
|
||||
>
|
||||
> [initialize.components.span_ruler.patterns]
|
||||
> @readers = "srsly.read_jsonl.v1"
|
||||
> path = "corpus/span_ruler_patterns.jsonl
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Not used by the `SpanRuler`. ~~Callable[[], Iterable[Example]]~~ |
|
||||
| _keyword-only_ | |
|
||||
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
|
||||
| `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ |
|
||||
|
||||
## SpanRuler.\_\len\_\_ {#len tag="method"}
|
||||
|
||||
The number of all patterns added to the span ruler.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> assert len(ruler) == 0
|
||||
> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}])
|
||||
> assert len(ruler) == 1
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ----------- | ------------------------------- |
|
||||
| **RETURNS** | The number of patterns. ~~int~~ |
|
||||
|
||||
## SpanRuler.\_\_contains\_\_ {#contains tag="method"}
|
||||
|
||||
Whether a label is present in the patterns.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}])
|
||||
> assert "ORG" in ruler
|
||||
> assert not "PERSON" in ruler
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ----------- | --------------------------------------------------- |
|
||||
| `label` | The label to check. ~~str~~ |
|
||||
| **RETURNS** | Whether the span ruler contains the label. ~~bool~~ |
|
||||
|
||||
## SpanRuler.\_\_call\_\_ {#call tag="method"}
|
||||
|
||||
Find matches in the `Doc` and add them to `doc.spans[span_key]` and/or
|
||||
`doc.ents`. Typically, this happens automatically after the component has been
|
||||
added to the pipeline using [`nlp.add_pipe`](/api/language#add_pipe). If the
|
||||
span ruler was initialized with `overwrite=True`, existing spans and entities
|
||||
will be removed.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}])
|
||||
>
|
||||
> doc = nlp("A text about Apple.")
|
||||
> spans = [(span.text, span.label_) for span in doc.spans["ruler"]]
|
||||
> assert spans == [("Apple", "ORG")]
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ----------- | -------------------------------------------------------------------- |
|
||||
| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ |
|
||||
| **RETURNS** | The modified `Doc` with added spans/entities. ~~Doc~~ |
|
||||
|
||||
## SpanRuler.add_patterns {#add_patterns tag="method"}
|
||||
|
||||
Add patterns to the span ruler. A pattern can either be a token pattern (list of
|
||||
dicts) or a phrase pattern (string). For more details, see the usage guide on
|
||||
[rule-based matching](/usage/rule-based-matching).
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> patterns = [
|
||||
> {"label": "ORG", "pattern": "Apple"},
|
||||
> {"label": "GPE", "pattern": [{"lower": "san"}, {"lower": "francisco"}]}
|
||||
> ]
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler.add_patterns(patterns)
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ---------- | ---------------------------------------------------------------- |
|
||||
| `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ |
|
||||
|
||||
## SpanRuler.remove {#remove tag="method"}
|
||||
|
||||
Remove patterns by label from the span ruler. A `ValueError` is raised if the
|
||||
label does not exist in any patterns.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}]
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler.add_patterns(patterns)
|
||||
> ruler.remove("ORG")
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ------- | -------------------------------------- |
|
||||
| `label` | The label of the pattern rule. ~~str~~ |
|
||||
|
||||
## SpanRuler.remove_by_id {#remove_by_id tag="method"}
|
||||
|
||||
Remove patterns by ID from the span ruler. A `ValueError` is raised if the ID
|
||||
does not exist in any patterns.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}]
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler.add_patterns(patterns)
|
||||
> ruler.remove_by_id("apple")
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ------------ | ----------------------------------- |
|
||||
| `pattern_id` | The ID of the pattern rule. ~~str~~ |
|
||||
|
||||
## SpanRuler.clear {#clear tag="method"}
|
||||
|
||||
Remove all patterns the span ruler.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}]
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler.add_patterns(patterns)
|
||||
> ruler.clear()
|
||||
> ```
|
||||
|
||||
## SpanRuler.to_disk {#to_disk tag="method"}
|
||||
|
||||
Save the span ruler patterns to a directory. The patterns will be saved as
|
||||
newline-delimited JSON (JSONL).
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler.to_disk("/path/to/span_ruler")
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
|
||||
|
||||
## SpanRuler.from_disk {#from_disk tag="method"}
|
||||
|
||||
Load the span ruler from a path.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler.from_disk("/path/to/span_ruler")
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ----------- | ----------------------------------------------------------------------------------------------- |
|
||||
| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
|
||||
| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ |
|
||||
|
||||
## SpanRuler.to_bytes {#to_bytes tag="method"}
|
||||
|
||||
Serialize the span ruler to a bytestring.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler_bytes = ruler.to_bytes()
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ----------- | ---------------------------------- |
|
||||
| **RETURNS** | The serialized patterns. ~~bytes~~ |
|
||||
|
||||
## SpanRuler.from_bytes {#from_bytes tag="method"}
|
||||
|
||||
Load the pipe from a bytestring. Modifies the object in place and returns it.
|
||||
|
||||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> ruler_bytes = ruler.to_bytes()
|
||||
> ruler = nlp.add_pipe("span_ruler")
|
||||
> ruler.from_bytes(ruler_bytes)
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| ------------ | ---------------------------------------------- |
|
||||
| `bytes_data` | The bytestring to load. ~~bytes~~ |
|
||||
| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ |
|
||||
|
||||
## SpanRuler.labels {#labels tag="property"}
|
||||
|
||||
All labels present in the match patterns.
|
||||
|
||||
| Name | Description |
|
||||
| ----------- | -------------------------------------- |
|
||||
| **RETURNS** | The string labels. ~~Tuple[str, ...]~~ |
|
||||
|
||||
## SpanRuler.ids {#ids tag="property"}
|
||||
|
||||
All IDs present in the `id` property of the match patterns.
|
||||
|
||||
| Name | Description |
|
||||
| ----------- | ----------------------------------- |
|
||||
| **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ |
|
||||
|
||||
## SpanRuler.patterns {#patterns tag="property"}
|
||||
|
||||
All patterns that were added to the span ruler.
|
||||
|
||||
| Name | Description |
|
||||
| ----------- | ---------------------------------------------------------------------------------------- |
|
||||
| **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ |
|
||||
|
||||
## Attributes {#attributes}
|
||||
|
||||
| Name | Description |
|
||||
| ---------------- | -------------------------------------------------------------------------------- |
|
||||
| `key` | The spans key that spans are saved under. ~~Optional[str]~~ |
|
||||
| `matcher` | The underlying matcher used to process token patterns. ~~Matcher~~ |
|
||||
| `phrase_matcher` | The underlying phrase matcher used to process phrase patterns. ~~PhraseMatcher~~ |
|
|
@ -161,7 +161,7 @@ Load state from a binary string.
|
|||
> #### Example
|
||||
>
|
||||
> ```python
|
||||
> fron spacy.strings import StringStore
|
||||
> from spacy.strings import StringStore
|
||||
> store_bytes = stringstore.to_bytes()
|
||||
> new_store = StringStore().from_bytes(store_bytes)
|
||||
> ```
|
||||
|
|
|
@ -221,7 +221,7 @@ dependency tree.
|
|||
|
||||
## Token.ancestors {#ancestors tag="property" model="parser"}
|
||||
|
||||
The rightmost token of this token's syntactic descendants.
|
||||
A sequence of the token's syntactic ancestors (parents, grandparents, etc).
|
||||
|
||||
> #### Example
|
||||
>
|
||||
|
|
|
@ -239,7 +239,7 @@ browser. Will run a simple web server.
|
|||
| Name | Description |
|
||||
| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ |
|
||||
| `style` | Visualization style, `"dep"` or `"ent"`. Defaults to `"dep"`. ~~str~~ |
|
||||
| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` <Tag variant="new">3.3</Tag>. Defaults to `"dep"`. ~~str~~ |
|
||||
| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ |
|
||||
| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ |
|
||||
| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ |
|
||||
|
@ -264,7 +264,7 @@ Render a dependency parse tree or named entity visualization.
|
|||
| Name | Description |
|
||||
| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span, dict]], Doc, Span, dict]~~ |
|
||||
| `style` | Visualization style, `"dep"` or `"ent"`. Defaults to `"dep"`. ~~str~~ |
|
||||
| `style` | Visualization style,`"dep"`, `"ent"` or `"span"` <Tag variant="new">3.3</Tag>. Defaults to `"dep"`. ~~str~~ |
|
||||
| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ |
|
||||
| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ |
|
||||
| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ |
|
||||
|
@ -320,7 +320,6 @@ If a setting is not present in the options, the default value will be used.
|
|||
| `template` <Tag variant="new">2.2</Tag> | Optional template to overwrite the HTML used to render entity spans. Should be a format string and can use `{bg}`, `{text}` and `{label}`. See [`templates.py`](%%GITHUB_SPACY/spacy/displacy/templates.py) for examples. ~~Optional[str]~~ |
|
||||
| `kb_url_template` <Tag variant="new">3.2.1</Tag> | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in. ~~Optional[str]~~ |
|
||||
|
||||
|
||||
#### Span Visualizer options {#displacy_options-span}
|
||||
|
||||
> #### Example
|
||||
|
@ -330,21 +329,19 @@ If a setting is not present in the options, the default value will be used.
|
|||
> displacy.serve(doc, style="span", options=options)
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `spans_key` | Which spans key to render spans from. Default is `"sc"`. ~~str~~ |
|
||||
| Name | Description |
|
||||
| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `spans_key` | Which spans key to render spans from. Default is `"sc"`. ~~str~~ |
|
||||
| `templates` | Dictionary containing the keys `"span"`, `"slice"`, and `"start"`. These dictate how the overall span, a span slice, and the starting token will be rendered. ~~Optional[Dict[str, str]~~ |
|
||||
| `kb_url_template` | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in ~~Optional[str]~~ |
|
||||
| `colors` | Color overrides. Entity types should be mapped to color names or values. ~~Dict[str, str]~~ |
|
||||
| `kb_url_template` | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in ~~Optional[str]~~ |
|
||||
| `colors` | Color overrides. Entity types should be mapped to color names or values. ~~Dict[str, str]~~ |
|
||||
|
||||
|
||||
By default, displaCy comes with colors for all entity types used by [spaCy's
|
||||
trained pipelines](/models) for both entity and span visualizer. If you're
|
||||
using custom entity types, you can use the `colors` setting to add your own
|
||||
colors for them. Your application or pipeline package can also expose a
|
||||
[`spacy_displacy_colors` entry
|
||||
point](/usage/saving-loading#entry-points-displacy) to add custom labels and
|
||||
their colors automatically.
|
||||
By default, displaCy comes with colors for all entity types used by
|
||||
[spaCy's trained pipelines](/models) for both entity and span visualizer. If
|
||||
you're using custom entity types, you can use the `colors` setting to add your
|
||||
own colors for them. Your application or pipeline package can also expose a
|
||||
[`spacy_displacy_colors` entry point](/usage/saving-loading#entry-points-displacy)
|
||||
to add custom labels and their colors automatically.
|
||||
|
||||
By default, displaCy links to `#` for entities without a `kb_id` set on their
|
||||
span. If you wish to link an entity to their URL then consider using the
|
||||
|
@ -354,7 +351,6 @@ span. If you wish to link an entity to their URL then consider using the
|
|||
should redirect you to their Wikidata page, in this case
|
||||
`https://www.wikidata.org/wiki/Q95`.
|
||||
|
||||
|
||||
## registry {#registry source="spacy/util.py" new="3"}
|
||||
|
||||
spaCy's function registry extends
|
||||
|
@ -443,8 +439,8 @@ and the accuracy scores on the development set.
|
|||
The built-in, default logger is the ConsoleLogger, which prints results to the
|
||||
console in tabular format. The
|
||||
[spacy-loggers](https://github.com/explosion/spacy-loggers) package, included as
|
||||
a dependency of spaCy, enables other loggers, such as one that
|
||||
sends results to a [Weights & Biases](https://www.wandb.com/) dashboard.
|
||||
a dependency of spaCy, enables other loggers, such as one that sends results to
|
||||
a [Weights & Biases](https://www.wandb.com/) dashboard.
|
||||
|
||||
Instead of using one of the built-in loggers, you can
|
||||
[implement your own](/usage/training#custom-logging).
|
||||
|
@ -583,14 +579,14 @@ the [`Corpus`](/api/corpus) class.
|
|||
> limit = 0
|
||||
> ```
|
||||
|
||||
| Name | Description |
|
||||
| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Union[str, Path]~~ |
|
||||
| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ |
|
||||
| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ |
|
||||
| **CREATES** | The corpus reader. ~~Corpus~~ |
|
||||
| Name | Description |
|
||||
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Union[str, Path]~~ |
|
||||
| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ |
|
||||
| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ |
|
||||
| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ |
|
||||
| **CREATES** | The corpus reader. ~~Corpus~~ |
|
||||
|
||||
#### spacy.JsonlCorpus.v1 {#jsonlcorpus tag="registered function"}
|
||||
|
||||
|
|
|
@ -115,7 +115,7 @@ The Finnish, Korean and Swedish `md` and `lg` pipelines use
|
|||
running a trained pipeline on texts and working with [`Doc`](/api/doc) objects,
|
||||
you shouldn't notice any difference with floret vectors. With floret vectors no
|
||||
tokens are out-of-vocabulary, so [`Token.is_oov`](/api/token#attributes) will
|
||||
return `True` for all tokens.
|
||||
return `False` for all tokens.
|
||||
|
||||
If you access vectors directly for similarity comparisons, there are a few
|
||||
differences because floret vectors don't include a fixed word list like the
|
||||
|
|
|
@ -530,7 +530,8 @@ models, which can **improve the accuracy** of your components.
|
|||
Word vectors in spaCy are "static" in the sense that they are not learned
|
||||
parameters of the statistical models, and spaCy itself does not feature any
|
||||
algorithms for learning word vector tables. You can train a word vectors table
|
||||
using tools such as [Gensim](https://radimrehurek.com/gensim/),
|
||||
using tools such as [floret](https://github.com/explosion/floret),
|
||||
[Gensim](https://radimrehurek.com/gensim/),
|
||||
[FastText](https://fasttext.cc/) or
|
||||
[GloVe](https://nlp.stanford.edu/projects/glove/), or download existing
|
||||
pretrained vectors. The [`init vectors`](/api/cli#init-vectors) command lets you
|
||||
|
|
|
@ -129,15 +129,14 @@ machine learning library, [Thinc](https://thinc.ai). For GPU support, we've been
|
|||
grateful to use the work of Chainer's [CuPy](https://cupy.chainer.org) module,
|
||||
which provides a numpy-compatible interface for GPU arrays.
|
||||
|
||||
spaCy can be installed on GPU by specifying `spacy[cuda]`, `spacy[cuda90]`,
|
||||
`spacy[cuda91]`, `spacy[cuda92]`, `spacy[cuda100]`, `spacy[cuda101]`,
|
||||
`spacy[cuda102]`, `spacy[cuda110]`, `spacy[cuda111]` or `spacy[cuda112]`. If you
|
||||
know your cuda version, using the more explicit specifier allows cupy to be
|
||||
installed via wheel, saving some compilation time. The specifiers should install
|
||||
spaCy can be installed for a CUDA-compatible GPU by specifying `spacy[cuda]`,
|
||||
`spacy[cuda102]`, `spacy[cuda112]`, `spacy[cuda113]`, etc. If you know your
|
||||
CUDA version, using the more explicit specifier allows CuPy to be installed via
|
||||
wheel, saving some compilation time. The specifiers should install
|
||||
[`cupy`](https://cupy.chainer.org).
|
||||
|
||||
```bash
|
||||
$ pip install -U %%SPACY_PKG_NAME[cuda92]%%SPACY_PKG_FLAGS
|
||||
$ pip install -U %%SPACY_PKG_NAME[cuda113]%%SPACY_PKG_FLAGS
|
||||
```
|
||||
|
||||
Once you have a GPU-enabled installation, the best way to activate it is to call
|
||||
|
|
|
@ -48,7 +48,7 @@ but do not change its part-of-speech. We say that a **lemma** (root form) is
|
|||
**inflected** (modified/combined) with one or more **morphological features** to
|
||||
create a surface form. Here are some examples:
|
||||
|
||||
| Context | Surface | Lemma | POS | Morphological Features |
|
||||
| Context | Surface | Lemma | POS | Morphological Features |
|
||||
| ---------------------------------------- | ------- | ----- | ------ | ---------------------------------------- |
|
||||
| I was reading the paper | reading | read | `VERB` | `VerbForm=Ger` |
|
||||
| I don't watch the news, I read the paper | read | read | `VERB` | `VerbForm=Fin`, `Mood=Ind`, `Tense=Pres` |
|
||||
|
@ -430,7 +430,7 @@ for token in doc:
|
|||
print(token.text, token.pos_, token.dep_, token.head.text)
|
||||
```
|
||||
|
||||
| Text | POS | Dep | Head text |
|
||||
| Text | POS | Dep | Head text |
|
||||
| ----------------------------------- | ------ | ------- | --------- |
|
||||
| Credit and mortgage account holders | `NOUN` | `nsubj` | submit |
|
||||
| must | `VERB` | `aux` | submit |
|
||||
|
|
|
@ -6,6 +6,7 @@ menu:
|
|||
- ['Phrase Matcher', 'phrasematcher']
|
||||
- ['Dependency Matcher', 'dependencymatcher']
|
||||
- ['Entity Ruler', 'entityruler']
|
||||
- ['Span Ruler', 'spanruler']
|
||||
- ['Models & Rules', 'models-rules']
|
||||
---
|
||||
|
||||
|
@ -158,23 +159,23 @@ The available token pattern keys correspond to a number of
|
|||
[`Token` attributes](/api/token#attributes). The supported attributes for
|
||||
rule-based matching are:
|
||||
|
||||
| Attribute | Description |
|
||||
| ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `ORTH` | The exact verbatim text of a token. ~~str~~ |
|
||||
| `TEXT` <Tag variant="new">2.1</Tag> | The exact verbatim text of a token. ~~str~~ |
|
||||
| `NORM` | The normalized form of the token text. ~~str~~ |
|
||||
| `LOWER` | The lowercase form of the token text. ~~str~~ |
|
||||
| `LENGTH` | The length of the token text. ~~int~~ |
|
||||
| `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ |
|
||||
| `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ |
|
||||
| `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ |
|
||||
| `IS_SENT_START` | Token is start of sentence. ~~bool~~ |
|
||||
| `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ |
|
||||
| `SPACY` | Token has a trailing space. ~~bool~~ |
|
||||
| `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. Note that the values of these attributes are case-sensitive. For a list of available part-of-speech tags and dependency labels, see the [Annotation Specifications](/api/annotation). ~~str~~ |
|
||||
| `ENT_TYPE` | The token's entity label. ~~str~~ |
|
||||
| `_` <Tag variant="new">2.1</Tag> | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ |
|
||||
| `OP` | [Operator or quantifier](#quantifiers) to determine how often to match a token pattern. ~~str~~ |
|
||||
| Attribute | Description |
|
||||
| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `ORTH` | The exact verbatim text of a token. ~~str~~ |
|
||||
| `TEXT` <Tag variant="new">2.1</Tag> | The exact verbatim text of a token. ~~str~~ |
|
||||
| `NORM` | The normalized form of the token text. ~~str~~ |
|
||||
| `LOWER` | The lowercase form of the token text. ~~str~~ |
|
||||
| `LENGTH` | The length of the token text. ~~int~~ |
|
||||
| `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ |
|
||||
| `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ |
|
||||
| `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ |
|
||||
| `IS_SENT_START` | Token is start of sentence. ~~bool~~ |
|
||||
| `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ |
|
||||
| `SPACY` | Token has a trailing space. ~~bool~~ |
|
||||
| `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. Note that the values of these attributes are case-sensitive. For a list of available part-of-speech tags and dependency labels, see the [Annotation Specifications](/api/annotation). ~~str~~ |
|
||||
| `ENT_TYPE` | The token's entity label. ~~str~~ |
|
||||
| `_` <Tag variant="new">2.1</Tag> | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ |
|
||||
| `OP` | [Operator or quantifier](#quantifiers) to determine how often to match a token pattern. ~~str~~ |
|
||||
|
||||
<Accordion title="Does it matter if the attribute names are uppercase or lowercase?">
|
||||
|
||||
|
@ -1446,6 +1447,108 @@ with nlp.select_pipes(enable="tagger"):
|
|||
ruler.add_patterns(patterns)
|
||||
```
|
||||
|
||||
## Rule-based span matching {#spanruler new="3.3.1"}
|
||||
|
||||
The [`SpanRuler`](/api/spanruler) is a generalized version of the entity ruler
|
||||
that lets you add spans to `doc.spans` or `doc.ents` based on pattern
|
||||
dictionaries, which makes it easy to combine rule-based and statistical pipeline
|
||||
components.
|
||||
|
||||
### Span patterns {#spanruler-patterns}
|
||||
|
||||
The [pattern format](#entityruler-patterns) is the same as for the entity ruler:
|
||||
|
||||
1. **Phrase patterns** for exact string matches (string).
|
||||
|
||||
```python
|
||||
{"label": "ORG", "pattern": "Apple"}
|
||||
```
|
||||
|
||||
2. **Token patterns** with one dictionary describing one token (list).
|
||||
|
||||
```python
|
||||
{"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]}
|
||||
```
|
||||
|
||||
### Using the span ruler {#spanruler-usage}
|
||||
|
||||
The [`SpanRuler`](/api/spanruler) is a pipeline component that's typically added
|
||||
via [`nlp.add_pipe`](/api/language#add_pipe). When the `nlp` object is called on
|
||||
a text, it will find matches in the `doc` and add them as spans to
|
||||
`doc.spans["ruler"]`, using the specified pattern label as the entity label.
|
||||
Unlike in `doc.ents`, overlapping matches are allowed in `doc.spans`, so no
|
||||
filtering is required, but optional filtering and sorting can be applied to the
|
||||
spans before they're saved.
|
||||
|
||||
```python
|
||||
### {executable="true"}
|
||||
import spacy
|
||||
|
||||
nlp = spacy.blank("en")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
patterns = [{"label": "ORG", "pattern": "Apple"},
|
||||
{"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]}]
|
||||
ruler.add_patterns(patterns)
|
||||
|
||||
doc = nlp("Apple is opening its first big office in San Francisco.")
|
||||
print([(span.text, span.label_) for span in doc.spans["ruler"]])
|
||||
```
|
||||
|
||||
The span ruler is designed to integrate with spaCy's existing pipeline
|
||||
components and enhance the [SpanCategorizer](/api/spancat) and
|
||||
[EntityRecognizer](/api/entityrecognizer). The `overwrite` setting determines
|
||||
whether the existing annotation in `doc.spans` or `doc.ents` is preserved.
|
||||
Because overlapping entities are not allowed for `doc.ents`, the entities are
|
||||
always filtered, using [`util.filter_spans`](/api/top-level#util.filter_spans)
|
||||
by default. See the [`SpanRuler` API docs](/api/spanruler) for more information
|
||||
about how to customize the sorting and filtering of matched spans.
|
||||
|
||||
```python
|
||||
### {executable="true"}
|
||||
import spacy
|
||||
|
||||
nlp = spacy.load("en_core_web_sm")
|
||||
# only annotate doc.ents, not doc.spans
|
||||
config = {"spans_key": None, "annotate_ents": True, "overwrite": False}
|
||||
ruler = nlp.add_pipe("span_ruler", config=config)
|
||||
patterns = [{"label": "ORG", "pattern": "MyCorp Inc."}]
|
||||
ruler.add_patterns(patterns)
|
||||
|
||||
doc = nlp("MyCorp Inc. is a company in the U.S.")
|
||||
print([(ent.text, ent.label_) for ent in doc.ents])
|
||||
```
|
||||
|
||||
### Using pattern files {#spanruler-files}
|
||||
|
||||
You can save patterns in a JSONL file (newline-delimited JSON) to load with
|
||||
[`SpanRuler.initialize`](/api/spanruler#initialize) or
|
||||
[`SpanRuler.add_patterns`](/api/spanruler#add_patterns).
|
||||
|
||||
```json
|
||||
### patterns.jsonl
|
||||
{"label": "ORG", "pattern": "Apple"}
|
||||
{"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]}
|
||||
```
|
||||
|
||||
```python
|
||||
import srsly
|
||||
|
||||
patterns = srsly.read_jsonl("patterns.jsonl")
|
||||
ruler = nlp.add_pipe("span_ruler")
|
||||
ruler.add_patterns(patterns)
|
||||
```
|
||||
|
||||
<Infobox title="Important note" variant="warning">
|
||||
|
||||
Unlike the entity ruler, the span ruler cannot load patterns on initialization
|
||||
with `SpanRuler(patterns=patterns)` or directly from a JSONL file path with
|
||||
`SpanRuler.from_disk(jsonl_path)`. Patterns should be loaded from the JSONL file
|
||||
separately and then added through
|
||||
[`SpanRuler.initialize`](/api/spanruler#initialize]) or
|
||||
[`SpanRuler.add_patterns`](/api/spanruler#add_patterns) as shown above.
|
||||
|
||||
</Infobox>
|
||||
|
||||
## Combining models and rules {#models-rules}
|
||||
|
||||
You can combine statistical and rule-based components in a variety of ways.
|
||||
|
|
|
@ -132,13 +132,13 @@ your own.
|
|||
> contributions for Catalan and to Kenneth Enevoldsen for Danish. For additional
|
||||
> Danish pipelines, check out [DaCy](https://github.com/KennethEnevoldsen/DaCy).
|
||||
|
||||
| Package | Language | UPOS | Parser LAS | NER F |
|
||||
| ------------------------------------------------- | -------- | ---: | ---------: | -----: |
|
||||
| [`ca_core_news_sm`](/models/ca#ca_core_news_sm) | Catalan | 98.2 | 87.4 | 79.8 |
|
||||
| [`ca_core_news_md`](/models/ca#ca_core_news_md) | Catalan | 98.3 | 88.2 | 84.0 |
|
||||
| [`ca_core_news_lg`](/models/ca#ca_core_news_lg) | Catalan | 98.5 | 88.4 | 84.2 |
|
||||
| [`ca_core_news_trf`](/models/ca#ca_core_news_trf) | Catalan | 98.9 | 93.0 | 91.2 |
|
||||
| [`da_core_news_trf`](/models/da#da_core_news_trf) | Danish | 98.0 | 85.0 | 82.9 |
|
||||
| Package | Language | UPOS | Parser LAS | NER F |
|
||||
| ------------------------------------------------- | -------- | ---: | ---------: | ----: |
|
||||
| [`ca_core_news_sm`](/models/ca#ca_core_news_sm) | Catalan | 98.2 | 87.4 | 79.8 |
|
||||
| [`ca_core_news_md`](/models/ca#ca_core_news_md) | Catalan | 98.3 | 88.2 | 84.0 |
|
||||
| [`ca_core_news_lg`](/models/ca#ca_core_news_lg) | Catalan | 98.5 | 88.4 | 84.2 |
|
||||
| [`ca_core_news_trf`](/models/ca#ca_core_news_trf) | Catalan | 98.9 | 93.0 | 91.2 |
|
||||
| [`da_core_news_trf`](/models/da#da_core_news_trf) | Danish | 98.0 | 85.0 | 82.9 |
|
||||
|
||||
### Resizable text classification architectures {#resizable-textcat}
|
||||
|
||||
|
|
|
@ -116,7 +116,7 @@ import Benchmarks from 'usage/\_benchmarks-models.md'
|
|||
> corpus that had both syntactic and entity annotations, so the transformer
|
||||
> models for those languages do not include NER.
|
||||
|
||||
| Package | Language | Transformer | Tagger | Parser | NER |
|
||||
| Package | Language | Transformer | Tagger | Parser | NER |
|
||||
| ------------------------------------------------ | -------- | --------------------------------------------------------------------------------------------- | -----: | -----: | ---: |
|
||||
| [`en_core_web_trf`](/models/en#en_core_web_trf) | English | [`roberta-base`](https://huggingface.co/roberta-base) | 97.8 | 95.2 | 89.9 |
|
||||
| [`de_dep_news_trf`](/models/de#de_dep_news_trf) | German | [`bert-base-german-cased`](https://huggingface.co/bert-base-german-cased) | 99.0 | 95.8 | - |
|
||||
|
@ -856,9 +856,9 @@ attribute ruler before training using the `[initialize]` block of your config.
|
|||
|
||||
### Using Lexeme Tables
|
||||
|
||||
To use tables like `lexeme_prob` when training a model from scratch, you need
|
||||
to add an entry to the `initialize` block in your config. Here's what that
|
||||
looks like for the existing trained pipelines:
|
||||
To use tables like `lexeme_prob` when training a model from scratch, you need to
|
||||
add an entry to the `initialize` block in your config. Here's what that looks
|
||||
like for the existing trained pipelines:
|
||||
|
||||
```ini
|
||||
[initialize.lookups]
|
||||
|
|
|
@ -103,6 +103,7 @@
|
|||
{ "text": "SentenceRecognizer", "url": "/api/sentencerecognizer" },
|
||||
{ "text": "Sentencizer", "url": "/api/sentencizer" },
|
||||
{ "text": "SpanCategorizer", "url": "/api/spancategorizer" },
|
||||
{ "text": "SpanRuler", "url": "/api/spanruler" },
|
||||
{ "text": "Tagger", "url": "/api/tagger" },
|
||||
{ "text": "TextCategorizer", "url": "/api/textcategorizer" },
|
||||
{ "text": "Tok2Vec", "url": "/api/tok2vec" },
|
||||
|
|
|
@ -1,5 +1,25 @@
|
|||
{
|
||||
"resources": [
|
||||
{
|
||||
"id": "spacy-report",
|
||||
"title": "spacy-report",
|
||||
"slogan": "Generates interactive reports for spaCy models.",
|
||||
"description": "The goal of spacy-report is to offer static reports for spaCy models that help users make better decisions on how the models can be used.",
|
||||
"github": "koaning/spacy-report",
|
||||
"pip": "spacy-report",
|
||||
"thumb": "https://github.com/koaning/spacy-report/raw/main/icon.png",
|
||||
"image": "https://raw.githubusercontent.com/koaning/spacy-report/main/gif.gif",
|
||||
"code_example": [
|
||||
"python -m spacy report textcat training/model-best/ corpus/train.spacy corpus/dev.spacy"
|
||||
],
|
||||
"category": ["visualizers", "research"],
|
||||
"author": "Vincent D. Warmerdam",
|
||||
"author_links": {
|
||||
"twitter": "fishnets88",
|
||||
"github": "koaning",
|
||||
"website": "https://koaning.io"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "scrubadub_spacy",
|
||||
"title": "scrubadub_spacy",
|
||||
|
@ -2799,13 +2819,13 @@
|
|||
"id": "holmes",
|
||||
"title": "Holmes",
|
||||
"slogan": "Information extraction from English and German texts based on predicate logic",
|
||||
"github": "msg-systems/holmes-extractor",
|
||||
"url": "https://github.com/msg-systems/holmes-extractor",
|
||||
"description": "Holmes is a Python 3 library that supports a number of use cases involving information extraction from English and German texts, including chatbot, structural extraction, topic matching and supervised document classification. There is a [website demonstrating intelligent search based on topic matching](https://holmes-demo.xt.msg.team).",
|
||||
"github": "explosion/holmes-extractor",
|
||||
"url": "https://github.com/explosion/holmes-extractor",
|
||||
"description": "Holmes is a Python 3 library that supports a number of use cases involving information extraction from English and German texts, including chatbot, structural extraction, topic matching and supervised document classification. There is a [website demonstrating intelligent search based on topic matching](https://demo.holmes.prod.demos.explosion.services).",
|
||||
"pip": "holmes-extractor",
|
||||
"category": ["conversational", "standalone"],
|
||||
"category": ["pipeline", "standalone"],
|
||||
"tags": ["chatbots", "text-processing"],
|
||||
"thumb": "https://raw.githubusercontent.com/msg-systems/holmes-extractor/master/docs/holmes_thumbnail.png",
|
||||
"thumb": "https://raw.githubusercontent.com/explosion/holmes-extractor/master/docs/holmes_thumbnail.png",
|
||||
"code_example": [
|
||||
"import holmes_extractor as holmes",
|
||||
"holmes_manager = holmes.Manager(model='en_core_web_lg')",
|
||||
|
|
|
@ -23,6 +23,8 @@ const CUDA = {
|
|||
'11.2': 'cuda112',
|
||||
'11.3': 'cuda113',
|
||||
'11.4': 'cuda114',
|
||||
'11.5': 'cuda115',
|
||||
'11.6': 'cuda116',
|
||||
}
|
||||
const LANG_EXTRAS = ['ja'] // only for languages with models
|
||||
|
||||
|
@ -48,7 +50,7 @@ const QuickstartInstall = ({ id, title }) => {
|
|||
const modelExtras = train ? selectedModels.filter(m => LANG_EXTRAS.includes(m)) : []
|
||||
const apple = os === 'mac' && platform === 'arm'
|
||||
const pipExtras = [
|
||||
hardware === 'gpu' && cuda,
|
||||
(hardware === 'gpu' && (platform !== 'arm' || os === 'linux')) && cuda,
|
||||
train && 'transformers',
|
||||
train && 'lookups',
|
||||
apple && 'apple',
|
||||
|
|
Loading…
Reference in New Issue
Block a user