explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-28 02:46:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #12096 from svlandeg/copy_v4

Browse Source 
Sync with latest from master
This commit is contained in:
Sofie Van Landeghem 2023-01-11 20:46:33 +01:00 committed by GitHub
commit 2c2e66e145
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
266 changed files with 27733 additions and 34513 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.cfg
View File

@ -66,7 +66,7 @@ console_scripts =
lookups =
spacy_lookups_data>=1.0.3,<1.1.0
transformers =
spacy_transformers>=1.1.2,<1.2.0
spacy_transformers>=1.1.2,<1.3.0
ray =
spacy_ray>=0.1.0,<1.0.0
cuda =

4
spacy/cli/_util.py
View File

@ -583,6 +583,10 @@ def setup_gpu(use_gpu: int, silent=None) -> None:
def walk_directory(path: Path, suffix: Optional[str] = None) -> List[Path]:
"""Given a directory and a suffix, recursively find all files matching the suffix.
Directories or files with names beginning with a . are ignored, but hidden flags on
filesystems are not checked.
When provided with a suffix `None`, there is no suffix-based filtering."""
if not path.is_dir():
return [path]
paths = [path]

26
spacy/cli/convert.py
View File

@ -28,6 +28,8 @@ CONVERTERS: Mapping[str, Callable[..., Iterable[Doc]]] = {
"json": json_to_docs,
}
AUTO = "auto"
# File types that can be written to stdout
FILE_TYPES_STDOUT = ("json",)
@ -49,7 +51,7 @@ def convert_cli(
model: Optional[str] = Opt(None, "--model", "--base", "-b", help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)"),
morphology: bool = Opt(False, "--morphology", "-m", help="Enable appending morphology to tags"),
merge_subtokens: bool = Opt(False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"),
converter: str = Opt("auto", "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"),
converter: str = Opt(AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"),
ner_map: Optional[Path] = Opt(None, "--ner-map", "-nm", help="NER tag mapping (as JSON-encoded dict of entity types)", exists=True),
lang: Optional[str] = Opt(None, "--lang", "-l", help="Language (if tokenizer required)"),
concatenate: bool = Opt(None, "--concatenate", "-C", help="Concatenate output to a single file"),
@ -70,8 +72,8 @@ def convert_cli(
output_dir: Union[str, Path] = "-" if output_dir == Path("-") else output_dir
silent = output_dir == "-"
msg = Printer(no_print=silent)
verify_cli_args(msg, input_path, output_dir, file_type.value, converter, ner_map)
converter = _get_converter(msg, converter, input_path)
verify_cli_args(msg, input_path, output_dir, file_type.value, converter, ner_map)
convert(
input_path,
output_dir,
@ -100,7 +102,7 @@ def convert(
model: Optional[str] = None,
morphology: bool = False,
merge_subtokens: bool = False,
converter: str = "auto",
converter: str,
ner_map: Optional[Path] = None,
lang: Optional[str] = None,
concatenate: bool = False,
@ -212,18 +214,22 @@ def verify_cli_args(
input_locs = walk_directory(input_path, converter)
if len(input_locs) == 0:
msg.fail("No input files in directory", input_path, exits=1)
file_types = list(set([loc.suffix[1:] for loc in input_locs]))
if converter == "auto" and len(file_types) >= 2:
file_types_str = ",".join(file_types)
msg.fail("All input files must be same type", file_types_str, exits=1)
if converter != "auto" and converter not in CONVERTERS:
if converter not in CONVERTERS:
msg.fail(f"Can't find converter for {converter}", exits=1)
def _get_converter(msg, converter, input_path: Path):
if input_path.is_dir():
input_path = walk_directory(input_path, converter)[0]
if converter == "auto":
if converter == AUTO:
input_locs = walk_directory(input_path, suffix=None)
file_types = list(set([loc.suffix[1:] for loc in input_locs]))
if len(file_types) >= 2:
file_types_str = ",".join(file_types)
msg.fail("All input files must be same type", file_types_str, exits=1)
input_path = input_locs[0]
else:
input_path = walk_directory(input_path, suffix=converter)[0]
if converter == AUTO:
converter = input_path.suffix[1:]
if converter == "ner" or converter == "iob":
with input_path.open(encoding="utf8") as file_:

11
spacy/displacy/__init__.py
View File

@ -11,6 +11,7 @@ from .render import DependencyRenderer, EntityRenderer, SpanRenderer
from ..tokens import Doc, Span
from ..errors import Errors, Warnings
from ..util import is_in_jupyter
from ..util import find_available_port
_html = {}
@ -36,7 +37,7 @@ def render(
jupyter (bool): Override Jupyter auto-detection.
options (dict): Visualiser-specific options, e.g. colors.
manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts.
RETURNS (str): Rendered HTML markup.
RETURNS (str): Rendered SVG or HTML markup.
DOCS: https://spacy.io/api/top-level#displacy.render
USAGE: https://spacy.io/usage/visualizers
@ -82,6 +83,7 @@ def serve(
manual: bool = False,
port: int = 5000,
host: str = "0.0.0.0",
auto_select_port: bool = False,
) -> None:
"""Serve displaCy visualisation.
@ -93,15 +95,20 @@ def serve(
manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts.
port (int): Port to serve visualisation.
host (str): Host to serve visualisation.
auto_select_port (bool): Automatically select a port if the specified port is in use.
DOCS: https://spacy.io/api/top-level#displacy.serve
USAGE: https://spacy.io/usage/visualizers
"""
from wsgiref import simple_server
port = find_available_port(port, host, auto_select_port)
if is_in_jupyter():
warnings.warn(Warnings.W011)
render(docs, style=style, page=page, minify=minify, options=options, manual=manual)
render(
docs, style=style, page=page, minify=minify, options=options, manual=manual
)
httpd = simple_server.make_server(host, port, app)
print(f"\nUsing the '{style}' visualizer")
print(f"Serving on http://{host}:{port} ...\n")

4
spacy/displacy/render.py
View File

@ -94,7 +94,7 @@ class SpanRenderer:
parsed (list): Dependency parses to render.
page (bool): Render parses wrapped as full HTML page.
minify (bool): Minify HTML markup.
RETURNS (str): Rendered HTML markup.
RETURNS (str): Rendered SVG or HTML markup.
"""
rendered = []
for i, p in enumerate(parsed):
@ -510,7 +510,7 @@ class EntityRenderer:
parsed (list): Dependency parses to render.
page (bool): Render parses wrapped as full HTML page.
minify (bool): Minify HTML markup.
RETURNS (str): Rendered HTML markup.
RETURNS (str): Rendered SVG or HTML markup.
"""
rendered = []
for i, p in enumerate(parsed):

5
spacy/errors.py
View File

@ -207,6 +207,7 @@ class Warnings(metaclass=ErrorsWithCodes):
"is a Cython extension type.")
W123 = ("Argument `enable` with value {enable} does not contain all values specified in the config option "
"`enabled` ({enabled}). Be aware that this might affect other components in your pipeline.")
W124 = ("{host}:{port} is already in use, using the nearest available port {serve_port} as an alternative.")
class Errors(metaclass=ErrorsWithCodes):
@ -945,6 +946,10 @@ class Errors(metaclass=ErrorsWithCodes):
"knowledge base, use `InMemoryLookupKB`.")
E1047 = ("`find_threshold()` only supports components with a `scorer` attribute.")
E1048 = ("Got '{unexpected}' as console progress bar type, but expected one of the following: {expected}")
E1049 = ("No available port found for displaCy on host {host}. Please specify an available port "
"with `displacy.serve(doc, port)`")
E1050 = ("Port {port} is already in use. Please specify an available port with `displacy.serve(doc, port)` "
"or use `auto_switch_port=True` to pick an available port automatically.")
# v4 error strings
E4000 = ("Expected a Doc as input, but got: '{type}'")

17
spacy/matcher/levenshtein.pyx
View File

@ -4,6 +4,8 @@ from libc.stdint cimport int64_t
from typing import Optional
from ..util import registry
cdef extern from "polyleven.c":
int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k)
@ -13,3 +15,18 @@ cpdef int64_t levenshtein(a: str, b: str, k: Optional[int] = None):
if k is None:
k = -1
return polyleven(<PyObject*>a, <PyObject*>b, k)
cpdef bint levenshtein_compare(input_text: str, pattern_text: str, fuzzy: int = -1):
if fuzzy >= 0:
max_edits = fuzzy
else:
# allow at least two edits (to allow at least one transposition) and up
# to 20% of the pattern string length
max_edits = max(2, round(0.3 * len(pattern_text)))
return levenshtein(input_text, pattern_text, max_edits) <= max_edits
@registry.misc("spacy.levenshtein_compare.v1")
def make_levenshtein_compare():
return levenshtein_compare

1
spacy/matcher/matcher.pxd
View File

@ -77,3 +77,4 @@ cdef class Matcher:
cdef public object _extensions
cdef public object _extra_predicates
cdef public object _seen_attrs
cdef public object _fuzzy_compare

3
spacy/matcher/matcher.pyi
View File

@ -5,7 +5,8 @@ from ..vocab import Vocab
from ..tokens import Doc, Span
class Matcher:
def __init__(self, vocab: Vocab, validate: bool = ...) -> None: ...
def __init__(self, vocab: Vocab, validate: bool = ...,
fuzzy_compare: Callable[[str, str, int], bool] = ...) -> None: ...
def __reduce__(self) -> Any: ...
def __len__(self) -> int: ...
def __contains__(self, key: str) -> bool: ...

170
spacy/matcher/matcher.pyx
View File

@ -1,4 +1,4 @@
# cython: infer_types=True, profile=True
# cython: binding=True, infer_types=True, profile=True
from typing import List, Iterable
from libcpp.vector cimport vector
@ -20,10 +20,12 @@ from ..tokens.token cimport Token
from ..tokens.morphanalysis cimport MorphAnalysis
from ..attrs cimport ID, attr_id_t, NULL_ATTR, ORTH, POS, TAG, DEP, LEMMA, MORPH, ENT_IOB
from .levenshtein import levenshtein_compare
from ..schemas import validate_token_pattern
from ..errors import Errors, MatchPatternError, Warnings
from ..strings cimport get_string_id
from ..attrs import IDS
from ..util import registry
DEF PADDING = 5
@ -36,11 +38,13 @@ cdef class Matcher:
USAGE: https://spacy.io/usage/rule-based-matching
"""
def __init__(self, vocab, validate=True):
def __init__(self, vocab, validate=True, *, fuzzy_compare=levenshtein_compare):
"""Create the Matcher.
vocab (Vocab): The vocabulary object, which must be shared with the
documents the matcher will operate on.
validate (bool): Validate all patterns added to this matcher.
fuzzy_compare (Callable[[str, str, int], bool]): The comparison method
for the FUZZY operators.
"""
self._extra_predicates = []
self._patterns = {}
@ -51,9 +55,10 @@ cdef class Matcher:
self.vocab = vocab
self.mem = Pool()
self.validate = validate
self._fuzzy_compare = fuzzy_compare
def __reduce__(self):
data = (self.vocab, self._patterns, self._callbacks)
data = (self.vocab, self._patterns, self._callbacks, self.validate, self._fuzzy_compare)
return (unpickle_matcher, data, None, None)
def __len__(self):
@ -128,7 +133,7 @@ cdef class Matcher:
for pattern in patterns:
try:
specs = _preprocess_pattern(pattern, self.vocab,
self._extensions, self._extra_predicates)
self._extensions, self._extra_predicates, self._fuzzy_compare)
self.patterns.push_back(init_pattern(self.mem, key, specs))
for spec in specs:
for attr, _ in spec[1]:
@ -327,8 +332,8 @@ cdef class Matcher:
return key
def unpickle_matcher(vocab, patterns, callbacks):
matcher = Matcher(vocab)
def unpickle_matcher(vocab, patterns, callbacks, validate, fuzzy_compare):
matcher = Matcher(vocab, validate=validate, fuzzy_compare=fuzzy_compare)
for key, pattern in patterns.items():
callback = callbacks.get(key, None)
matcher.add(key, pattern, on_match=callback)
@ -755,7 +760,7 @@ cdef attr_t get_ent_id(const TokenPatternC* pattern) nogil:
return id_attr.value
def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates):
def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates, fuzzy_compare):
"""This function interprets the pattern, converting the various bits of
syntactic sugar before we compile it into a struct with init_pattern.
@ -782,7 +787,7 @@ def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates):
ops = _get_operators(spec)
attr_values = _get_attr_values(spec, string_store)
extensions = _get_extensions(spec, string_store, extensions_table)
predicates = _get_extra_predicates(spec, extra_predicates, vocab)
predicates = _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare)
for op in ops:
tokens.append((op, list(attr_values), list(extensions), list(predicates), token_idx))
return tokens
@ -827,16 +832,45 @@ def _get_attr_values(spec, string_store):
# These predicate helper classes are used to match the REGEX, IN, >= etc
# extensions to the matcher introduced in #3173.
class _FuzzyPredicate:
operators = ("FUZZY", "FUZZY1", "FUZZY2", "FUZZY3", "FUZZY4", "FUZZY5",
"FUZZY6", "FUZZY7", "FUZZY8", "FUZZY9")
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None,
regex=False, fuzzy=None, fuzzy_compare=None):
self.i = i
self.attr = attr
self.value = value
self.predicate = predicate
self.is_extension = is_extension
if self.predicate not in self.operators:
raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate))
fuzz = self.predicate[len("FUZZY"):] # number after prefix
self.fuzzy = int(fuzz) if fuzz else -1
self.fuzzy_compare = fuzzy_compare
self.key = (self.attr, self.fuzzy, self.predicate, srsly.json_dumps(value, sort_keys=True))
def __call__(self, Token token):
if self.is_extension:
value = token._.get(self.attr)
else:
value = token.vocab.strings[get_token_attr_for_matcher(token.c, self.attr)]
if self.value == value:
return True
return self.fuzzy_compare(value, self.value, self.fuzzy)
class _RegexPredicate:
operators = ("REGEX",)
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None):
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None,
regex=False, fuzzy=None, fuzzy_compare=None):
self.i = i
self.attr = attr
self.value = re.compile(value)
self.predicate = predicate
self.is_extension = is_extension
self.key = (attr, self.predicate, srsly.json_dumps(value, sort_keys=True))
self.key = (self.attr, self.predicate, srsly.json_dumps(value, sort_keys=True))
if self.predicate not in self.operators:
raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate))
@ -851,18 +885,28 @@ class _RegexPredicate:
class _SetPredicate:
operators = ("IN", "NOT_IN", "IS_SUBSET", "IS_SUPERSET", "INTERSECTS")
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None):
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None,
regex=False, fuzzy=None, fuzzy_compare=None):
self.i = i
self.attr = attr
self.vocab = vocab
self.regex = regex
self.fuzzy = fuzzy
self.fuzzy_compare = fuzzy_compare
if self.attr == MORPH:
# normalize morph strings
self.value = set(self.vocab.morphology.add(v) for v in value)
else:
self.value = set(get_string_id(v) for v in value)
if self.regex:
self.value = set(re.compile(v) for v in value)
elif self.fuzzy is not None:
# add to string store
self.value = set(self.vocab.strings.add(v) for v in value)
else:
self.value = set(get_string_id(v) for v in value)
self.predicate = predicate
self.is_extension = is_extension
self.key = (attr, self.predicate, srsly.json_dumps(value, sort_keys=True))
self.key = (self.attr, self.regex, self.fuzzy, self.predicate, srsly.json_dumps(value, sort_keys=True))
if self.predicate not in self.operators:
raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate))
@ -890,9 +934,29 @@ class _SetPredicate:
return False
if self.predicate == "IN":
return value in self.value
if self.regex:
value = self.vocab.strings[value]
return any(bool(v.search(value)) for v in self.value)
elif self.fuzzy is not None:
value = self.vocab.strings[value]
return any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy)
for v in self.value)
elif value in self.value:
return True
else:
return False
elif self.predicate == "NOT_IN":
return value not in self.value
if self.regex:
value = self.vocab.strings[value]
return not any(bool(v.search(value)) for v in self.value)
elif self.fuzzy is not None:
value = self.vocab.strings[value]
return not any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy)
for v in self.value)
elif value in self.value:
return False
else:
return True
elif self.predicate == "IS_SUBSET":
return value <= self.value
elif self.predicate == "IS_SUPERSET":
@ -907,13 +971,14 @@ class _SetPredicate:
class _ComparisonPredicate:
operators = ("==", "!=", ">=", "<=", ">", "<")
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None):
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None,
regex=False, fuzzy=None, fuzzy_compare=None):
self.i = i
self.attr = attr
self.value = value
self.predicate = predicate
self.is_extension = is_extension
self.key = (attr, self.predicate, srsly.json_dumps(value, sort_keys=True))
self.key = (self.attr, self.predicate, srsly.json_dumps(value, sort_keys=True))
if self.predicate not in self.operators:
raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate))
@ -936,7 +1001,7 @@ class _ComparisonPredicate:
return value < self.value
def _get_extra_predicates(spec, extra_predicates, vocab):
def _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare):
predicate_types = {
"REGEX": _RegexPredicate,
"IN": _SetPredicate,
@ -950,6 +1015,16 @@ def _get_extra_predicates(spec, extra_predicates, vocab):
"<=": _ComparisonPredicate,
">": _ComparisonPredicate,
"<": _ComparisonPredicate,
"FUZZY": _FuzzyPredicate,
"FUZZY1": _FuzzyPredicate,
"FUZZY2": _FuzzyPredicate,
"FUZZY3": _FuzzyPredicate,
"FUZZY4": _FuzzyPredicate,
"FUZZY5": _FuzzyPredicate,
"FUZZY6": _FuzzyPredicate,
"FUZZY7": _FuzzyPredicate,
"FUZZY8": _FuzzyPredicate,
"FUZZY9": _FuzzyPredicate,
}
seen_predicates = {pred.key: pred.i for pred in extra_predicates}
output = []
@ -967,22 +1042,47 @@ def _get_extra_predicates(spec, extra_predicates, vocab):
attr = "ORTH"
attr = IDS.get(attr.upper())
if isinstance(value, dict):
processed = False
value_with_upper_keys = {k.upper(): v for k, v in value.items()}
for type_, cls in predicate_types.items():
if type_ in value_with_upper_keys:
predicate = cls(len(extra_predicates), attr, value_with_upper_keys[type_], type_, vocab=vocab)
# Don't create a redundant predicates.
# This helps with efficiency, as we're caching the results.
if predicate.key in seen_predicates:
output.append(seen_predicates[predicate.key])
else:
extra_predicates.append(predicate)
output.append(predicate.i)
seen_predicates[predicate.key] = predicate.i
processed = True
if not processed:
warnings.warn(Warnings.W035.format(pattern=value))
output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types,
extra_predicates, seen_predicates, fuzzy_compare=fuzzy_compare))
return output
def _get_extra_predicates_dict(attr, value_dict, vocab, predicate_types,
extra_predicates, seen_predicates, regex=False, fuzzy=None, fuzzy_compare=None):
output = []
for type_, value in value_dict.items():
type_ = type_.upper()
cls = predicate_types.get(type_)
if cls is None:
warnings.warn(Warnings.W035.format(pattern=value_dict))
# ignore unrecognized predicate type
continue
elif cls == _RegexPredicate:
if isinstance(value, dict):
# add predicates inside regex operator
output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types,
extra_predicates, seen_predicates,
regex=True))
continue
elif cls == _FuzzyPredicate:
if isinstance(value, dict):
# add predicates inside fuzzy operator
fuzz = type_[len("FUZZY"):] # number after prefix
fuzzy_val = int(fuzz) if fuzz else -1
output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types,
extra_predicates, seen_predicates,
fuzzy=fuzzy_val, fuzzy_compare=fuzzy_compare))
continue
predicate = cls(len(extra_predicates), attr, value, type_, vocab=vocab,
regex=regex, fuzzy=fuzzy, fuzzy_compare=fuzzy_compare)
# Don't create redundant predicates.
# This helps with efficiency, as we're caching the results.
if predicate.key in seen_predicates:
output.append(seen_predicates[predicate.key])
else:
extra_predicates.append(predicate)
output.append(predicate.i)
seen_predicates[predicate.key] = predicate.i
return output

0
spacy/pipeline/entityruler.py Normal file
View File

18
spacy/pipeline/span_ruler.py
View File

@ -13,6 +13,7 @@ from ..util import ensure_path, SimpleFrozenList, registry
from ..tokens import Doc, Span
from ..scorer import Scorer, get_ner_prf
from ..matcher import Matcher, PhraseMatcher
from ..matcher.levenshtein import levenshtein_compare
from .. import util
PatternType = Dict[str, Union[str, List[Dict[str, Any]]]]
@ -28,6 +29,7 @@ DEFAULT_SPANS_KEY = "ruler"
"overwrite_ents": False,
"scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"},
"ent_id_sep": "__unused__",
"matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"},
},
default_score_weights={
"ents_f": 1.0,
@ -40,6 +42,7 @@ def make_entity_ruler(
nlp: Language,
name: str,
phrase_matcher_attr: Optional[Union[int, str]],
matcher_fuzzy_compare: Callable,
validate: bool,
overwrite_ents: bool,
scorer: Optional[Callable],
@ -57,6 +60,7 @@ def make_entity_ruler(
annotate_ents=True,
ents_filter=ents_filter,
phrase_matcher_attr=phrase_matcher_attr,
matcher_fuzzy_compare=matcher_fuzzy_compare,
validate=validate,
overwrite=False,
scorer=scorer,
@ -81,6 +85,7 @@ def make_entity_ruler_scorer():
"annotate_ents": False,
"ents_filter": {"@misc": "spacy.first_longest_spans_filter.v1"},
"phrase_matcher_attr": None,
"matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"},
"validate": False,
"overwrite": True,
"scorer": {
@ -103,6 +108,7 @@ def make_span_ruler(
annotate_ents: bool,
ents_filter: Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]],
phrase_matcher_attr: Optional[Union[int, str]],
matcher_fuzzy_compare: Callable,
validate: bool,
overwrite: bool,
scorer: Optional[Callable],
@ -115,6 +121,7 @@ def make_span_ruler(
annotate_ents=annotate_ents,
ents_filter=ents_filter,
phrase_matcher_attr=phrase_matcher_attr,
matcher_fuzzy_compare=matcher_fuzzy_compare,
validate=validate,
overwrite=overwrite,
scorer=scorer,
@ -225,6 +232,7 @@ class SpanRuler(Pipe):
[Iterable[Span], Iterable[Span]], Iterable[Span]
] = util.filter_chain_spans,
phrase_matcher_attr: Optional[Union[int, str]] = None,
matcher_fuzzy_compare: Callable = levenshtein_compare,
validate: bool = False,
overwrite: bool = False,
scorer: Optional[Callable] = partial(
@ -255,6 +263,9 @@ class SpanRuler(Pipe):
phrase_matcher_attr (Optional[Union[int, str]]): Token attribute to
match on, passed to the internal PhraseMatcher as `attr`. Defaults
to `None`.
matcher_fuzzy_compare (Callable): The fuzzy comparison method for the
internal Matcher. Defaults to
spacy.matcher.levenshtein.levenshtein_compare.
validate (bool): Whether patterns should be validated, passed to
Matcher and PhraseMatcher as `validate`.
overwrite (bool): Whether to remove any existing spans under this spans
@ -275,6 +286,7 @@ class SpanRuler(Pipe):
self.spans_filter = spans_filter
self.ents_filter = ents_filter
self.scorer = scorer
self.matcher_fuzzy_compare = matcher_fuzzy_compare
self._match_label_id_map: Dict[int, Dict[str, str]] = {}
self.clear()
@ -460,7 +472,11 @@ class SpanRuler(Pipe):
DOCS: https://spacy.io/api/spanruler#clear
"""
self._patterns: List[PatternType] = []
self.matcher: Matcher = Matcher(self.nlp.vocab, validate=self.validate)
self.matcher: Matcher = Matcher(
self.nlp.vocab,
validate=self.validate,
fuzzy_compare=self.matcher_fuzzy_compare,
)
self.phrase_matcher: PhraseMatcher = PhraseMatcher(
self.nlp.vocab,
attr=self.phrase_matcher_attr,

4
spacy/pipeline/textcat_multilabel.py
View File

@ -74,7 +74,7 @@ subword_features = true
default_config={
"threshold": 0.5,
"model": DEFAULT_MULTI_TEXTCAT_MODEL,
"scorer": {"@scorers": "spacy.textcat_multilabel_scorer.v1"},
"scorer": {"@scorers": "spacy.textcat_multilabel_scorer.v2"},
"save_activations": False,
},
default_score_weights={
@ -127,7 +127,7 @@ def textcat_multilabel_score(examples: Iterable[Example], **kwargs) -> Dict[str,
)
@registry.scorers("spacy.textcat_multilabel_scorer.v1")
@registry.scorers("spacy.textcat_multilabel_scorer.v2")
def make_textcat_multilabel_scorer():
return textcat_multilabel_score

12
spacy/schemas.py
View File

@ -156,12 +156,22 @@ def validate_token_pattern(obj: list) -> List[str]:
class TokenPatternString(BaseModel):
REGEX: Optional[StrictStr] = Field(None, alias="regex")
REGEX: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="regex")
IN: Optional[List[StrictStr]] = Field(None, alias="in")
NOT_IN: Optional[List[StrictStr]] = Field(None, alias="not_in")
IS_SUBSET: Optional[List[StrictStr]] = Field(None, alias="is_subset")
IS_SUPERSET: Optional[List[StrictStr]] = Field(None, alias="is_superset")
INTERSECTS: Optional[List[StrictStr]] = Field(None, alias="intersects")
FUZZY: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy")
FUZZY1: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy1")
FUZZY2: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy2")
FUZZY3: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy3")
FUZZY4: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy4")
FUZZY5: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy5")
FUZZY6: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy6")
FUZZY7: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy7")
FUZZY8: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy8")
FUZZY9: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy9")
class Config:
extra = "forbid"

8
spacy/scorer.py
View File

@ -174,7 +174,7 @@ class Scorer:
prf_score.score_set(pred_spans, gold_spans)
if len(acc_score) > 0:
return {
"token_acc": acc_score.fscore,
"token_acc": acc_score.precision,
"token_p": prf_score.precision,
"token_r": prf_score.recall,
"token_f": prf_score.fscore,
@ -476,14 +476,12 @@ class Scorer:
f_per_type = {label: PRFScore() for label in labels}
auc_per_type = {label: ROCAUCScore() for label in labels}
labels = set(labels)
if labels:
for eg in examples:
labels.update(eg.predicted.cats.keys())
labels.update(eg.reference.cats.keys())
for example in examples:
# Through this loop, None in the gold_cats indicates missing label.
pred_cats = getter(example.predicted, attr)
pred_cats = {k: v for k, v in pred_cats.items() if k in labels}
gold_cats = getter(example.reference, attr)
gold_cats = {k: v for k, v in gold_cats.items() if k in labels}
for label in labels:
pred_score = pred_cats.get(label, 0.0)

29
spacy/tests/matcher/test_levenshtein.py
View File

@ -1,5 +1,6 @@
import pytest
from spacy.matcher import levenshtein
from spacy.matcher.levenshtein import levenshtein_compare
# empty string plus 10 random ASCII, 10 random unicode, and 2 random long tests
@ -42,3 +43,31 @@ from spacy.matcher import levenshtein
)
def test_levenshtein(dist, a, b):
assert levenshtein(a, b) == dist
@pytest.mark.parametrize(
"a,b,fuzzy,expected",
[
("a", "a", 1, True),
("a", "a", 0, True),
("a", "a", -1, True),
("a", "ab", 1, True),
("a", "ab", 0, False),
("a", "ab", -1, True),
("ab", "ac", 1, True),
("ab", "ac", -1, True),
("abc", "cde", 4, True),
("abc", "cde", -1, False),
("abcdef", "cdefgh", 4, True),
("abcdef", "cdefgh", 3, False),
("abcdef", "cdefgh", -1, False), # default (2 for length 6)
("abcdefgh", "cdefghijk", 5, True),
("abcdefgh", "cdefghijk", 4, False),
("abcdefgh", "cdefghijk", -1, False), # default (2)
("abcdefgh", "cdefghijkl", 6, True),
("abcdefgh", "cdefghijkl", 5, False),
("abcdefgh", "cdefghijkl", -1, False), # default (2)
],
)
def test_levenshtein_compare(a, b, fuzzy, expected):
assert levenshtein_compare(a, b, fuzzy) == expected

173
spacy/tests/matcher/test_matcher_api.py
View File

@ -115,6 +115,155 @@ def test_matcher_match_multi(matcher):
]
@pytest.mark.parametrize(
"rules,match_locs",
[
(
{
"GoogleNow": [[{"ORTH": {"FUZZY": "Google"}}, {"ORTH": "Now"}]],
},
[(2, 4)],
),
(
{
"Java": [[{"LOWER": {"FUZZY": "java"}}]],
},
[(5, 6)],
),
(
{
"JS": [[{"ORTH": {"FUZZY": "JavaScript"}}]],
"GoogleNow": [[{"ORTH": {"FUZZY": "Google"}}, {"ORTH": "Now"}]],
"Java": [[{"LOWER": {"FUZZY": "java"}}]],
},
[(2, 4), (5, 6), (8, 9)],
),
# only the second pattern matches (check that predicate keys used for
# caching don't collide)
(
{
"A": [[{"ORTH": {"FUZZY": "Javascripts"}}]],
"B": [[{"ORTH": {"FUZZY5": "Javascripts"}}]],
},
[(8, 9)],
),
],
)
def test_matcher_match_fuzzy(en_vocab, rules, match_locs):
words = ["They", "like", "Goggle", "Now", "and", "Jav", "but", "not", "JvvaScrpt"]
doc = Doc(en_vocab, words=words)
matcher = Matcher(en_vocab)
for key, patterns in rules.items():
matcher.add(key, patterns)
assert match_locs == [(start, end) for m_id, start, end in matcher(doc)]
@pytest.mark.parametrize("set_op", ["IN", "NOT_IN"])
def test_matcher_match_fuzzy_set_op_longest(en_vocab, set_op):
rules = {
"GoogleNow": [[{"ORTH": {"FUZZY": {set_op: ["Google", "Now"]}}, "OP": "+"}]]
}
matcher = Matcher(en_vocab)
for key, patterns in rules.items():
matcher.add(key, patterns, greedy="LONGEST")
words = ["They", "like", "Goggle", "Noo"]
doc = Doc(en_vocab, words=words)
assert len(matcher(doc)) == 1
def test_matcher_match_fuzzy_set_multiple(en_vocab):
rules = {
"GoogleNow": [
[
{
"ORTH": {"FUZZY": {"IN": ["Google", "Now"]}, "NOT_IN": ["Goggle"]},
"OP": "+",
}
]
]
}
matcher = Matcher(en_vocab)
for key, patterns in rules.items():
matcher.add(key, patterns, greedy="LONGEST")
words = ["They", "like", "Goggle", "Noo"]
doc = Doc(matcher.vocab, words=words)
assert matcher(doc) == [
(doc.vocab.strings["GoogleNow"], 3, 4),
]
@pytest.mark.parametrize("fuzzyn", range(1, 10))
def test_matcher_match_fuzzyn_all_insertions(en_vocab, fuzzyn):
matcher = Matcher(en_vocab)
matcher.add("GoogleNow", [[{"ORTH": {f"FUZZY{fuzzyn}": "GoogleNow"}}]])
# words with increasing edit distance
words = ["GoogleNow" + "a" * i for i in range(0, 10)]
doc = Doc(en_vocab, words)
assert len(matcher(doc)) == fuzzyn + 1
@pytest.mark.parametrize("fuzzyn", range(1, 6))
def test_matcher_match_fuzzyn_various_edits(en_vocab, fuzzyn):
matcher = Matcher(en_vocab)
matcher.add("GoogleNow", [[{"ORTH": {f"FUZZY{fuzzyn}": "GoogleNow"}}]])
# words with increasing edit distance of different edit types
words = [
"GoogleNow",
"GoogleNuw",
"GoogleNuew",
"GoogleNoweee",
"GiggleNuw3",
"gouggle5New",
]
doc = Doc(en_vocab, words)
assert len(matcher(doc)) == fuzzyn + 1
@pytest.mark.parametrize("greedy", ["FIRST", "LONGEST"])
@pytest.mark.parametrize("set_op", ["IN", "NOT_IN"])
def test_matcher_match_fuzzyn_set_op_longest(en_vocab, greedy, set_op):
rules = {
"GoogleNow": [[{"ORTH": {"FUZZY2": {set_op: ["Google", "Now"]}}, "OP": "+"}]]
}
matcher = Matcher(en_vocab)
for key, patterns in rules.items():
matcher.add(key, patterns, greedy=greedy)
words = ["They", "like", "Goggle", "Noo"]
doc = Doc(matcher.vocab, words=words)
spans = matcher(doc, as_spans=True)
assert len(spans) == 1
if set_op == "IN":
assert spans[0].text == "Goggle Noo"
else:
assert spans[0].text == "They like"
def test_matcher_match_fuzzyn_set_multiple(en_vocab):
rules = {
"GoogleNow": [
[
{
"ORTH": {"FUZZY1": {"IN": ["Google", "Now"]}, "NOT_IN": ["Goggle"]},
"OP": "+",
}
]
]
}
matcher = Matcher(en_vocab)
for key, patterns in rules.items():
matcher.add(key, patterns, greedy="LONGEST")
words = ["They", "like", "Goggle", "Noo"]
doc = Doc(matcher.vocab, words=words)
assert matcher(doc) == [
(doc.vocab.strings["GoogleNow"], 3, 4),
]
def test_matcher_empty_dict(en_vocab):
"""Test matcher allows empty token specs, meaning match on any token."""
matcher = Matcher(en_vocab)
@ -434,6 +583,30 @@ def test_matcher_regex(en_vocab):
assert len(matches) == 0
def test_matcher_regex_set_in(en_vocab):
matcher = Matcher(en_vocab)
pattern = [{"ORTH": {"REGEX": {"IN": [r"(?:a)", r"(?:an)"]}}}]
matcher.add("A_OR_AN", [pattern])
doc = Doc(en_vocab, words=["an", "a", "hi"])
matches = matcher(doc)
assert len(matches) == 2
doc = Doc(en_vocab, words=["bye"])
matches = matcher(doc)
assert len(matches) == 0
def test_matcher_regex_set_not_in(en_vocab):
matcher = Matcher(en_vocab)
pattern = [{"ORTH": {"REGEX": {"NOT_IN": [r"(?:a)", r"(?:an)"]}}}]
matcher.add("A_OR_AN", [pattern])
doc = Doc(en_vocab, words=["an", "a", "hi"])
matches = matcher(doc)
assert len(matches) == 1
doc = Doc(en_vocab, words=["bye"])
matches = matcher(doc)
assert len(matches) == 1
def test_matcher_regex_shape(en_vocab):
matcher = Matcher(en_vocab)
pattern = [{"SHAPE": {"REGEX": r"^[^x]+$"}}]

33
spacy/tests/pipeline/test_entity_ruler.py
View File

@ -353,6 +353,39 @@ def test_entity_ruler_overlapping_spans(nlp):
assert doc.ents[0].label_ == "FOOBAR"
def test_entity_ruler_fuzzy_pipe(nlp):
ruler = nlp.add_pipe("entity_ruler")
patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}]
ruler.add_patterns(patterns)
doc = nlp("helloo")
assert len(doc.ents) == 1
assert doc.ents[0].label_ == "HELLO"
def test_entity_ruler_fuzzy(nlp):
ruler = nlp.add_pipe("entity_ruler")
patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}]
ruler.add_patterns(patterns)
doc = nlp("helloo")
assert len(doc.ents) == 1
assert doc.ents[0].label_ == "HELLO"
def test_entity_ruler_fuzzy_disabled(nlp):
@registry.misc("test_fuzzy_compare_disabled")
def make_test_fuzzy_compare_disabled():
return lambda x, y, z: False
ruler = nlp.add_pipe(
"entity_ruler",
config={"matcher_fuzzy_compare": {"@misc": "test_fuzzy_compare_disabled"}},
)
patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}]
ruler.add_patterns(patterns)
doc = nlp("helloo")
assert len(doc.ents) == 0
@pytest.mark.parametrize("n_process", [1, 2])
def test_entity_ruler_multiprocessing(nlp, n_process):
if isinstance(get_current_ops, NumpyOps) or n_process < 2:

6
spacy/tests/pipeline/test_textcat.py
View File

@ -937,7 +937,11 @@ def test_save_activations_multi():
@pytest.mark.parametrize(
"component_name,scorer", [("textcat", "spacy.textcat_scorer.v1")]
"component_name,scorer",
[
("textcat", "spacy.textcat_scorer.v1"),
("textcat_multilabel", "spacy.textcat_multilabel_scorer.v1"),
],
)
def test_textcat_legacy_scorers(component_name, scorer):
"""Check that legacy scorers are registered and produce the expected score

26
spacy/tests/test_cli.py
View File

@ -4,6 +4,7 @@ from collections import Counter
from typing import Tuple, List, Dict, Any
import pkg_resources
import time
from pathlib import Path
import spacy
import numpy
@ -15,7 +16,7 @@ from thinc.api import Config, ConfigValidationError
from spacy import about
from spacy.cli import info
from spacy.cli._util import is_subpath_of, load_project_config
from spacy.cli._util import is_subpath_of, load_project_config, walk_directory
from spacy.cli._util import parse_config_overrides, string_to_list
from spacy.cli._util import substitute_project_variables
from spacy.cli._util import validate_project_commands
@ -1185,3 +1186,26 @@ def test_upload_download_local_file():
download_file(remote_file, local_file)
with local_file.open(mode="r") as file_:
assert file_.read() == content
def test_walk_directory():
with make_tempdir() as d:
files = [
"data1.iob",
"data2.iob",
"data3.json",
"data4.conll",
"data5.conll",
"data6.conll",
"data7.txt",
]
for f in files:
Path(d / f).touch()
assert (len(walk_directory(d))) == 7
assert (len(walk_directory(d, suffix=None))) == 7
assert (len(walk_directory(d, suffix="json"))) == 1
assert (len(walk_directory(d, suffix="iob"))) == 2
assert (len(walk_directory(d, suffix="conll"))) == 3
assert (len(walk_directory(d, suffix="pdf"))) == 0

33
spacy/tests/test_cli_app.py Normal file
View File

@ -0,0 +1,33 @@
import os
from pathlib import Path
from typer.testing import CliRunner
from spacy.cli._util import app
from .util import make_tempdir
def test_convert_auto():
with make_tempdir() as d_in, make_tempdir() as d_out:
for f in ["data1.iob", "data2.iob", "data3.iob"]:
Path(d_in / f).touch()
# ensure that "automatic" suffix detection works
result = CliRunner().invoke(app, ["convert", str(d_in), str(d_out)])
assert "Generated output file" in result.stdout
out_files = os.listdir(d_out)
assert len(out_files) == 3
assert "data1.spacy" in out_files
assert "data2.spacy" in out_files
assert "data3.spacy" in out_files
def test_convert_auto_conflict():
with make_tempdir() as d_in, make_tempdir() as d_out:
for f in ["data1.iob", "data2.iob", "data3.json"]:
Path(d_in / f).touch()
# ensure that "automatic" suffix detection warns when there are different file types
result = CliRunner().invoke(app, ["convert", str(d_in), str(d_out)])
assert "All input files must be same type" in result.stdout
out_files = os.listdir(d_out)
assert len(out_files) == 0

107
spacy/tests/test_language.py
View File

@ -3,6 +3,7 @@ import logging
from unittest import mock
import pytest
from spacy.language import Language
from spacy.scorer import Scorer
from spacy.tokens import Doc, Span
from spacy.vocab import Vocab
from spacy.training import Example
@ -126,6 +127,112 @@ def test_evaluate_no_pipe(nlp):
nlp.evaluate([Example.from_dict(doc, annots)])
def test_evaluate_textcat_multilabel(en_vocab):
"""Test that evaluate works with a multilabel textcat pipe."""
nlp = Language(en_vocab)
textcat_multilabel = nlp.add_pipe("textcat_multilabel")
for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"):
textcat_multilabel.add_label(label)
nlp.initialize()
annots = {"cats": {"FEATURE": 1.0, "QUESTION": 1.0}}
doc = nlp.make_doc("hello world")
example = Example.from_dict(doc, annots)
scores = nlp.evaluate([example])
labels = nlp.get_pipe("textcat_multilabel").labels
for label in labels:
assert scores["cats_f_per_type"].get(label) is not None
for key in example.reference.cats.keys():
if key not in labels:
assert scores["cats_f_per_type"].get(key) is None
def test_evaluate_multiple_textcat_final(en_vocab):
"""Test that evaluate evaluates the final textcat component in a pipeline
with more than one textcat or textcat_multilabel."""
nlp = Language(en_vocab)
textcat = nlp.add_pipe("textcat")
for label in ("POSITIVE", "NEGATIVE"):
textcat.add_label(label)
textcat_multilabel = nlp.add_pipe("textcat_multilabel")
for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"):
textcat_multilabel.add_label(label)
nlp.initialize()
annots = {
"cats": {
"POSITIVE": 1.0,
"NEGATIVE": 0.0,
"FEATURE": 1.0,
"QUESTION": 1.0,
"POSITIVE": 1.0,
"NEGATIVE": 0.0,
}
}
doc = nlp.make_doc("hello world")
example = Example.from_dict(doc, annots)
scores = nlp.evaluate([example])
# get the labels from the final pipe
labels = nlp.get_pipe(nlp.pipe_names[-1]).labels
for label in labels:
assert scores["cats_f_per_type"].get(label) is not None
for key in example.reference.cats.keys():
if key not in labels:
assert scores["cats_f_per_type"].get(key) is None
def test_evaluate_multiple_textcat_separate(en_vocab):
"""Test that evaluate can evaluate multiple textcat components separately
with custom scorers."""
def custom_textcat_score(examples, **kwargs):
scores = Scorer.score_cats(
examples,
"cats",
multi_label=False,
**kwargs,
)
return {f"custom_{k}": v for k, v in scores.items()}
@spacy.registry.scorers("test_custom_textcat_scorer")
def make_custom_textcat_scorer():
return custom_textcat_score
nlp = Language(en_vocab)
textcat = nlp.add_pipe(
"textcat",
config={"scorer": {"@scorers": "test_custom_textcat_scorer"}},
)
for label in ("POSITIVE", "NEGATIVE"):
textcat.add_label(label)
textcat_multilabel = nlp.add_pipe("textcat_multilabel")
for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"):
textcat_multilabel.add_label(label)
nlp.initialize()
annots = {
"cats": {
"POSITIVE": 1.0,
"NEGATIVE": 0.0,
"FEATURE": 1.0,
"QUESTION": 1.0,
"POSITIVE": 1.0,
"NEGATIVE": 0.0,
}
}
doc = nlp.make_doc("hello world")
example = Example.from_dict(doc, annots)
scores = nlp.evaluate([example])
# check custom scores for the textcat pipe
assert "custom_cats_f_per_type" in scores
labels = nlp.get_pipe("textcat").labels
assert set(scores["custom_cats_f_per_type"].keys()) == set(labels)
# check default scores for the textcat_multilabel pipe
assert "cats_f_per_type" in scores
labels = nlp.get_pipe("textcat_multilabel").labels
assert set(scores["cats_f_per_type"].keys()) == set(labels)
def vector_modification_pipe(doc):
doc.vector += 1
return doc

15
spacy/tests/test_misc.py
View File

@ -8,7 +8,7 @@ from spacy import prefer_gpu, require_gpu, require_cpu
from spacy.ml._precomputable_affine import PrecomputableAffine
from spacy.ml._precomputable_affine import _backprop_precomputable_affine_padding
from spacy.util import dot_to_object, SimpleFrozenList, import_file
from spacy.util import to_ternary_int
from spacy.util import to_ternary_int, find_available_port
from thinc.api import Config, Optimizer, ConfigValidationError
from thinc.api import get_current_ops, set_current_ops, NumpyOps, CupyOps, MPSOps
from thinc.compat import has_cupy_gpu, has_torch_mps_gpu
@ -434,3 +434,16 @@ def test_to_ternary_int():
assert to_ternary_int(-10) == -1
assert to_ternary_int("string") == -1
assert to_ternary_int([0, "string"]) == -1
def test_find_available_port():
host = "0.0.0.0"
port = 5000
assert find_available_port(port, host) == port, "Port 5000 isn't free"
from wsgiref.simple_server import make_server, demo_app
with make_server(host, port, demo_app) as httpd:
with pytest.warns(UserWarning, match="already in use"):
found_port = find_available_port(port, host, auto_select=True)
assert found_port == port + 1, "Didn't find next port"

2
spacy/tests/test_scorer.py
View File

@ -110,7 +110,7 @@ def test_tokenization(sented_doc):
)
example.predicted[1].is_sent_start = False
scores = scorer.score([example])
assert scores["token_acc"] == approx(0.66666666)
assert scores["token_acc"] == 0.5
assert scores["token_p"] == 0.5
assert scores["token_r"] == approx(0.33333333)
assert scores["token_f"] == 0.4

48
spacy/util.py
View File

@ -31,6 +31,7 @@ import shlex
import inspect
import pkgutil
import logging
import socket
try:
import cupy.random
@ -1728,3 +1729,50 @@ def all_equal(iterable):
(or if the input is an empty sequence), False otherwise."""
g = itertools.groupby(iterable)
return next(g, True) and not next(g, False)
def _is_port_in_use(port: int, host: str = "localhost") -> bool:
"""Check if 'host:port' is in use. Return True if it is, False otherwise.
port (int): the port to check
host (str): the host to check (default "localhost")
RETURNS (bool): Whether 'host:port' is in use.
"""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind((host, port))
return False
except socket.error:
return True
finally:
s.close()
def find_available_port(start: int, host: str, auto_select: bool = False) -> int:
"""Given a starting port and a host, handle finding a port.
If `auto_select` is False, a busy port will raise an error.
If `auto_select` is True, the next free higher port will be used.
start (int): the port to start looking from
host (str): the host to find a port on
auto_select (bool): whether to automatically select a new port if the given port is busy (default False)
RETURNS (int): The port to use.
"""
if not _is_port_in_use(start, host):
return start
port = start
if not auto_select:
raise ValueError(Errors.E1050.format(port=port))
while _is_port_in_use(port, host) and port < 65535:
port += 1
if port == 65535 and _is_port_in_use(port, host):
raise ValueError(Errors.E1049.format(host=host))
# if we get here, the port changed
warnings.warn(Warnings.W124.format(host=host, port=start, serve_port=port))
return port

3
website/.eslintrc.json Normal file
View File

@ -0,0 +1,3 @@
{
"extends": "next/core-web-vitals"
}

44
website/.gitignore vendored Normal file
View File

@ -0,0 +1,44 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.js
# testing
/coverage
# next.js
/.next/
/out/
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# local env files
.env*.local
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts
!.vscode/extensions.json
!public
public/robots.txt
public/sitemap*
public/sw.js*
public/workbox*

1
website/.nvmrc Normal file
View File

@ -0,0 +1 @@
18

1
website/.prettierignore Normal file
View File

@ -0,0 +1 @@
.next

5
website/.prettierrc
View File

@ -20,12 +20,11 @@
}
},
{
"files": "*.md",
"files": ["package.json", "package-lock.json"],
"options": {
"tabWidth": 2,
"printWidth": 80,
"proseWrap": "always",
"htmlWhitespaceSensitivity": "strict"
"proseWrap": "always"
}
},
{

8
website/.vscode/extensions.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"recommendations": [
"dbaeumer.vscode-eslint",
"unifiedjs.vscode-mdx",
"esbenp.prettier-vscode",
"syler.sass-indented"
]
}

28
website/README.md
View File

@ -7,17 +7,16 @@ The styleguide for the spaCy website is available at
## Setup and installation
Before running the setup, make sure your versions of
[Node](https://nodejs.org/en/) and [npm](https://www.npmjs.com/) are up to date.
Node v10.15 or later is required.
```bash
# Clone the repository
git clone https://github.com/explosion/spaCy
cd spaCy/website
# Install Gatsby's command-line tool
npm install --global gatsby-cli
# Switch to the correct Node version
#
# If you don't have NVM and don't want to use it, you can manually switch to the Node version
# stated in /.nvmrc and skip this step
nvm use
# Install the dependencies
npm install
@ -36,8 +35,7 @@ file in the root defines the settings used in this codebase.
## Building & developing the site with Docker
Sometimes it's hard to get a local environment working due to rapid updates to
node dependencies, so it may be easier to use docker for building the docs.
While it shouldn't be necessary and is not recommended you can run this site in a Docker container.
If you'd like to do this, **be sure you do _not_ include your local
`node_modules` folder**, since there are some dependencies that need to be built
@ -76,12 +74,14 @@ bit of time.
```yaml
├── docs # the actual markdown content
├── meta # JSON-formatted site metadata
| ├── dynamicMeta.js # At build time generated meta data
| ├── languages.json # supported languages and statistical models
| ├── sidebars.json # sidebar navigations for different sections
| ├── site.json # general site metadata
| ├── type-annotations.json # Type annotations
| └── universe.json # data for the spaCy universe section
├── public # compiled site
├── pages # Next router pages
├── public # static images and other assets
├── setup # Jinja setup
├── src # source
| ├── components # React components
@ -96,9 +96,11 @@ bit of time.
| | └── universe.js # layout templates for universe
| └── widgets # non-reusable components with content, e.g. changelog
├── .eslintrc.json # ESLint config file
├── .nvmrc # NVM config file
| # (to support "nvm use" to switch to correct Node version)
|
├── .prettierrc # Prettier config file
├── gatsby-browser.js # browser-specific hooks for Gatsby
├── gatsby-config.js # Gatsby configuration
├── gatsby-node.js # Node-specific hooks for Gatsby
└── package.json # package settings and dependencies
├── next.config.mjs # Next config file
├── package.json # package settings and dependencies
└── tsconfig.json # TypeScript config file
```

78
website/UNIVERSE.md
View File

@ -2,42 +2,52 @@
# spaCy Universe
The [spaCy Universe](https://spacy.io/universe) collects the many great resources developed with or for spaCy. It
includes standalone packages, plugins, extensions, educational materials,
operational utilities and bindings for other languages.
The [spaCy Universe](https://spacy.io/universe) collects the many great
resources developed with or for spaCy. It includes standalone packages, plugins,
extensions, educational materials, operational utilities and bindings for other
languages.
If you have a project that you want the spaCy community to make use of, you can
suggest it by submitting a pull request to this repository. The Universe
database is open-source and collected in a simple JSON file.
Looking for inspiration for your own spaCy plugin or extension? Check out the
[`project ideas`](https://github.com/explosion/spaCy/discussions?discussions_q=category%3A%22New+Features+%26+Project+Ideas%22)
[`project ideas`](https://github.com/explosion/spaCy/discussions?discussions_q=category%3A%22New+Features+%26+Project+Ideas%22)
discussion forum.
## Checklist
### Projects
✅ Libraries and packages should be **open-source** (with a user-friendly license) and at least somewhat **documented** (e.g. a simple `README` with usage instructions).
✅ Libraries and packages should be **open-source** (with a user-friendly
license) and at least somewhat **documented** (e.g. a simple `README` with usage
instructions).
✅ We're happy to include work in progress and prereleases, but we'd like to keep the emphasis on projects that should be useful to the community **right away**.
✅ We're happy to include work in progress and prereleases, but we'd like to
keep the emphasis on projects that should be useful to the community **right
away**.
✅ Demos and visualizers should be available via a **public URL**.
### Educational Materials
✅ Books should be **available for purchase or download** (not just pre-order). Ebooks and self-published books are fine, too, if they include enough substantial content.
✅ Books should be **available for purchase or download** (not just pre-order).
Ebooks and self-published books are fine, too, if they include enough
substantial content.
✅ The `"url"` of book entries should either point to the publisher's website or a reseller of your choice (ideally one that ships worldwide or as close as possible).
✅ The `"url"` of book entries should either point to the publisher's website or
a reseller of your choice (ideally one that ships worldwide or as close as
possible).
✅ If an online course is only available behind a paywall, it should at least have a **free excerpt** or chapter available, so users know what to expect.
✅ If an online course is only available behind a paywall, it should at least
have a **free excerpt** or chapter available, so users know what to expect.
## JSON format
To add a project, fork this repository, edit the [`universe.json`](meta/universe.json)
and add an object of the following format to the list of `"resources"`. Before
you submit your pull request, make sure to use a linter to verify that your
markup is correct.
To add a project, fork this repository, edit the
[`universe.json`](meta/universe.json) and add an object of the following format
to the list of `"resources"`. Before you submit your pull request, make sure to
use a linter to verify that your markup is correct.
```json
{
@ -69,26 +79,26 @@ markup is correct.
}
```
| Field | Type | Description |
| --- | --- | --- |
| `id` | string | Unique ID of the project. |
| `title` | string | Project title. If not set, the `id` will be used as the display title. |
| `slogan` | string | A short description of the project. Displayed in the overview and under the title. |
| `description` | string | A longer description of the project. Markdown is allowed, but should be limited to basic formatting like bold, italics, code or links. |
| `github` | string | Associated GitHub repo in the format `user/repo`. Will be displayed as a link and used for release, license and star badges. |
| `pip` | string | Package name on pip. If available, the installation command will be displayed. |
| `cran` | string | For R packages: package name on CRAN. If available, the installation command will be displayed. |
| `code_example` | array | Short example that shows how to use the project. Formatted as an array with one string per line. |
| `code_language` | string | Defaults to `'python'`. Optional code language used for syntax highlighting with [Prism](http://prismjs.com/). |
| `url` | string | Optional project link to display as button. |
| `thumb` | string | Optional URL to project thumbnail to display in overview and project header. Recommended size is 100x100px. |
| `image` | string | Optional URL to project image to display with description. |
| `author` | string | Name(s) of project author(s). |
| `author_links` | object | Usernames and links to display as icons to author info. Currently supports `twitter` and `github` usernames, as well as `website` link. |
| `category` | list | One or more categories to assign to project. Must be one of the available options. |
| `tags` | list | Still experimental and not used for filtering: one or more tags to assign to project. |
| Field | Type | Description |
| --------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------- |
| `id` | string | Unique ID of the project. |
| `title` | string | Project title. If not set, the `id` will be used as the display title. |
| `slogan` | string | A short description of the project. Displayed in the overview and under the title. |
| `description` | string | A longer description of the project. Markdown is allowed, but should be limited to basic formatting like bold, italics, code or links. |
| `github` | string | Associated GitHub repo in the format `user/repo`. Will be displayed as a link and used for release, license and star badges. |
| `pip` | string | Package name on pip. If available, the installation command will be displayed. |
| `cran` | string | For R packages: package name on CRAN. If available, the installation command will be displayed. |
| `code_example` | array | Short example that shows how to use the project. Formatted as an array with one string per line. |
| `code_language` | string | Defaults to `'python'`. Optional code language used for syntax highlighting with [Prism](http://prismjs.com/). |
| `url` | string | Optional project link to display as button. |
| `thumb` | string | Optional URL to project thumbnail to display in overview and project header. Recommended size is 100x100px. |
| `image` | string | Optional URL to project image to display with description. |
| `author` | string | Name(s) of project author(s). |
| `author_links` | object | Usernames and links to display as icons to author info. Currently supports `twitter` and `github` usernames, as well as `website` link. |
| `category` | list | One or more categories to assign to project. Must be one of the available options. |
| `tags` | list | Still experimental and not used for filtering: one or more tags to assign to project. |
To separate them from the projects, educational materials also specify
`"type": "education`. Books can also set a `"cover"` field containing a URL
to a cover image. If available, it's used in the overview and displayed on
the individual book page.
`"type": "education`. Books can also set a `"cover"` field containing a URL to a
cover image. If available, it's used in the overview and displayed on the
individual book page.

78
website/docs/api/architectures.md → website/docs/api/architectures.mdx
View File

@ -26,9 +26,9 @@ part of the [training config](/usage/training#custom-functions). Also see the
usage documentation on
[layers and model architectures](/usage/layers-architectures).
## Tok2Vec architectures {#tok2vec-arch source="spacy/ml/models/tok2vec.py"}
## Tok2Vec architectures {id="tok2vec-arch",source="spacy/ml/models/tok2vec.py"}
### spacy.Tok2Vec.v2 {#Tok2Vec}
### spacy.Tok2Vec.v2 {id="Tok2Vec"}
> #### Example config
>
@ -56,7 +56,7 @@ blog post for background.
| `encode` | Encode context into the embeddings, using an architecture such as a CNN, BiLSTM or transformer. For example, [MaxoutWindowEncoder](/api/architectures#MaxoutWindowEncoder). ~~Model[List[Floats2d], List[Floats2d]]~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ |
### spacy.HashEmbedCNN.v2 {#HashEmbedCNN}
### spacy.HashEmbedCNN.v2 {id="HashEmbedCNN"}
> #### Example Config
>
@ -89,7 +89,7 @@ consisting of a CNN and a layer-normalized maxout activation function.
| `pretrained_vectors` | Whether to also use static vectors. ~~bool~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ |
### spacy.Tok2VecListener.v1 {#Tok2VecListener}
### spacy.Tok2VecListener.v1 {id="Tok2VecListener"}
> #### Example config
>
@ -139,7 +139,7 @@ the `Tok2Vec` component.
| `upstream` | A string to identify the "upstream" `Tok2Vec` component to communicate with. By default, the upstream name is the wildcard string `"*"`, but you could also specify the name of the `Tok2Vec` component. You'll almost never have multiple upstream `Tok2Vec` components, so the wildcard string will almost always be fine. ~~str~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ |
### spacy.MultiHashEmbed.v2 {#MultiHashEmbed}
### spacy.MultiHashEmbed.v2 {id="MultiHashEmbed"}
> #### Example config
>
@ -170,7 +170,7 @@ updated).
| `include_static_vectors` | Whether to also use static word vectors. Requires a vectors table to be loaded in the [`Doc`](/api/doc) objects' vocab. ~~bool~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ |
### spacy.CharacterEmbed.v2 {#CharacterEmbed}
### spacy.CharacterEmbed.v2 {id="CharacterEmbed"}
> #### Example config
>
@ -207,7 +207,7 @@ network to construct a single vector to represent the information.
| `nC` | The number of UTF-8 bytes to embed per word. Recommended values are between `3` and `8`, although it may depend on the length of words in the language. ~~int~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ |
### spacy.MaxoutWindowEncoder.v2 {#MaxoutWindowEncoder}
### spacy.MaxoutWindowEncoder.v2 {id="MaxoutWindowEncoder"}
> #### Example config
>
@ -231,7 +231,7 @@ and residual connections.
| `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ |
### spacy.MishWindowEncoder.v2 {#MishWindowEncoder}
### spacy.MishWindowEncoder.v2 {id="MishWindowEncoder"}
> #### Example config
>
@ -254,7 +254,7 @@ and residual connections.
| `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ |
### spacy.TorchBiLSTMEncoder.v1 {#TorchBiLSTMEncoder}
### spacy.TorchBiLSTMEncoder.v1 {id="TorchBiLSTMEncoder"}
> #### Example config
>
@ -276,7 +276,7 @@ Encode context using bidirectional LSTM layers. Requires
| `dropout` | Creates a Dropout layer on the outputs of each LSTM layer except the last layer. Set to 0.0 to disable this functionality. ~~float~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ |
### spacy.StaticVectors.v2 {#StaticVectors}
### spacy.StaticVectors.v2 {id="StaticVectors"}
> #### Example config
>
@ -306,7 +306,7 @@ mapped to a zero vector. See the documentation on
| `key_attr` | Defaults to `"ORTH"`. ~~str~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], Ragged]~~ |
### spacy.FeatureExtractor.v1 {#FeatureExtractor}
### spacy.FeatureExtractor.v1 {id="FeatureExtractor"}
> #### Example config
>
@ -324,7 +324,7 @@ of feature names to extract, which should refer to token attributes.
| `columns` | The token attributes to extract. ~~List[Union[int, str]]~~ |
| **CREATES** | The created feature extraction layer. ~~Model[List[Doc], List[Ints2d]]~~ |
## Transformer architectures {#transformers source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/architectures.py"}
## Transformer architectures {id="transformers",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/architectures.py"}
The following architectures are provided by the package
[`spacy-transformers`](https://github.com/explosion/spacy-transformers). See the
@ -341,7 +341,7 @@ for details and system requirements.
</Infobox>
### spacy-transformers.TransformerModel.v3 {#TransformerModel}
### spacy-transformers.TransformerModel.v3 {id="TransformerModel"}
> #### Example Config
>
@ -390,7 +390,7 @@ in other components, see
| | |
<Infobox title="Mixed precision support" variant="warning">
Mixed-precision support is currently an experimental feature.
Mixed-precision support is currently an experimental feature.
</Infobox>
<Accordion title="Previous versions of spacy-transformers.TransformerModel" spaced>
@ -404,7 +404,7 @@ The other arguments are shared between all versions.
</Accordion>
### spacy-transformers.TransformerListener.v1 {#TransformerListener}
### spacy-transformers.TransformerListener.v1 {id="TransformerListener"}
> #### Example Config
>
@ -434,7 +434,7 @@ a single token vector given zero or more wordpiece vectors.
| `upstream` | A string to identify the "upstream" `Transformer` component to communicate with. By default, the upstream name is the wildcard string `"*"`, but you could also specify the name of the `Transformer` component. You'll almost never have multiple upstream `Transformer` components, so the wildcard string will almost always be fine. ~~str~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ |
### spacy-transformers.Tok2VecTransformer.v3 {#Tok2VecTransformer}
### spacy-transformers.Tok2VecTransformer.v3 {id="Tok2VecTransformer"}
> #### Example Config
>
@ -467,7 +467,7 @@ one component.
| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ |
<Infobox title="Mixed precision support" variant="warning">
Mixed-precision support is currently an experimental feature.
Mixed-precision support is currently an experimental feature.
</Infobox>
<Accordion title="Previous versions of spacy-transformers.Tok2VecTransformer" spaced>
@ -481,7 +481,7 @@ The other arguments are shared between all versions.
</Accordion>
## Pretraining architectures {#pretrain source="spacy/ml/models/multi_task.py"}
## Pretraining architectures {id="pretrain",source="spacy/ml/models/multi_task.py"}
The spacy `pretrain` command lets you initialize a `Tok2Vec` layer in your
pipeline with information from raw text. To this end, additional layers are
@ -494,7 +494,7 @@ BERT.
For more information, see the section on
[pretraining](/usage/embeddings-transformers#pretraining).
### spacy.PretrainVectors.v1 {#pretrain_vectors}
### spacy.PretrainVectors.v1 {id="pretrain_vectors"}
> #### Example config
>
@ -525,7 +525,7 @@ vectors.
| `loss` | The loss function can be either "cosine" or "L2". We typically recommend to use "cosine". ~~~str~~ |
| **CREATES** | A callable function that can create the Model, given the `vocab` of the pipeline and the `tok2vec` layer to pretrain. ~~Callable[[Vocab, Model], Model]~~ |
### spacy.PretrainCharacters.v1 {#pretrain_chars}
### spacy.PretrainCharacters.v1 {id="pretrain_chars"}
> #### Example config
>
@ -551,9 +551,9 @@ for a Tok2Vec layer.
| `n_characters` | The window of characters - e.g. if `n_characters = 2`, the model will try to predict the first two and last two characters of the word. ~~int~~ |
| **CREATES** | A callable function that can create the Model, given the `vocab` of the pipeline and the `tok2vec` layer to pretrain. ~~Callable[[Vocab, Model], Model]~~ |
## Parser & NER architectures {#parser}
## Parser & NER architectures {id="parser"}
### spacy.TransitionBasedParser.v2 {#TransitionBasedParser source="spacy/ml/models/parser.py"}
### spacy.TransitionBasedParser.v2 {id="TransitionBasedParser",source="spacy/ml/models/parser.py"}
> #### Example Config
>
@ -612,9 +612,9 @@ same signature, but the `use_upper` argument was `True` by default.
</Accordion>
## Tagging architectures {#tagger source="spacy/ml/models/tagger.py"}
## Tagging architectures {id="tagger",source="spacy/ml/models/tagger.py"}
### spacy.Tagger.v2 {#Tagger}
### spacy.Tagger.v2 {id="Tagger"}
> #### Example Config
>
@ -648,7 +648,7 @@ The other arguments are shared between all versions.
</Accordion>
## Text classification architectures {#textcat source="spacy/ml/models/textcat.py"}
## Text classification architectures {id="textcat",source="spacy/ml/models/textcat.py"}
A text classification architecture needs to take a [`Doc`](/api/doc) as input,
and produce a score for each potential label class. Textcat challenges can be
@ -672,7 +672,7 @@ single-label use-cases where `exclusive_classes = true`, while the
</Infobox>
### spacy.TextCatEnsemble.v2 {#TextCatEnsemble}
### spacy.TextCatEnsemble.v2 {id="TextCatEnsemble"}
> #### Example Config
>
@ -737,7 +737,7 @@ but used an internal `tok2vec` instead of taking it as argument:
</Accordion>
### spacy.TextCatCNN.v2 {#TextCatCNN}
### spacy.TextCatCNN.v2 {id="TextCatCNN"}
> #### Example Config
>
@ -777,7 +777,7 @@ after training.
</Accordion>
### spacy.TextCatBOW.v2 {#TextCatBOW}
### spacy.TextCatBOW.v2 {id="TextCatBOW"}
> #### Example Config
>
@ -809,9 +809,9 @@ after training.
</Accordion>
## Span classification architectures {#spancat source="spacy/ml/models/spancat.py"}
## Span classification architectures {id="spancat",source="spacy/ml/models/spancat.py"}
### spacy.SpanCategorizer.v1 {#SpanCategorizer}
### spacy.SpanCategorizer.v1 {id="SpanCategorizer"}
> #### Example Config
>
@ -848,7 +848,7 @@ single vector, and a scorer model to map the vectors to probabilities.
| `scorer` | The scorer model. ~~Model[Floats2d, Floats2d]~~ |
| **CREATES** | The model using the architecture. ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ |
### spacy.mean_max_reducer.v1 {#mean_max_reducer}
### spacy.mean_max_reducer.v1 {id="mean_max_reducer"}
Reduce sequences by concatenating their mean and max pooled vectors, and then
combine the concatenated vectors with a hidden layer.
@ -857,7 +857,7 @@ combine the concatenated vectors with a hidden layer.
| ------------- | ------------------------------------- |
| `hidden_size` | The size of the hidden layer. ~~int~~ |
## Entity linking architectures {#entitylinker source="spacy/ml/models/entity_linker.py"}
## Entity linking architectures {id="entitylinker",source="spacy/ml/models/entity_linker.py"}
An [`EntityLinker`](/api/entitylinker) component disambiguates textual mentions
(tagged as named entities) to unique identifiers, grounding the named entities
@ -870,7 +870,7 @@ into the "real world". This requires 3 main components:
- A machine learning [`Model`](https://thinc.ai/docs/api-model) that picks the
most plausible ID from the set of candidates.
### spacy.EntityLinker.v2 {#EntityLinker}
### spacy.EntityLinker.v2 {id="EntityLinker"}
> #### Example Config
>
@ -899,7 +899,7 @@ The `EntityLinker` model architecture is a Thinc `Model` with a
| `nO` | Output dimension, determined by the length of the vectors encoding each entity in the KB. If the `nO` dimension is not set, the entity linking component will set it when `initialize` is called. ~~Optional[int]~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ |
### spacy.EmptyKB.v1 {#EmptyKB}
### spacy.EmptyKB.v1 {id="EmptyKB"}
A function that creates an empty `KnowledgeBase` from a [`Vocab`](/api/vocab)
instance. This is the default when a new entity linker component is created.
@ -908,7 +908,7 @@ instance. This is the default when a new entity linker component is created.
| ---------------------- | ----------------------------------------------------------------------------------- |
| `entity_vector_length` | The length of the vectors encoding each entity in the KB. Defaults to `64`. ~~int~~ |
### spacy.KBFromFile.v1 {#KBFromFile}
### spacy.KBFromFile.v1 {id="KBFromFile"}
A function that reads an existing `KnowledgeBase` from file.
@ -916,7 +916,7 @@ A function that reads an existing `KnowledgeBase` from file.
| --------- | -------------------------------------------------------- |
| `kb_path` | The location of the KB that was stored to file. ~~Path~~ |
### spacy.CandidateGenerator.v1 {#CandidateGenerator}
### spacy.CandidateGenerator.v1 {id="CandidateGenerator"}
A function that takes as input a [`KnowledgeBase`](/api/kb) and a
[`Span`](/api/span) object denoting a named entity, and returns a list of
@ -924,7 +924,7 @@ plausible [`Candidate`](/api/kb/#candidate) objects. The default
`CandidateGenerator` uses the text of a mention to find its potential aliases in
the `KnowledgeBase`. Note that this function is case-dependent.
## Coreference {#coref-architectures tag="experimental"}
## Coreference {id="coref-architectures",tag="experimental"}
A [`CoreferenceResolver`](/api/coref) component identifies tokens that refer to
the same entity. A [`SpanResolver`](/api/span-resolver) component infers spans
@ -932,7 +932,7 @@ from single tokens. Together these components can be used to reproduce
traditional coreference models. You can also omit the `SpanResolver` if working
with only token-level clusters is acceptable.
### spacy-experimental.Coref.v1 {#Coref tag="experimental"}
### spacy-experimental.Coref.v1 {id="Coref",tag="experimental"}
> #### Example Config
>
@ -967,7 +967,7 @@ The `Coref` model architecture is a Thinc `Model`.
| `antecedent_batch_size` | Internal batch size. ~~int~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ |
### spacy-experimental.SpanResolver.v1 {#SpanResolver tag="experimental"}
### spacy-experimental.SpanResolver.v1 {id="SpanResolver",tag="experimental"}
> #### Example Config
>

30
website/docs/api/attributeruler.md → website/docs/api/attributeruler.mdx
View File

@ -2,7 +2,7 @@
title: AttributeRuler
tag: class
source: spacy/pipeline/attribute_ruler.py
new: 3
version: 3
teaser: 'Pipeline component for rule-based token attribute assignment'
api_string_name: attribute_ruler
api_trainable: false
@ -15,7 +15,7 @@ between attributes such as mapping fine-grained POS tags to coarse-grained POS
tags. See the [usage guide](/usage/linguistic-features/#mappings-exceptions) for
examples.
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -37,7 +37,7 @@ how the component should be configured. You can override its settings via the
%%GITHUB_SPACY/spacy/pipeline/attribute_ruler.py
```
## AttributeRuler.\_\_init\_\_ {#init tag="method"}
## AttributeRuler.\_\_init\_\_ {id="init",tag="method"}
Initialize the attribute ruler.
@ -56,7 +56,7 @@ Initialize the attribute ruler.
| `validate` | Whether patterns should be validated (passed to the [`Matcher`](/api/matcher#init)). Defaults to `False`. ~~bool~~ |
| `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"tag`", `"pos"`, `"morph"` and `"lemma"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ |
## AttributeRuler.\_\_call\_\_ {#call tag="method"}
## AttributeRuler.\_\_call\_\_ {id="call",tag="method"}
Apply the attribute ruler to a `Doc`, setting token attributes for tokens
matched by the provided patterns.
@ -66,7 +66,7 @@ matched by the provided patterns.
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## AttributeRuler.add {#add tag="method"}
## AttributeRuler.add {id="add",tag="method"}
Add patterns to the attribute ruler. The patterns are a list of `Matcher`
patterns and the attributes are a dict of attributes to set on the matched
@ -89,7 +89,7 @@ may be negative to index from the end of the span.
| `attrs` | The attributes to assign to the target token in the matched span. ~~Dict[str, Any]~~ |
| `index` | The index of the token in the matched span to modify. May be negative to index from the end of the span. Defaults to `0`. ~~int~~ |
## AttributeRuler.add_patterns {#add_patterns tag="method"}
## AttributeRuler.add_patterns {id="add_patterns",tag="method"}
> #### Example
>
@ -116,7 +116,7 @@ keys `"patterns"`, `"attrs"` and `"index"`, which match the arguments of
| ---------- | -------------------------------------------------------------------------- |
| `patterns` | The patterns to add. ~~Iterable[Dict[str, Union[List[dict], dict, int]]]~~ |
## AttributeRuler.patterns {#patterns tag="property"}
## AttributeRuler.patterns {id="patterns",tag="property"}
Get all patterns that have been added to the attribute ruler in the
`patterns_dict` format accepted by
@ -126,7 +126,7 @@ Get all patterns that have been added to the attribute ruler in the
| ----------- | -------------------------------------------------------------------------------------------- |
| **RETURNS** | The patterns added to the attribute ruler. ~~List[Dict[str, Union[List[dict], dict, int]]]~~ |
## AttributeRuler.initialize {#initialize tag="method"}
## AttributeRuler.initialize {id="initialize",tag="method"}
Initialize the component with data and used before training to load in rules
from a file. This method is typically called by
@ -160,7 +160,7 @@ config.
| `tag_map` | The tag map that maps fine-grained tags to coarse-grained tags and morphological features. Defaults to `None`. ~~Optional[Dict[str, Dict[Union[int, str], Union[int, str]]]]~~ |
| `morph_rules` | The morph rules that map token text and fine-grained tags to coarse-grained tags, lemmas and morphological features. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]]~~ |
## AttributeRuler.load_from_tag_map {#load_from_tag_map tag="method"}
## AttributeRuler.load_from_tag_map {id="load_from_tag_map",tag="method"}
Load attribute ruler patterns from a tag map.
@ -168,7 +168,7 @@ Load attribute ruler patterns from a tag map.
| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
| `tag_map` | The tag map that maps fine-grained tags to coarse-grained tags and morphological features. ~~Dict[str, Dict[Union[int, str], Union[int, str]]]~~ |
## AttributeRuler.load_from_morph_rules {#load_from_morph_rules tag="method"}
## AttributeRuler.load_from_morph_rules {id="load_from_morph_rules",tag="method"}
Load attribute ruler patterns from morph rules.
@ -176,7 +176,7 @@ Load attribute ruler patterns from morph rules.
| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `morph_rules` | The morph rules that map token text and fine-grained tags to coarse-grained tags, lemmas and morphological features. ~~Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]~~ |
## AttributeRuler.to_disk {#to_disk tag="method"}
## AttributeRuler.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -193,7 +193,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## AttributeRuler.from_disk {#from_disk tag="method"}
## AttributeRuler.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -211,7 +211,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `AttributeRuler` object. ~~AttributeRuler~~ |
## AttributeRuler.to_bytes {#to_bytes tag="method"}
## AttributeRuler.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -228,7 +228,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `AttributeRuler` object. ~~bytes~~ |
## AttributeRuler.from_bytes {#from_bytes tag="method"}
## AttributeRuler.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -247,7 +247,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `AttributeRuler` object. ~~AttributeRuler~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

7
website/docs/api/attributes.md → website/docs/api/attributes.mdx
View File

@ -41,10 +41,9 @@ from string attribute names to internal attribute IDs is stored in
The corresponding [`Token` object attributes](/api/token#attributes) can be
accessed using the same names in lowercase, e.g. `token.orth` or `token.length`.
For attributes that represent string values, the internal integer ID is
accessed as `Token.attr`, e.g. `token.dep`, while the string value can be
retrieved by appending `_` as in `token.dep_`.
For attributes that represent string values, the internal integer ID is accessed
as `Token.attr`, e.g. `token.dep`, while the string value can be retrieved by
appending `_` as in `token.dep_`.
| Attribute | Description |
| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |

177
website/docs/api/cli.md → website/docs/api/cli.mdx
View File

@ -26,7 +26,7 @@ a list of available commands, you can type `python -m spacy --help`. You can
also add the `--help` flag to any command or subcommand to see the description,
available arguments and usage.
## download {#download tag="command"}
## download {id="download",tag="command"}
Download [trained pipelines](/usage/models) for spaCy. The downloader finds the
best-matching compatible version and uses `pip install` to download the Python
@ -44,7 +44,7 @@ pipeline name to be specified with its version (e.g. `en_core_web_sm-3.0.0`).
> will also allow you to add it as a versioned package dependency to your
> project.
```cli
```bash
$ python -m spacy download [model] [--direct] [--sdist] [pip_args]
```
@ -57,24 +57,24 @@ $ python -m spacy download [model] [--direct] [--sdist] [pip_args]
| pip args | Additional installation options to be passed to `pip install` when installing the pipeline package. For example, `--user` to install to the user home directory or `--no-deps` to not install package dependencies. ~~Any (option/flag)~~ |
| **CREATES** | The installed pipeline package in your `site-packages` directory. |
## info {#info tag="command"}
## info {id="info",tag="command"}
Print information about your spaCy installation, trained pipelines and local
setup, and generate [Markdown](https://en.wikipedia.org/wiki/Markdown)-formatted
markup to copy-paste into
[GitHub issues](https://github.com/explosion/spaCy/issues).
```cli
```bash
$ python -m spacy info [--markdown] [--silent] [--exclude]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy info en_core_web_lg --markdown
> ```
```cli
```bash
$ python -m spacy info [model] [--markdown] [--silent] [--exclude]
```
@ -88,7 +88,7 @@ $ python -m spacy info [model] [--markdown] [--silent] [--exclude]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **PRINTS** | Information about your spaCy installation. |
## validate {#validate new="2" tag="command"}
## validate {id="validate",version="2",tag="command"}
Find all trained pipeline packages installed in the current environment and
check whether they are compatible with the currently installed version of spaCy.
@ -103,7 +103,7 @@ compatible versions and command for updating are shown.
> suite, to ensure all packages are up to date before proceeding. If
> incompatible packages are found, it will return `1`.
```cli
```bash
$ python -m spacy validate
```
@ -111,12 +111,12 @@ $ python -m spacy validate
| ---------- | -------------------------------------------------------------------- |
| **PRINTS** | Details about the compatibility of your installed pipeline packages. |
## init {#init new="3"}
## init {id="init",version="3"}
The `spacy init` CLI includes helpful commands for initializing training config
files and pipeline directories.
### init config {#init-config new="3" tag="command"}
### init config {id="init-config",version="3",tag="command"}
Initialize and save a [`config.cfg` file](/usage/training#config) using the
**recommended settings** for your use case. It works just like the
@ -128,11 +128,11 @@ customize those settings in your config file later.
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy init config config.cfg --lang en --pipeline ner,textcat --optimize accuracy
> ```
```cli
```bash
$ python -m spacy init config [output_file] [--lang] [--pipeline] [--optimize] [--gpu] [--pretraining] [--force]
```
@ -148,7 +148,7 @@ $ python -m spacy init config [output_file] [--lang] [--pipeline] [--optimize] [
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | The config file for training. |
### init fill-config {#init-fill-config new="3"}
### init fill-config {id="init-fill-config",version="3"}
Auto-fill a partial [.cfg file](/usage/training#config) with **all default
values**, e.g. a config generated with the
@ -162,15 +162,15 @@ validation error with more details.
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy init fill-config base.cfg config.cfg --diff
> ```
>
> #### Example diff
>
> ![Screenshot of visual diff in terminal](../images/cli_init_fill-config_diff.jpg)
> ![Screenshot of visual diff in terminal](/images/cli_init_fill-config_diff.jpg)
```cli
```bash
$ python -m spacy init fill-config [base_path] [output_file] [--diff]
```
@ -184,7 +184,7 @@ $ python -m spacy init fill-config [base_path] [output_file] [--diff]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | Complete and auto-filled config file for training. |
### init vectors {#init-vectors new="3" tag="command"}
### init vectors {id="init-vectors",version="3",tag="command"}
Convert [word vectors](/usage/linguistic-features#vectors-similarity) for use
with spaCy. Will export an `nlp` object that you can use in the
@ -199,7 +199,7 @@ This functionality was previously available as part of the command `init-model`.
</Infobox>
```cli
```bash
$ python -m spacy init vectors [lang] [vectors_loc] [output_dir] [--prune] [--truncate] [--name] [--verbose]
```
@ -216,7 +216,7 @@ $ python -m spacy init vectors [lang] [vectors_loc] [output_dir] [--prune] [--tr
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | A spaCy pipeline directory containing the vocab and vectors. |
### init labels {#init-labels new="3" tag="command"}
### init labels {id="init-labels",version="3",tag="command"}
Generate JSON files for the labels in the data. This helps speed up the training
process, since spaCy won't have to preprocess the data to extract the labels.
@ -234,7 +234,7 @@ After generating the labels, you can provide them to components that accept a
> path = "corpus/labels/ner.json
> ```
```cli
```bash
$ python -m spacy init labels [config_path] [output_path] [--code] [--verbose] [--gpu-id] [overrides]
```
@ -249,7 +249,7 @@ $ python -m spacy init labels [config_path] [output_path] [--code] [--verbose] [
| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ |
| **CREATES** | The label files. |
## convert {#convert tag="command"}
## convert {id="convert",tag="command"}
Convert files into spaCy's
[binary training data format](/api/data-formats#binary-training), a serialized
@ -257,7 +257,7 @@ Convert files into spaCy's
management functions. The converter can be specified on the command line, or
chosen based on the file extension of the input file.
```cli
```bash
$ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type] [--n-sents] [--seg-sents] [--base] [--morphology] [--merge-subtokens] [--ner-map] [--lang]
```
@ -278,7 +278,7 @@ $ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | Binary [`DocBin`](/api/docbin) training data that can be used with [`spacy train`](/api/cli#train). |
### Converters {#converters}
### Converters {id="converters"}
| ID | Description |
| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
@ -288,12 +288,12 @@ $ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type]
| `ner` / `conll` | NER with IOB/IOB2/BILUO tags, one token per line with columns separated by whitespace. The first column is the token and the final column is the NER tag. Sentences are separated by blank lines and documents are separated by the line `-DOCSTART- -X- O O`. Supports CoNLL 2003 NER format. See [sample data](%%GITHUB_SPACY/extra/example_data/ner_example_data). |
| `iob` | NER with IOB/IOB2/BILUO tags, one sentence per line with tokens separated by whitespace and annotation separated by `\|`, either `word\|B-ENT`or`word\|POS\|B-ENT`. See [sample data](%%GITHUB_SPACY/extra/example_data/ner_example_data). |
## debug {#debug new="3"}
## debug {id="debug",version="3"}
The `spacy debug` CLI includes helpful commands for debugging and profiling your
configs, data and implementations.
### debug config {#debug-config new="3" tag="command"}
### debug config {id="debug-config",version="3",tag="command"}
Debug a [`config.cfg` file](/usage/training#config) and show validation errors.
The command will create all objects in the tree and validate them. Note that
@ -303,13 +303,13 @@ errors at once and some issues are only shown once previous errors have been
fixed. To auto-fill a partial config and save the result, you can use the
[`init fill-config`](/api/cli#init-fill-config) command.
```cli
```bash
$ python -m spacy debug config [config_path] [--code] [--show-functions] [--show-variables] [overrides]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy debug config config.cfg
> ```
@ -333,7 +333,7 @@ python -m spacy init fill-config tmp/starter-config_invalid.cfg tmp/starter-conf
<Accordion title="Example output (valid config and all options)" spaced>
```cli
```bash
$ python -m spacy debug config ./config.cfg --show-functions --show-variables
```
@ -453,7 +453,7 @@ File /path/to/thinc/thinc/schedules.py (line 91)
| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ |
| **PRINTS** | Config validation errors, if available. |
### debug data {#debug-data tag="command"}
### debug data {id="debug-data",tag="command"}
Analyze, debug and validate your training and development data. Get useful
stats, and find problems like invalid entity annotations, cyclic dependencies,
@ -479,13 +479,13 @@ the token distributions. To learn more, you can check out Papay et al.'s work on
</Infobox>
```cli
```bash
$ python -m spacy debug data [config_path] [--code] [--ignore-warnings] [--verbose] [--no-format] [overrides]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy debug data ./config.cfg
> ```
@ -639,7 +639,7 @@ will not be available.
| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ |
| **PRINTS** | Debugging information. |
### debug diff-config {#debug-diff tag="command"}
### debug diff-config {id="debug-diff",tag="command"}
Show a diff of a config file with respect to spaCy's defaults or another config
file. If additional settings were used in the creation of the config file, then
@ -647,13 +647,13 @@ you must supply these as extra parameters to the command when comparing to the
default settings. The generated diff can also be used when posting to the
discussion forum to provide more information for the maintainers.
```cli
```bash
$ python -m spacy debug diff-config [config_path] [--compare-to] [--optimize] [--gpu] [--pretraining] [--markdown]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy debug diff-config ./config.cfg
> ```
@ -868,7 +868,7 @@ after_init = null
| `markdown`, `-md` | Generate Markdown for Github issues. Defaults to `False`. ~~bool (flag)~~ |
| **PRINTS** | Diff between the two config files. |
### debug profile {#debug-profile tag="command"}
### debug profile {id="debug-profile",tag="command"}
Profile which functions take the most time in a spaCy pipeline. Input should be
formatted as one JSON object per line with a key `"text"`. It can either be
@ -882,7 +882,7 @@ The `profile` command is now available as a subcommand of `spacy debug`.
</Infobox>
```cli
```bash
$ python -m spacy debug profile [model] [inputs] [--n-texts]
```
@ -894,12 +894,12 @@ $ python -m spacy debug profile [model] [inputs] [--n-texts]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **PRINTS** | Profiling information for the pipeline. |
### debug model {#debug-model new="3" tag="command"}
### debug model {id="debug-model",version="3",tag="command"}
Debug a Thinc [`Model`](https://thinc.ai/docs/api-model) by running it on a
sample text and checking how it updates its internal weights and parameters.
```cli
```bash
$ python -m spacy debug model [config_path] [component] [--layers] [--dimensions] [--parameters] [--gradients] [--attributes] [--print-step0] [--print-step1] [--print-step2] [--print-step3] [--gpu-id]
```
@ -910,7 +910,7 @@ model ("Step 0"), which helps us to understand the internal structure of the
Neural Network, and to focus on specific layers that we want to inspect further
(see next example).
```cli
```bash
$ python -m spacy debug model ./config.cfg tagger -P0
```
@ -956,7 +956,7 @@ an all-zero matrix determined by the `nO` and `nI` dimensions. After a first
training step (Step 2), this matrix has clearly updated its values through the
training feedback loop.
```cli
```bash
$ python -m spacy debug model ./config.cfg tagger -l "5,15" -DIM -PAR -P0 -P1 -P2
```
@ -1017,7 +1017,7 @@ $ python -m spacy debug model ./config.cfg tagger -l "5,15" -DIM -PAR -P0 -P1 -P
| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ |
| **PRINTS** | Debugging information. |
## train {#train tag="command"}
## train {id="train",tag="command"}
Train a pipeline. Expects data in spaCy's
[binary format](/api/data-formats#training) and a
@ -1043,11 +1043,11 @@ in the section `[paths]`.
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy train config.cfg --output ./output --paths.train ./train --paths.dev ./dev
> ```
```cli
```bash
$ python -m spacy train [config_path] [--output] [--code] [--verbose] [--gpu-id] [overrides]
```
@ -1062,7 +1062,7 @@ $ python -m spacy train [config_path] [--output] [--code] [--verbose] [--gpu-id]
| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ |
| **CREATES** | The final trained pipeline and the best trained pipeline. |
### Calling the training function from Python {#train-function new="3.2"}
### Calling the training function from Python {id="train-function",version="3.2"}
The training CLI exposes a `train` helper function that lets you run the
training just like `spacy train`. Usually it's easier to use the command line
@ -1085,7 +1085,7 @@ directly, but if you need to kick off training from code this is how to do it.
| `use_gpu` | Which GPU to use. Defaults to -1 for no GPU. ~~int~~ |
| `overrides` | Values to override config settings. ~~Dict[str, Any]~~ |
## pretrain {#pretrain new="2.1" tag="command,experimental"}
## pretrain {id="pretrain",version="2.1",tag="command,experimental"}
Pretrain the "token to vector" ([`Tok2vec`](/api/tok2vec)) layer of pipeline
components on raw text, using an approximate language-modeling objective.
@ -1113,11 +1113,11 @@ auto-generated by setting `--pretraining` on
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy pretrain config.cfg ./output_pretrain --paths.raw_text ./data.jsonl
> ```
```cli
```bash
$ python -m spacy pretrain [config_path] [output_dir] [--code] [--resume-path] [--epoch-resume] [--gpu-id] [overrides]
```
@ -1133,7 +1133,7 @@ $ python -m spacy pretrain [config_path] [output_dir] [--code] [--resume-path] [
| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--training.dropout 0.2`. ~~Any (option/flag)~~ |
| **CREATES** | The pretrained weights that can be used to initialize `spacy train`. |
## evaluate {#evaluate new="2" tag="command"}
## evaluate {id="evaluate",version="2",tag="command"}
Evaluate a trained pipeline. Expects a loadable spaCy pipeline (package name or
path) and evaluation data in the
@ -1146,7 +1146,7 @@ skew. To render a sample of dependency parses in a HTML file using the
[displaCy visualizations](/usage/visualizers), set as output directory as the
`--displacy-path` argument.
```cli
```bash
$ python -m spacy evaluate [model] [data_path] [--output] [--code] [--gold-preproc] [--gpu-id] [--displacy-path] [--displacy-limit]
```
@ -1163,7 +1163,7 @@ $ python -m spacy evaluate [model] [data_path] [--output] [--code] [--gold-prepr
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | Training results and optional metrics and visualizations. |
## apply {#apply new="3.5" tag="command"}
## apply {id="apply", version="3.5", tag="command"}
Applies a trained pipeline to data and stores the resulting annotated documents
in a `DocBin`. The input can be a single file or a directory. The recognized
@ -1194,7 +1194,8 @@ $ python -m spacy apply [model] [data-path] [output-file] [--code] [--text-key]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | A `DocBin` with the annotations from the `model` for all the files found in `data-path`. |
## find-threshold {#find-threshold new="3.5" tag="command"}
## find-threshold {id="find-threshold",version="3.5",tag="command"}
Runs prediction trials for a trained model with varying tresholds to maximize
the specified metric. The search space for the threshold is traversed linearly
@ -1209,12 +1210,12 @@ be provided.
> #### Examples
>
> ```cli
> ```bash
> # For textcat_multilabel:
> $ python -m spacy find-threshold my_nlp data.spacy textcat_multilabel threshold cats_macro_f
> ```
>
> ```cli
> ```bash
> # For spancat:
> $ python -m spacy find-threshold my_nlp data.spacy spancat threshold spans_sc_f
> ```
@ -1233,7 +1234,7 @@ be provided.
| `--silent`, `-V`, `-VV` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ |
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
## assemble {#assemble tag="command"}
## assemble {id="assemble",tag="command"}
Assemble a pipeline from a config file without additional training. Expects a
[config file](/api/data-formats#config) with all settings and hyperparameters.
@ -1243,11 +1244,11 @@ config.
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy assemble config.cfg ./output
> ```
```cli
```bash
$ python -m spacy assemble [config_path] [output_dir] [--code] [--verbose] [overrides]
```
@ -1261,7 +1262,7 @@ $ python -m spacy assemble [config_path] [output_dir] [--code] [--verbose] [over
| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.data ./data`. ~~Any (option/flag)~~ |
| **CREATES** | The final assembled pipeline. |
## package {#package tag="command"}
## package {id="package",tag="command"}
Generate an installable [Python package](/usage/training#models-generating) from
an existing pipeline data directory. All data files are copied over. If
@ -1287,13 +1288,13 @@ the sdist and wheel by setting `--build sdist,wheel`.
</Infobox>
```cli
```bash
$ python -m spacy package [input_dir] [output_dir] [--code] [--meta-path] [--create-meta] [--build] [--name] [--version] [--force]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy package /input /output
> $ cd /output/en_pipeline-0.0.0
> $ pip install dist/en_pipeline-0.0.0.tar.gz
@ -1313,13 +1314,13 @@ $ python -m spacy package [input_dir] [output_dir] [--code] [--meta-path] [--cre
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | A Python package containing the spaCy pipeline. |
## project {#project new="3"}
## project {id="project",version="3"}
The `spacy project` CLI includes subcommands for working with
[spaCy projects](/usage/projects), end-to-end workflows for building and
deploying custom spaCy pipelines.
### project clone {#project-clone tag="command"}
### project clone {id="project-clone",tag="command"}
Clone a project template from a Git repository. Calls into `git` under the hood
and can use the sparse checkout feature if available, so you're only downloading
@ -1328,19 +1329,19 @@ what you need. By default, spaCy's
can provide any other repo (public or private) that you have access to using the
`--repo` option.
```cli
```bash
$ python -m spacy project clone [name] [dest] [--repo] [--branch] [--sparse]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy project clone pipelines/ner_wikiner
> ```
>
> Clone from custom repo:
>
> ```cli
> ```bash
> $ python -m spacy project clone template --repo https://github.com/your_org/your_repo
> ```
@ -1354,7 +1355,7 @@ $ python -m spacy project clone [name] [dest] [--repo] [--branch] [--sparse]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | The cloned [project directory](/usage/projects#project-files). |
### project assets {#project-assets tag="command"}
### project assets {id="project-assets",tag="command"}
Fetch project assets like datasets and pretrained weights. Assets are defined in
the `assets` section of the [`project.yml`](/usage/projects#project-yml). If a
@ -1365,13 +1366,13 @@ considered "private" and you have to take care of putting them into the
destination directory yourself. If a local path is provided, the asset is copied
into the current project.
```cli
```bash
$ python -m spacy project assets [project_dir]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy project assets [--sparse]
> ```
@ -1382,7 +1383,7 @@ $ python -m spacy project assets [project_dir]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | Downloaded or copied assets defined in the `project.yml`. |
### project run {#project-run tag="command"}
### project run {id="project-run",tag="command"}
Run a named command or workflow defined in the
[`project.yml`](/usage/projects#project-yml). If a workflow name is specified,
@ -1391,13 +1392,13 @@ all commands in the workflow are run, in order. If commands define
re-run if state has changed. For example, if the input dataset changes, a
preprocessing command that depends on those files will be re-run.
```cli
```bash
$ python -m spacy project run [subcommand] [project_dir] [--force] [--dry]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy project run train
> ```
@ -1410,7 +1411,7 @@ $ python -m spacy project run [subcommand] [project_dir] [--force] [--dry]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **EXECUTES** | The command defined in the `project.yml`. |
### project push {#project-push tag="command"}
### project push {id="project-push",tag="command"}
Upload all available files or directories listed as in the `outputs` section of
commands to a remote storage. Outputs are archived and compressed prior to
@ -1430,13 +1431,13 @@ remote storages, so you can use any protocol that `Pathy` supports, including
filesystem, although you may need to install extra dependencies to use certain
protocols.
```cli
```bash
$ python -m spacy project push [remote] [project_dir]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy project push my_bucket
> ```
>
@ -1453,7 +1454,7 @@ $ python -m spacy project push [remote] [project_dir]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **UPLOADS** | All project outputs that exist and are not already stored in the remote. |
### project pull {#project-pull tag="command"}
### project pull {id="project-pull",tag="command"}
Download all files or directories listed as `outputs` for commands, unless they
are not already present locally. When searching for files in the remote, `pull`
@ -1475,13 +1476,13 @@ remote storages, so you can use any protocol that `Pathy` supports, including
filesystem, although you may need to install extra dependencies to use certain
protocols.
```cli
```bash
$ python -m spacy project pull [remote] [project_dir]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy project pull my_bucket
> ```
>
@ -1498,7 +1499,7 @@ $ python -m spacy project pull [remote] [project_dir]
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **DOWNLOADS** | All project outputs that do not exist locally and can be found in the remote. |
### project document {#project-document tag="command"}
### project document {id="project-document",tag="command"}
Auto-generate a pretty Markdown-formatted `README` for your project, based on
its [`project.yml`](/usage/projects#project-yml). Will create sections that
@ -1507,13 +1508,13 @@ content will be placed between two hidden markers, so you can add your own
custom content before or after the auto-generated documentation. When you re-run
the `project document` command, only the auto-generated part is replaced.
```cli
```bash
$ python -m spacy project document [project_dir] [--output] [--no-emoji]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy project document --output README.md
> ```
@ -1522,7 +1523,7 @@ $ python -m spacy project document [project_dir] [--output] [--no-emoji]
For more examples, see the templates in our
[`projects`](https://github.com/explosion/projects) repo.
![Screenshot of auto-generated Markdown Readme](../images/project_document.jpg)
![Screenshot of auto-generated Markdown Readme](/images/project_document.jpg)
</Accordion>
@ -1533,7 +1534,7 @@ For more examples, see the templates in our
| `--no-emoji`, `-NE` | Don't use emoji in the titles. ~~bool (flag)~~ |
| **CREATES** | The Markdown-formatted project documentation. |
### project dvc {#project-dvc tag="command"}
### project dvc {id="project-dvc",tag="command"}
Auto-generate [Data Version Control](https://dvc.org) (DVC) config file. Calls
[`dvc run`](https://dvc.org/doc/command-reference/run) with `--no-exec` under
@ -1553,13 +1554,13 @@ You'll also need to add the assets you want to track with
</Infobox>
```cli
```bash
$ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [--quiet]
```
> #### Example
>
> ```cli
> ```bash
> $ git init
> $ dvc init
> $ python -m spacy project dvc all
@ -1575,14 +1576,14 @@ $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [--
| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ |
| **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. |
## huggingface-hub {#huggingface-hub new="3.1"}
## huggingface-hub {id="huggingface-hub",version="3.1"}
The `spacy huggingface-cli` CLI includes commands for uploading your trained
spaCy pipelines to the [Hugging Face Hub](https://huggingface.co/).
> #### Installation
>
> ```cli
> ```bash
> $ pip install spacy-huggingface-hub
> $ huggingface-cli login
> ```
@ -1596,19 +1597,19 @@ package installed. Installing the package will automatically add the
</Infobox>
### huggingface-hub push {#huggingface-hub-push tag="command"}
### huggingface-hub push {id="huggingface-hub-push",tag="command"}
Push a spaCy pipeline to the Hugging Face Hub. Expects a `.whl` file packaged
with [`spacy package`](/api/cli#package) and `--build wheel`. For more details,
see the spaCy project [integration](/usage/projects#huggingface_hub).
```cli
```bash
$ python -m spacy huggingface-hub push [whl_path] [--org] [--msg] [--local-repo] [--verbose]
```
> #### Example
>
> ```cli
> ```bash
> $ python -m spacy huggingface-hub push en_ner_fashion-0.0.0-py3-none-any.whl
> ```

32
website/docs/api/coref.md → website/docs/api/coref.mdx
View File

@ -34,7 +34,7 @@ same thing. Clusters are represented as SpanGroups that start with a prefix
A `CoreferenceResolver` component can be paired with a
[`SpanResolver`](/api/span-resolver) to expand single tokens to spans.
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predictions will be saved to `Doc.spans` as a [`SpanGroup`](/api/spangroup). The
span key will be a prefix plus a serial number referring to the coreference
@ -47,7 +47,7 @@ parameter.
| ------------------------------------------ | ------------------------------------------------------------------------------------------------------- |
| `Doc.spans[prefix + "_" + cluster_number]` | One coreference cluster, represented as single-token spans. Cluster numbers start from 1. ~~SpanGroup~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -73,7 +73,7 @@ details on the architectures and their arguments and hyperparameters.
| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [Coref](/api/architectures#Coref). ~~Model~~ |
| `span_cluster_prefix` | The prefix for the keys for clusters saved to `doc.spans`. Defaults to `coref_clusters`. ~~str~~ |
## CoreferenceResolver.\_\_init\_\_ {#init tag="method"}
## CoreferenceResolver.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -102,7 +102,7 @@ shortcut for this and instantiate the component using its string name and
| _keyword-only_ | |
| `span_cluster_prefix` | The prefix for the key for saving clusters of spans. ~~bool~~ |
## CoreferenceResolver.\_\_call\_\_ {#call tag="method"}
## CoreferenceResolver.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -125,7 +125,7 @@ and all pipeline components are applied to the `Doc` in order. Both
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## CoreferenceResolver.pipe {#pipe tag="method"}
## CoreferenceResolver.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -148,7 +148,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/coref#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## CoreferenceResolver.initialize {#initialize tag="method"}
## CoreferenceResolver.initialize {id="initialize",tag="method"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -172,7 +172,7 @@ by [`Language.initialize`](/api/language#initialize).
| _keyword-only_ | |
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
## CoreferenceResolver.predict {#predict tag="method"}
## CoreferenceResolver.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them. Clusters are returned as a list of `MentionClusters`, one for
@ -192,7 +192,7 @@ to token indices.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ |
## CoreferenceResolver.set_annotations {#set_annotations tag="method"}
## CoreferenceResolver.set_annotations {id="set_annotations",tag="method"}
Modify a batch of documents, saving coreference clusters in `Doc.spans`.
@ -209,7 +209,7 @@ Modify a batch of documents, saving coreference clusters in `Doc.spans`.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `clusters` | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ |
## CoreferenceResolver.update {#update tag="method"}
## CoreferenceResolver.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects. Delegates to
[`predict`](/api/coref#predict).
@ -231,7 +231,7 @@ Learn from a batch of [`Example`](/api/example) objects. Delegates to
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## CoreferenceResolver.create_optimizer {#create_optimizer tag="method"}
## CoreferenceResolver.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -246,7 +246,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## CoreferenceResolver.use_params {#use_params tag="method, contextmanager"}
## CoreferenceResolver.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -263,7 +263,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## CoreferenceResolver.to_disk {#to_disk tag="method"}
## CoreferenceResolver.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -280,7 +280,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## CoreferenceResolver.from_disk {#from_disk tag="method"}
## CoreferenceResolver.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -298,7 +298,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `CoreferenceResolver` object. ~~CoreferenceResolver~~ |
## CoreferenceResolver.to_bytes {#to_bytes tag="method"}
## CoreferenceResolver.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -315,7 +315,7 @@ Serialize the pipe to a bytestring, including the `KnowledgeBase`.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `CoreferenceResolver` object. ~~bytes~~ |
## CoreferenceResolver.from_bytes {#from_bytes tag="method"}
## CoreferenceResolver.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -334,7 +334,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `CoreferenceResolver` object. ~~CoreferenceResolver~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

17
website/docs/api/corpus.md → website/docs/api/corpus.mdx
View File

@ -3,7 +3,7 @@ title: Corpus
teaser: An annotated corpus
tag: class
source: spacy/training/corpus.py
new: 3
version: 3
---
This class manages annotated corpora and can be used for training and
@ -13,7 +13,7 @@ customize the data loading during training, you can register your own
see the usage guide on [data utilities](/usage/training#data) for more details
and examples.
## Config and implementation {#config}
## Config and implementation {id="config"}
`spacy.Corpus.v1` is a registered function that creates a `Corpus` of training
or evaluation data. It takes the same arguments as the `Corpus` class and
@ -49,7 +49,7 @@ streaming.
%%GITHUB_SPACY/spacy/training/corpus.py
```
## Corpus.\_\_init\_\_ {#init tag="method"}
## Corpus.\_\_init\_\_ {id="init",tag="method"}
Create a `Corpus` for iterating [Example](/api/example) objects from a file or
directory of [`.spacy` data files](/api/data-formats#binary-training). The
@ -81,7 +81,7 @@ train/test skew.
| `augmenter` | Optional data augmentation callback. ~~Callable[[Language, Example], Iterable[Example]]~~ |
| `shuffle` | Whether to shuffle the examples. Defaults to `False`. ~~bool~~ |
## Corpus.\_\_call\_\_ {#call tag="method"}
## Corpus.\_\_call\_\_ {id="call",tag="method"}
Yield examples from the data.
@ -101,7 +101,7 @@ Yield examples from the data.
| `nlp` | The current `nlp` object. ~~Language~~ |
| **YIELDS** | The examples. ~~Example~~ |
## JsonlCorpus {#jsonlcorpus tag="class"}
## JsonlCorpus {id="jsonlcorpus",tag="class"}
Iterate Doc objects from a file or directory of JSONL (newline-delimited JSON)
formatted raw text files. Can be used to read the raw text corpus for language
@ -120,14 +120,13 @@ file.
> srsly.write_jsonl("/path/to/text.jsonl", data)
> ```
```json
### Example
```json {title="Example"}
{"text": "Can I ask where you work now and what you do, and if you enjoy it?"}
{"text": "They may just pull out of the Seattle market completely, at least until they have autonomous vehicles."}
{"text": "My cynical view on this is that it will never be free to the public. Reason: what would be the draw of joining the military? Right now their selling point is free Healthcare and Education. Ironically both are run horribly and most, that I've talked to, come out wishing they never went in."}
```
### JsonlCorpus.\_\init\_\_ {#jsonlcorpus tag="method"}
### JsonlCorpus.\_\_init\_\_ {id="jsonlcorpus",tag="method"}
Initialize the reader.
@ -157,7 +156,7 @@ Initialize the reader.
| `max_length` | Maximum document length (in tokens). Longer documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ |
| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ |
### JsonlCorpus.\_\_call\_\_ {#jsonlcorpus-call tag="method"}
### JsonlCorpus.\_\_call\_\_ {id="jsonlcorpus-call",tag="method"}
Yield examples from the data.

32
website/docs/api/cython-classes.md → website/docs/api/cython-classes.mdx
View File

@ -9,7 +9,7 @@ menu:
- ['StringStore', 'stringstore']
---
## Doc {#doc tag="cdef class" source="spacy/tokens/doc.pxd"}
## Doc {id="doc",tag="cdef class",source="spacy/tokens/doc.pxd"}
The `Doc` object holds an array of [`TokenC`](/api/cython-structs#tokenc)
structs.
@ -21,7 +21,7 @@ accessed from Python. For the Python documentation, see [`Doc`](/api/doc).
</Infobox>
### Attributes {#doc_attributes}
### Attributes {id="doc_attributes"}
| Name | Description |
| ------------ | -------------------------------------------------------------------------------------------------------- |
@ -31,7 +31,7 @@ accessed from Python. For the Python documentation, see [`Doc`](/api/doc).
| `length` | The number of tokens in the document. ~~int~~ |
| `max_length` | The underlying size of the `Doc.c` array. ~~int~~ |
### Doc.push_back {#doc_push_back tag="method"}
### Doc.push_back {id="doc_push_back",tag="method"}
Append a token to the `Doc`. The token can be provided as a
[`LexemeC`](/api/cython-structs#lexemec) or
@ -55,7 +55,7 @@ Append a token to the `Doc`. The token can be provided as a
| `lex_or_tok` | The word to append to the `Doc`. ~~LexemeOrToken~~ |
| `has_space` | Whether the word has trailing whitespace. ~~bint~~ |
## Token {#token tag="cdef class" source="spacy/tokens/token.pxd"}
## Token {id="token",tag="cdef class",source="spacy/tokens/token.pxd"}
A Cython class providing access and methods for a
[`TokenC`](/api/cython-structs#tokenc) struct. Note that the `Token` object does
@ -68,7 +68,7 @@ accessed from Python. For the Python documentation, see [`Token`](/api/token).
</Infobox>
### Attributes {#token_attributes}
### Attributes {id="token_attributes"}
| Name | Description |
| ------- | -------------------------------------------------------------------------- |
@ -77,7 +77,7 @@ accessed from Python. For the Python documentation, see [`Token`](/api/token).
| `i` | The offset of the token within the document. ~~int~~ |
| `doc` | The parent document. ~~Doc~~ |
### Token.cinit {#token_cinit tag="method"}
### Token.cinit {id="token_cinit",tag="method"}
Create a `Token` object from a `TokenC*` pointer.
@ -94,7 +94,7 @@ Create a `Token` object from a `TokenC*` pointer.
| `offset` | The offset of the token within the document. ~~int~~ |
| `doc` | The parent document. ~~int~~ |
## Span {#span tag="cdef class" source="spacy/tokens/span.pxd"}
## Span {id="span",tag="cdef class",source="spacy/tokens/span.pxd"}
A Cython class providing access and methods for a slice of a `Doc` object.
@ -105,7 +105,7 @@ accessed from Python. For the Python documentation, see [`Span`](/api/span).
</Infobox>
### Attributes {#span_attributes}
### Attributes {id="span_attributes"}
| Name | Description |
| ------------ | ----------------------------------------------------------------------------- |
@ -116,7 +116,7 @@ accessed from Python. For the Python documentation, see [`Span`](/api/span).
| `end_char` | The index of the last character of the span. ~~int~~ |
| `label` | A label to attach to the span, e.g. for named entities. ~~attr_t (uint64_t)~~ |
## Lexeme {#lexeme tag="cdef class" source="spacy/lexeme.pxd"}
## Lexeme {id="lexeme",tag="cdef class",source="spacy/lexeme.pxd"}
A Cython class providing access and methods for an entry in the vocabulary.
@ -127,7 +127,7 @@ accessed from Python. For the Python documentation, see [`Lexeme`](/api/lexeme).
</Infobox>
### Attributes {#lexeme_attributes}
### Attributes {id="lexeme_attributes"}
| Name | Description |
| ------- | ----------------------------------------------------------------------------- |
@ -135,7 +135,7 @@ accessed from Python. For the Python documentation, see [`Lexeme`](/api/lexeme).
| `vocab` | A reference to the shared `Vocab` object. ~~Vocab~~ |
| `orth` | ID of the verbatim text content. ~~attr_t (uint64_t)~~ |
## Vocab {#vocab tag="cdef class" source="spacy/vocab.pxd"}
## Vocab {id="vocab",tag="cdef class",source="spacy/vocab.pxd"}
A Cython class providing access and methods for a vocabulary and other data
shared across a language.
@ -147,7 +147,7 @@ accessed from Python. For the Python documentation, see [`Vocab`](/api/vocab).
</Infobox>
### Attributes {#vocab_attributes}
### Attributes {id="vocab_attributes"}
| Name | Description |
| --------- | ---------------------------------------------------------------------------------------------------------- |
@ -155,7 +155,7 @@ accessed from Python. For the Python documentation, see [`Vocab`](/api/vocab).
| `strings` | A `StringStore` that maps string to hash values and vice versa. ~~StringStore~~ |
| `length` | The number of entries in the vocabulary. ~~int~~ |
### Vocab.get {#vocab_get tag="method"}
### Vocab.get {id="vocab_get",tag="method"}
Retrieve a [`LexemeC*`](/api/cython-structs#lexemec) pointer from the
vocabulary.
@ -172,7 +172,7 @@ vocabulary.
| `string` | The string of the word to look up. ~~str~~ |
| **RETURNS** | The lexeme in the vocabulary. ~~const LexemeC\*~~ |
### Vocab.get_by_orth {#vocab_get_by_orth tag="method"}
### Vocab.get_by_orth {id="vocab_get_by_orth",tag="method"}
Retrieve a [`LexemeC*`](/api/cython-structs#lexemec) pointer from the
vocabulary.
@ -189,7 +189,7 @@ vocabulary.
| `orth` | ID of the verbatim text content. ~~attr_t (uint64_t)~~ |
| **RETURNS** | The lexeme in the vocabulary. ~~const LexemeC\*~~ |
## StringStore {#stringstore tag="cdef class" source="spacy/strings.pxd"}
## StringStore {id="stringstore",tag="cdef class",source="spacy/strings.pxd"}
A lookup table to retrieve strings by 64-bit hashes.
@ -201,7 +201,7 @@ accessed from Python. For the Python documentation, see
</Infobox>
### Attributes {#stringstore_attributes}
### Attributes {id="stringstore_attributes"}
| Name | Description |
| ------ | ---------------------------------------------------------------------------------------------------------------- |

22
website/docs/api/cython-structs.md → website/docs/api/cython-structs.mdx
View File

@ -7,7 +7,7 @@ menu:
- ['LexemeC', 'lexemec']
---
## TokenC {#tokenc tag="C struct" source="spacy/structs.pxd"}
## TokenC {id="tokenc",tag="C struct",source="spacy/structs.pxd"}
Cython data container for the `Token` object.
@ -39,7 +39,7 @@ Cython data container for the `Token` object.
| `ent_type` | Named entity type. ~~attr_t (uint64_t)~~ |
| `ent_id` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~attr_t (uint64_t)~~ |
### Token.get_struct_attr {#token_get_struct_attr tag="staticmethod, nogil" source="spacy/tokens/token.pxd"}
### Token.get_struct_attr {id="token_get_struct_attr",tag="staticmethod, nogil",source="spacy/tokens/token.pxd"}
Get the value of an attribute from the `TokenC` struct by attribute ID.
@ -58,7 +58,7 @@ Get the value of an attribute from the `TokenC` struct by attribute ID.
| `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ |
| **RETURNS** | The value of the attribute. ~~attr_t (uint64_t)~~ |
### Token.set_struct_attr {#token_set_struct_attr tag="staticmethod, nogil" source="spacy/tokens/token.pxd"}
### Token.set_struct_attr {id="token_set_struct_attr",tag="staticmethod, nogil",source="spacy/tokens/token.pxd"}
Set the value of an attribute of the `TokenC` struct by attribute ID.
@ -78,7 +78,7 @@ Set the value of an attribute of the `TokenC` struct by attribute ID.
| `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ |
| `value` | The value to set. ~~attr_t (uint64_t)~~ |
### token_by_start {#token_by_start tag="function" source="spacy/tokens/doc.pxd"}
### token_by_start {id="token_by_start",tag="function",source="spacy/tokens/doc.pxd"}
Find a token in a `TokenC*` array by the offset of its first character.
@ -100,7 +100,7 @@ Find a token in a `TokenC*` array by the offset of its first character.
| `start_char` | The start index to search for. ~~int~~ |
| **RETURNS** | The index of the token in the array or `-1` if not found. ~~int~~ |
### token_by_end {#token_by_end tag="function" source="spacy/tokens/doc.pxd"}
### token_by_end {id="token_by_end",tag="function",source="spacy/tokens/doc.pxd"}
Find a token in a `TokenC*` array by the offset of its final character.
@ -122,7 +122,7 @@ Find a token in a `TokenC*` array by the offset of its final character.
| `end_char` | The end index to search for. ~~int~~ |
| **RETURNS** | The index of the token in the array or `-1` if not found. ~~int~~ |
### set_children_from_heads {#set_children_from_heads tag="function" source="spacy/tokens/doc.pxd"}
### set_children_from_heads {id="set_children_from_heads",tag="function",source="spacy/tokens/doc.pxd"}
Set attributes that allow lookup of syntactic children on a `TokenC*` array.
This function must be called after making changes to the `TokenC.head`
@ -148,7 +148,7 @@ attribute, in order to make the parse tree navigation consistent.
| `tokens` | A `TokenC*` array. ~~const TokenC\*~~ |
| `length` | The number of tokens in the array. ~~int~~ |
## LexemeC {#lexemec tag="C struct" source="spacy/structs.pxd"}
## LexemeC {id="lexemec",tag="C struct",source="spacy/structs.pxd"}
Struct holding information about a lexical type. `LexemeC` structs are usually
owned by the `Vocab`, and accessed through a read-only pointer on the `TokenC`
@ -172,7 +172,7 @@ struct.
| `prefix` | Length-N substring from the start of the lexeme. Defaults to `N=1`. ~~attr_t (uint64_t)~~ |
| `suffix` | Length-N substring from the end of the lexeme. Defaults to `N=3`. ~~attr_t (uint64_t)~~ |
### Lexeme.get_struct_attr {#lexeme_get_struct_attr tag="staticmethod, nogil" source="spacy/lexeme.pxd"}
### Lexeme.get_struct_attr {id="lexeme_get_struct_attr",tag="staticmethod, nogil",source="spacy/lexeme.pxd"}
Get the value of an attribute from the `LexemeC` struct by attribute ID.
@ -192,7 +192,7 @@ Get the value of an attribute from the `LexemeC` struct by attribute ID.
| `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ |
| **RETURNS** | The value of the attribute. ~~attr_t (uint64_t)~~ |
### Lexeme.set_struct_attr {#lexeme_set_struct_attr tag="staticmethod, nogil" source="spacy/lexeme.pxd"}
### Lexeme.set_struct_attr {id="lexeme_set_struct_attr",tag="staticmethod, nogil",source="spacy/lexeme.pxd"}
Set the value of an attribute of the `LexemeC` struct by attribute ID.
@ -212,7 +212,7 @@ Set the value of an attribute of the `LexemeC` struct by attribute ID.
| `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ |
| `value` | The value to set. ~~attr_t (uint64_t)~~ |
### Lexeme.c_check_flag {#lexeme_c_check_flag tag="staticmethod, nogil" source="spacy/lexeme.pxd"}
### Lexeme.c_check_flag {id="lexeme_c_check_flag",tag="staticmethod, nogil",source="spacy/lexeme.pxd"}
Check the value of a binary flag attribute.
@ -232,7 +232,7 @@ Check the value of a binary flag attribute.
| `flag_id` | The ID of the flag to look up. The flag IDs are enumerated in `spacy.typedefs`. ~~attr_id_t~~ |
| **RETURNS** | The boolean value of the flag. ~~bint~~ |
### Lexeme.c_set_flag {#lexeme_c_set_flag tag="staticmethod, nogil" source="spacy/lexeme.pxd"}
### Lexeme.c_set_flag {id="lexeme_c_set_flag",tag="staticmethod, nogil",source="spacy/lexeme.pxd"}
Set the value of a binary flag attribute.

4
website/docs/api/cython.md → website/docs/api/cython.mdx
View File

@ -6,7 +6,7 @@ menu:
- ['Conventions', 'conventions']
---
## Overview {#overview hidden="true"}
## Overview {id="overview",hidden="true"}
> #### What's Cython?
>
@ -37,7 +37,7 @@ class holds a [`LexemeC`](/api/cython-structs#lexemec) struct, at `Lexeme.c`.
This lets you shed the Python container, and pass a pointer to the underlying
data into C-level functions.
## Conventions {#conventions}
## Conventions {id="conventions"}
spaCy's core data structures are implemented as [Cython](http://cython.org/)
`cdef` classes. Memory is managed through the

46
website/docs/api/data-formats.md → website/docs/api/data-formats.mdx
View File

@ -14,7 +14,7 @@ vocabulary data. For an overview of label schemes used by the models, see the
[models directory](/models). Each trained pipeline documents the label schemes
used in its components, depending on the data it was trained on.
## Training config {#config new="3"}
## Training config {id="config",version="3"}
Config files define the training process and pipeline and can be passed to
[`spacy train`](/api/cli#train). They use
@ -52,7 +52,7 @@ your config and check that it's valid, you can run the
</Infobox>
### nlp {#config-nlp tag="section"}
### nlp {id="config-nlp",tag="section"}
> #### Example
>
@ -83,7 +83,7 @@ Defines the `nlp` object, its tokenizer and
| `tokenizer` | The tokenizer to use. Defaults to [`Tokenizer`](/api/tokenizer). ~~Callable[[str], Doc]~~ |
| `batch_size` | Default batch size for [`Language.pipe`](/api/language#pipe) and [`Language.evaluate`](/api/language#evaluate). ~~int~~ |
### components {#config-components tag="section"}
### components {id="config-components",tag="section"}
> #### Example
>
@ -106,7 +106,7 @@ function to use to create component) or a `source` (name of path of trained
pipeline to copy components from). See the docs on
[defining pipeline components](/usage/training#config-components) for details.
### paths, system {#config-variables tag="variables"}
### paths, system {id="config-variables",tag="variables"}
These sections define variables that can be referenced across the other sections
as variables. For example `${paths.train}` uses the value of `train` defined in
@ -116,11 +116,11 @@ need paths, you can define them here. All config values can also be
[`spacy train`](/api/cli#train), which is especially relevant for data paths
that you don't want to hard-code in your config file.
```cli
```bash
$ python -m spacy train config.cfg --paths.train ./corpus/train.spacy
```
### corpora {#config-corpora tag="section"}
### corpora {id="config-corpora",tag="section"}
> #### Example
>
@ -176,7 +176,7 @@ single corpus once and then divide it up into `train` and `dev` partitions.
| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `corpora` | A dictionary keyed by string names, mapped to corpus functions that receive the current `nlp` object and return an iterator of [`Example`](/api/example) objects. ~~Dict[str, Callable[[Language], Iterator[Example]]]~~ |
### training {#config-training tag="section"}
### training {id="config-training",tag="section"}
This section defines settings and controls for the training and evaluation
process that are used when you run [`spacy train`](/api/cli#train).
@ -186,7 +186,7 @@ process that are used when you run [`spacy train`](/api/cli#train).
| `accumulate_gradient` | Whether to divide the batch up into substeps. Defaults to `1`. ~~int~~ |
| `batcher` | Callable that takes an iterator of [`Doc`](/api/doc) objects and yields batches of `Doc`s. Defaults to [`batch_by_words`](/api/top-level#batch_by_words). ~~Callable[[Iterator[Doc], Iterator[List[Doc]]]]~~ |
| `before_to_disk` | Optional callback to modify `nlp` object right before it is saved to disk during and after training. Can be used to remove or reset config values or disable components. Defaults to `null`. ~~Optional[Callable[[Language], Language]]~~ |
| `before_update` | Optional callback that is invoked at the start of each training step with the `nlp` object and a `Dict` containing the following entries: `step`, `epoch`. Can be used to make deferred changes to components. Defaults to `null`. ~~Optional[Callable[[Language, Dict[str, Any]], None]]~~ |
| `before_update` <Tag variant="new">3.5</Tag> | Optional callback that is invoked at the start of each training step with the `nlp` object and a `Dict` containing the following entries: `step`, `epoch`. Can be used to make deferred changes to components. Defaults to `null`. ~~Optional[Callable[[Language, Dict[str, Any]], None]]~~ |
| `dev_corpus` | Dot notation of the config location defining the dev corpus. Defaults to `corpora.dev`. ~~str~~ |
| `dropout` | The dropout rate. Defaults to `0.1`. ~~float~~ |
| `eval_frequency` | How often to evaluate during training (steps). Defaults to `200`. ~~int~~ |
@ -202,7 +202,7 @@ process that are used when you run [`spacy train`](/api/cli#train).
| `seed` | The random seed. Defaults to variable `${system.seed}`. ~~int~~ |
| `train_corpus` | Dot notation of the config location defining the train corpus. Defaults to `corpora.train`. ~~str~~ |
### pretraining {#config-pretraining tag="section,optional"}
### pretraining {id="config-pretraining",tag="section,optional"}
This section is optional and defines settings and controls for
[language model pretraining](/usage/embeddings-transformers#pretraining). It's
@ -220,7 +220,7 @@ used when you run [`spacy pretrain`](/api/cli#pretrain).
| `component` | Component name to identify the layer with the model to pretrain. Defaults to `"tok2vec"`. ~~str~~ |
| `layer` | The specific layer of the model to pretrain. If empty, the whole model will be used. ~~str~~ |
### initialize {#config-initialize tag="section"}
### initialize {id="config-initialize",tag="section"}
This config block lets you define resources for **initializing the pipeline**.
It's used by [`Language.initialize`](/api/language#initialize) and typically
@ -255,9 +255,9 @@ Also see the usage guides on the
| `vectors` | Name or path of pipeline containing pretrained word vectors to use, e.g. created with [`init vectors`](/api/cli#init-vectors). Defaults to `null`. ~~Optional[str]~~ |
| `vocab_data` | Path to JSONL-formatted [vocabulary file](/api/data-formats#vocab-jsonl) to initialize vocabulary. ~~Optional[str]~~ |
## Training data {#training}
## Training data {id="training"}
### Binary training format {#binary-training new="3"}
### Binary training format {id="binary-training",version="3"}
> #### Example
>
@ -288,7 +288,7 @@ Note that while this is the format used to save training data, you do not have
to understand the internal details to use it or create training data. See the
section on [preparing training data](/usage/training#training-data).
### JSON training format {#json-input tag="deprecated"}
### JSON training format {id="json-input",tag="deprecated"}
<Infobox variant="warning" title="Changed in v3.0">
@ -300,7 +300,7 @@ objects to JSON, you can now serialize them directly using the
[`spacy convert`](/api/cli) lets you convert your JSON data to the new `.spacy`
format:
```cli
```bash
$ python -m spacy convert ./data.json .
```
@ -317,8 +317,7 @@ $ python -m spacy convert ./data.json .
> [`offsets_to_biluo_tags`](/api/top-level#offsets_to_biluo_tags) function can
> help you convert entity offsets to the right format.
```python
### Example structure
```python {title="Example structure"}
[{
"id": int, # ID of the document within the corpus
"paragraphs": [{ # list of paragraphs in the corpus
@ -357,7 +356,7 @@ https://github.com/explosion/spaCy/blob/v2.3.x/examples/training/training-data.j
</Accordion>
### Annotation format for creating training examples {#dict-input}
### Annotation format for creating training examples {id="dict-input"}
An [`Example`](/api/example) object holds the information for one training
instance. It stores two [`Doc`](/api/doc) objects: one for holding the
@ -436,8 +435,7 @@ file to keep track of your settings and hyperparameters and your own
</Infobox>
```python
### Examples
```python {title="Examples"}
# Training data for a part-of-speech tagger
doc = Doc(vocab, words=["I", "like", "stuff"])
gold_dict = {"tags": ["NOUN", "VERB", "NOUN"]}
@ -466,7 +464,7 @@ gold_dict = {"entities": [(0, 12, "PERSON")],
example = Example.from_dict(doc, gold_dict)
```
## Lexical data for vocabulary {#vocab-jsonl new="2"}
## Lexical data for vocabulary {id="vocab-jsonl",version="2"}
This data file can be provided via the `vocab_data` setting in the
`[initialize]` block of the training config to pre-define the lexical data to
@ -483,13 +481,11 @@ spaCy's [`Lexeme`](/api/lexeme#attributes) object.
> vocab_data = "/path/to/vocab-data.jsonl"
> ```
```python
### First line
```python {title="First line"}
{"lang": "en", "settings": {"oov_prob": -20.502029418945312}}
```
```python
### Entry structure
```python {title="Entry structure"}
{
"orth": string, # the word text
"id": int, # can correspond to row in vectors table
@ -526,7 +522,7 @@ Here's an example of the 20 most frequent lexemes in the English training data:
%%GITHUB_SPACY/extra/example_data/vocab-data.jsonl
```
## Pipeline meta {#meta}
## Pipeline meta {id="meta"}
The pipeline meta is available as the file `meta.json` and exported
automatically when you save an `nlp` object to disk. Its contents are available

21
website/docs/api/dependencymatcher.md → website/docs/api/dependencymatcher.mdx
View File

@ -2,7 +2,7 @@
title: DependencyMatcher
teaser: Match subtrees within a dependency parse
tag: class
new: 3
version: 3
source: spacy/matcher/dependencymatcher.pyx
---
@ -14,7 +14,7 @@ It requires a pretrained [`DependencyParser`](/api/parser) or other component
that sets the `Token.dep` and `Token.head` attributes. See the
[usage guide](/usage/rule-based-matching#dependencymatcher) for examples.
## Pattern format {#patterns}
## Pattern format {id="patterns"}
> ```python
> ### Example
@ -62,7 +62,7 @@ of relations, see the usage guide on
</Infobox>
### Operators {#operators}
### Operators {id="operators"}
The following operators are supported by the `DependencyMatcher`, most of which
come directly from
@ -87,8 +87,7 @@ come directly from
| `A <++ B` | `B` is a right parent of `A`, i.e. `A` is a child of `B` and `A.i < B.i` _(not in Semgrex)_. |
| `A <-- B` | `B` is a left parent of `A`, i.e. `A` is a child of `B` and `A.i > B.i` _(not in Semgrex)_. |
## DependencyMatcher.\_\_init\_\_ {#init tag="method"}
## DependencyMatcher.\_\_init\_\_ {id="init",tag="method"}
Create a `DependencyMatcher`.
@ -105,7 +104,7 @@ Create a `DependencyMatcher`.
| _keyword-only_ | |
| `validate` | Validate all patterns added to this matcher. ~~bool~~ |
## DependencyMatcher.\_\call\_\_ {#call tag="method"}
## DependencyMatcher.\_\_call\_\_ {id="call",tag="method"}
Find all tokens matching the supplied patterns on the `Doc` or `Span`.
@ -127,7 +126,7 @@ Find all tokens matching the supplied patterns on the `Doc` or `Span`.
| `doclike` | The `Doc` or `Span` to match over. ~~Union[Doc, Span]~~ |
| **RETURNS** | A list of `(match_id, token_ids)` tuples, describing the matches. The `match_id` is the ID of the match pattern and `token_ids` is a list of token indices matched by the pattern, where the position of each token in the list corresponds to the position of the node specification in the pattern. ~~List[Tuple[int, List[int]]]~~ |
## DependencyMatcher.\_\_len\_\_ {#len tag="method"}
## DependencyMatcher.\_\_len\_\_ {id="len",tag="method"}
Get the number of rules added to the dependency matcher. Note that this only
returns the number of rules (identical with the number of IDs), not the number
@ -148,7 +147,7 @@ of individual patterns.
| ----------- | ---------------------------- |
| **RETURNS** | The number of rules. ~~int~~ |
## DependencyMatcher.\_\_contains\_\_ {#contains tag="method"}
## DependencyMatcher.\_\_contains\_\_ {id="contains",tag="method"}
Check whether the matcher contains rules for a match ID.
@ -166,7 +165,7 @@ Check whether the matcher contains rules for a match ID.
| `key` | The match ID. ~~str~~ |
| **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ |
## DependencyMatcher.add {#add tag="method"}
## DependencyMatcher.add {id="add",tag="method"}
Add a rule to the matcher, consisting of an ID key, one or more patterns, and an
optional callback function to act on the matches. The callback function will
@ -191,7 +190,7 @@ will be overwritten.
| _keyword-only_ | |
| `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[DependencyMatcher, Doc, int, List[Tuple], Any]]~~ |
## DependencyMatcher.get {#get tag="method"}
## DependencyMatcher.get {id="get",tag="method"}
Retrieve the pattern stored for a key. Returns the rule as an
`(on_match, patterns)` tuple containing the callback and available patterns.
@ -208,7 +207,7 @@ Retrieve the pattern stored for a key. Returns the rule as an
| `key` | The ID of the match rule. ~~str~~ |
| **RETURNS** | The rule, as an `(on_match, patterns)` tuple. ~~Tuple[Optional[Callable], List[List[Union[Dict, Tuple]]]]~~ |
## DependencyMatcher.remove {#remove tag="method"}
## DependencyMatcher.remove {id="remove",tag="method"}
Remove a rule from the dependency matcher. A `KeyError` is raised if the match
ID does not exist.

42
website/docs/api/dependencyparser.md → website/docs/api/dependencyparser.mdx
View File

@ -25,7 +25,7 @@ current state. The weights are updated such that the scores assigned to the set
of optimal actions is increased, while scores assigned to other actions are
decreased. Note that more than one action may be optimal for a given state.
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Dependency predictions are assigned to the `Token.dep` and `Token.head` fields.
Beside the dependencies themselves, the parser decides sentence boundaries,
@ -39,7 +39,7 @@ which are saved in `Token.is_sent_start` and accessible via `Doc.sents`.
| `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. After the parser runs this will be `True` or `False` for all tokens. ~~bool~~ |
| `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -74,7 +74,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/dep_parser.pyx
```
## DependencyParser.\_\_init\_\_ {#init tag="method"}
## DependencyParser.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -107,7 +107,7 @@ shortcut for this and instantiate the component using its string name and
| `min_action_freq` | The minimum frequency of labelled actions to retain. Rarer labelled actions have their label backed-off to "dep". While this primarily affects the label accuracy, it can also affect the attachment structure, as the labels are used to represent the pseudo-projectivity transformation. ~~int~~ |
| `scorer` | The scoring method. Defaults to [`Scorer.score_deps`](/api/scorer#score_deps) for the attribute `"dep"` ignoring the labels `p` and `punct` and [`Scorer.score_spans`](/api/scorer/#score_spans) for the attribute `"sents"`. ~~Optional[Callable]~~ |
## DependencyParser.\_\_call\_\_ {#call tag="method"}
## DependencyParser.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -131,7 +131,7 @@ and all pipeline components are applied to the `Doc` in order. Both
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## DependencyParser.pipe {#pipe tag="method"}
## DependencyParser.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -155,7 +155,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/dependencyparser#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## DependencyParser.initialize {#initialize tag="method" new="3"}
## DependencyParser.initialize {id="initialize",tag="method",version="3"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -192,7 +192,7 @@ config.
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ |
## DependencyParser.predict {#predict tag="method"}
## DependencyParser.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them.
@ -209,7 +209,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | A helper class for the parse state (internal). ~~StateClass~~ |
## DependencyParser.set_annotations {#set_annotations tag="method"}
## DependencyParser.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
@ -226,7 +226,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `DependencyParser.predict`. Returns an internal helper class for the parse state. ~~List[StateClass]~~ |
## DependencyParser.update {#update tag="method"}
## DependencyParser.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects, updating the pipe's
model. Delegates to [`predict`](/api/dependencyparser#predict) and
@ -249,7 +249,7 @@ model. Delegates to [`predict`](/api/dependencyparser#predict) and
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## DependencyParser.get_loss {#get_loss tag="method"}
## DependencyParser.get_loss {id="get_loss",tag="method"}
Find the loss and gradient of loss for the batch of documents and their
predicted scores.
@ -268,7 +268,7 @@ predicted scores.
| `scores` | Scores representing the model's predictions. ~~StateClass~~ |
| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ |
## DependencyParser.create_optimizer {#create_optimizer tag="method"}
## DependencyParser.create_optimizer {id="create_optimizer",tag="method"}
Create an [`Optimizer`](https://thinc.ai/docs/api-optimizers) for the pipeline
component.
@ -284,7 +284,7 @@ component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## DependencyParser.use_params {#use_params tag="method, contextmanager"}
## DependencyParser.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -301,7 +301,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## DependencyParser.add_label {#add_label tag="method"}
## DependencyParser.add_label {id="add_label",tag="method"}
Add a new label to the pipe. Note that you don't have to call this method if you
provide a **representative data sample** to the [`initialize`](#initialize)
@ -321,7 +321,7 @@ to the model, and the output dimension will be
| `label` | The label to add. ~~str~~ |
| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ |
## DependencyParser.set_output {#set_output tag="method"}
## DependencyParser.set_output {id="set_output",tag="method"}
Change the output dimension of the component's model by calling the model's
attribute `resize_output`. This is a function that takes the original model and
@ -340,7 +340,7 @@ forgetting" problem.
| ---- | --------------------------------- |
| `nO` | The new output dimension. ~~int~~ |
## DependencyParser.to_disk {#to_disk tag="method"}
## DependencyParser.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -357,7 +357,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## DependencyParser.from_disk {#from_disk tag="method"}
## DependencyParser.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -375,7 +375,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `DependencyParser` object. ~~DependencyParser~~ |
## DependencyParser.to_bytes {#to_bytes tag="method"}
## DependencyParser.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -392,7 +392,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `DependencyParser` object. ~~bytes~~ |
## DependencyParser.from_bytes {#from_bytes tag="method"}
## DependencyParser.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -411,7 +411,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `DependencyParser` object. ~~DependencyParser~~ |
## DependencyParser.labels {#labels tag="property"}
## DependencyParser.labels {id="labels",tag="property"}
The labels currently added to the component.
@ -426,7 +426,7 @@ The labels currently added to the component.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ |
## DependencyParser.label_data {#label_data tag="property" new="3"}
## DependencyParser.label_data {id="label_data",tag="property",version="3"}
The labels currently added to the component and their internal meta information.
This is the data generated by [`init labels`](/api/cli#init-labels) and used by
@ -444,7 +444,7 @@ the model with a pre-defined label set.
| ----------- | ------------------------------------------------------------------------------- |
| **RETURNS** | The label data added to the component. ~~Dict[str, Dict[str, Dict[str, int]]]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

72
website/docs/api/doc.md → website/docs/api/doc.mdx
View File

@ -12,7 +12,7 @@ compressed binary strings. The `Doc` object holds an array of
[`Span`](/api/span) objects are views of this array, i.e. they don't own the
data themselves.
## Doc.\_\_init\_\_ {#init tag="method"}
## Doc.\_\_init\_\_ {id="init",tag="method"}
Construct a `Doc` object. The most common way to get a `Doc` object is via the
`nlp` object.
@ -47,7 +47,7 @@ Construct a `Doc` object. The most common way to get a `Doc` object is via the
| `sent_starts` <Tag variant="new">3</Tag> | A list of values, of the same length as `words`, to assign as `token.is_sent_start`. Will be overridden by heads if `heads` is provided. Defaults to `None`. ~~Optional[List[Union[bool, int, None]]]~~ |
| `ents` <Tag variant="new">3</Tag> | A list of strings, of the same length of `words`, to assign the token-based IOB tag. Defaults to `None`. ~~Optional[List[str]]~~ |
## Doc.\_\_getitem\_\_ {#getitem tag="method"}
## Doc.\_\_getitem\_\_ {id="getitem",tag="method"}
Get a [`Token`](/api/token) object at position `i`, where `i` is an integer.
Negative indexing is supported, and follows the usual Python semantics, i.e.
@ -80,7 +80,7 @@ semantics.
| `start_end` | The slice of the document to get. ~~Tuple[int, int]~~ |
| **RETURNS** | The span at `doc[start:end]`. ~~Span~~ |
## Doc.\_\_iter\_\_ {#iter tag="method"}
## Doc.\_\_iter\_\_ {id="iter",tag="method"}
Iterate over `Token` objects, from which the annotations can be easily accessed.
@ -100,7 +100,7 @@ underlying C data directly from Cython.
| ---------- | --------------------------- |
| **YIELDS** | A `Token` object. ~~Token~~ |
## Doc.\_\_len\_\_ {#len tag="method"}
## Doc.\_\_len\_\_ {id="len",tag="method"}
Get the number of tokens in the document.
@ -115,7 +115,7 @@ Get the number of tokens in the document.
| ----------- | --------------------------------------------- |
| **RETURNS** | The number of tokens in the document. ~~int~~ |
## Doc.set_extension {#set_extension tag="classmethod" new="2"}
## Doc.set_extension {id="set_extension",tag="classmethod",version="2"}
Define a custom attribute on the `Doc` which becomes available via `Doc._`. For
details, see the documentation on
@ -140,7 +140,7 @@ details, see the documentation on
| `setter` | Setter function that takes the `Doc` and a value, and modifies the object. Is called when the user writes to the `Doc._` attribute. ~~Optional[Callable[[Doc, Any], None]]~~ |
| `force` | Force overwriting existing attribute. ~~bool~~ |
## Doc.get_extension {#get_extension tag="classmethod" new="2"}
## Doc.get_extension {id="get_extension",tag="classmethod",version="2"}
Look up a previously registered extension by name. Returns a 4-tuple
`(default, method, getter, setter)` if the extension is registered. Raises a
@ -160,7 +160,7 @@ Look up a previously registered extension by name. Returns a 4-tuple
| `name` | Name of the extension. ~~str~~ |
| **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ |
## Doc.has_extension {#has_extension tag="classmethod" new="2"}
## Doc.has_extension {id="has_extension",tag="classmethod",version="2"}
Check whether an extension has been registered on the `Doc` class.
@ -177,7 +177,7 @@ Check whether an extension has been registered on the `Doc` class.
| `name` | Name of the extension to check. ~~str~~ |
| **RETURNS** | Whether the extension has been registered. ~~bool~~ |
## Doc.remove_extension {#remove_extension tag="classmethod" new="2.0.12"}
## Doc.remove_extension {id="remove_extension",tag="classmethod",version="2.0.12"}
Remove a previously registered extension.
@ -195,7 +195,7 @@ Remove a previously registered extension.
| `name` | Name of the extension. ~~str~~ |
| **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ |
## Doc.char_span {#char_span tag="method" new="2"}
## Doc.char_span {id="char_span",tag="method",version="2"}
Create a `Span` object from the slice `doc.text[start_idx:end_idx]`. Returns
`None` if the character indices don't map to a valid span using the default
@ -219,7 +219,7 @@ alignment mode `"strict".
| `alignment_mode` | How character indices snap to token boundaries. Options: `"strict"` (no snapping), `"contract"` (span of all tokens completely within the character span), `"expand"` (span of all tokens at least partially covered by the character span). Defaults to `"strict"`. ~~str~~ |
| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ |
## Doc.set_ents {#set_ents tag="method" new="3"}
## Doc.set_ents {id="set_ents",tag="method",version="3"}
Set the named entities in the document.
@ -243,7 +243,7 @@ Set the named entities in the document.
| `outside` | Spans outside of entities (O in IOB). ~~Optional[List[Span]]~~ |
| `default` | How to set entity annotation for tokens outside of any provided spans. Options: `"blocked"`, `"missing"`, `"outside"` and `"unmodified"` (preserve current state). Defaults to `"outside"`. ~~str~~ |
## Doc.similarity {#similarity tag="method" model="vectors"}
## Doc.similarity {id="similarity",tag="method",model="vectors"}
Make a semantic similarity estimate. The default estimate is cosine similarity
using an average of word vectors.
@ -263,7 +263,7 @@ using an average of word vectors.
| `other` | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ |
| **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ |
## Doc.count_by {#count_by tag="method"}
## Doc.count_by {id="count_by",tag="method"}
Count the frequencies of a given attribute. Produces a dict of
`{attr (int): count (ints)}` frequencies, keyed by the values of the given
@ -284,7 +284,7 @@ attribute ID.
| `attr_id` | The attribute ID. ~~int~~ |
| **RETURNS** | A dictionary mapping attributes to integer counts. ~~Dict[int, int]~~ |
## Doc.get_lca_matrix {#get_lca_matrix tag="method"}
## Doc.get_lca_matrix {id="get_lca_matrix",tag="method"}
Calculates the lowest common ancestor matrix for a given `Doc`. Returns LCA
matrix containing the integer index of the ancestor, or `-1` if no common
@ -302,7 +302,7 @@ ancestor is found, e.g. if span excludes a necessary ancestor.
| ----------- | -------------------------------------------------------------------------------------- |
| **RETURNS** | The lowest common ancestor matrix of the `Doc`. ~~numpy.ndarray[ndim=2, dtype=int32]~~ |
## Doc.has_annotation {#has_annotation tag="method"}
## Doc.has_annotation {id="has_annotation",tag="method"}
Check whether the doc contains annotation on a
[`Token` attribute](/api/token#attributes).
@ -327,7 +327,7 @@ doc = nlp("This is a text")
| `require_complete` | Whether to check that the attribute is set on every token in the doc. Defaults to `False`. ~~bool~~ |
| **RETURNS** | Whether specified annotation is present in the doc. ~~bool~~ |
## Doc.to_array {#to_array tag="method"}
## Doc.to_array {id="to_array",tag="method"}
Export given token attributes to a numpy `ndarray`. If `attr_ids` is a sequence
of `M` attributes, the output array will be of shape `(N, M)`, where `N` is the
@ -355,7 +355,7 @@ Returns a 2D array with one row per token and one column per attribute (when
| `attr_ids` | A list of attributes (int IDs or string names) or a single attribute (int ID or string name). ~~Union[int, str, List[Union[int, str]]]~~ |
| **RETURNS** | The exported attributes as a numpy array. ~~Union[numpy.ndarray[ndim=2, dtype=uint64], numpy.ndarray[ndim=1, dtype=uint64]]~~ |
## Doc.from_array {#from_array tag="method"}
## Doc.from_array {id="from_array",tag="method"}
Load attributes from a numpy array. Write to a `Doc` object, from an `(M, N)`
array of attributes.
@ -379,7 +379,7 @@ array of attributes.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Doc` itself. ~~Doc~~ |
## Doc.from_docs {#from_docs tag="staticmethod" new="3"}
## Doc.from_docs {id="from_docs",tag="staticmethod",version="3"}
Concatenate multiple `Doc` objects to form a new one. Raises an error if the
`Doc` objects do not all share the same `Vocab`.
@ -408,7 +408,7 @@ Concatenate multiple `Doc` objects to form a new one. Raises an error if the
| `exclude` <Tag variant="new">3.3</Tag> | String names of Doc attributes to exclude. Supported: `spans`, `tensor`, `user_data`. ~~Iterable[str]~~ |
| **RETURNS** | The new `Doc` object that is containing the other docs or `None`, if `docs` is empty or `None`. ~~Optional[Doc]~~ |
## Doc.to_disk {#to_disk tag="method" new="2"}
## Doc.to_disk {id="to_disk",tag="method",version="2"}
Save the current state to a directory.
@ -424,7 +424,7 @@ Save the current state to a directory.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## Doc.from_disk {#from_disk tag="method" new="2"}
## Doc.from_disk {id="from_disk",tag="method",version="2"}
Loads state from a directory. Modifies the object in place and returns it.
@ -443,7 +443,7 @@ Loads state from a directory. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `Doc` object. ~~Doc~~ |
## Doc.to_bytes {#to_bytes tag="method"}
## Doc.to_bytes {id="to_bytes",tag="method"}
Serialize, i.e. export the document contents to a binary string.
@ -460,7 +460,7 @@ Serialize, i.e. export the document contents to a binary string.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | A losslessly serialized copy of the `Doc`, including all annotations. ~~bytes~~ |
## Doc.from_bytes {#from_bytes tag="method"}
## Doc.from_bytes {id="from_bytes",tag="method"}
Deserialize, i.e. import the document contents from a binary string.
@ -481,7 +481,7 @@ Deserialize, i.e. import the document contents from a binary string.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Doc` object. ~~Doc~~ |
## Doc.to_json {#to_json tag="method"}
## Doc.to_json {id="to_json",tag="method"}
Serializes a document to JSON. Note that this is format differs from the
deprecated [`JSON training format`](/api/data-formats#json-input).
@ -498,7 +498,7 @@ deprecated [`JSON training format`](/api/data-formats#json-input).
| `underscore` | Optional list of string names of custom `Doc` attributes. Attribute values need to be JSON-serializable. Values will be added to an `"_"` key in the data, e.g. `"_": {"foo": "bar"}`. ~~Optional[List[str]]~~ |
| **RETURNS** | The data in JSON format. ~~Dict[str, Any]~~ |
## Doc.from_json {#from_json tag="method" new="3.3.1"}
## Doc.from_json {id="from_json",tag="method",version="3.3.1"}
Deserializes a document from JSON, i.e. generates a document from the provided
JSON data as generated by [`Doc.to_json()`](/api/doc#to_json).
@ -520,7 +520,7 @@ JSON data as generated by [`Doc.to_json()`](/api/doc#to_json).
| `validate` | Whether to validate the JSON input against the expected schema for detailed debugging. Defaults to `False`. ~~bool~~ |
| **RETURNS** | A `Doc` corresponding to the provided JSON. ~~Doc~~ |
## Doc.retokenize {#retokenize tag="contextmanager" new="2.1"}
## Doc.retokenize {id="retokenize",tag="contextmanager",version="2.1"}
Context manager to handle retokenization of the `Doc`. Modifications to the
`Doc`'s tokenization are stored, and then made all at once when the context
@ -540,7 +540,7 @@ invalidated, although they may accidentally continue to work.
| ----------- | -------------------------------- |
| **RETURNS** | The retokenizer. ~~Retokenizer~~ |
### Retokenizer.merge {#retokenizer.merge tag="method"}
### Retokenizer.merge {id="retokenizer.merge",tag="method"}
Mark a span for merging. The `attrs` will be applied to the resulting token (if
they're context-dependent token attributes like `LEMMA` or `DEP`) or to the
@ -563,7 +563,7 @@ values.
| `span` | The span to merge. ~~Span~~ |
| `attrs` | Attributes to set on the merged token. ~~Dict[Union[str, int], Any]~~ |
### Retokenizer.split {#retokenizer.split tag="method"}
### Retokenizer.split {id="retokenizer.split",tag="method"}
Mark a token for splitting, into the specified `orths`. The `heads` are required
to specify how the new subtokens should be integrated into the dependency tree.
@ -599,7 +599,7 @@ underlying lexeme (if they're context-independent lexical attributes like
| `heads` | List of `token` or `(token, subtoken)` tuples specifying the tokens to attach the newly split subtokens to. ~~List[Union[Token, Tuple[Token, int]]]~~ |
| `attrs` | Attributes to set on all split tokens. Attribute names mapped to list of per-token attribute values. ~~Dict[Union[str, int], List[Any]]~~ |
## Doc.ents {#ents tag="property" model="NER"}
## Doc.ents {id="ents",tag="property",model="NER"}
The named entities in the document. Returns a tuple of named entity `Span`
objects, if the entity recognizer has been applied.
@ -617,7 +617,7 @@ objects, if the entity recognizer has been applied.
| ----------- | ---------------------------------------------------------------- |
| **RETURNS** | Entities in the document, one `Span` per entity. ~~Tuple[Span]~~ |
## Doc.spans {#spans tag="property"}
## Doc.spans {id="spans",tag="property"}
A dictionary of named span groups, to store and access additional span
annotations. You can write to it by assigning a list of [`Span`](/api/span)
@ -634,7 +634,7 @@ objects or a [`SpanGroup`](/api/spangroup) to a given key.
| ----------- | ------------------------------------------------------------------ |
| **RETURNS** | The span groups assigned to the document. ~~Dict[str, SpanGroup]~~ |
## Doc.cats {#cats tag="property" model="text classifier"}
## Doc.cats {id="cats",tag="property",model="text classifier"}
Maps a label to a score for categories applied to the document. Typically set by
the [`TextCategorizer`](/api/textcategorizer).
@ -650,7 +650,7 @@ the [`TextCategorizer`](/api/textcategorizer).
| ----------- | ---------------------------------------------------------- |
| **RETURNS** | The text categories mapped to scores. ~~Dict[str, float]~~ |
## Doc.noun_chunks {#noun_chunks tag="property" model="parser"}
## Doc.noun_chunks {id="noun_chunks",tag="property",model="parser"}
Iterate over the base noun phrases in the document. Yields base noun-phrase
`Span` objects, if the document has been syntactically parsed. A base noun
@ -677,7 +677,7 @@ implemented for the given language, a `NotImplementedError` is raised.
| ---------- | ------------------------------------- |
| **YIELDS** | Noun chunks in the document. ~~Span~~ |
## Doc.sents {#sents tag="property" model="sentences"}
## Doc.sents {id="sents",tag="property",model="sentences"}
Iterate over the sentences in the document. Sentence spans have no label.
@ -699,7 +699,7 @@ will raise an error otherwise.
| ---------- | ----------------------------------- |
| **YIELDS** | Sentences in the document. ~~Span~~ |
## Doc.has_vector {#has_vector tag="property" model="vectors"}
## Doc.has_vector {id="has_vector",tag="property",model="vectors"}
A boolean value indicating whether a word vector is associated with the object.
@ -714,7 +714,7 @@ A boolean value indicating whether a word vector is associated with the object.
| ----------- | --------------------------------------------------------- |
| **RETURNS** | Whether the document has a vector data attached. ~~bool~~ |
## Doc.vector {#vector tag="property" model="vectors"}
## Doc.vector {id="vector",tag="property",model="vectors"}
A real-valued meaning representation. Defaults to an average of the token
vectors.
@ -731,7 +731,7 @@ vectors.
| ----------- | -------------------------------------------------------------------------------------------------- |
| **RETURNS** | A 1-dimensional array representing the document's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
## Doc.vector_norm {#vector_norm tag="property" model="vectors"}
## Doc.vector_norm {id="vector_norm",tag="property",model="vectors"}
The L2 norm of the document's vector representation.
@ -749,7 +749,7 @@ The L2 norm of the document's vector representation.
| ----------- | --------------------------------------------------- |
| **RETURNS** | The L2 norm of the vector representation. ~~float~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| ------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- |
@ -768,7 +768,7 @@ The L2 norm of the document's vector representation.
| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ |
| `activations` <Tag variant="new">4.0</Tag> | A dictionary of activations per trainable pipe (available when the `save_activations` option of a pipe is enabled). ~~Dict[str, Option[Any]]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

23
website/docs/api/docbin.md → website/docs/api/docbin.mdx
View File

@ -1,7 +1,7 @@
---
title: DocBin
tag: class
new: 2.2
version: 2.2
teaser: Pack Doc objects for binary serialization
source: spacy/tokens/_serialize.py
---
@ -15,8 +15,7 @@ notable downside to this format is that you can't easily extract just one
document from the `DocBin`. The serialization format is gzipped msgpack, where
the msgpack object has the following structure:
```python
### msgpack object structure
```python {title="msgpack object structure"}
{
"version": str, # DocBin version number
"attrs": List[uint64], # e.g. [TAG, HEAD, ENT_IOB, ENT_TYPE]
@ -33,7 +32,7 @@ object. This means the storage is more efficient if you pack more documents
together, because you have less duplication in the strings. For usage examples,
see the docs on [serializing `Doc` objects](/usage/saving-loading#docs).
## DocBin.\_\_init\_\_ {#init tag="method"}
## DocBin.\_\_init\_\_ {id="init",tag="method"}
Create a `DocBin` object to hold serialized annotations.
@ -50,7 +49,7 @@ Create a `DocBin` object to hold serialized annotations.
| `store_user_data` | Whether to write the `Doc.user_data` and the values of custom extension attributes to file/bytes. Defaults to `False`. ~~bool~~ |
| `docs` | `Doc` objects to add on initialization. ~~Iterable[Doc]~~ |
## DocBin.\_\len\_\_ {#len tag="method"}
## DocBin.\_\_len\_\_ {id="len",tag="method"}
Get the number of `Doc` objects that were added to the `DocBin`.
@ -67,7 +66,7 @@ Get the number of `Doc` objects that were added to the `DocBin`.
| ----------- | --------------------------------------------------- |
| **RETURNS** | The number of `Doc`s added to the `DocBin`. ~~int~~ |
## DocBin.add {#add tag="method"}
## DocBin.add {id="add",tag="method"}
Add a `Doc`'s annotations to the `DocBin` for serialization.
@ -83,7 +82,7 @@ Add a `Doc`'s annotations to the `DocBin` for serialization.
| -------- | -------------------------------- |
| `doc` | The `Doc` object to add. ~~Doc~~ |
## DocBin.get_docs {#get_docs tag="method"}
## DocBin.get_docs {id="get_docs",tag="method"}
Recover `Doc` objects from the annotations, using the given vocab.
@ -98,7 +97,7 @@ Recover `Doc` objects from the annotations, using the given vocab.
| `vocab` | The shared vocab. ~~Vocab~~ |
| **YIELDS** | The `Doc` objects. ~~Doc~~ |
## DocBin.merge {#merge tag="method"}
## DocBin.merge {id="merge",tag="method"}
Extend the annotations of this `DocBin` with the annotations from another. Will
raise an error if the pre-defined `attrs` of the two `DocBin`s don't match.
@ -118,7 +117,7 @@ raise an error if the pre-defined `attrs` of the two `DocBin`s don't match.
| -------- | ------------------------------------------------------ |
| `other` | The `DocBin` to merge into the current bin. ~~DocBin~~ |
## DocBin.to_bytes {#to_bytes tag="method"}
## DocBin.to_bytes {id="to_bytes",tag="method"}
Serialize the `DocBin`'s annotations to a bytestring.
@ -134,7 +133,7 @@ Serialize the `DocBin`'s annotations to a bytestring.
| ----------- | ---------------------------------- |
| **RETURNS** | The serialized `DocBin`. ~~bytes~~ |
## DocBin.from_bytes {#from_bytes tag="method"}
## DocBin.from_bytes {id="from_bytes",tag="method"}
Deserialize the `DocBin`'s annotations from a bytestring.
@ -150,7 +149,7 @@ Deserialize the `DocBin`'s annotations from a bytestring.
| `bytes_data` | The data to load from. ~~bytes~~ |
| **RETURNS** | The loaded `DocBin`. ~~DocBin~~ |
## DocBin.to_disk {#to_disk tag="method" new="3"}
## DocBin.to_disk {id="to_disk",tag="method",version="3"}
Save the serialized `DocBin` to a file. Typically uses the `.spacy` extension
and the result can be used as the input data for
@ -168,7 +167,7 @@ and the result can be used as the input data for
| -------- | -------------------------------------------------------------------------- |
| `path` | The file path, typically with the `.spacy` extension. ~~Union[str, Path]~~ |
## DocBin.from_disk {#from_disk tag="method" new="3"}
## DocBin.from_disk {id="from_disk",tag="method",version="3"}
Load a serialized `DocBin` from a file. Typically uses the `.spacy` extension.

40
website/docs/api/edittreelemmatizer.md → website/docs/api/edittreelemmatizer.mdx
View File

@ -2,7 +2,7 @@
title: EditTreeLemmatizer
tag: class
source: spacy/pipeline/edit_tree_lemmatizer.py
new: 3.3
version: 3.3
teaser: 'Pipeline component for lemmatization'
api_base_class: /api/pipe
api_string_name: trainable_lemmatizer
@ -18,7 +18,7 @@ and construction method used by this lemmatizer were proposed in
For a lookup and rule-based lemmatizer, see [`Lemmatizer`](/api/lemmatizer).
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predictions are assigned to `Token.lemma`.
@ -27,7 +27,7 @@ Predictions are assigned to `Token.lemma`.
| `Token.lemma` | The lemma (hash). ~~int~~ |
| `Token.lemma_` | The lemma. ~~str~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -58,7 +58,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/edit_tree_lemmatizer.py
```
## EditTreeLemmatizer.\_\_init\_\_ {#init tag="method"}
## EditTreeLemmatizer.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -91,7 +91,7 @@ shortcut for this and instantiate the component using its string name and
| `top_k` | The number of most probable edit trees to try before resorting to `backoff`. Defaults to `1`. ~~int~~ |
| `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"lemma"`. ~~Optional[Callable]~~ |
## EditTreeLemmatizer.\_\_call\_\_ {#call tag="method"}
## EditTreeLemmatizer.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -115,7 +115,7 @@ and all pipeline components are applied to the `Doc` in order. Both
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## EditTreeLemmatizer.pipe {#pipe tag="method"}
## EditTreeLemmatizer.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -139,7 +139,7 @@ and [`pipe`](/api/edittreelemmatizer#pipe) delegate to the
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## EditTreeLemmatizer.initialize {#initialize tag="method" new="3"}
## EditTreeLemmatizer.initialize {id="initialize",tag="method",version="3"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -176,7 +176,7 @@ config.
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ |
## EditTreeLemmatizer.predict {#predict tag="method"}
## EditTreeLemmatizer.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them.
@ -193,7 +193,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The model's prediction for each document. |
## EditTreeLemmatizer.set_annotations {#set_annotations tag="method"}
## EditTreeLemmatizer.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects, using pre-computed tree
identifiers.
@ -211,7 +211,7 @@ identifiers.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `tree_ids` | The identifiers of the edit trees to apply, produced by `EditTreeLemmatizer.predict`. |
## EditTreeLemmatizer.update {#update tag="method"}
## EditTreeLemmatizer.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects containing the
predictions and gold-standard annotations, and update the component's model.
@ -235,7 +235,7 @@ Delegates to [`predict`](/api/edittreelemmatizer#predict) and
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## EditTreeLemmatizer.get_loss {#get_loss tag="method"}
## EditTreeLemmatizer.get_loss {id="get_loss",tag="method"}
Find the loss and gradient of loss for the batch of documents and their
predicted scores.
@ -254,7 +254,7 @@ predicted scores.
| `scores` | Scores representing the model's predictions. |
| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ |
## EditTreeLemmatizer.create_optimizer {#create_optimizer tag="method"}
## EditTreeLemmatizer.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -269,7 +269,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## EditTreeLemmatizer.use_params {#use_params tag="method, contextmanager"}
## EditTreeLemmatizer.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -286,7 +286,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## EditTreeLemmatizer.to_disk {#to_disk tag="method"}
## EditTreeLemmatizer.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -303,7 +303,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## EditTreeLemmatizer.from_disk {#from_disk tag="method"}
## EditTreeLemmatizer.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -321,7 +321,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `EditTreeLemmatizer` object. ~~EditTreeLemmatizer~~ |
## EditTreeLemmatizer.to_bytes {#to_bytes tag="method"}
## EditTreeLemmatizer.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -338,7 +338,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `EditTreeLemmatizer` object. ~~bytes~~ |
## EditTreeLemmatizer.from_bytes {#from_bytes tag="method"}
## EditTreeLemmatizer.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -357,7 +357,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `EditTreeLemmatizer` object. ~~EditTreeLemmatizer~~ |
## EditTreeLemmatizer.labels {#labels tag="property"}
## EditTreeLemmatizer.labels {id="labels",tag="property"}
The labels currently added to the component.
@ -372,7 +372,7 @@ identifiers of edit trees.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ |
## EditTreeLemmatizer.label_data {#label_data tag="property" new="3"}
## EditTreeLemmatizer.label_data {id="label_data",tag="property",version="3"}
The labels currently added to the component and their internal meta information.
This is the data generated by [`init labels`](/api/cli#init-labels) and used by
@ -390,7 +390,7 @@ initialize the model with a pre-defined label set.
| ----------- | ---------------------------------------------------------- |
| **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

36
website/docs/api/entitylinker.md → website/docs/api/entitylinker.mdx
View File

@ -2,7 +2,7 @@
title: EntityLinker
tag: class
source: spacy/pipeline/entity_linker.py
new: 2.2
version: 2.2
teaser: 'Pipeline component for named entity linking and disambiguation'
api_base_class: /api/pipe
api_string_name: entity_linker
@ -17,7 +17,7 @@ and a machine learning model to pick the right candidate, given the local
context of the mention. `EntityLinker` defaults to using the
[`InMemoryLookupKB`](/api/kb_in_memory) implementation.
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predictions, in the form of knowledge base IDs, will be assigned to
`Token.ent_kb_id_`.
@ -27,7 +27,7 @@ Predictions, in the form of knowledge base IDs, will be assigned to
| `Token.ent_kb_id` | Knowledge base ID (hash). ~~int~~ |
| `Token.ent_kb_id_` | Knowledge base ID. ~~str~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -72,7 +72,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/entity_linker.py
```
## EntityLinker.\_\_init\_\_ {#init tag="method"}
## EntityLinker.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -115,7 +115,7 @@ custom knowledge base, you should either call
| `scorer` <Tag variant="new">3.2</Tag> | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ |
| `threshold` <Tag variant="new">3.4</Tag> | Confidence threshold for entity predictions. The default of `None` implies that all predictions are accepted, otherwise those with a score beneath the treshold are discarded. If there are no predictions with scores above the threshold, the linked entity is `NIL`. ~~Optional[float]~~ |
## EntityLinker.\_\_call\_\_ {#call tag="method"}
## EntityLinker.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -138,7 +138,7 @@ delegate to the [`predict`](/api/entitylinker#predict) and
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## EntityLinker.pipe {#pipe tag="method"}
## EntityLinker.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -162,7 +162,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/entitylinker#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## EntityLinker.set_kb {#set_kb tag="method" new="3"}
## EntityLinker.set_kb {id="set_kb",tag="method",version="3"}
The `kb_loader` should be a function that takes a `Vocab` instance and creates
the `KnowledgeBase`, ensuring that the strings of the knowledge base are synced
@ -184,7 +184,7 @@ with the current vocab.
| ----------- | ---------------------------------------------------------------------------------------------------------------- |
| `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ |
## EntityLinker.initialize {#initialize tag="method" new="3"}
## EntityLinker.initialize {id="initialize",tag="method",version="3"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -214,7 +214,7 @@ are synced with the current vocab.
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
| `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ |
## EntityLinker.predict {#predict tag="method"}
## EntityLinker.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them. Returns the KB IDs for each entity in each doc, including `NIL`
@ -232,7 +232,7 @@ if there is no prediction.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The predicted KB identifiers for the entities in the `docs`. ~~List[str]~~ |
## EntityLinker.set_annotations {#set_annotations tag="method"}
## EntityLinker.set_annotations {id="set_annotations",tag="method"}
Modify a batch of documents, using pre-computed entity IDs for a list of named
entities.
@ -250,7 +250,7 @@ entities.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `kb_ids` | The knowledge base identifiers for the entities in the docs, predicted by `EntityLinker.predict`. ~~List[str]~~ |
## EntityLinker.update {#update tag="method"}
## EntityLinker.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects, updating both the
pipe's entity linking model and context encoder. Delegates to
@ -273,7 +273,7 @@ pipe's entity linking model and context encoder. Delegates to
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## EntityLinker.create_optimizer {#create_optimizer tag="method"}
## EntityLinker.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -288,7 +288,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## EntityLinker.use_params {#use_params tag="method, contextmanager"}
## EntityLinker.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -305,7 +305,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## EntityLinker.to_disk {#to_disk tag="method"}
## EntityLinker.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -322,7 +322,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## EntityLinker.from_disk {#from_disk tag="method"}
## EntityLinker.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -340,7 +340,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `EntityLinker` object. ~~EntityLinker~~ |
## EntityLinker.to_bytes {#to_bytes tag="method"}
## EntityLinker.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -357,7 +357,7 @@ Serialize the pipe to a bytestring, including the `KnowledgeBase`.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `EntityLinker` object. ~~bytes~~ |
## EntityLinker.from_bytes {#from_bytes tag="method"}
## EntityLinker.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -376,7 +376,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `EntityLinker` object. ~~EntityLinker~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

42
website/docs/api/entityrecognizer.md → website/docs/api/entityrecognizer.mdx
View File

@ -20,7 +20,7 @@ your entities will be close to their initial tokens. If your entities are long
and characterized by tokens in their middle, the component will likely not be a
good fit for your task.
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predictions will be saved to `Doc.ents` as a tuple. Each label will also be
reflected to each underlying token, where it is saved in the `Token.ent_type`
@ -38,7 +38,7 @@ non-overlapping, or an error will be thrown.
| `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ |
| `Token.ent_type_` | The label part of the named entity tag. ~~str~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -72,7 +72,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/ner.pyx
```
## EntityRecognizer.\_\_init\_\_ {#init tag="method"}
## EntityRecognizer.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -103,7 +103,7 @@ shortcut for this and instantiate the component using its string name and
| `update_with_oracle_cut_size` | During training, cut long sequences into shorter segments by creating intermediate states based on the gold-standard history. The model is not very sensitive to this parameter, so you usually won't need to change it. Defaults to `100`. ~~int~~ |
| `incorrect_spans_key` | Identifies spans that are known to be incorrect entity annotations. The incorrect entity annotations can be stored in the span group in [`Doc.spans`](/api/doc#spans), under this key. Defaults to `None`. ~~Optional[str]~~ |
## EntityRecognizer.\_\_call\_\_ {#call tag="method"}
## EntityRecognizer.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -127,7 +127,7 @@ and all pipeline components are applied to the `Doc` in order. Both
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## EntityRecognizer.pipe {#pipe tag="method"}
## EntityRecognizer.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -151,7 +151,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/entityrecognizer#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## EntityRecognizer.initialize {#initialize tag="method" new="3"}
## EntityRecognizer.initialize {id="initialize",tag="method",version="3"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -188,7 +188,7 @@ config.
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ |
## EntityRecognizer.predict {#predict tag="method"}
## EntityRecognizer.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them.
@ -205,7 +205,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | A helper class for the parse state (internal). ~~StateClass~~ |
## EntityRecognizer.set_annotations {#set_annotations tag="method"}
## EntityRecognizer.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
@ -222,7 +222,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `EntityRecognizer.predict`. Returns an internal helper class for the parse state. ~~List[StateClass]~~ |
## EntityRecognizer.update {#update tag="method"}
## EntityRecognizer.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects, updating the pipe's
model. Delegates to [`predict`](/api/entityrecognizer#predict) and
@ -245,7 +245,7 @@ model. Delegates to [`predict`](/api/entityrecognizer#predict) and
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## EntityRecognizer.get_loss {#get_loss tag="method"}
## EntityRecognizer.get_loss {id="get_loss",tag="method"}
Find the loss and gradient of loss for the batch of documents and their
predicted scores.
@ -264,7 +264,7 @@ predicted scores.
| `scores` | Scores representing the model's predictions. ~~StateClass~~ |
| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ |
## EntityRecognizer.create_optimizer {#create_optimizer tag="method"}
## EntityRecognizer.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -279,7 +279,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## EntityRecognizer.use_params {#use_params tag="method, contextmanager"}
## EntityRecognizer.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -296,7 +296,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## EntityRecognizer.add_label {#add_label tag="method"}
## EntityRecognizer.add_label {id="add_label",tag="method"}
Add a new label to the pipe. Note that you don't have to call this method if you
provide a **representative data sample** to the [`initialize`](#initialize)
@ -316,7 +316,7 @@ to the model, and the output dimension will be
| `label` | The label to add. ~~str~~ |
| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ |
## EntityRecognizer.set_output {#set_output tag="method"}
## EntityRecognizer.set_output {id="set_output",tag="method"}
Change the output dimension of the component's model by calling the model's
attribute `resize_output`. This is a function that takes the original model and
@ -335,7 +335,7 @@ forgetting" problem.
| ---- | --------------------------------- |
| `nO` | The new output dimension. ~~int~~ |
## EntityRecognizer.to_disk {#to_disk tag="method"}
## EntityRecognizer.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -352,7 +352,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## EntityRecognizer.from_disk {#from_disk tag="method"}
## EntityRecognizer.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -370,7 +370,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `EntityRecognizer` object. ~~EntityRecognizer~~ |
## EntityRecognizer.to_bytes {#to_bytes tag="method"}
## EntityRecognizer.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -387,7 +387,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `EntityRecognizer` object. ~~bytes~~ |
## EntityRecognizer.from_bytes {#from_bytes tag="method"}
## EntityRecognizer.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -406,7 +406,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `EntityRecognizer` object. ~~EntityRecognizer~~ |
## EntityRecognizer.labels {#labels tag="property"}
## EntityRecognizer.labels {id="labels",tag="property"}
The labels currently added to the component.
@ -421,7 +421,7 @@ The labels currently added to the component.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ |
## EntityRecognizer.label_data {#label_data tag="property" new="3"}
## EntityRecognizer.label_data {id="label_data",tag="property",version="3"}
The labels currently added to the component and their internal meta information.
This is the data generated by [`init labels`](/api/cli#init-labels) and used by
@ -439,7 +439,7 @@ the model with a pre-defined label set.
| ----------- | ------------------------------------------------------------------------------- |
| **RETURNS** | The label data added to the component. ~~Dict[str, Dict[str, Dict[str, int]]]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

23
website/docs/api/entityruler.md → website/docs/api/entityruler.mdx
View File

@ -1,6 +1,6 @@
---
title: EntityRuler
new: 2.1
version: 2.1
teaser: 'Pipeline component for rule-based named entity recognition'
api_string_name: entity_ruler
api_trainable: false
@ -26,7 +26,7 @@ used on its own to implement a purely rule-based entity recognition system. For
usage examples, see the docs on
[rule-based entity recognition](/usage/rule-based-matching#entityruler).
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
This component assigns predictions basically the same way as the
[`EntityRecognizer`](/api/entityrecognizer).
@ -47,7 +47,7 @@ non-overlapping, or an error will be thrown.
| `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ |
| `Token.ent_type_` | The label part of the named entity tag. ~~str~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -66,15 +66,16 @@ how the component should be configured. You can override its settings via the
> nlp.add_pipe("entity_ruler", config=config)
> ```
| Setting | Description |
| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ |
| `validate` | Whether patterns should be validated (passed to the `Matcher` and `PhraseMatcher`). Defaults to `False`. ~~bool~~ |
| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ |
| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ |
| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ |
| Setting | Description |
| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ |
| `matcher_fuzzy_compare` <Tag variant="new">3.5</Tag> | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ |
| `validate` | Whether patterns should be validated (passed to the `Matcher` and `PhraseMatcher`). Defaults to `False`. ~~bool~~ |
| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ |
| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ |
| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ |
## Migrating from v3 {#migrating}
## Migrating from v3 {id="migrating"}
### Loading patterns

40
website/docs/api/example.md → website/docs/api/example.mdx
View File

@ -3,7 +3,7 @@ title: Example
teaser: A training instance
tag: class
source: spacy/training/example.pyx
new: 3.0
version: 3.0
---
An `Example` holds the information for one training instance. It stores two
@ -12,7 +12,7 @@ holding the predictions of the pipeline. An
[`Alignment`](/api/example#alignment-object) object stores the alignment between
these two documents, as they can differ in tokenization.
## Example.\_\_init\_\_ {#init tag="method"}
## Example.\_\_init\_\_ {id="init",tag="method"}
Construct an `Example` object from the `predicted` document and the `reference`
document. If `alignment` is `None`, it will be initialized from the words in
@ -40,7 +40,7 @@ both documents.
| _keyword-only_ | |
| `alignment` | An object holding the alignment between the tokens of the `predicted` and `reference` documents. ~~Optional[Alignment]~~ |
## Example.from_dict {#from_dict tag="classmethod"}
## Example.from_dict {id="from_dict",tag="classmethod"}
Construct an `Example` object from the `predicted` document and the reference
annotations provided as a dictionary. For more details on the required format,
@ -64,7 +64,7 @@ see the [training format documentation](/api/data-formats#dict-input).
| `example_dict` | The gold-standard annotations as a dictionary. Cannot be `None`. ~~Dict[str, Any]~~ |
| **RETURNS** | The newly constructed object. ~~Example~~ |
## Example.text {#text tag="property"}
## Example.text {id="text",tag="property"}
The text of the `predicted` document in this `Example`.
@ -78,7 +78,7 @@ The text of the `predicted` document in this `Example`.
| ----------- | --------------------------------------------- |
| **RETURNS** | The text of the `predicted` document. ~~str~~ |
## Example.predicted {#predicted tag="property"}
## Example.predicted {id="predicted",tag="property"}
The `Doc` holding the predictions. Occasionally also referred to as `example.x`.
@ -94,7 +94,7 @@ The `Doc` holding the predictions. Occasionally also referred to as `example.x`.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | The document containing (partial) predictions. ~~Doc~~ |
## Example.reference {#reference tag="property"}
## Example.reference {id="reference",tag="property"}
The `Doc` holding the gold-standard annotations. Occasionally also referred to
as `example.y`.
@ -111,7 +111,7 @@ as `example.y`.
| ----------- | ---------------------------------------------------------- |
| **RETURNS** | The document containing gold-standard annotations. ~~Doc~~ |
## Example.alignment {#alignment tag="property"}
## Example.alignment {id="alignment",tag="property"}
The [`Alignment`](/api/example#alignment-object) object mapping the tokens of
the `predicted` document to those of the `reference` document.
@ -131,7 +131,7 @@ the `predicted` document to those of the `reference` document.
| ----------- | ---------------------------------------------------------------- |
| **RETURNS** | The document containing gold-standard annotations. ~~Alignment~~ |
## Example.get_aligned {#get_aligned tag="method"}
## Example.get_aligned {id="get_aligned",tag="method"}
Get the aligned view of a certain token attribute, denoted by its int ID or
string name.
@ -152,7 +152,7 @@ string name.
| `as_string` | Whether or not to return the list of values as strings. Defaults to `False`. ~~bool~~ |
| **RETURNS** | List of integer values, or string values if `as_string` is `True`. ~~Union[List[int], List[str]]~~ |
## Example.get_aligned_parse {#get_aligned_parse tag="method"}
## Example.get_aligned_parse {id="get_aligned_parse",tag="method"}
Get the aligned view of the dependency parse. If `projectivize` is set to
`True`, non-projective dependency trees are made projective through the
@ -172,7 +172,7 @@ Pseudo-Projective Dependency Parsing algorithm by Nivre and Nilsson (2005).
| `projectivize` | Whether or not to projectivize the dependency trees. Defaults to `True`. ~~bool~~ |
| **RETURNS** | List of integer values, or string values if `as_string` is `True`. ~~Union[List[int], List[str]]~~ |
## Example.get_aligned_ner {#get_aligned_ner tag="method"}
## Example.get_aligned_ner {id="get_aligned_ner",tag="method"}
Get the aligned view of the NER
[BILUO](/usage/linguistic-features#accessing-ner) tags.
@ -193,7 +193,7 @@ Get the aligned view of the NER
| ----------- | ------------------------------------------------------------------------------------------------- |
| **RETURNS** | List of BILUO values, denoting whether tokens are part of an NER annotation or not. ~~List[str]~~ |
## Example.get_aligned_spans_y2x {#get_aligned_spans_y2x tag="method"}
## Example.get_aligned_spans_y2x {id="get_aligned_spans_y2x",tag="method"}
Get the aligned view of any set of [`Span`](/api/span) objects defined over
[`Example.reference`](/api/example#reference). The resulting span indices will
@ -219,7 +219,7 @@ align to the tokenization in [`Example.predicted`](/api/example#predicted).
| `allow_overlap` | Whether the resulting `Span` objects may overlap or not. Set to `False` by default. ~~bool~~ |
| **RETURNS** | `Span` objects aligned to the tokenization of `predicted`. ~~List[Span]~~ |
## Example.get_aligned_spans_x2y {#get_aligned_spans_x2y tag="method"}
## Example.get_aligned_spans_x2y {id="get_aligned_spans_x2y",tag="method"}
Get the aligned view of any set of [`Span`](/api/span) objects defined over
[`Example.predicted`](/api/example#predicted). The resulting span indices will
@ -247,7 +247,7 @@ against the original gold-standard annotation.
| `allow_overlap` | Whether the resulting `Span` objects may overlap or not. Set to `False` by default. ~~bool~~ |
| **RETURNS** | `Span` objects aligned to the tokenization of `reference`. ~~List[Span]~~ |
## Example.to_dict {#to_dict tag="method"}
## Example.to_dict {id="to_dict",tag="method"}
Return a [dictionary representation](/api/data-formats#dict-input) of the
reference annotation contained in this `Example`.
@ -262,7 +262,7 @@ reference annotation contained in this `Example`.
| ----------- | ------------------------------------------------------------------------- |
| **RETURNS** | Dictionary representation of the reference annotation. ~~Dict[str, Any]~~ |
## Example.split_sents {#split_sents tag="method"}
## Example.split_sents {id="split_sents",tag="method"}
Split one `Example` into multiple `Example` objects, one for each sentence.
@ -282,15 +282,15 @@ Split one `Example` into multiple `Example` objects, one for each sentence.
| ----------- | ---------------------------------------------------------------------------- |
| **RETURNS** | List of `Example` objects, one for each original sentence. ~~List[Example]~~ |
## Alignment {#alignment-object new="3"}
## Alignment {id="alignment-object",version="3"}
Calculate alignment tables between two tokenizations.
### Alignment attributes {#alignment-attributes"}
### Alignment attributes {id="alignment-attributes"}
Alignment attributes are managed using `AlignmentArray`, which is a
simplified version of Thinc's [Ragged](https://thinc.ai/docs/api-types#ragged)
type that only supports the `data` and `length` attributes.
Alignment attributes are managed using `AlignmentArray`, which is a simplified
version of Thinc's [Ragged](https://thinc.ai/docs/api-types#ragged) type that
only supports the `data` and `length` attributes.
| Name | Description |
| ----- | ------------------------------------------------------------------------------------- |
@ -321,7 +321,7 @@ tokenizations add up to the same string. For example, you'll be able to align
> If `a2b.data[1] == a2b.data[2] == 1`, that means that `A[1]` (`"'"`) and
> `A[2]` (`"s"`) both align to `B[1]` (`"'s"`).
### Alignment.from_strings {#classmethod tag="function"}
### Alignment.from_strings {id="classmethod",tag="function"}
| Name | Description |
| ----------- | ------------------------------------------------------------- |

2
website/docs/api/index.md → website/docs/api/index.mdx
View File

@ -3,6 +3,4 @@ title: Library Architecture
next: /api/architectures
---
import Architecture101 from 'usage/101/\_architecture.md'
<Architecture101 />

28
website/docs/api/kb.md → website/docs/api/kb.mdx
View File

@ -5,7 +5,7 @@ teaser:
(ontology)
tag: class
source: spacy/kb/kb.pyx
new: 2.2
version: 2.2
---
The `KnowledgeBase` object is an abstract class providing a method to generate
@ -26,7 +26,7 @@ onwards.
</Infobox>
## KnowledgeBase.\_\_init\_\_ {#init tag="method"}
## KnowledgeBase.\_\_init\_\_ {id="init",tag="method"}
`KnowledgeBase` is an abstract class and cannot be instantiated. Its child
classes should call `__init__()` to set up some necessary attributes.
@ -50,7 +50,7 @@ classes should call `__init__()` to set up some necessary attributes.
| `vocab` | The shared vocabulary. ~~Vocab~~ |
| `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ |
## KnowledgeBase.entity_vector_length {#entity_vector_length tag="property"}
## KnowledgeBase.entity_vector_length {id="entity_vector_length",tag="property"}
The length of the fixed-size entity vectors in the knowledge base.
@ -58,7 +58,7 @@ The length of the fixed-size entity vectors in the knowledge base.
| ----------- | ------------------------------------------------ |
| **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ |
## KnowledgeBase.get_candidates {#get_candidates tag="method"}
## KnowledgeBase.get_candidates {id="get_candidates",tag="method"}
Given a certain textual mention as input, retrieve a list of candidate entities
of type [`Candidate`](/api/kb#candidate).
@ -77,7 +77,7 @@ of type [`Candidate`](/api/kb#candidate).
| `mention` | The textual mention or alias. ~~Span~~ |
| **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ |
## KnowledgeBase.get_candidates_batch {#get_candidates_batch tag="method"}
## KnowledgeBase.get_candidates_batch {id="get_candidates_batch",tag="method"}
Same as [`get_candidates()`](/api/kb#get_candidates), but for an arbitrary
number of mentions. The [`EntityLinker`](/api/entitylinker) component will call
@ -103,10 +103,10 @@ to you.
| `mentions` | The textual mention or alias. ~~Iterable[Span]~~ |
| **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ |
## KnowledgeBase.get_alias_candidates {#get_alias_candidates tag="method"}
## KnowledgeBase.get_alias_candidates {id="get_alias_candidates",tag="method"}
<Infobox variant="warning">
This method is _not_ available from spaCy 3.5 onwards.
This method is _not_ available from spaCy 3.5 onwards.
</Infobox>
From spaCy 3.5 on `KnowledgeBase` is an abstract class (with
@ -119,7 +119,7 @@ Note: [`InMemoryLookupKB.get_candidates()`](/api/kb_in_memory#get_candidates)
defaults to
[`InMemoryLookupKB.get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates).
## KnowledgeBase.get_vector {#get_vector tag="method"}
## KnowledgeBase.get_vector {id="get_vector",tag="method"}
Given a certain entity ID, retrieve its pretrained entity vector.
@ -134,7 +134,7 @@ Given a certain entity ID, retrieve its pretrained entity vector.
| `entity` | The entity ID. ~~str~~ |
| **RETURNS** | The entity vector. ~~Iterable[float]~~ |
## KnowledgeBase.get_vectors {#get_vectors tag="method"}
## KnowledgeBase.get_vectors {id="get_vectors",tag="method"}
Same as [`get_vector()`](/api/kb#get_vector), but for an arbitrary number of
entity IDs.
@ -154,7 +154,7 @@ entities at once, if performance is of concern to you.
| `entities` | The entity IDs. ~~Iterable[str]~~ |
| **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ |
## KnowledgeBase.to_disk {#to_disk tag="method"}
## KnowledgeBase.to_disk {id="to_disk",tag="method"}
Save the current state of the knowledge base to a directory.
@ -169,7 +169,7 @@ Save the current state of the knowledge base to a directory.
| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
| `exclude` | List of components to exclude. ~~Iterable[str]~~ |
## KnowledgeBase.from_disk {#from_disk tag="method"}
## KnowledgeBase.from_disk {id="from_disk",tag="method"}
Restore the state of the knowledge base from a given directory. Note that the
[`Vocab`](/api/vocab) should also be the same as the one used to create the KB.
@ -189,7 +189,7 @@ Restore the state of the knowledge base from a given directory. Note that the
| `exclude` | List of components to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `KnowledgeBase` object. ~~KnowledgeBase~~ |
## Candidate {#candidate tag="class"}
## Candidate {id="candidate",tag="class"}
A `Candidate` object refers to a textual mention (alias) that may or may not be
resolved to a specific entity from a `KnowledgeBase`. This will be used as input
@ -197,7 +197,7 @@ for the entity linking algorithm which will disambiguate the various candidates
to the correct one. Each candidate `(alias, entity)` pair is assigned to a
certain prior probability.
### Candidate.\_\_init\_\_ {#candidate-init tag="method"}
### Candidate.\_\_init\_\_ {id="candidate-init",tag="method"}
Construct a `Candidate` object. Usually this constructor is not called directly,
but instead these objects are returned by the `get_candidates` method of the
@ -218,7 +218,7 @@ but instead these objects are returned by the `get_candidates` method of the
| `alias_hash` | The hash of the textual mention or alias. ~~int~~ |
| `prior_prob` | The prior probability of the `alias` referring to the `entity`. ~~float~~ |
## Candidate attributes {#candidate-attributes}
## Candidate attributes {id="candidate-attributes"}
| Name | Description |
| --------------- | ------------------------------------------------------------------------ |

36
website/docs/api/kb_in_memory.md → website/docs/api/kb_in_memory.mdx
View File

@ -5,7 +5,7 @@ teaser:
information in-memory.
tag: class
source: spacy/kb/kb_in_memory.pyx
new: 3.5
version: 3.5
---
The `InMemoryLookupKB` class inherits from [`KnowledgeBase`](/api/kb) and
@ -14,7 +14,7 @@ implements all of its methods. It stores all KB data in-memory and generates
entity names. It's highly optimized for both a low memory footprint and speed of
retrieval.
## InMemoryLookupKB.\_\_init\_\_ {#init tag="method"}
## InMemoryLookupKB.\_\_init\_\_ {id="init",tag="method"}
Create the knowledge base.
@ -31,7 +31,7 @@ Create the knowledge base.
| `vocab` | The shared vocabulary. ~~Vocab~~ |
| `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ |
## InMemoryLookupKB.entity_vector_length {#entity_vector_length tag="property"}
## InMemoryLookupKB.entity_vector_length {id="entity_vector_length",tag="property"}
The length of the fixed-size entity vectors in the knowledge base.
@ -39,7 +39,7 @@ The length of the fixed-size entity vectors in the knowledge base.
| ----------- | ------------------------------------------------ |
| **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ |
## InMemoryLookupKB.add_entity {#add_entity tag="method"}
## InMemoryLookupKB.add_entity {id="add_entity",tag="method"}
Add an entity to the knowledge base, specifying its corpus frequency and entity
vector, which should be of length
@ -58,7 +58,7 @@ vector, which should be of length
| `freq` | The frequency of the entity in a typical corpus. ~~float~~ |
| `entity_vector` | The pretrained vector of the entity. ~~numpy.ndarray~~ |
## InMemoryLookupKB.set_entities {#set_entities tag="method"}
## InMemoryLookupKB.set_entities {id="set_entities",tag="method"}
Define the full list of entities in the knowledge base, specifying the corpus
frequency and entity vector for each entity.
@ -75,7 +75,7 @@ frequency and entity vector for each entity.
| `freq_list` | List of entity frequencies. ~~Iterable[int]~~ |
| `vector_list` | List of entity vectors. ~~Iterable[numpy.ndarray]~~ |
## InMemoryLookupKB.add_alias {#add_alias tag="method"}
## InMemoryLookupKB.add_alias {id="add_alias",tag="method"}
Add an alias or mention to the knowledge base, specifying its potential KB
identifiers and their prior probabilities. The entity identifiers should refer
@ -96,7 +96,7 @@ alias.
| `entities` | The potential entities that the alias may refer to. ~~Iterable[Union[str, int]]~~ |
| `probabilities` | The prior probabilities of each entity. ~~Iterable[float]~~ |
## InMemoryLookupKB.\_\_len\_\_ {#len tag="method"}
## InMemoryLookupKB.\_\_len\_\_ {id="len",tag="method"}
Get the total number of entities in the knowledge base.
@ -110,7 +110,7 @@ Get the total number of entities in the knowledge base.
| ----------- | ----------------------------------------------------- |
| **RETURNS** | The number of entities in the knowledge base. ~~int~~ |
## InMemoryLookupKB.get_entity_strings {#get_entity_strings tag="method"}
## InMemoryLookupKB.get_entity_strings {id="get_entity_strings",tag="method"}
Get a list of all entity IDs in the knowledge base.
@ -124,7 +124,7 @@ Get a list of all entity IDs in the knowledge base.
| ----------- | --------------------------------------------------------- |
| **RETURNS** | The list of entities in the knowledge base. ~~List[str]~~ |
## InMemoryLookupKB.get_size_aliases {#get_size_aliases tag="method"}
## InMemoryLookupKB.get_size_aliases {id="get_size_aliases",tag="method"}
Get the total number of aliases in the knowledge base.
@ -138,7 +138,7 @@ Get the total number of aliases in the knowledge base.
| ----------- | ---------------------------------------------------- |
| **RETURNS** | The number of aliases in the knowledge base. ~~int~~ |
## InMemoryLookupKB.get_alias_strings {#get_alias_strings tag="method"}
## InMemoryLookupKB.get_alias_strings {id="get_alias_strings",tag="method"}
Get a list of all aliases in the knowledge base.
@ -152,7 +152,7 @@ Get a list of all aliases in the knowledge base.
| ----------- | -------------------------------------------------------- |
| **RETURNS** | The list of aliases in the knowledge base. ~~List[str]~~ |
## InMemoryLookupKB.get_candidates {#get_candidates tag="method"}
## InMemoryLookupKB.get_candidates {id="get_candidates",tag="method"}
Given a certain textual mention as input, retrieve a list of candidate entities
of type [`Candidate`](/api/kb#candidate). Wraps
@ -172,7 +172,7 @@ of type [`Candidate`](/api/kb#candidate). Wraps
| `mention` | The textual mention or alias. ~~Span~~ |
| **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ |
## InMemoryLookupKB.get_candidates_batch {#get_candidates_batch tag="method"}
## InMemoryLookupKB.get_candidates_batch {id="get_candidates_batch",tag="method"}
Same as [`get_candidates()`](/api/kb_in_memory#get_candidates), but for an
arbitrary number of mentions. The [`EntityLinker`](/api/entitylinker) component
@ -198,7 +198,7 @@ to you.
| `mentions` | The textual mention or alias. ~~Iterable[Span]~~ |
| **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ |
## InMemoryLookupKB.get_alias_candidates {#get_alias_candidates tag="method"}
## InMemoryLookupKB.get_alias_candidates {id="get_alias_candidates",tag="method"}
Given a certain textual mention as input, retrieve a list of candidate entities
of type [`Candidate`](/api/kb#candidate).
@ -214,7 +214,7 @@ of type [`Candidate`](/api/kb#candidate).
| `alias` | The textual mention or alias. ~~str~~ |
| **RETURNS** | The list of relevant `Candidate` objects. ~~List[Candidate]~~ |
## InMemoryLookupKB.get_vector {#get_vector tag="method"}
## InMemoryLookupKB.get_vector {id="get_vector",tag="method"}
Given a certain entity ID, retrieve its pretrained entity vector.
@ -229,7 +229,7 @@ Given a certain entity ID, retrieve its pretrained entity vector.
| `entity` | The entity ID. ~~str~~ |
| **RETURNS** | The entity vector. ~~numpy.ndarray~~ |
## InMemoryLookupKB.get_vectors {#get_vectors tag="method"}
## InMemoryLookupKB.get_vectors {id="get_vectors",tag="method"}
Same as [`get_vector()`](/api/kb_in_memory#get_vector), but for an arbitrary
number of entity IDs.
@ -249,7 +249,7 @@ entities at once, if performance is of concern to you.
| `entities` | The entity IDs. ~~Iterable[str]~~ |
| **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ |
## InMemoryLookupKB.get_prior_prob {#get_prior_prob tag="method"}
## InMemoryLookupKB.get_prior_prob {id="get_prior_prob",tag="method"}
Given a certain entity ID and a certain textual mention, retrieve the prior
probability of the fact that the mention links to the entity ID.
@ -266,7 +266,7 @@ probability of the fact that the mention links to the entity ID.
| `alias` | The textual mention or alias. ~~str~~ |
| **RETURNS** | The prior probability of the `alias` referring to the `entity`. ~~float~~ |
## InMemoryLookupKB.to_disk {#to_disk tag="method"}
## InMemoryLookupKB.to_disk {id="to_disk",tag="method"}
Save the current state of the knowledge base to a directory.
@ -281,7 +281,7 @@ Save the current state of the knowledge base to a directory.
| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
| `exclude` | List of components to exclude. ~~Iterable[str]~~ |
## InMemoryLookupKB.from_disk {#from_disk tag="method"}
## InMemoryLookupKB.from_disk {id="from_disk",tag="method"}
Restore the state of the knowledge base from a given directory. Note that the
[`Vocab`](/api/vocab) should also be the same as the one used to create the KB.

88
website/docs/api/language.md → website/docs/api/language.mdx
View File

@ -15,7 +15,7 @@ the tagger or parser that are called on a document in order. You can also add
your own processing pipeline components that take a `Doc` object, modify it and
return it.
## Language.\_\_init\_\_ {#init tag="method"}
## Language.\_\_init\_\_ {id="init",tag="method"}
Initialize a `Language` object. Note that the `meta` is only used for meta
information in [`Language.meta`](/api/language#meta) and not to configure the
@ -44,7 +44,7 @@ information in [`Language.meta`](/api/language#meta) and not to configure the
| `create_tokenizer` | Optional function that receives the `nlp` object and returns a tokenizer. ~~Callable[[Language], Callable[[str], Doc]]~~ |
| `batch_size` | Default batch size for [`pipe`](#pipe) and [`evaluate`](#evaluate). Defaults to `1000`. ~~int~~ |
## Language.from_config {#from_config tag="classmethod" new="3"}
## Language.from_config {id="from_config",tag="classmethod",version="3"}
Create a `Language` object from a loaded config. Will set up the tokenizer and
language data, add pipeline components based on the pipeline and add pipeline
@ -76,7 +76,7 @@ spaCy loads a model under the hood based on its
| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ |
| **RETURNS** | The initialized object. ~~Language~~ |
## Language.component {#component tag="classmethod" new="3"}
## Language.component {id="component",tag="classmethod",version="3"}
Register a custom pipeline component under a given name. This allows
initializing the component by name using
@ -112,7 +112,7 @@ decorator. For more details and examples, see the
| `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~ |
| `func` | Optional function if not used as a decorator. ~~Optional[Callable[[Doc], Doc]]~~ |
## Language.factory {#factory tag="classmethod"}
## Language.factory {id="factory",tag="classmethod"}
Register a custom pipeline component factory under a given name. This allows
initializing the component by name using
@ -159,7 +159,7 @@ examples, see the
| `default_score_weights` | The scores to report during training, and their default weight towards the final score used to select the best model. Weights should sum to `1.0` per component and will be combined and normalized for the whole pipeline. If a weight is set to `None`, the score will not be logged or weighted. ~~Dict[str, Optional[float]]~~ |
| `func` | Optional function if not used as a decorator. ~~Optional[Callable[[...], Callable[[Doc], Doc]]]~~ |
## Language.\_\_call\_\_ {#call tag="method"}
## Language.\_\_call\_\_ {id="call",tag="method"}
Apply the pipeline to some text. The text can span multiple sentences, and can
contain arbitrary whitespace. Alignment into the original string is preserved.
@ -182,7 +182,7 @@ skipped, but the rest of the pipeline is run.
| `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ |
| **RETURNS** | A container for accessing the annotations. ~~Doc~~ |
## Language.pipe {#pipe tag="method"}
## Language.pipe {id="pipe",tag="method"}
Process texts as a stream, and yield `Doc` objects in order. This is usually
more efficient than processing texts one-by-one.
@ -209,7 +209,7 @@ tokenization is skipped but the rest of the pipeline is run.
| `n_process` | Number of processors to use. Defaults to `1`. ~~int~~ |
| **YIELDS** | Documents in the order of the original text. ~~Doc~~ |
## Language.set_error_handler {#set_error_handler tag="method" new="3"}
## Language.set_error_handler {id="set_error_handler",tag="method",version="3"}
Define a callback that will be invoked when an error is thrown during processing
of one or more documents. Specifically, this function will call
@ -231,7 +231,7 @@ being processed, and the original error.
| --------------- | -------------------------------------------------------------------------------------------------------------- |
| `error_handler` | A function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ |
## Language.initialize {#initialize tag="method" new="3"}
## Language.initialize {id="initialize",tag="method",version="3"}
Initialize the pipeline for training and return an
[`Optimizer`](https://thinc.ai/docs/api-optimizers). Under the hood, it uses the
@ -273,7 +273,7 @@ the tagger or textcat).
| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## Language.resume_training {#resume_training tag="method,experimental" new="3"}
## Language.resume_training {id="resume_training",tag="method,experimental",version="3"}
Continue training a trained pipeline. Create and return an optimizer, and
initialize "rehearsal" for any pipeline component that has a `rehearse` method.
@ -295,7 +295,7 @@ a batch of [Example](/api/example) objects.
| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## Language.update {#update tag="method"}
## Language.update {id="update",tag="method"}
Update the models in the pipeline.
@ -333,7 +333,7 @@ and custom registered functions if needed. See the
| `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## Language.rehearse {#rehearse tag="method,experimental" new="3"}
## Language.rehearse {id="rehearse",tag="method,experimental",version="3"}
Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the
current model to make predictions similar to an initial model, to try to address
@ -355,7 +355,7 @@ the "catastrophic forgetting" problem. This feature is experimental.
| `losses` | Dictionary to update with the loss, keyed by pipeline component. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## Language.evaluate {#evaluate tag="method"}
## Language.evaluate {id="evaluate",tag="method"}
Evaluate a pipeline's components.
@ -383,7 +383,7 @@ objects instead of tuples of `Doc` and `GoldParse` objects.
| `scorer_cfg` | Optional dictionary of keyword arguments for the `Scorer`. Defaults to `None`. ~~Optional[Dict[str, Any]]~~ |
| **RETURNS** | A dictionary of evaluation scores. ~~Dict[str, Union[float, Dict[str, float]]]~~ |
## Language.use_params {#use_params tag="contextmanager, method"}
## Language.use_params {id="use_params",tag="contextmanager, method"}
Replace weights of models in the pipeline with those provided in the params
dictionary. Can be used as a context manager, in which case, models go back to
@ -400,7 +400,7 @@ their original weights after the block.
| -------- | ------------------------------------------------------ |
| `params` | A dictionary of parameters keyed by model ID. ~~dict~~ |
## Language.add_pipe {#add_pipe tag="method" new="2"}
## Language.add_pipe {id="add_pipe",tag="method",version="2"}
Add a component to the processing pipeline. Expects a name that maps to a
component factory registered using
@ -449,7 +449,7 @@ component, adds it to the pipeline and returns it.
| `validate` <Tag variant="new">3</Tag> | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ |
| **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ |
## Language.create_pipe {#create_pipe tag="method" new="2"}
## Language.create_pipe {id="create_pipe",tag="method",version="2"}
Create a pipeline component from a factory.
@ -478,7 +478,7 @@ To create a component and add it to the pipeline, you should always use
| `validate` <Tag variant="new">3</Tag> | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ |
| **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ |
## Language.has_factory {#has_factory tag="classmethod" new="3"}
## Language.has_factory {id="has_factory",tag="classmethod",version="3"}
Check whether a factory name is registered on the `Language` class or subclass.
Will check for
@ -505,7 +505,7 @@ the `Language` base class, available to all subclasses.
| `name` | Name of the pipeline factory to check. ~~str~~ |
| **RETURNS** | Whether a factory of that name is registered on the class. ~~bool~~ |
## Language.has_pipe {#has_pipe tag="method" new="2"}
## Language.has_pipe {id="has_pipe",tag="method",version="2"}
Check whether a component is present in the pipeline. Equivalent to
`name in nlp.pipe_names`.
@ -527,7 +527,7 @@ Check whether a component is present in the pipeline. Equivalent to
| `name` | Name of the pipeline component to check. ~~str~~ |
| **RETURNS** | Whether a component of that name exists in the pipeline. ~~bool~~ |
## Language.get_pipe {#get_pipe tag="method" new="2"}
## Language.get_pipe {id="get_pipe",tag="method",version="2"}
Get a pipeline component for a given component name.
@ -543,7 +543,7 @@ Get a pipeline component for a given component name.
| `name` | Name of the pipeline component to get. ~~str~~ |
| **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ |
## Language.replace_pipe {#replace_pipe tag="method" new="2"}
## Language.replace_pipe {id="replace_pipe",tag="method",version="2"}
Replace a component in the pipeline and return the new component.
@ -571,7 +571,7 @@ and instead expects the **name of a component factory** registered using
| `validate` <Tag variant="new">3</Tag> | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ |
| **RETURNS** | The new pipeline component. ~~Callable[[Doc], Doc]~~ |
## Language.rename_pipe {#rename_pipe tag="method" new="2"}
## Language.rename_pipe {id="rename_pipe",tag="method",version="2"}
Rename a component in the pipeline. Useful to create custom names for
pre-defined and pre-loaded components. To change the default name of a component
@ -589,7 +589,7 @@ added to the pipeline, you can also use the `name` argument on
| `old_name` | Name of the component to rename. ~~str~~ |
| `new_name` | New name of the component. ~~str~~ |
## Language.remove_pipe {#remove_pipe tag="method" new="2"}
## Language.remove_pipe {id="remove_pipe",tag="method",version="2"}
Remove a component from the pipeline. Returns the removed component name and
component function.
@ -606,7 +606,7 @@ component function.
| `name` | Name of the component to remove. ~~str~~ |
| **RETURNS** | A `(name, component)` tuple of the removed component. ~~Tuple[str, Callable[[Doc], Doc]]~~ |
## Language.disable_pipe {#disable_pipe tag="method" new="3"}
## Language.disable_pipe {id="disable_pipe",tag="method",version="3"}
Temporarily disable a pipeline component so it's not run as part of the
pipeline. Disabled components are listed in
@ -632,7 +632,7 @@ does nothing.
| ------ | ----------------------------------------- |
| `name` | Name of the component to disable. ~~str~~ |
## Language.enable_pipe {#enable_pipe tag="method" new="3"}
## Language.enable_pipe {id="enable_pipe",tag="method",version="3"}
Enable a previously disabled component (e.g. via
[`Language.disable_pipes`](/api/language#disable_pipes)) so it's run as part of
@ -654,7 +654,7 @@ already enabled, this method does nothing.
| ------ | ---------------------------------------- |
| `name` | Name of the component to enable. ~~str~~ |
## Language.select_pipes {#select_pipes tag="contextmanager, method" new="3"}
## Language.select_pipes {id="select_pipes",tag="contextmanager, method",version="3"}
Disable one or more pipeline components. If used as a context manager, the
pipeline will be restored to the initial state at the end of the block.
@ -697,7 +697,7 @@ As of spaCy v3.0, the `disable_pipes` method has been renamed to `select_pipes`:
| `enable` | Name(s) of pipeline component(s) that will not be disabled. ~~Optional[Union[str, Iterable[str]]]~~ |
| **RETURNS** | The disabled pipes that can be restored by calling the object's `.restore()` method. ~~DisabledPipes~~ |
## Language.get_factory_meta {#get_factory_meta tag="classmethod" new="3"}
## Language.get_factory_meta {id="get_factory_meta",tag="classmethod",version="3"}
Get the factory meta information for a given pipeline component name. Expects
the name of the component **factory**. The factory meta is an instance of the
@ -719,7 +719,7 @@ information about the component and its default provided by the
| `name` | The factory name. ~~str~~ |
| **RETURNS** | The factory meta. ~~FactoryMeta~~ |
## Language.get_pipe_meta {#get_pipe_meta tag="method" new="3"}
## Language.get_pipe_meta {id="get_pipe_meta",tag="method",version="3"}
Get the factory meta information for a given pipeline component name. Expects
the name of the component **instance** in the pipeline. The factory meta is an
@ -742,7 +742,7 @@ contains the information about the component and its default provided by the
| `name` | The pipeline component name. ~~str~~ |
| **RETURNS** | The factory meta. ~~FactoryMeta~~ |
## Language.analyze_pipes {#analyze_pipes tag="method" new="3"}
## Language.analyze_pipes {id="analyze_pipes",tag="method",version="3"}
Analyze the current pipeline components and show a summary of the attributes
they assign and require, and the scores they set. The data is based on the
@ -771,8 +771,7 @@ doesn't, the pipeline analysis won't catch that.
<Accordion title="Example output" spaced>
```json
### Structured
```json {title="Structured"}
{
"summary": {
"tagger": {
@ -790,7 +789,12 @@ doesn't, the pipeline analysis won't catch that.
},
"problems": {
"tagger": [],
"entity_linker": ["doc.ents", "doc.sents", "token.ent_iob", "token.ent_type"]
"entity_linker": [
"doc.ents",
"doc.sents",
"token.ent_iob",
"token.ent_type"
]
},
"attrs": {
"token.ent_iob": { "assigns": [], "requires": ["entity_linker"] },
@ -831,7 +835,7 @@ token.ent_iob, token.ent_type
| `pretty` | Pretty-print the results as a table. Defaults to `False`. ~~bool~~ |
| **RETURNS** | Dictionary containing the pipe analysis, keyed by `"summary"` (component meta by pipe), `"problems"` (attribute names by pipe) and `"attrs"` (pipes that assign and require an attribute, keyed by attribute). ~~Optional[Dict[str, Any]]~~ |
## Language.replace_listeners {#replace_listeners tag="method" new="3"}
## Language.replace_listeners {id="replace_listeners",tag="method",version="3"}
Find [listener layers](/usage/embeddings-transformers#embedding-layers)
(connecting to a shared token-to-vector embedding component) of a given pipeline
@ -876,7 +880,7 @@ when loading a config with
| `pipe_name` | Name of pipeline component to replace listeners for. ~~str~~ |
| `listeners` | The paths to the listeners, relative to the component config, e.g. `["model.tok2vec"]`. Typically, implementations will only connect to one tok2vec component, `model.tok2vec`, but in theory, custom models can use multiple listeners. The value here can either be an empty list to not replace any listeners, or a _complete_ list of the paths to all listener layers used by the model that should be replaced.~~Iterable[str]~~ |
## Language.meta {#meta tag="property"}
## Language.meta {id="meta",tag="property"}
Meta data for the `Language` class, including name, version, data sources,
license, author information and more. If a trained pipeline is loaded, this
@ -902,7 +906,7 @@ information is expressed in the [`config.cfg`](/api/data-formats#config).
| ----------- | --------------------------------- |
| **RETURNS** | The meta data. ~~Dict[str, Any]~~ |
## Language.config {#config tag="property" new="3"}
## Language.config {id="config",tag="property",version="3"}
Export a trainable [`config.cfg`](/api/data-formats#config) for the current
`nlp` object. Includes the current pipeline, all configs used to create the
@ -923,7 +927,7 @@ subclass of the built-in `dict`. It supports the additional methods `to_disk`
| ----------- | ---------------------- |
| **RETURNS** | The config. ~~Config~~ |
## Language.to_disk {#to_disk tag="method" new="2"}
## Language.to_disk {id="to_disk",tag="method",version="2"}
Save the current state to a directory. Under the hood, this method delegates to
the `to_disk` methods of the individual pipeline components, if available. This
@ -942,7 +946,7 @@ will be saved to disk.
| _keyword-only_ | |
| `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## Language.from_disk {#from_disk tag="method" new="2"}
## Language.from_disk {id="from_disk",tag="method",version="2"}
Loads state from a directory, including all data that was saved with the
`Language` object. Modifies the object in place and returns it.
@ -975,7 +979,7 @@ you want to load a serialized pipeline from a directory, you should use
| `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `Language` object. ~~Language~~ |
## Language.to_bytes {#to_bytes tag="method"}
## Language.to_bytes {id="to_bytes",tag="method"}
Serialize the current state to a binary string.
@ -991,7 +995,7 @@ Serialize the current state to a binary string.
| `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~iterable~~ |
| **RETURNS** | The serialized form of the `Language` object. ~~bytes~~ |
## Language.from_bytes {#from_bytes tag="method"}
## Language.from_bytes {id="from_bytes",tag="method"}
Load state from a binary string. Note that this method is commonly used via the
subclasses like `English` or `German` to make language-specific functionality
@ -1019,7 +1023,7 @@ details.
| `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Language` object. ~~Language~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
@ -1037,7 +1041,7 @@ details.
| `disabled` <Tag variant="new">3</Tag> | Names of components that are currently disabled and don't run as part of the pipeline. ~~List[str]~~ |
| `path` | Path to the pipeline data directory, if a pipeline is loaded from a path or package. Otherwise `None`. ~~Optional[Path]~~ |
## Class attributes {#class-attributes}
## Class attributes {id="class-attributes"}
| Name | Description |
| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
@ -1045,7 +1049,7 @@ details.
| `lang` | [IETF language tag](https://www.w3.org/International/articles/language-tags/), such as 'en' for English. ~~str~~ |
| `default_config` | Base [config](/usage/training#config) to use for [Language.config](/api/language#config). Defaults to [`default_config.cfg`](%%GITHUB_SPACY/spacy/default_config.cfg). ~~Config~~ |
## Defaults {#defaults}
## Defaults {id="defaults"}
The following attributes can be set on the `Language.Defaults` class to
customize the default language data:
@ -1088,7 +1092,7 @@ customize the default language data:
| `writing_system` | Information about the language's writing system, available via `Vocab.writing_system`. Defaults to: `{"direction": "ltr", "has_case": True, "has_letters": True}.`.<br />**Example:** [`zh/__init__.py`](%%GITHUB_SPACY/spacy/lang/zh/__init__.py) ~~Dict[str, Any]~~ |
| `config` | Default [config](/usage/training#config) added to `nlp.config`. This can include references to custom tokenizers or lemmatizers.<br />**Example:** [`zh/__init__.py`](%%GITHUB_SPACY/spacy/lang/zh/__init__.py) ~~Config~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from
@ -1108,7 +1112,7 @@ serialization by passing in the string names via the `exclude` argument.
| `meta` | The meta data, available as [`Language.meta`](/api/language#meta). |
| ... | String names of pipeline components, e.g. `"ner"`. |
## FactoryMeta {#factorymeta new="3" tag="dataclass"}
## FactoryMeta {id="factorymeta",version="3",tag="dataclass"}
The `FactoryMeta` contains the information about the component and its default
provided by the [`@Language.component`](/api/language#component) or

32
website/docs/api/legacy.md → website/docs/api/legacy.mdx
View File

@ -12,11 +12,11 @@ functions that may still be used in projects.
You can find the detailed documentation of each such legacy function on this
page.
## Architectures {#architectures}
## Architectures {id="architectures"}
These functions are available from `@spacy.registry.architectures`.
### spacy.Tok2Vec.v1 {#Tok2Vec_v1}
### spacy.Tok2Vec.v1 {id="Tok2Vec_v1"}
The `spacy.Tok2Vec.v1` architecture was expecting an `encode` model of type
`Model[Floats2D, Floats2D]` such as `spacy.MaxoutWindowEncoder.v1` or
@ -48,7 +48,7 @@ blog post for background.
| `encode` | Encode context into the embeddings, using an architecture such as a CNN, BiLSTM or transformer. For example, [MaxoutWindowEncoder.v1](/api/legacy#MaxoutWindowEncoder_v1). ~~Model[Floats2d, Floats2d]~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ |
### spacy.MaxoutWindowEncoder.v1 {#MaxoutWindowEncoder_v1}
### spacy.MaxoutWindowEncoder.v1 {id="MaxoutWindowEncoder_v1"}
The `spacy.MaxoutWindowEncoder.v1` architecture was producing a model of type
`Model[Floats2D, Floats2D]`. Since `spacy.MaxoutWindowEncoder.v2`, this has been
@ -76,7 +76,7 @@ and residual connections.
| `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ |
| **CREATES** | The model using the architecture. ~~Model[Floats2d, Floats2d]~~ |
### spacy.MishWindowEncoder.v1 {#MishWindowEncoder_v1}
### spacy.MishWindowEncoder.v1 {id="MishWindowEncoder_v1"}
The `spacy.MishWindowEncoder.v1` architecture was producing a model of type
`Model[Floats2D, Floats2D]`. Since `spacy.MishWindowEncoder.v2`, this has been
@ -103,24 +103,24 @@ and residual connections.
| `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ |
| **CREATES** | The model using the architecture. ~~Model[Floats2d, Floats2d]~~ |
### spacy.HashEmbedCNN.v1 {#HashEmbedCNN_v1}
### spacy.HashEmbedCNN.v1 {id="HashEmbedCNN_v1"}
Identical to [`spacy.HashEmbedCNN.v2`](/api/architectures#HashEmbedCNN) except
using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included.
### spacy.MultiHashEmbed.v1 {#MultiHashEmbed_v1}
### spacy.MultiHashEmbed.v1 {id="MultiHashEmbed_v1"}
Identical to [`spacy.MultiHashEmbed.v2`](/api/architectures#MultiHashEmbed)
except with [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are
included.
### spacy.CharacterEmbed.v1 {#CharacterEmbed_v1}
### spacy.CharacterEmbed.v1 {id="CharacterEmbed_v1"}
Identical to [`spacy.CharacterEmbed.v2`](/api/architectures#CharacterEmbed)
except using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are
included.
### spacy.TextCatEnsemble.v1 {#TextCatEnsemble_v1}
### spacy.TextCatEnsemble.v1 {id="TextCatEnsemble_v1"}
The `spacy.TextCatEnsemble.v1` architecture built an internal `tok2vec` and
`linear_model`. Since `spacy.TextCatEnsemble.v2`, this has been refactored so
@ -158,7 +158,7 @@ network has an internal CNN Tok2Vec layer and uses attention.
| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ |
### spacy.TextCatCNN.v1 {#TextCatCNN_v1}
### spacy.TextCatCNN.v1 {id="TextCatCNN_v1"}
Since `spacy.TextCatCNN.v2`, this architecture has become resizable, which means
that you can add labels to a previously trained textcat. `TextCatCNN` v1 did not
@ -194,7 +194,7 @@ architecture is usually less accurate than the ensemble, but runs faster.
| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ |
### spacy.TextCatBOW.v1 {#TextCatBOW_v1}
### spacy.TextCatBOW.v1 {id="TextCatBOW_v1"}
Since `spacy.TextCatBOW.v2`, this architecture has become resizable, which means
that you can add labels to a previously trained textcat. `TextCatBOW` v1 did not
@ -222,17 +222,17 @@ the others, but may not be as accurate, especially if texts are short.
| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ |
| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ |
### spacy.TransitionBasedParser.v1 {#TransitionBasedParser_v1}
### spacy.TransitionBasedParser.v1 {id="TransitionBasedParser_v1"}
Identical to
[`spacy.TransitionBasedParser.v2`](/api/architectures#TransitionBasedParser)
except the `use_upper` was set to `True` by default.
## Layers {#layers}
## Layers {id="layers"}
These functions are available from `@spacy.registry.layers`.
### spacy.StaticVectors.v1 {#StaticVectors_v1}
### spacy.StaticVectors.v1 {id="StaticVectors_v1"}
Identical to [`spacy.StaticVectors.v2`](/api/architectures#StaticVectors) except
for the handling of tokens without vectors.
@ -246,11 +246,11 @@ added to an existing vectors table. See more details in
</Infobox>
## Loggers {#loggers}
## Loggers {id="loggers"}
These functions are available from `@spacy.registry.loggers`.
### spacy.ConsoleLogger.v1 {#ConsoleLogger_v1}
### spacy.ConsoleLogger.v1 {id="ConsoleLogger_v1"}
> #### Example config
>
@ -264,7 +264,7 @@ Writes the results of a training step to the console in a tabular format.
<Accordion title="Example console output" spaced>
```cli
```bash
$ python -m spacy train config.cfg
```

32
website/docs/api/lemmatizer.md → website/docs/api/lemmatizer.mdx
View File

@ -2,7 +2,7 @@
title: Lemmatizer
tag: class
source: spacy/pipeline/lemmatizer.py
new: 3
version: 3
teaser: 'Pipeline component for lemmatization'
api_string_name: lemmatizer
api_trainable: false
@ -32,7 +32,7 @@ available in the pipeline and runs _before_ the lemmatizer.
</Infobox>
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Lemmas generated by rules or predicted will be saved to `Token.lemma`.
@ -94,7 +94,7 @@ libraries (`pymorphy3`).
%%GITHUB_SPACY/spacy/pipeline/lemmatizer.py
```
## Lemmatizer.\_\_init\_\_ {#init tag="method"}
## Lemmatizer.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -120,7 +120,7 @@ shortcut for this and instantiate the component using its string name and
| mode | The lemmatizer mode, e.g. `"lookup"` or `"rule"`. Defaults to `"lookup"`. ~~str~~ |
| overwrite | Whether to overwrite existing lemmas. ~~bool~~ |
## Lemmatizer.\_\_call\_\_ {#call tag="method"}
## Lemmatizer.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -140,7 +140,7 @@ and all pipeline components are applied to the `Doc` in order.
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## Lemmatizer.pipe {#pipe tag="method"}
## Lemmatizer.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -161,7 +161,7 @@ applied to the `Doc` in order.
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## Lemmatizer.initialize {#initialize tag="method"}
## Lemmatizer.initialize {id="initialize",tag="method"}
Initialize the lemmatizer and load any data resources. This method is typically
called by [`Language.initialize`](/api/language#initialize) and lets you
@ -192,7 +192,7 @@ training. At runtime, all data is loaded from disk.
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
| `lookups` | The lookups object containing the tables such as `"lemma_rules"`, `"lemma_index"`, `"lemma_exc"` and `"lemma_lookup"`. If `None`, default tables are loaded from [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data). Defaults to `None`. ~~Optional[Lookups]~~ |
## Lemmatizer.lookup_lemmatize {#lookup_lemmatize tag="method"}
## Lemmatizer.lookup_lemmatize {id="lookup_lemmatize",tag="method"}
Lemmatize a token using a lookup-based approach. If no lemma is found, the
original string is returned.
@ -202,7 +202,7 @@ original string is returned.
| `token` | The token to lemmatize. ~~Token~~ |
| **RETURNS** | A list containing one or more lemmas. ~~List[str]~~ |
## Lemmatizer.rule_lemmatize {#rule_lemmatize tag="method"}
## Lemmatizer.rule_lemmatize {id="rule_lemmatize",tag="method"}
Lemmatize a token using a rule-based approach. Typically relies on POS tags.
@ -211,7 +211,7 @@ Lemmatize a token using a rule-based approach. Typically relies on POS tags.
| `token` | The token to lemmatize. ~~Token~~ |
| **RETURNS** | A list containing one or more lemmas. ~~List[str]~~ |
## Lemmatizer.is_base_form {#is_base_form tag="method"}
## Lemmatizer.is_base_form {id="is_base_form",tag="method"}
Check whether we're dealing with an uninflected paradigm, so we can avoid
lemmatization entirely.
@ -221,7 +221,7 @@ lemmatization entirely.
| `token` | The token to analyze. ~~Token~~ |
| **RETURNS** | Whether the token's attributes (e.g., part-of-speech tag, morphological features) describe a base form. ~~bool~~ |
## Lemmatizer.get_lookups_config {#get_lookups_config tag="classmethod"}
## Lemmatizer.get_lookups_config {id="get_lookups_config",tag="classmethod"}
Returns the lookups configuration settings for a given mode for use in
[`Lemmatizer.load_lookups`](/api/lemmatizer#load_lookups).
@ -231,7 +231,7 @@ Returns the lookups configuration settings for a given mode for use in
| `mode` | The lemmatizer mode. ~~str~~ |
| **RETURNS** | The required table names and the optional table names. ~~Tuple[List[str], List[str]]~~ |
## Lemmatizer.to_disk {#to_disk tag="method"}
## Lemmatizer.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -248,7 +248,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## Lemmatizer.from_disk {#from_disk tag="method"}
## Lemmatizer.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -266,7 +266,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `Lemmatizer` object. ~~Lemmatizer~~ |
## Lemmatizer.to_bytes {#to_bytes tag="method"}
## Lemmatizer.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -283,7 +283,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `Lemmatizer` object. ~~bytes~~ |
## Lemmatizer.from_bytes {#from_bytes tag="method"}
## Lemmatizer.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -302,7 +302,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Lemmatizer` object. ~~Lemmatizer~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| --------- | ------------------------------------------- |
@ -310,7 +310,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `lookups` | The lookups object. ~~Lookups~~ |
| `mode` | The lemmatizer mode. ~~str~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

16
website/docs/api/lexeme.md → website/docs/api/lexeme.mdx
View File

@ -9,7 +9,7 @@ A `Lexeme` has no string context – it's a word type, as opposed to a word toke
It therefore has no part-of-speech tag, dependency parse, or lemma (if
lemmatization depends on the part-of-speech tag).
## Lexeme.\_\_init\_\_ {#init tag="method"}
## Lexeme.\_\_init\_\_ {id="init",tag="method"}
Create a `Lexeme` object.
@ -18,7 +18,7 @@ Create a `Lexeme` object.
| `vocab` | The parent vocabulary. ~~Vocab~~ |
| `orth` | The orth id of the lexeme. ~~int~~ |
## Lexeme.set_flag {#set_flag tag="method"}
## Lexeme.set_flag {id="set_flag",tag="method"}
Change the value of a boolean flag.
@ -34,7 +34,7 @@ Change the value of a boolean flag.
| `flag_id` | The attribute ID of the flag to set. ~~int~~ |
| `value` | The new value of the flag. ~~bool~~ |
## Lexeme.check_flag {#check_flag tag="method"}
## Lexeme.check_flag {id="check_flag",tag="method"}
Check the value of a boolean flag.
@ -51,7 +51,7 @@ Check the value of a boolean flag.
| `flag_id` | The attribute ID of the flag to query. ~~int~~ |
| **RETURNS** | The value of the flag. ~~bool~~ |
## Lexeme.similarity {#similarity tag="method" model="vectors"}
## Lexeme.similarity {id="similarity",tag="method",model="vectors"}
Compute a semantic similarity estimate. Defaults to cosine over vectors.
@ -70,7 +70,7 @@ Compute a semantic similarity estimate. Defaults to cosine over vectors.
| other | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ |
| **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ |
## Lexeme.has_vector {#has_vector tag="property" model="vectors"}
## Lexeme.has_vector {id="has_vector",tag="property",model="vectors"}
A boolean value indicating whether a word vector is associated with the lexeme.
@ -85,7 +85,7 @@ A boolean value indicating whether a word vector is associated with the lexeme.
| ----------- | ------------------------------------------------------- |
| **RETURNS** | Whether the lexeme has a vector data attached. ~~bool~~ |
## Lexeme.vector {#vector tag="property" model="vectors"}
## Lexeme.vector {id="vector",tag="property",model="vectors"}
A real-valued meaning representation.
@ -101,7 +101,7 @@ A real-valued meaning representation.
| ----------- | ------------------------------------------------------------------------------------------------ |
| **RETURNS** | A 1-dimensional array representing the lexeme's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
## Lexeme.vector_norm {#vector_norm tag="property" model="vectors"}
## Lexeme.vector_norm {id="vector_norm",tag="property",model="vectors"}
The L2 norm of the lexeme's vector representation.
@ -119,7 +119,7 @@ The L2 norm of the lexeme's vector representation.
| ----------- | --------------------------------------------------- |
| **RETURNS** | The L2 norm of the vector representation. ~~float~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |

40
website/docs/api/lookups.md → website/docs/api/lookups.mdx
View File

@ -3,7 +3,7 @@ title: Lookups
teaser: A container for large lookup tables and dictionaries
tag: class
source: spacy/lookups.py
new: 2.2
version: 2.2
---
This class allows convenient access to large lookup tables and dictionaries,
@ -13,7 +13,7 @@ can be accessed before the pipeline components are applied (e.g. in the
tokenizer and lemmatizer), as well as within the pipeline components via
`doc.vocab.lookups`.
## Lookups.\_\_init\_\_ {#init tag="method"}
## Lookups.\_\_init\_\_ {id="init",tag="method"}
Create a `Lookups` object.
@ -24,7 +24,7 @@ Create a `Lookups` object.
> lookups = Lookups()
> ```
## Lookups.\_\_len\_\_ {#len tag="method"}
## Lookups.\_\_len\_\_ {id="len",tag="method"}
Get the current number of tables in the lookups.
@ -39,7 +39,7 @@ Get the current number of tables in the lookups.
| ----------- | -------------------------------------------- |
| **RETURNS** | The number of tables in the lookups. ~~int~~ |
## Lookups.\_\contains\_\_ {#contains tag="method"}
## Lookups.\_\_contains\_\_ {id="contains",tag="method"}
Check if the lookups contain a table of a given name. Delegates to
[`Lookups.has_table`](/api/lookups#has_table).
@ -57,7 +57,7 @@ Check if the lookups contain a table of a given name. Delegates to
| `name` | Name of the table. ~~str~~ |
| **RETURNS** | Whether a table of that name is in the lookups. ~~bool~~ |
## Lookups.tables {#tables tag="property"}
## Lookups.tables {id="tables",tag="property"}
Get the names of all tables in the lookups.
@ -73,7 +73,7 @@ Get the names of all tables in the lookups.
| ----------- | ------------------------------------------------- |
| **RETURNS** | Names of the tables in the lookups. ~~List[str]~~ |
## Lookups.add_table {#add_table tag="method"}
## Lookups.add_table {id="add_table",tag="method"}
Add a new table with optional data to the lookups. Raises an error if the table
exists.
@ -91,7 +91,7 @@ exists.
| `data` | Optional data to add to the table. ~~dict~~ |
| **RETURNS** | The newly added table. ~~Table~~ |
## Lookups.get_table {#get_table tag="method"}
## Lookups.get_table {id="get_table",tag="method"}
Get a table from the lookups. Raises an error if the table doesn't exist.
@ -109,7 +109,7 @@ Get a table from the lookups. Raises an error if the table doesn't exist.
| `name` | Name of the table. ~~str~~ |
| **RETURNS** | The table. ~~Table~~ |
## Lookups.remove_table {#remove_table tag="method"}
## Lookups.remove_table {id="remove_table",tag="method"}
Remove a table from the lookups. Raises an error if the table doesn't exist.
@ -127,7 +127,7 @@ Remove a table from the lookups. Raises an error if the table doesn't exist.
| `name` | Name of the table to remove. ~~str~~ |
| **RETURNS** | The removed table. ~~Table~~ |
## Lookups.has_table {#has_table tag="method"}
## Lookups.has_table {id="has_table",tag="method"}
Check if the lookups contain a table of a given name. Equivalent to
[`Lookups.__contains__`](/api/lookups#contains).
@ -145,7 +145,7 @@ Check if the lookups contain a table of a given name. Equivalent to
| `name` | Name of the table. ~~str~~ |
| **RETURNS** | Whether a table of that name is in the lookups. ~~bool~~ |
## Lookups.to_bytes {#to_bytes tag="method"}
## Lookups.to_bytes {id="to_bytes",tag="method"}
Serialize the lookups to a bytestring.
@ -159,7 +159,7 @@ Serialize the lookups to a bytestring.
| ----------- | --------------------------------- |
| **RETURNS** | The serialized lookups. ~~bytes~~ |
## Lookups.from_bytes {#from_bytes tag="method"}
## Lookups.from_bytes {id="from_bytes",tag="method"}
Load the lookups from a bytestring.
@ -176,7 +176,7 @@ Load the lookups from a bytestring.
| `bytes_data` | The data to load from. ~~bytes~~ |
| **RETURNS** | The loaded lookups. ~~Lookups~~ |
## Lookups.to_disk {#to_disk tag="method"}
## Lookups.to_disk {id="to_disk",tag="method"}
Save the lookups to a directory as `lookups.bin`. Expects a path to a directory,
which will be created if it doesn't exist.
@ -191,7 +191,7 @@ which will be created if it doesn't exist.
| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ |
| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
## Lookups.from_disk {#from_disk tag="method"}
## Lookups.from_disk {id="from_disk",tag="method"}
Load lookups from a directory containing a `lookups.bin`. Will skip loading if
the file doesn't exist.
@ -209,7 +209,7 @@ the file doesn't exist.
| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
| **RETURNS** | The loaded lookups. ~~Lookups~~ |
## Table {#table tag="class, ordererddict"}
## Table {id="table",tag="class, ordererddict"}
A table in the lookups. Subclass of `OrderedDict` that implements a slightly
more consistent and unified API and includes a Bloom filter to speed up missed
@ -218,7 +218,7 @@ lookups. Supports **all other methods and attributes** of `OrderedDict` /
accept both integers and strings (which will be hashed before being added to the
table).
### Table.\_\_init\_\_ {#table.init tag="method"}
### Table.\_\_init\_\_ {id="table.init",tag="method"}
Initialize a new table.
@ -236,7 +236,7 @@ Initialize a new table.
| ------ | ------------------------------------------ |
| `name` | Optional table name for reference. ~~str~~ |
### Table.from_dict {#table.from_dict tag="classmethod"}
### Table.from_dict {id="table.from_dict",tag="classmethod"}
Initialize a new table from a dict.
@ -254,7 +254,7 @@ Initialize a new table from a dict.
| `name` | Optional table name for reference. ~~str~~ |
| **RETURNS** | The newly constructed object. ~~Table~~ |
### Table.set {#table.set tag="method"}
### Table.set {id="table.set",tag="method"}
Set a new key / value pair. String keys will be hashed. Same as
`table[key] = value`.
@ -273,7 +273,7 @@ Set a new key / value pair. String keys will be hashed. Same as
| `key` | The key. ~~Union[str, int]~~ |
| `value` | The value. |
### Table.to_bytes {#table.to_bytes tag="method"}
### Table.to_bytes {id="table.to_bytes",tag="method"}
Serialize the table to a bytestring.
@ -287,7 +287,7 @@ Serialize the table to a bytestring.
| ----------- | ------------------------------- |
| **RETURNS** | The serialized table. ~~bytes~~ |
### Table.from_bytes {#table.from_bytes tag="method"}
### Table.from_bytes {id="table.from_bytes",tag="method"}
Load a table from a bytestring.
@ -304,7 +304,7 @@ Load a table from a bytestring.
| `bytes_data` | The data to load. ~~bytes~~ |
| **RETURNS** | The loaded table. ~~Table~~ |
### Attributes {#table-attributes}
### Attributes {id="table-attributes"}
| Name | Description |
| -------------- | ------------------------------------------------------------- |

47
website/docs/api/matcher.md → website/docs/api/matcher.mdx
View File

@ -13,7 +13,7 @@ tokens in context. For in-depth examples and workflows for combining rules and
statistical models, see the [usage guide](/usage/rule-based-matching) on
rule-based matching.
## Pattern format {#patterns}
## Pattern format {id="patterns"}
> ```json
> ### Example
@ -86,16 +86,22 @@ it compares to another value.
> ]
> ```
| Attribute | Description |
| -------------------------- | -------------------------------------------------------------------------------------------------------- |
| `IN` | Attribute value is member of a list. ~~Any~~ |
| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ |
| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ |
| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ |
| `INTERSECTS` | Attribute value (for `MORPH` or custom list attribute) has a non-empty intersection with a list. ~~Any~~ |
| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ |
| Attribute | Description |
| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `REGEX` | Attribute value matches the regular expression at any position in the string. ~~Any~~ |
| `FUZZY` | Attribute value matches if the `fuzzy_compare` method matches for `(value, pattern, -1)`. The default method allows a Levenshtein edit distance of at least 2 and up to 30% of the pattern string length. ~~Any~~ |
| `FUZZY1`, `FUZZY2`, ... `FUZZY9` | Attribute value matches if the `fuzzy_compare` method matches for `(value, pattern, N)`. The default method allows a Levenshtein edit distance of at most N (1-9). ~~Any~~ |
| `IN` | Attribute value is member of a list. ~~Any~~ |
| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ |
| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ |
| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ |
| `INTERSECTS` | Attribute value (for `MORPH` or custom list attribute) has a non-empty intersection with a list. ~~Any~~ |
| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ |
## Matcher.\_\_init\_\_ {#init tag="method"}
As of spaCy v3.5, `REGEX` and `FUZZY` can be used in combination with `IN` and
`NOT_IN`.
## Matcher.\_\_init\_\_ {id="init",tag="method"}
Create the rule-based `Matcher`. If `validate=True` is set, all patterns added
to the matcher will be validated against a JSON schema and a `MatchPatternError`
@ -109,12 +115,13 @@ string where an integer is expected) or unexpected property names.
> matcher = Matcher(nlp.vocab)
> ```
| Name | Description |
| ---------- | ----------------------------------------------------------------------------------------------------- |
| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ |
| `validate` | Validate all patterns added to this matcher. ~~bool~~ |
| Name | Description |
| --------------- | ----------------------------------------------------------------------------------------------------- |
| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ |
| `validate` | Validate all patterns added to this matcher. ~~bool~~ |
| `fuzzy_compare` | The comparison method used for the `FUZZY` operators. ~~Callable[[str, str, int], bool]~~ |
## Matcher.\_\_call\_\_ {#call tag="method"}
## Matcher.\_\_call\_\_ {id="call",tag="method"}
Find all token sequences matching the supplied patterns on the `Doc` or `Span`.
@ -143,7 +150,7 @@ the match.
| `with_alignments` <Tag variant="new">3.0.6</Tag> | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token in the pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ |
| **RETURNS** | A list of `(match_id, start, end)` tuples, describing the matches. A match tuple describes a span `doc[start:end`]. The `match_id` is the ID of the added match pattern. If `as_spans` is set to `True`, a list of `Span` objects is returned instead. ~~Union[List[Tuple[int, int, int]], List[Span]]~~ |
## Matcher.\_\_len\_\_ {#len tag="method" new="2"}
## Matcher.\_\_len\_\_ {id="len",tag="method",version="2"}
Get the number of rules added to the matcher. Note that this only returns the
number of rules (identical with the number of IDs), not the number of individual
@ -162,7 +169,7 @@ patterns.
| ----------- | ---------------------------- |
| **RETURNS** | The number of rules. ~~int~~ |
## Matcher.\_\_contains\_\_ {#contains tag="method" new="2"}
## Matcher.\_\_contains\_\_ {id="contains",tag="method",version="2"}
Check whether the matcher contains rules for a match ID.
@ -180,7 +187,7 @@ Check whether the matcher contains rules for a match ID.
| `key` | The match ID. ~~str~~ |
| **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ |
## Matcher.add {#add tag="method" new="2"}
## Matcher.add {id="add",tag="method",version="2"}
Add a rule to the matcher, consisting of an ID key, one or more patterns, and an
optional callback function to act on the matches. The callback function will
@ -212,7 +219,7 @@ will be overwritten.
| `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[Matcher, Doc, int, List[tuple], Any]]~~ |
| `greedy` <Tag variant="new">3</Tag> | Optional filter for greedy matches. Can either be `"FIRST"` or `"LONGEST"`. ~~Optional[str]~~ |
## Matcher.remove {#remove tag="method" new="2"}
## Matcher.remove {id="remove",tag="method",version="2"}
Remove a rule from the matcher. A `KeyError` is raised if the match ID does not
exist.
@ -230,7 +237,7 @@ exist.
| ----- | --------------------------------- |
| `key` | The ID of the match rule. ~~str~~ |
## Matcher.get {#get tag="method" new="2"}
## Matcher.get {id="get",tag="method",version="2"}
Retrieve the pattern stored for a key. Returns the rule as an
`(on_match, patterns)` tuple containing the callback and available patterns.

42
website/docs/api/morphologizer.md → website/docs/api/morphologizer.mdx
View File

@ -2,7 +2,7 @@
title: Morphologizer
tag: class
source: spacy/pipeline/morphologizer.pyx
new: 3
version: 3
teaser: 'Pipeline component for predicting morphological features'
api_base_class: /api/tagger
api_string_name: morphologizer
@ -15,7 +15,7 @@ coarse-grained POS tags following the Universal Dependencies
[FEATS](https://universaldependencies.org/format.html#morphological-annotation)
annotation guidelines.
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predictions are saved to `Token.morph` and `Token.pos`.
@ -25,7 +25,7 @@ Predictions are saved to `Token.morph` and `Token.pos`.
| `Token.pos_` | The UPOS part of speech. ~~str~~ |
| `Token.morph` | Morphological features. ~~MorphAnalysis~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -54,7 +54,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/morphologizer.pyx
```
## Morphologizer.\_\_init\_\_ {#init tag="method"}
## Morphologizer.\_\_init\_\_ {id="init",tag="method"}
Create a new pipeline instance. In your application, you would normally use a
shortcut for this and instantiate the component using its string name and
@ -98,7 +98,7 @@ annotation `C=E|X=Y`):
| `extend` <Tag variant="new">3.2</Tag> | Whether existing feature types (whose values may or may not be overwritten depending on `overwrite`) are preserved. Defaults to `False`. ~~bool~~ |
| `scorer` <Tag variant="new">3.2</Tag> | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"pos"` and `"morph"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ |
## Morphologizer.\_\_call\_\_ {#call tag="method"}
## Morphologizer.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -121,7 +121,7 @@ delegate to the [`predict`](/api/morphologizer#predict) and
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## Morphologizer.pipe {#pipe tag="method"}
## Morphologizer.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -145,7 +145,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/morphologizer#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## Morphologizer.initialize {#initialize tag="method"}
## Morphologizer.initialize {id="initialize",tag="method"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -182,7 +182,7 @@ config.
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[dict]~~ |
## Morphologizer.predict {#predict tag="method"}
## Morphologizer.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them.
@ -199,7 +199,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The model's prediction for each document. |
## Morphologizer.set_annotations {#set_annotations tag="method"}
## Morphologizer.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
@ -216,7 +216,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `Morphologizer.predict`. |
## Morphologizer.update {#update tag="method"}
## Morphologizer.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects containing the
predictions and gold-standard annotations, and update the component's model.
@ -240,7 +240,7 @@ Delegates to [`predict`](/api/morphologizer#predict) and
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## Morphologizer.get_loss {#get_loss tag="method"}
## Morphologizer.get_loss {id="get_loss",tag="method"}
Find the loss and gradient of loss for the batch of documents and their
predicted scores.
@ -259,7 +259,7 @@ predicted scores.
| `scores` | Scores representing the model's predictions. |
| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ |
## Morphologizer.create_optimizer {#create_optimizer tag="method"}
## Morphologizer.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -274,7 +274,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## Morphologizer.use_params {#use_params tag="method, contextmanager"}
## Morphologizer.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -291,7 +291,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## Morphologizer.add_label {#add_label tag="method"}
## Morphologizer.add_label {id="add_label",tag="method"}
Add a new label to the pipe. If the `Morphologizer` should set annotations for
both `pos` and `morph`, the label should include the UPOS as the feature `POS`.
@ -314,7 +314,7 @@ will be automatically added to the model, and the output dimension will be
| `label` | The label to add. ~~str~~ |
| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ |
## Morphologizer.to_disk {#to_disk tag="method"}
## Morphologizer.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -331,7 +331,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## Morphologizer.from_disk {#from_disk tag="method"}
## Morphologizer.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -349,7 +349,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `Morphologizer` object. ~~Morphologizer~~ |
## Morphologizer.to_bytes {#to_bytes tag="method"}
## Morphologizer.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -366,7 +366,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `Morphologizer` object. ~~bytes~~ |
## Morphologizer.from_bytes {#from_bytes tag="method"}
## Morphologizer.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -385,7 +385,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Morphologizer` object. ~~Morphologizer~~ |
## Morphologizer.labels {#labels tag="property"}
## Morphologizer.labels {id="labels",tag="property"}
The labels currently added to the component in the Universal Dependencies
[FEATS](https://universaldependencies.org/format.html#morphological-annotation)
@ -404,7 +404,7 @@ coarse-grained POS as the feature `POS`.
| ----------- | --------------------------------------------------------- |
| **RETURNS** | The labels added to the component. ~~Iterable[str, ...]~~ |
## Morphologizer.label_data {#label_data tag="property" new="3"}
## Morphologizer.label_data {id="label_data",tag="property",version="3"}
The labels currently added to the component and their internal meta information.
This is the data generated by [`init labels`](/api/cli#init-labels) and used by
@ -422,7 +422,7 @@ model with a pre-defined label set.
| ----------- | ----------------------------------------------- |
| **RETURNS** | The label data added to the component. ~~dict~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

40
website/docs/api/morphology.md → website/docs/api/morphology.mdx
View File

@ -10,7 +10,7 @@ morphological analysis, so queries of morphological attributes are delegated to
this class. See [`MorphAnalysis`](/api/morphology#morphanalysis) for the
container storing a single morphological analysis.
## Morphology.\_\_init\_\_ {#init tag="method"}
## Morphology.\_\_init\_\_ {id="init",tag="method"}
Create a `Morphology` object.
@ -26,7 +26,7 @@ Create a `Morphology` object.
| --------- | --------------------------------- |
| `strings` | The string store. ~~StringStore~~ |
## Morphology.add {#add tag="method"}
## Morphology.add {id="add",tag="method"}
Insert a morphological analysis in the morphology table, if not already present.
The morphological analysis may be provided in the Universal Dependencies
@ -46,7 +46,7 @@ new analysis.
| ---------- | ------------------------------------------------ |
| `features` | The morphological features. ~~Union[Dict, str]~~ |
## Morphology.get {#get tag="method"}
## Morphology.get {id="get",tag="method"}
> #### Example
>
@ -64,7 +64,7 @@ string for the hash of the morphological analysis.
| ------- | ----------------------------------------------- |
| `morph` | The hash of the morphological analysis. ~~int~~ |
## Morphology.feats_to_dict {#feats_to_dict tag="staticmethod"}
## Morphology.feats_to_dict {id="feats_to_dict",tag="staticmethod"}
Convert a string
[FEATS](https://universaldependencies.org/format.html#morphological-annotation)
@ -84,7 +84,7 @@ tag map.
| `feats` | The morphological features in Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ |
| **RETURNS** | The morphological features as a dictionary. ~~Dict[str, str]~~ |
## Morphology.dict_to_feats {#dict_to_feats tag="staticmethod"}
## Morphology.dict_to_feats {id="dict_to_feats",tag="staticmethod"}
Convert a dictionary of features and values to a string
[FEATS](https://universaldependencies.org/format.html#morphological-annotation)
@ -103,19 +103,19 @@ representation.
| `feats_dict` | The morphological features as a dictionary. ~~Dict[str, str]~~ |
| **RETURNS** | The morphological features in Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| ------------- | ------------------------------------------------------------------------------------------------------------------------------ |
| `FEATURE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) feature separator. Default is `|`. ~~str~~ |
| `FIELD_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) field separator. Default is `=`. ~~str~~ |
| `VALUE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) value separator. Default is `,`. ~~str~~ |
| Name | Description |
| ------------- | ------------------------------------------------------------------------------------------------------------------------------- |
| `FEATURE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) feature separator. Default is `\|`. ~~str~~ |
| `FIELD_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) field separator. Default is `=`. ~~str~~ |
| `VALUE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) value separator. Default is `,`. ~~str~~ |
## MorphAnalysis {#morphanalysis tag="class" source="spacy/tokens/morphanalysis.pyx"}
## MorphAnalysis {id="morphanalysis",tag="class",source="spacy/tokens/morphanalysis.pyx"}
Stores a single morphological analysis.
### MorphAnalysis.\_\_init\_\_ {#morphanalysis-init tag="method"}
### MorphAnalysis.\_\_init\_\_ {id="morphanalysis-init",tag="method"}
Initialize a MorphAnalysis object from a Universal Dependencies
[FEATS](https://universaldependencies.org/format.html#morphological-annotation)
@ -135,7 +135,7 @@ string or a dictionary of morphological features.
| `vocab` | The vocab. ~~Vocab~~ |
| `features` | The morphological features. ~~Union[Dict[str, str], str]~~ |
### MorphAnalysis.\_\_contains\_\_ {#morphanalysis-contains tag="method"}
### MorphAnalysis.\_\_contains\_\_ {id="morphanalysis-contains",tag="method"}
Whether a feature/value pair is in the analysis.
@ -151,7 +151,7 @@ Whether a feature/value pair is in the analysis.
| ----------- | --------------------------------------------- |
| **RETURNS** | A feature/value pair in the analysis. ~~str~~ |
### MorphAnalysis.\_\_iter\_\_ {#morphanalysis-iter tag="method"}
### MorphAnalysis.\_\_iter\_\_ {id="morphanalysis-iter",tag="method"}
Iterate over the feature/value pairs in the analysis.
@ -167,7 +167,7 @@ Iterate over the feature/value pairs in the analysis.
| ---------- | --------------------------------------------- |
| **YIELDS** | A feature/value pair in the analysis. ~~str~~ |
### MorphAnalysis.\_\_len\_\_ {#morphanalysis-len tag="method"}
### MorphAnalysis.\_\_len\_\_ {id="morphanalysis-len",tag="method"}
Returns the number of features in the analysis.
@ -183,7 +183,7 @@ Returns the number of features in the analysis.
| ----------- | ----------------------------------------------- |
| **RETURNS** | The number of features in the analysis. ~~int~~ |
### MorphAnalysis.\_\_str\_\_ {#morphanalysis-str tag="method"}
### MorphAnalysis.\_\_str\_\_ {id="morphanalysis-str",tag="method"}
Returns the morphological analysis in the Universal Dependencies
[FEATS](https://universaldependencies.org/format.html#morphological-annotation)
@ -201,7 +201,7 @@ string format.
| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
| **RETURNS** | The analysis in the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ |
### MorphAnalysis.get {#morphanalysis-get tag="method"}
### MorphAnalysis.get {id="morphanalysis-get",tag="method"}
Retrieve values for a feature by field.
@ -218,7 +218,7 @@ Retrieve values for a feature by field.
| `field` | The field to retrieve. ~~str~~ |
| **RETURNS** | A list of the individual features. ~~List[str]~~ |
### MorphAnalysis.to_dict {#morphanalysis-to_dict tag="method"}
### MorphAnalysis.to_dict {id="morphanalysis-to_dict",tag="method"}
Produce a dict representation of the analysis, in the same format as the tag
map.
@ -235,7 +235,7 @@ map.
| ----------- | ----------------------------------------------------------- |
| **RETURNS** | The dict representation of the analysis. ~~Dict[str, str]~~ |
### MorphAnalysis.from_id {#morphanalysis-from_id tag="classmethod"}
### MorphAnalysis.from_id {id="morphanalysis-from_id",tag="classmethod"}
Create a morphological analysis from a given hash ID.

14
website/docs/api/phrasematcher.md → website/docs/api/phrasematcher.mdx
View File

@ -3,7 +3,7 @@ title: PhraseMatcher
teaser: Match sequences of tokens, based on documents
tag: class
source: spacy/matcher/phrasematcher.pyx
new: 2
version: 2
---
The `PhraseMatcher` lets you efficiently match large terminology lists. While
@ -12,7 +12,7 @@ descriptions, the `PhraseMatcher` accepts match patterns in the form of `Doc`
objects. See the [usage guide](/usage/rule-based-matching#phrasematcher) for
examples.
## PhraseMatcher.\_\_init\_\_ {#init tag="method"}
## PhraseMatcher.\_\_init\_\_ {id="init",tag="method"}
Create the rule-based `PhraseMatcher`. Setting a different `attr` to match on
will change the token attributes that will be compared to determine a match. By
@ -42,7 +42,7 @@ be shown.
| `attr` | The token attribute to match on. Defaults to `ORTH`, i.e. the verbatim token text. ~~Union[int, str]~~ |
| `validate` | Validate patterns added to the matcher. ~~bool~~ |
## PhraseMatcher.\_\_call\_\_ {#call tag="method"}
## PhraseMatcher.\_\_call\_\_ {id="call",tag="method"}
Find all token sequences matching the supplied patterns on the `Doc` or `Span`.
@ -76,7 +76,7 @@ match_id_string = nlp.vocab.strings[match_id]
</Infobox>
## PhraseMatcher.\_\_len\_\_ {#len tag="method"}
## PhraseMatcher.\_\_len\_\_ {id="len",tag="method"}
Get the number of rules added to the matcher. Note that this only returns the
number of rules (identical with the number of IDs), not the number of individual
@ -95,7 +95,7 @@ patterns.
| ----------- | ---------------------------- |
| **RETURNS** | The number of rules. ~~int~~ |
## PhraseMatcher.\_\_contains\_\_ {#contains tag="method"}
## PhraseMatcher.\_\_contains\_\_ {id="contains",tag="method"}
Check whether the matcher contains rules for a match ID.
@ -113,7 +113,7 @@ Check whether the matcher contains rules for a match ID.
| `key` | The match ID. ~~str~~ |
| **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ |
## PhraseMatcher.add {#add tag="method"}
## PhraseMatcher.add {id="add",tag="method"}
Add a rule to the matcher, consisting of an ID key, one or more patterns, and a
optional callback function to act on the matches. The callback function will
@ -141,7 +141,7 @@ will be overwritten.
| _keyword-only_ | |
| `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[Matcher, Doc, int, List[tuple], Any]]~~ |
## PhraseMatcher.remove {#remove tag="method" new="2.2"}
## PhraseMatcher.remove {id="remove",tag="method",version="2.2"}
Remove a rule from the matcher by match ID. A `KeyError` is raised if the key
does not exist.

50
website/docs/api/pipe.md → website/docs/api/pipe.mdx
View File

@ -12,7 +12,7 @@ spaCy pipeline. See the docs on
[writing trainable components](/usage/processing-pipelines#trainable-components)
for how to use the `TrainablePipe` base class to implement custom components.
<!-- TODO: Pipe vs TrainablePipe, check methods below (all renamed to TrainablePipe for now) -->
{/* TODO: Pipe vs TrainablePipe, check methods below (all renamed to TrainablePipe for now) */}
> #### Why is it implemented in Cython?
>
@ -27,7 +27,7 @@ for how to use the `TrainablePipe` base class to implement custom components.
%%GITHUB_SPACY/spacy/pipeline/trainable_pipe.pyx
```
## TrainablePipe.\_\_init\_\_ {#init tag="method"}
## TrainablePipe.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -54,7 +54,7 @@ shortcut for this and instantiate the component using its string name and
| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ |
| `**cfg` | Additional config parameters and settings. Will be available as the dictionary `cfg` and is serialized with the component. |
## TrainablePipe.\_\_call\_\_ {#call tag="method"}
## TrainablePipe.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -77,7 +77,7 @@ and all pipeline components are applied to the `Doc` in order. Both
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## TrainablePipe.pipe {#pipe tag="method"}
## TrainablePipe.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -100,7 +100,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/pipe#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## TrainablePipe.set_error_handler {#set_error_handler tag="method" new="3"}
## TrainablePipe.set_error_handler {id="set_error_handler",tag="method",version="3"}
Define a callback that will be invoked when an error is thrown during processing
of one or more documents with either [`__call__`](/api/pipe#call) or
@ -122,7 +122,7 @@ processed, and the original error.
| --------------- | -------------------------------------------------------------------------------------------------------------- |
| `error_handler` | A function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ |
## TrainablePipe.get_error_handler {#get_error_handler tag="method" new="3"}
## TrainablePipe.get_error_handler {id="get_error_handler",tag="method",version="3"}
Retrieve the callback that performs error handling for this component's
[`__call__`](/api/pipe#call) and [`pipe`](/api/pipe#pipe) methods. If no custom
@ -141,7 +141,7 @@ returned that simply reraises the exception.
| ----------- | ---------------------------------------------------------------------------------------------------------------- |
| **RETURNS** | The function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ |
## TrainablePipe.initialize {#initialize tag="method" new="3"}
## TrainablePipe.initialize {id="initialize",tag="method",version="3"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. The data examples are
@ -165,7 +165,7 @@ by [`Language.initialize`](/api/language#initialize).
| _keyword-only_ | |
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
## TrainablePipe.predict {#predict tag="method"}
## TrainablePipe.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them.
@ -188,7 +188,7 @@ This method needs to be overwritten with your own custom `predict` method.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The model's prediction for each document. |
## TrainablePipe.set_annotations {#set_annotations tag="method"}
## TrainablePipe.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
@ -212,7 +212,7 @@ method.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `Tagger.predict`. |
## TrainablePipe.update {#update tag="method"}
## TrainablePipe.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects containing the
predictions and gold-standard annotations, and update the component's model.
@ -234,7 +234,7 @@ predictions and gold-standard annotations, and update the component's model.
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## TrainablePipe.rehearse {#rehearse tag="method,experimental" new="3"}
## TrainablePipe.rehearse {id="rehearse",tag="method,experimental",version="3"}
Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the
current model to make predictions similar to an initial model, to try to address
@ -256,7 +256,7 @@ the "catastrophic forgetting" problem. This feature is experimental.
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## TrainablePipe.get_loss {#get_loss tag="method"}
## TrainablePipe.get_loss {id="get_loss",tag="method"}
Find the loss and gradient of loss for the batch of documents and their
predicted scores.
@ -281,7 +281,7 @@ This method needs to be overwritten with your own custom `get_loss` method.
| `scores` | Scores representing the model's predictions. |
| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ |
## TrainablePipe.score {#score tag="method" new="3"}
## TrainablePipe.score {id="score",tag="method",version="3"}
Score a batch of examples.
@ -298,7 +298,7 @@ Score a batch of examples.
| `\*\*kwargs` | Any additional settings to pass on to the scorer. ~~Any~~ |
| **RETURNS** | The scores, e.g. produced by the [`Scorer`](/api/scorer). ~~Dict[str, Union[float, Dict[str, float]]]~~ |
## TrainablePipe.create_optimizer {#create_optimizer tag="method"}
## TrainablePipe.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component. Defaults to
[`Adam`](https://thinc.ai/docs/api-optimizers#adam) with default settings.
@ -314,7 +314,7 @@ Create an optimizer for the pipeline component. Defaults to
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## TrainablePipe.use_params {#use_params tag="method, contextmanager"}
## TrainablePipe.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -331,7 +331,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## TrainablePipe.finish_update {#finish_update tag="method"}
## TrainablePipe.finish_update {id="finish_update",tag="method"}
Update parameters using the current parameter gradients. Defaults to calling
[`self.model.finish_update`](https://thinc.ai/docs/api-model#finish_update).
@ -349,7 +349,7 @@ Update parameters using the current parameter gradients. Defaults to calling
| ----- | ------------------------------------- |
| `sgd` | An optimizer. ~~Optional[Optimizer]~~ |
## TrainablePipe.add_label {#add_label tag="method"}
## TrainablePipe.add_label {id="add_label",tag="method"}
> #### Example
>
@ -384,7 +384,7 @@ case, all labels found in the sample will be automatically added to the model,
and the output dimension will be
[inferred](/usage/layers-architectures#thinc-shape-inference) automatically.
## TrainablePipe.is_resizable {#is_resizable tag="property"}
## TrainablePipe.is_resizable {id="is_resizable",tag="property"}
> #### Example
>
@ -415,7 +415,7 @@ as an attribute to the component's model.
| ----------- | ---------------------------------------------------------------------------------------------- |
| **RETURNS** | Whether or not the output dimension of the model can be changed after initialization. ~~bool~~ |
## TrainablePipe.set_output {#set_output tag="method"}
## TrainablePipe.set_output {id="set_output",tag="method"}
Change the output dimension of the component's model. If the component is not
[resizable](#is_resizable), this method will raise a `NotImplementedError`. If a
@ -435,7 +435,7 @@ care should be taken to avoid the "catastrophic forgetting" problem.
| ---- | --------------------------------- |
| `nO` | The new output dimension. ~~int~~ |
## TrainablePipe.to_disk {#to_disk tag="method"}
## TrainablePipe.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -452,7 +452,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## TrainablePipe.from_disk {#from_disk tag="method"}
## TrainablePipe.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -470,7 +470,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified pipe. ~~TrainablePipe~~ |
## TrainablePipe.to_bytes {#to_bytes tag="method"}
## TrainablePipe.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -487,7 +487,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the pipe. ~~bytes~~ |
## TrainablePipe.from_bytes {#from_bytes tag="method"}
## TrainablePipe.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -506,7 +506,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The pipe. ~~TrainablePipe~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| ------- | --------------------------------------------------------------------------------------------------------------------------------- |
@ -515,7 +515,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `name` | The name of the component instance in the pipeline. Can be used in the losses. ~~str~~ |
| `cfg` | Keyword arguments passed to [`TrainablePipe.__init__`](/api/pipe#init). Will be serialized with the component. ~~Dict[str, Any]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

12
website/docs/api/pipeline-functions.md → website/docs/api/pipeline-functions.mdx
View File

@ -10,7 +10,7 @@ menu:
- ['doc_cleaner', 'doc_cleaner']
---
## merge_noun_chunks {#merge_noun_chunks tag="function"}
## merge_noun_chunks {id="merge_noun_chunks",tag="function"}
Merge noun chunks into a single token. Also available via the string name
`"merge_noun_chunks"`.
@ -40,7 +40,7 @@ all other components.
| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ |
| **RETURNS** | The modified `Doc` with merged noun chunks. ~~Doc~~ |
## merge_entities {#merge_entities tag="function"}
## merge_entities {id="merge_entities",tag="function"}
Merge named entities into a single token. Also available via the string name
`"merge_entities"`.
@ -70,7 +70,7 @@ components to the end of the pipeline and after all other components.
| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ |
| **RETURNS** | The modified `Doc` with merged entities. ~~Doc~~ |
## merge_subtokens {#merge_subtokens tag="function" new="2.1"}
## merge_subtokens {id="merge_subtokens",tag="function",version="2.1"}
Merge subtokens into a single token. Also available via the string name
`"merge_subtokens"`. As of v2.1, the parser is able to predict "subtokens" that
@ -110,7 +110,7 @@ end of the pipeline and after all other components.
| `label` | The subtoken dependency label. Defaults to `"subtok"`. ~~str~~ |
| **RETURNS** | The modified `Doc` with merged subtokens. ~~Doc~~ |
## token_splitter {#token_splitter tag="function" new="3.0"}
## token_splitter {id="token_splitter",tag="function",version="3.0"}
Split tokens longer than a minimum length into shorter tokens. Intended for use
with transformer pipelines where long spaCy tokens lead to input text that
@ -132,7 +132,7 @@ exceed the transformer model max length.
| `split_length` | The length of the split tokens. Defaults to `5`. ~~int~~ |
| **RETURNS** | The modified `Doc` with the split tokens. ~~Doc~~ |
## doc_cleaner {#doc_cleaner tag="function" new="3.2.1"}
## doc_cleaner {id="doc_cleaner",tag="function",version="3.2.1"}
Clean up `Doc` attributes. Intended for use at the end of pipelines with
`tok2vec` or `transformer` pipeline components that store tensors and other
@ -154,7 +154,7 @@ whole pipeline has run.
| `silent` | If `False`, show warnings if attributes aren't found or can't be set. Defaults to `True`. ~~bool~~ |
| **RETURNS** | The modified `Doc` with the modified attributes. ~~Doc~~ |
## span_cleaner {#span_cleaner tag="function,experimental"}
## span_cleaner {id="span_cleaner",tag="function,experimental"}
Remove `SpanGroup`s from `doc.spans` based on a key prefix. This is used to
clean up after the [`CoreferenceResolver`](/api/coref) when it's paired with a

26
website/docs/api/scorer.md → website/docs/api/scorer.mdx
View File

@ -10,7 +10,7 @@ The `Scorer` computes evaluation scores. It's typically created by
provides a number of evaluation methods for evaluating [`Token`](/api/token) and
[`Doc`](/api/doc) attributes.
## Scorer.\_\_init\_\_ {#init tag="method"}
## Scorer.\_\_init\_\_ {id="init",tag="method"}
Create a new `Scorer`.
@ -35,7 +35,7 @@ Create a new `Scorer`.
| _keyword-only_ | |
| `\*\*kwargs` | Any additional settings to pass on to the individual scoring methods. ~~Any~~ |
## Scorer.score {#score tag="method"}
## Scorer.score {id="score",tag="method"}
Calculate the scores for a list of [`Example`](/api/example) objects using the
scoring methods provided by the components in the pipeline.
@ -72,11 +72,11 @@ core pipeline components, the individual score names start with the `Token` or
| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ |
| **RETURNS** | A dictionary of scores. ~~Dict[str, Union[float, Dict[str, float]]]~~ |
## Scorer.score_tokenization {#score_tokenization tag="staticmethod" new="3"}
## Scorer.score_tokenization {id="score_tokenization",tag="staticmethod",version="3"}
Scores the tokenization:
- `token_acc`: number of correct tokens / number of gold tokens
- `token_acc`: number of correct tokens / number of predicted tokens
- `token_p`, `token_r`, `token_f`: precision, recall and F-score for token
character spans
@ -93,7 +93,7 @@ Docs with `has_unknown_spaces` are skipped during scoring.
| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ |
| **RETURNS** | `Dict` | A dictionary containing the scores `token_acc`, `token_p`, `token_r`, `token_f`. ~~Dict[str, float]]~~ |
## Scorer.score_token_attr {#score_token_attr tag="staticmethod" new="3"}
## Scorer.score_token_attr {id="score_token_attr",tag="staticmethod",version="3"}
Scores a single token attribute. Tokens with missing values in the reference doc
are skipped during scoring.
@ -114,7 +114,7 @@ are skipped during scoring.
| `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ |
| **RETURNS** | A dictionary containing the score `{attr}_acc`. ~~Dict[str, float]~~ |
## Scorer.score_token_attr_per_feat {#score_token_attr_per_feat tag="staticmethod" new="3"}
## Scorer.score_token_attr_per_feat {id="score_token_attr_per_feat",tag="staticmethod",version="3"}
Scores a single token attribute per feature for a token attribute in the
Universal Dependencies
@ -138,7 +138,7 @@ scoring.
| `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ |
| **RETURNS** | A dictionary containing the micro PRF scores under the key `{attr}_micro_p/r/f` and the per-feature PRF scores under `{attr}_per_feat`. ~~Dict[str, Dict[str, float]]~~ |
## Scorer.score_spans {#score_spans tag="staticmethod" new="3"}
## Scorer.score_spans {id="score_spans",tag="staticmethod",version="3"}
Returns PRF scores for labeled or unlabeled spans.
@ -160,7 +160,7 @@ Returns PRF scores for labeled or unlabeled spans.
| `allow_overlap` | Defaults to `False`. Whether or not to allow overlapping spans. If set to `False`, the alignment will automatically resolve conflicts. ~~bool~~ |
| **RETURNS** | A dictionary containing the PRF scores under the keys `{attr}_p`, `{attr}_r`, `{attr}_f` and the per-type PRF scores under `{attr}_per_type`. ~~Dict[str, Union[float, Dict[str, float]]]~~ |
## Scorer.score_deps {#score_deps tag="staticmethod" new="3"}
## Scorer.score_deps {id="score_deps",tag="staticmethod",version="3"}
Calculate the UAS, LAS, and LAS per type scores for dependency parses. Tokens
with missing values for the `attr` (typically `dep`) are skipped during scoring.
@ -194,7 +194,7 @@ with missing values for the `attr` (typically `dep`) are skipped during scoring.
| `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ |
| **RETURNS** | A dictionary containing the scores: `{attr}_uas`, `{attr}_las`, and `{attr}_las_per_type`. ~~Dict[str, Union[float, Dict[str, float]]]~~ |
## Scorer.score_cats {#score_cats tag="staticmethod" new="3"}
## Scorer.score_cats {id="score_cats",tag="staticmethod",version="3"}
Calculate PRF and ROC AUC scores for a doc-level attribute that is a dict
containing scores for each label like `Doc.cats`. The returned dictionary
@ -241,7 +241,7 @@ The reported `{attr}_score` depends on the classification properties:
| `threshold` | Cutoff to consider a prediction "positive". Defaults to `0.5` for multi-label, and `0.0` (i.e. whatever's highest scoring) otherwise. ~~float~~ |
| **RETURNS** | A dictionary containing the scores, with inapplicable scores as `None`. ~~Dict[str, Optional[float]]~~ |
## Scorer.score_links {#score_links tag="staticmethod" new="3"}
## Scorer.score_links {id="score_links",tag="staticmethod",version="3"}
Returns PRF for predicted links on the entity level. To disentangle the
performance of the NEL from the NER, this method only evaluates NEL links for
@ -264,7 +264,7 @@ entities that overlap between the gold reference and the predictions.
| `negative_labels` | The string values that refer to no annotation (e.g. "NIL"). ~~Iterable[str]~~ |
| **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ |
## get_ner_prf {#get_ner_prf new="3"}
## get_ner_prf {id="get_ner_prf",version="3"}
Compute micro-PRF and per-entity PRF scores.
@ -272,7 +272,7 @@ Compute micro-PRF and per-entity PRF scores.
| ---------- | ------------------------------------------------------------------------------------------------------------------- |
| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ |
## score_coref_clusters {#score_coref_clusters tag="experimental"}
## score_coref_clusters {id="score_coref_clusters",tag="experimental"}
Returns LEA ([Moosavi and Strube, 2016](https://aclanthology.org/P16-1060/)) PRF
scores for coreference clusters.
@ -301,7 +301,7 @@ the [CoreferenceResolver](/api/coref) docs.
| `span_cluster_prefix` | The prefix used for spans representing coreference clusters. ~~str~~ |
| **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ |
## score_span_predictions {#score_span_predictions tag="experimental"}
## score_span_predictions {id="score_span_predictions",tag="experimental"}
Return accuracy for reconstructions of spans from single tokens. Only exactly
correct predictions are counted as correct, there is no partial credit for near

38
website/docs/api/sentencerecognizer.md → website/docs/api/sentencerecognizer.mdx
View File

@ -2,7 +2,7 @@
title: SentenceRecognizer
tag: class
source: spacy/pipeline/senter.pyx
new: 3
version: 3
teaser: 'Pipeline component for sentence segmentation'
api_base_class: /api/tagger
api_string_name: senter
@ -12,7 +12,7 @@ api_trainable: true
A trainable pipeline component for sentence segmentation. For a simpler,
rule-based strategy, see the [`Sentencizer`](/api/sentencizer).
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predicted values will be assigned to `Token.is_sent_start`. The resulting
sentences can be accessed using `Doc.sents`.
@ -22,7 +22,7 @@ sentences can be accessed using `Doc.sents`.
| `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. This will be either `True` or `False` for all tokens. ~~bool~~ |
| `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -50,7 +50,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/senter.pyx
```
## SentenceRecognizer.\_\_init\_\_ {#init tag="method"}
## SentenceRecognizer.\_\_init\_\_ {id="init",tag="method"}
Initialize the sentence recognizer.
@ -82,7 +82,7 @@ shortcut for this and instantiate the component using its string name and
| `overwrite` <Tag variant="new">3.2</Tag> | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ |
| `scorer` <Tag variant="new">3.2</Tag> | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"`. ~~Optional[Callable]~~ |
## SentenceRecognizer.\_\_call\_\_ {#call tag="method"}
## SentenceRecognizer.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -106,7 +106,7 @@ and all pipeline components are applied to the `Doc` in order. Both
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## SentenceRecognizer.pipe {#pipe tag="method"}
## SentenceRecognizer.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -130,7 +130,7 @@ and [`pipe`](/api/sentencerecognizer#pipe) delegate to the
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## SentenceRecognizer.initialize {#initialize tag="method"}
## SentenceRecognizer.initialize {id="initialize",tag="method"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -154,7 +154,7 @@ by [`Language.initialize`](/api/language#initialize).
| _keyword-only_ | |
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
## SentenceRecognizer.predict {#predict tag="method"}
## SentenceRecognizer.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them.
@ -171,7 +171,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The model's prediction for each document. |
## SentenceRecognizer.set_annotations {#set_annotations tag="method"}
## SentenceRecognizer.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
@ -188,7 +188,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `SentenceRecognizer.predict`. |
## SentenceRecognizer.update {#update tag="method"}
## SentenceRecognizer.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects containing the
predictions and gold-standard annotations, and update the component's model.
@ -212,7 +212,7 @@ Delegates to [`predict`](/api/sentencerecognizer#predict) and
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## SentenceRecognizer.rehearse {#rehearse tag="method,experimental" new="3"}
## SentenceRecognizer.rehearse {id="rehearse",tag="method,experimental",version="3"}
Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the
current model to make predictions similar to an initial model to try to address
@ -235,7 +235,7 @@ the "catastrophic forgetting" problem. This feature is experimental.
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## SentenceRecognizer.get_loss {#get_loss tag="method"}
## SentenceRecognizer.get_loss {id="get_loss",tag="method"}
Find the loss and gradient of loss for the batch of documents and their
predicted scores.
@ -254,7 +254,7 @@ predicted scores.
| `scores` | Scores representing the model's predictions. |
| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ |
## SentenceRecognizer.create_optimizer {#create_optimizer tag="method"}
## SentenceRecognizer.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -269,7 +269,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## SentenceRecognizer.use_params {#use_params tag="method, contextmanager"}
## SentenceRecognizer.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -286,7 +286,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## SentenceRecognizer.to_disk {#to_disk tag="method"}
## SentenceRecognizer.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -303,7 +303,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## SentenceRecognizer.from_disk {#from_disk tag="method"}
## SentenceRecognizer.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -321,7 +321,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `SentenceRecognizer` object. ~~SentenceRecognizer~~ |
## SentenceRecognizer.to_bytes {#to_bytes tag="method"}
## SentenceRecognizer.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -338,7 +338,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `SentenceRecognizer` object. ~~bytes~~ |
## SentenceRecognizer.from_bytes {#from_bytes tag="method"}
## SentenceRecognizer.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -357,7 +357,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `SentenceRecognizer` object. ~~SentenceRecognizer~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

23
website/docs/api/sentencizer.md → website/docs/api/sentencizer.mdx
View File

@ -13,7 +13,7 @@ performed by the [`DependencyParser`](/api/dependencyparser), so the
`Sentencizer` lets you implement a simpler, rule-based strategy that doesn't
require a statistical model to be loaded.
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Calculated values will be assigned to `Token.is_sent_start`. The resulting
sentences can be accessed using `Doc.sents`.
@ -23,7 +23,7 @@ sentences can be accessed using `Doc.sents`.
| `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. This will be either `True` or `False` for all tokens. ~~bool~~ |
| `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -39,7 +39,7 @@ how the component should be configured. You can override its settings via the
| Setting | Description |
| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `punct_chars` | Optional custom list of punctuation characters that mark sentence ends. See below for defaults if not set. Defaults to `None`. ~~Optional[List[str]]~~ | `None` |
| `punct_chars` | Optional custom list of punctuation characters that mark sentence ends. See below for defaults if not set. Defaults to `None`. ~~Optional[List[str]]~~ |
| `overwrite` <Tag variant="new">3.2</Tag> | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ |
| `scorer` <Tag variant="new">3.2</Tag> | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"` ~~Optional[Callable]~~ |
@ -47,7 +47,7 @@ how the component should be configured. You can override its settings via the
%%GITHUB_SPACY/spacy/pipeline/sentencizer.pyx
```
## Sentencizer.\_\_init\_\_ {#init tag="method"}
## Sentencizer.\_\_init\_\_ {id="init",tag="method"}
Initialize the sentencizer.
@ -69,8 +69,7 @@ Initialize the sentencizer.
| `overwrite` <Tag variant="new">3.2</Tag> | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ |
| `scorer` <Tag variant="new">3.2</Tag> | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"` ~~Optional[Callable]~~ |
```python
### punct_chars defaults
```python {title="punct_chars defaults"}
['!', '.', '?', '։', '؟', '۔', '܀', '܁', '܂', '߹', '।', '॥', '၊', '။', '።',
'፧', '፨', '᙮', '᜵', '᜶', '᠃', '᠉', '᥄', '᥅', '᪨', '᪩', '᪪', '᪫',
'᭚', '᭛', '᭞', '᭟', '᰻', '᰼', '᱾', '᱿', '‼', '‽', '⁇', '⁈', '⁉',
@ -83,7 +82,7 @@ Initialize the sentencizer.
'𑪜', '𑱁', '𑱂', '𖩮', '𖩯', '𖫵', '𖬷', '𖬸', '𖭄', '𛲟', '𝪈', '｡', '。']
```
## Sentencizer.\_\_call\_\_ {#call tag="method"}
## Sentencizer.\_\_call\_\_ {id="call",tag="method"}
Apply the sentencizer on a `Doc`. Typically, this happens automatically after
the component has been added to the pipeline using
@ -105,7 +104,7 @@ the component has been added to the pipeline using
| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ |
| **RETURNS** | The modified `Doc` with added sentence boundaries. ~~Doc~~ |
## Sentencizer.pipe {#pipe tag="method"}
## Sentencizer.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -126,7 +125,7 @@ applied to the `Doc` in order.
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## Sentencizer.to_disk {#to_disk tag="method"}
## Sentencizer.to_disk {id="to_disk",tag="method"}
Save the sentencizer settings (punctuation characters) to a directory. Will
create a file `sentencizer.json`. This also happens automatically when you save
@ -144,7 +143,7 @@ an `nlp` object with a sentencizer added to its pipeline.
| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ |
| `path` | A path to a JSON file, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
## Sentencizer.from_disk {#from_disk tag="method"}
## Sentencizer.from_disk {id="from_disk",tag="method"}
Load the sentencizer settings from a file. Expects a JSON file. This also
happens automatically when you load an `nlp` object or model with a sentencizer
@ -162,7 +161,7 @@ added to its pipeline.
| `path` | A path to a JSON file. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
| **RETURNS** | The modified `Sentencizer` object. ~~Sentencizer~~ |
## Sentencizer.to_bytes {#to_bytes tag="method"}
## Sentencizer.to_bytes {id="to_bytes",tag="method"}
Serialize the sentencizer settings to a bytestring.
@ -178,7 +177,7 @@ Serialize the sentencizer settings to a bytestring.
| ----------- | ------------------------------ |
| **RETURNS** | The serialized data. ~~bytes~~ |
## Sentencizer.from_bytes {#from_bytes tag="method"}
## Sentencizer.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.

32
website/docs/api/span-resolver.md → website/docs/api/span-resolver.mdx
View File

@ -33,7 +33,7 @@ use case is as a post-processing step on word-level
[coreference resolution](/api/coref). The input and output keys used to store
`Span` objects are configurable.
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predictions will be saved to `Doc.spans` as [`SpanGroup`s](/api/spangroup).
@ -46,7 +46,7 @@ prefixes are configurable.
| ------------------------------------------------- | ------------------------------------------------------------------------- |
| `Doc.spans[output_prefix + "_" + cluster_number]` | One group of predicted spans. Cluster number starts from 1. ~~SpanGroup~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -74,7 +74,7 @@ details on the architectures and their arguments and hyperparameters.
| `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ |
| `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ |
## SpanResolver.\_\_init\_\_ {#init tag="method"}
## SpanResolver.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -104,7 +104,7 @@ shortcut for this and instantiate the component using its string name and
| `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ |
| `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ |
## SpanResolver.\_\_call\_\_ {#call tag="method"}
## SpanResolver.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -126,7 +126,7 @@ and [`set_annotations`](#set_annotations) methods.
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## SpanResolver.pipe {#pipe tag="method"}
## SpanResolver.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -150,7 +150,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/span-resolver#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## SpanResolver.initialize {#initialize tag="method"}
## SpanResolver.initialize {id="initialize",tag="method"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -174,7 +174,7 @@ by [`Language.initialize`](/api/language#initialize).
| _keyword-only_ | |
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
## SpanResolver.predict {#predict tag="method"}
## SpanResolver.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them. Predictions are returned as a list of `MentionClusters`, one for
@ -194,7 +194,7 @@ correspond to token indices.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The predicted spans for the `Doc`s. ~~List[MentionClusters]~~ |
## SpanResolver.set_annotations {#set_annotations tag="method"}
## SpanResolver.set_annotations {id="set_annotations",tag="method"}
Modify a batch of documents, saving predictions using the output prefix in
`Doc.spans`.
@ -212,7 +212,7 @@ Modify a batch of documents, saving predictions using the output prefix in
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `spans` | The predicted spans for the `docs`. ~~List[MentionClusters]~~ |
## SpanResolver.update {#update tag="method"}
## SpanResolver.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects. Delegates to
[`predict`](/api/span-resolver#predict).
@ -234,7 +234,7 @@ Learn from a batch of [`Example`](/api/example) objects. Delegates to
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## SpanResolver.create_optimizer {#create_optimizer tag="method"}
## SpanResolver.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -249,7 +249,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## SpanResolver.use_params {#use_params tag="method, contextmanager"}
## SpanResolver.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -266,7 +266,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## SpanResolver.to_disk {#to_disk tag="method"}
## SpanResolver.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -283,7 +283,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## SpanResolver.from_disk {#from_disk tag="method"}
## SpanResolver.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -301,7 +301,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `SpanResolver` object. ~~SpanResolver~~ |
## SpanResolver.to_bytes {#to_bytes tag="method"}
## SpanResolver.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -318,7 +318,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `SpanResolver` object. ~~bytes~~ |
## SpanResolver.from_bytes {#from_bytes tag="method"}
## SpanResolver.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -337,7 +337,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `SpanResolver` object. ~~SpanResolver~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

56
website/docs/api/span.md → website/docs/api/span.mdx
View File

@ -6,7 +6,7 @@ source: spacy/tokens/span.pyx
A slice from a [`Doc`](/api/doc) object.
## Span.\_\_init\_\_ {#init tag="method"}
## Span.\_\_init\_\_ {id="init",tag="method"}
Create a `Span` object from the slice `doc[start : end]`.
@ -29,7 +29,7 @@ Create a `Span` object from the slice `doc[start : end]`.
| `kb_id` | A knowledge base ID to attach to the span, e.g. for named entities. ~~Union[str, int]~~ |
| `span_id` | An ID to associate with the span. ~~Union[str, int]~~ |
## Span.\_\_getitem\_\_ {#getitem tag="method"}
## Span.\_\_getitem\_\_ {id="getitem",tag="method"}
Get a `Token` object.
@ -61,7 +61,7 @@ Get a `Span` object.
| `start_end` | The slice of the span to get. ~~Tuple[int, int]~~ |
| **RETURNS** | The span at `span[start : end]`. ~~Span~~ |
## Span.\_\_iter\_\_ {#iter tag="method"}
## Span.\_\_iter\_\_ {id="iter",tag="method"}
Iterate over `Token` objects.
@ -77,7 +77,7 @@ Iterate over `Token` objects.
| ---------- | --------------------------- |
| **YIELDS** | A `Token` object. ~~Token~~ |
## Span.\_\_len\_\_ {#len tag="method"}
## Span.\_\_len\_\_ {id="len",tag="method"}
Get the number of tokens in the span.
@ -93,7 +93,7 @@ Get the number of tokens in the span.
| ----------- | ----------------------------------------- |
| **RETURNS** | The number of tokens in the span. ~~int~~ |
## Span.set_extension {#set_extension tag="classmethod" new="2"}
## Span.set_extension {id="set_extension",tag="classmethod",version="2"}
Define a custom attribute on the `Span` which becomes available via `Span._`.
For details, see the documentation on
@ -118,7 +118,7 @@ For details, see the documentation on
| `setter` | Setter function that takes the `Span` and a value, and modifies the object. Is called when the user writes to the `Span._` attribute. ~~Optional[Callable[[Span, Any], None]]~~ |
| `force` | Force overwriting existing attribute. ~~bool~~ |
## Span.get_extension {#get_extension tag="classmethod" new="2"}
## Span.get_extension {id="get_extension",tag="classmethod",version="2"}
Look up a previously registered extension by name. Returns a 4-tuple
`(default, method, getter, setter)` if the extension is registered. Raises a
@ -138,7 +138,7 @@ Look up a previously registered extension by name. Returns a 4-tuple
| `name` | Name of the extension. ~~str~~ |
| **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ |
## Span.has_extension {#has_extension tag="classmethod" new="2"}
## Span.has_extension {id="has_extension",tag="classmethod",version="2"}
Check whether an extension has been registered on the `Span` class.
@ -155,7 +155,7 @@ Check whether an extension has been registered on the `Span` class.
| `name` | Name of the extension to check. ~~str~~ |
| **RETURNS** | Whether the extension has been registered. ~~bool~~ |
## Span.remove_extension {#remove_extension tag="classmethod" new="2.0.12"}
## Span.remove_extension {id="remove_extension",tag="classmethod",version="2.0.12"}
Remove a previously registered extension.
@ -173,7 +173,7 @@ Remove a previously registered extension.
| `name` | Name of the extension. ~~str~~ |
| **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ |
## Span.char_span {#char_span tag="method" new="2.2.4"}
## Span.char_span {id="char_span",tag="method",version="2.2.4"}
Create a `Span` object from the slice `span.text[start:end]`. Returns `None` if
the character indices don't map to a valid span.
@ -195,7 +195,7 @@ the character indices don't map to a valid span.
| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ |
## Span.similarity {#similarity tag="method" model="vectors"}
## Span.similarity {id="similarity",tag="method",model="vectors"}
Make a semantic similarity estimate. The default estimate is cosine similarity
using an average of word vectors.
@ -216,7 +216,7 @@ using an average of word vectors.
| `other` | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ |
| **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ |
## Span.get_lca_matrix {#get_lca_matrix tag="method"}
## Span.get_lca_matrix {id="get_lca_matrix",tag="method"}
Calculates the lowest common ancestor matrix for a given `Span`. Returns LCA
matrix containing the integer index of the ancestor, or `-1` if no common
@ -235,7 +235,7 @@ ancestor is found, e.g. if span excludes a necessary ancestor.
| ----------- | --------------------------------------------------------------------------------------- |
| **RETURNS** | The lowest common ancestor matrix of the `Span`. ~~numpy.ndarray[ndim=2, dtype=int32]~~ |
## Span.to_array {#to_array tag="method" new="2"}
## Span.to_array {id="to_array",tag="method",version="2"}
Given a list of `M` attribute IDs, export the tokens to a numpy `ndarray` of
shape `(N, M)`, where `N` is the length of the document. The values will be
@ -256,7 +256,7 @@ shape `(N, M)`, where `N` is the length of the document. The values will be
| `attr_ids` | A list of attributes (int IDs or string names) or a single attribute (int ID or string name). ~~Union[int, str, List[Union[int, str]]]~~ |
| **RETURNS** | The exported attributes as a numpy array. ~~Union[numpy.ndarray[ndim=2, dtype=uint64], numpy.ndarray[ndim=1, dtype=uint64]]~~ |
## Span.ents {#ents tag="property" new="2.0.13" model="ner"}
## Span.ents {id="ents",tag="property",version="2.0.13",model="ner"}
The named entities that fall completely within the span. Returns a tuple of
`Span` objects.
@ -276,7 +276,7 @@ The named entities that fall completely within the span. Returns a tuple of
| ----------- | ----------------------------------------------------------------- |
| **RETURNS** | Entities in the span, one `Span` per entity. ~~Tuple[Span, ...]~~ |
## Span.noun_chunks {#noun_chunks tag="property" model="parser"}
## Span.noun_chunks {id="noun_chunks",tag="property",model="parser"}
Iterate over the base noun phrases in the span. Yields base noun-phrase `Span`
objects, if the document has been syntactically parsed. A base noun phrase, or
@ -302,7 +302,7 @@ raised.
| ---------- | --------------------------------- |
| **YIELDS** | Noun chunks in the span. ~~Span~~ |
## Span.as_doc {#as_doc tag="method"}
## Span.as_doc {id="as_doc",tag="method"}
Create a new `Doc` object corresponding to the `Span`, with a copy of the data.
@ -326,7 +326,7 @@ time.
| `array` | Precomputed array version of the original doc as generated by [`Doc.to_array`](/api/doc#to_array). ~~numpy.ndarray~~ |
| **RETURNS** | A `Doc` object of the `Span`'s content. ~~Doc~~ |
## Span.root {#root tag="property" model="parser"}
## Span.root {id="root",tag="property",model="parser"}
The token with the shortest path to the root of the sentence (or the root
itself). If multiple tokens are equally high in the tree, the first token is
@ -347,7 +347,7 @@ taken.
| ----------- | ------------------------- |
| **RETURNS** | The root token. ~~Token~~ |
## Span.conjuncts {#conjuncts tag="property" model="parser"}
## Span.conjuncts {id="conjuncts",tag="property",model="parser"}
A tuple of tokens coordinated to `span.root`.
@ -363,7 +363,7 @@ A tuple of tokens coordinated to `span.root`.
| ----------- | --------------------------------------------- |
| **RETURNS** | The coordinated tokens. ~~Tuple[Token, ...]~~ |
## Span.lefts {#lefts tag="property" model="parser"}
## Span.lefts {id="lefts",tag="property",model="parser"}
Tokens that are to the left of the span, whose heads are within the span.
@ -379,7 +379,7 @@ Tokens that are to the left of the span, whose heads are within the span.
| ---------- | ---------------------------------------------- |
| **YIELDS** | A left-child of a token of the span. ~~Token~~ |
## Span.rights {#rights tag="property" model="parser"}
## Span.rights {id="rights",tag="property",model="parser"}
Tokens that are to the right of the span, whose heads are within the span.
@ -395,7 +395,7 @@ Tokens that are to the right of the span, whose heads are within the span.
| ---------- | ----------------------------------------------- |
| **YIELDS** | A right-child of a token of the span. ~~Token~~ |
## Span.n_lefts {#n_lefts tag="property" model="parser"}
## Span.n_lefts {id="n_lefts",tag="property",model="parser"}
The number of tokens that are to the left of the span, whose heads are within
the span.
@ -411,7 +411,7 @@ the span.
| ----------- | ---------------------------------------- |
| **RETURNS** | The number of left-child tokens. ~~int~~ |
## Span.n_rights {#n_rights tag="property" model="parser"}
## Span.n_rights {id="n_rights",tag="property",model="parser"}
The number of tokens that are to the right of the span, whose heads are within
the span.
@ -427,7 +427,7 @@ the span.
| ----------- | ----------------------------------------- |
| **RETURNS** | The number of right-child tokens. ~~int~~ |
## Span.subtree {#subtree tag="property" model="parser"}
## Span.subtree {id="subtree",tag="property",model="parser"}
Tokens within the span and tokens which descend from them.
@ -443,7 +443,7 @@ Tokens within the span and tokens which descend from them.
| ---------- | ----------------------------------------------------------- |
| **YIELDS** | A token within the span, or a descendant from it. ~~Token~~ |
## Span.has_vector {#has_vector tag="property" model="vectors"}
## Span.has_vector {id="has_vector",tag="property",model="vectors"}
A boolean value indicating whether a word vector is associated with the object.
@ -458,7 +458,7 @@ A boolean value indicating whether a word vector is associated with the object.
| ----------- | ----------------------------------------------------- |
| **RETURNS** | Whether the span has a vector data attached. ~~bool~~ |
## Span.vector {#vector tag="property" model="vectors"}
## Span.vector {id="vector",tag="property",model="vectors"}
A real-valued meaning representation. Defaults to an average of the token
vectors.
@ -475,7 +475,7 @@ vectors.
| ----------- | ----------------------------------------------------------------------------------------------- |
| **RETURNS** | A 1-dimensional array representing the span's vector. ~~`numpy.ndarray[ndim=1, dtype=float32]~~ |
## Span.vector_norm {#vector_norm tag="property" model="vectors"}
## Span.vector_norm {id="vector_norm",tag="property",model="vectors"}
The L2 norm of the span's vector representation.
@ -492,7 +492,7 @@ The L2 norm of the span's vector representation.
| ----------- | --------------------------------------------------- |
| **RETURNS** | The L2 norm of the vector representation. ~~float~~ |
## Span.sent {#sent tag="property" model="sentences"}
## Span.sent {id="sent",tag="property",model="sentences"}
The sentence span that this span is a part of. This property is only available
when [sentence boundaries](/usage/linguistic-features#sbd) have been set on the
@ -520,7 +520,7 @@ sent = doc[sent.start : max(sent.end, span.end)]
| ----------- | ------------------------------------------------------- |
| **RETURNS** | The sentence span that this span is a part of. ~~Span~~ |
## Span.sents {#sents tag="property" model="sentences" new="3.2.1"}
## Span.sents {id="sents",tag="property",model="sentences",version="3.2.1"}
Returns a generator over the sentences the span belongs to. This property is
only available when [sentence boundaries](/usage/linguistic-features#sbd) have
@ -542,7 +542,7 @@ overlaps with will be returned.
| ----------- | -------------------------------------------------------------------------- |
| **RETURNS** | A generator yielding sentences this `Span` is a part of ~~Iterable[Span]~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| -------------- | ----------------------------------------------------------------------------------------------------------------------------- |

50
website/docs/api/spancategorizer.md → website/docs/api/spancategorizer.mdx
View File

@ -2,7 +2,7 @@
title: SpanCategorizer
tag: class,experimental
source: spacy/pipeline/spancat.py
new: 3.1
version: 3.1
teaser: 'Pipeline component for labeling potentially overlapping spans of text'
api_base_class: /api/pipe
api_string_name: spancat
@ -16,7 +16,7 @@ that predicts zero or more labels for each candidate.
Predicted spans will be saved in a [`SpanGroup`](/api/spangroup) on the doc.
Individual span scores can be found in `spangroup.attrs["scores"]`.
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predictions will be saved to `Doc.spans[spans_key]` as a
[`SpanGroup`](/api/spangroup). The scores for the spans in the `SpanGroup` will
@ -29,7 +29,7 @@ be saved in `SpanGroup.attrs["scores"]`.
| `Doc.spans[spans_key]` | The annotated spans. ~~SpanGroup~~ |
| `Doc.spans[spans_key].attrs["scores"]` | The score for each span in the `SpanGroup`. ~~Floats1d~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -66,7 +66,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/spancat.py
```
## SpanCategorizer.\_\_init\_\_ {#init tag="method"}
## SpanCategorizer.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -98,7 +98,7 @@ shortcut for this and instantiate the component using its string name and
| `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to `0.5`. ~~float~~ |
| `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. ~~Optional[int]~~ |
## SpanCategorizer.\_\_call\_\_ {#call tag="method"}
## SpanCategorizer.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -121,7 +121,7 @@ delegate to the [`predict`](/api/spancategorizer#predict) and
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## SpanCategorizer.pipe {#pipe tag="method"}
## SpanCategorizer.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -145,7 +145,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/spancategorizer#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## SpanCategorizer.initialize {#initialize tag="method"}
## SpanCategorizer.initialize {id="initialize",tag="method"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -182,7 +182,7 @@ config.
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ |
## SpanCategorizer.predict {#predict tag="method"}
## SpanCategorizer.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects without
modifying them.
@ -199,7 +199,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The model's prediction for each document. |
## SpanCategorizer.set_annotations {#set_annotations tag="method"}
## SpanCategorizer.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores.
@ -216,7 +216,7 @@ Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `SpanCategorizer.predict`. |
## SpanCategorizer.update {#update tag="method"}
## SpanCategorizer.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects containing the
predictions and gold-standard annotations, and update the component's model.
@ -240,7 +240,7 @@ Delegates to [`predict`](/api/spancategorizer#predict) and
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## SpanCategorizer.set_candidates {#set_candidates tag="method", new="3.3"}
## SpanCategorizer.set_candidates {id="set_candidates",tag="method", version="3.3"}
Use the suggester to add a list of [`Span`](/api/span) candidates to a list of
[`Doc`](/api/doc) objects. This method is intended to be used for debugging
@ -258,7 +258,7 @@ purposes.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `candidates_key` | Key of the Doc.spans dict to save the candidate spans under. ~~str~~ |
## SpanCategorizer.get_loss {#get_loss tag="method"}
## SpanCategorizer.get_loss {id="get_loss",tag="method"}
Find the loss and gradient of loss for the batch of documents and their
predicted scores.
@ -277,7 +277,7 @@ predicted scores.
| `spans_scores` | Scores representing the model's predictions. ~~Tuple[Ragged, Floats2d]~~ |
| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ |
## SpanCategorizer.create_optimizer {#create_optimizer tag="method"}
## SpanCategorizer.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -292,7 +292,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## SpanCategorizer.use_params {#use_params tag="method, contextmanager"}
## SpanCategorizer.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model to use the given parameter values.
@ -308,7 +308,7 @@ Modify the pipe's model to use the given parameter values.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## SpanCategorizer.add_label {#add_label tag="method"}
## SpanCategorizer.add_label {id="add_label",tag="method"}
Add a new label to the pipe. Raises an error if the output dimension is already
set, or if the model has already been fully [initialized](#initialize). Note
@ -330,7 +330,7 @@ automatically.
| `label` | The label to add. ~~str~~ |
| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ |
## SpanCategorizer.to_disk {#to_disk tag="method"}
## SpanCategorizer.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -347,7 +347,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## SpanCategorizer.from_disk {#from_disk tag="method"}
## SpanCategorizer.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -365,7 +365,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `SpanCategorizer` object. ~~SpanCategorizer~~ |
## SpanCategorizer.to_bytes {#to_bytes tag="method"}
## SpanCategorizer.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -382,7 +382,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `SpanCategorizer` object. ~~bytes~~ |
## SpanCategorizer.from_bytes {#from_bytes tag="method"}
## SpanCategorizer.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -401,7 +401,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `SpanCategorizer` object. ~~SpanCategorizer~~ |
## SpanCategorizer.labels {#labels tag="property"}
## SpanCategorizer.labels {id="labels",tag="property"}
The labels currently added to the component.
@ -416,7 +416,7 @@ The labels currently added to the component.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ |
## SpanCategorizer.label_data {#label_data tag="property"}
## SpanCategorizer.label_data {id="label_data",tag="property"}
The labels currently added to the component and their internal meta information.
This is the data generated by [`init labels`](/api/cli#init-labels) and used by
@ -434,7 +434,7 @@ the model with a pre-defined label set.
| ----------- | ---------------------------------------------------------- |
| **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from
@ -452,9 +452,9 @@ serialization by passing in the string names via the `exclude` argument.
| `cfg` | The config file. You usually don't want to exclude this. |
| `model` | The binary model data. You usually don't want to exclude this. |
## Suggesters {#suggesters tag="registered functions" source="spacy/pipeline/spancat.py"}
## Suggesters {id="suggesters",tag="registered functions",source="spacy/pipeline/spancat.py"}
### spacy.ngram_suggester.v1 {#ngram_suggester}
### spacy.ngram_suggester.v1 {id="ngram_suggester"}
> #### Example Config
>
@ -472,7 +472,7 @@ integers. The array has two columns, indicating the start and end position.
| `sizes` | The phrase lengths to suggest. For example, `[1, 2]` will suggest phrases consisting of 1 or 2 tokens. ~~List[int]~~ |
| **CREATES** | The suggester function. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ |
### spacy.ngram_range_suggester.v1 {#ngram_range_suggester}
### spacy.ngram_range_suggester.v1 {id="ngram_range_suggester"}
> #### Example Config
>

33
website/docs/api/spangroup.md → website/docs/api/spangroup.mdx
View File

@ -2,7 +2,7 @@
title: SpanGroup
tag: class
source: spacy/tokens/span_group.pyx
new: 3
version: 3
---
A group of arbitrary, potentially overlapping [`Span`](/api/span) objects that
@ -13,7 +13,7 @@ into a `SpanGroup` object for you automatically on assignment. `SpanGroup`
objects behave similar to `list`s, so you can append `Span` objects to them or
access a member at a given index.
## SpanGroup.\_\_init\_\_ {#init tag="method"}
## SpanGroup.\_\_init\_\_ {id="init",tag="method"}
Create a `SpanGroup`.
@ -42,7 +42,7 @@ Create a `SpanGroup`.
| `attrs` | Optional JSON-serializable attributes to attach to the span group. ~~Dict[str, Any]~~ |
| `spans` | The spans to add to the span group. ~~Iterable[Span]~~ |
## SpanGroup.doc {#doc tag="property"}
## SpanGroup.doc {id="doc",tag="property"}
The [`Doc`](/api/doc) object the span group is referring to.
@ -68,7 +68,7 @@ the scope of your function.
| ----------- | ------------------------------- |
| **RETURNS** | The reference document. ~~Doc~~ |
## SpanGroup.has_overlap {#has_overlap tag="property"}
## SpanGroup.has_overlap {id="has_overlap",tag="property"}
Check whether the span group contains overlapping spans.
@ -86,7 +86,7 @@ Check whether the span group contains overlapping spans.
| ----------- | -------------------------------------------------- |
| **RETURNS** | Whether the span group contains overlaps. ~~bool~~ |
## SpanGroup.\_\_len\_\_ {#len tag="method"}
## SpanGroup.\_\_len\_\_ {id="len",tag="method"}
Get the number of spans in the group.
@ -102,7 +102,7 @@ Get the number of spans in the group.
| ----------- | ----------------------------------------- |
| **RETURNS** | The number of spans in the group. ~~int~~ |
## SpanGroup.\_\_getitem\_\_ {#getitem tag="method"}
## SpanGroup.\_\_getitem\_\_ {id="getitem",tag="method"}
Get a span from the group. Note that a copy of the span is returned, so if any
changes are made to this span, they are not reflected in the corresponding
@ -125,7 +125,7 @@ changes to be reflected in the span group.
| `i` | The item index. ~~int~~ |
| **RETURNS** | The span at the given index. ~~Span~~ |
## SpanGroup.\_\_setitem\_\_ {#setitem tag="method", new="3.3"}
## SpanGroup.\_\_setitem\_\_ {id="setitem",tag="method", version="3.3"}
Set a span in the span group.
@ -144,7 +144,7 @@ Set a span in the span group.
| `i` | The item index. ~~int~~ |
| `span` | The new value. ~~Span~~ |
## SpanGroup.\_\_delitem\_\_ {#delitem tag="method", new="3.3"}
## SpanGroup.\_\_delitem\_\_ {id="delitem",tag="method", version="3.3"}
Delete a span from the span group.
@ -161,7 +161,7 @@ Delete a span from the span group.
| ---- | ----------------------- |
| `i` | The item index. ~~int~~ |
## SpanGroup.\_\_add\_\_ {#add tag="method", new="3.3"}
## SpanGroup.\_\_add\_\_ {id="add",tag="method", version="3.3"}
Concatenate the current span group with another span group and return the result
in a new span group. Any `attrs` from the first span group will have precedence
@ -182,7 +182,7 @@ over `attrs` in the second.
| `other` | The span group or spans to concatenate. ~~Union[SpanGroup, Iterable[Span]]~~ |
| **RETURNS** | The new span group. ~~SpanGroup~~ |
## SpanGroup.\_\_iadd\_\_ {#iadd tag="method", new="3.3"}
## SpanGroup.\_\_iadd\_\_ {id="iadd",tag="method", version="3.3"}
Append an iterable of spans or the content of a span group to the current span
group. Any `attrs` in the other span group will be added for keys that are not
@ -202,7 +202,7 @@ already present in the current span group.
| `other` | The span group or spans to append. ~~Union[SpanGroup, Iterable[Span]]~~ |
| **RETURNS** | The span group. ~~SpanGroup~~ |
## SpanGroup.\_\_iter\_\_ {#iter tag="method" new="3.5"}
## SpanGroup.\_\_iter\_\_ {id="iter",tag="method",version="3.5"}
Iterate over the spans in this span group.
@ -219,7 +219,8 @@ Iterate over the spans in this span group.
| ---------- | ----------------------------------- |
| **YIELDS** | A span in this span group. ~~Span~~ |
## SpanGroup.append {#append tag="method"}
## SpanGroup.append {id="append",tag="method"}
Add a [`Span`](/api/span) object to the group. The span must refer to the same
[`Doc`](/api/doc) object as the span group.
@ -237,7 +238,7 @@ Add a [`Span`](/api/span) object to the group. The span must refer to the same
| ------ | ---------------------------- |
| `span` | The span to append. ~~Span~~ |
## SpanGroup.extend {#extend tag="method"}
## SpanGroup.extend {id="extend",tag="method"}
Add multiple [`Span`](/api/span) objects or contents of another `SpanGroup` to
the group. All spans must refer to the same [`Doc`](/api/doc) object as the span
@ -258,7 +259,7 @@ group.
| ------- | -------------------------------------------------------- |
| `spans` | The spans to add. ~~Union[SpanGroup, Iterable["Span"]]~~ |
## SpanGroup.copy {#copy tag="method", new="3.3"}
## SpanGroup.copy {id="copy",tag="method", version="3.3"}
Return a copy of the span group.
@ -277,7 +278,7 @@ Return a copy of the span group.
| `doc` | The document to which the copy is bound. Defaults to `None` for the current doc. ~~Optional[Doc]~~ |
| **RETURNS** | A copy of the `SpanGroup` object. ~~SpanGroup~~ |
## SpanGroup.to_bytes {#to_bytes tag="method"}
## SpanGroup.to_bytes {id="to_bytes",tag="method"}
Serialize the span group to a bytestring.
@ -293,7 +294,7 @@ Serialize the span group to a bytestring.
| ----------- | ------------------------------------- |
| **RETURNS** | The serialized `SpanGroup`. ~~bytes~~ |
## SpanGroup.from_bytes {#from_bytes tag="method"}
## SpanGroup.from_bytes {id="from_bytes",tag="method"}
Load the span group from a bytestring. Modifies the object in place and returns
it.

88
website/docs/api/spanruler.md → website/docs/api/spanruler.mdx
View File

@ -2,7 +2,7 @@
title: SpanRuler
tag: class
source: spacy/pipeline/span_ruler.py
new: 3.3
version: 3.3
teaser: 'Pipeline component for rule-based span and named entity recognition'
api_string_name: span_ruler
api_trainable: false
@ -24,7 +24,7 @@ component.
</Infobox>
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Matches will be saved to `Doc.spans[spans_key]` as a
[`SpanGroup`](/api/spangroup) and/or to `Doc.ents`, where the annotation is
@ -39,7 +39,7 @@ saved in the `Token.ent_type` and `Token.ent_iob` fields.
| `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ |
| `Token.ent_type_` | The label part of the named entity tag. ~~str~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -57,22 +57,23 @@ how the component should be configured. You can override its settings via the
> nlp.add_pipe("span_ruler", config=config)
> ```
| Setting | Description |
| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ |
| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ |
| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ |
| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ |
| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ |
| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ |
| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ |
| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ |
| Setting | Description |
| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ |
| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ |
| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ |
| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ |
| `phrase_matcher_attr` | Token attribute to match on, passed to the internal `PhraseMatcher` as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ |
| `matcher_fuzzy_compare` <Tag variant="new">3.5</Tag> | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ |
| `validate` | Whether patterns should be validated, passed to `Matcher` and `PhraseMatcher` as `validate`. Defaults to `False`. ~~bool~~ |
| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ |
| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ |
```python
%%GITHUB_SPACY/spacy/pipeline/span_ruler.py
```
## SpanRuler.\_\_init\_\_ {#init tag="method"}
## SpanRuler.\_\_init\_\_ {id="init",tag="method"}
Initialize the span ruler. If patterns are supplied here, they need to be a list
of dictionaries with a `"label"` and `"pattern"` key. A pattern can either be a
@ -90,21 +91,22 @@ token pattern (list) or a phrase pattern (string). For example:
> ruler = SpanRuler(nlp, overwrite=True)
> ```
| Name | Description |
| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ |
| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ |
| _keyword-only_ | |
| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ |
| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ |
| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ |
| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ |
| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ |
| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ |
| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ |
| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ |
| Name | Description |
| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ |
| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ |
| _keyword-only_ | |
| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ |
| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ |
| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ |
| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ |
| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ |
| `matcher_fuzzy_compare` <Tag variant="new">3.5</Tag> | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ |
| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ |
| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ |
| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ |
## SpanRuler.initialize {#initialize tag="method"}
## SpanRuler.initialize {id="initialize",tag="method"}
Initialize the component with data and used before training to load in rules
from a [pattern file](/usage/rule-based-matching/#spanruler-files). This method
@ -136,7 +138,7 @@ config. Any existing patterns are removed on initialization.
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
| `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ |
## SpanRuler.\_\len\_\_ {#len tag="method"}
## SpanRuler.\_\_len\_\_ {id="len",tag="method"}
The number of all patterns added to the span ruler.
@ -153,7 +155,7 @@ The number of all patterns added to the span ruler.
| ----------- | ------------------------------- |
| **RETURNS** | The number of patterns. ~~int~~ |
## SpanRuler.\_\_contains\_\_ {#contains tag="method"}
## SpanRuler.\_\_contains\_\_ {id="contains",tag="method"}
Whether a label is present in the patterns.
@ -171,7 +173,7 @@ Whether a label is present in the patterns.
| `label` | The label to check. ~~str~~ |
| **RETURNS** | Whether the span ruler contains the label. ~~bool~~ |
## SpanRuler.\_\_call\_\_ {#call tag="method"}
## SpanRuler.\_\_call\_\_ {id="call",tag="method"}
Find matches in the `Doc` and add them to `doc.spans[span_key]` and/or
`doc.ents`. Typically, this happens automatically after the component has been
@ -195,7 +197,7 @@ will be removed.
| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ |
| **RETURNS** | The modified `Doc` with added spans/entities. ~~Doc~~ |
## SpanRuler.add_patterns {#add_patterns tag="method"}
## SpanRuler.add_patterns {id="add_patterns",tag="method"}
Add patterns to the span ruler. A pattern can either be a token pattern (list of
dicts) or a phrase pattern (string). For more details, see the usage guide on
@ -216,7 +218,7 @@ dicts) or a phrase pattern (string). For more details, see the usage guide on
| ---------- | ---------------------------------------------------------------- |
| `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ |
## SpanRuler.remove {#remove tag="method"}
## SpanRuler.remove {id="remove",tag="method"}
Remove patterns by label from the span ruler. A `ValueError` is raised if the
label does not exist in any patterns.
@ -234,7 +236,7 @@ label does not exist in any patterns.
| ------- | -------------------------------------- |
| `label` | The label of the pattern rule. ~~str~~ |
## SpanRuler.remove_by_id {#remove_by_id tag="method"}
## SpanRuler.remove_by_id {id="remove_by_id",tag="method"}
Remove patterns by ID from the span ruler. A `ValueError` is raised if the ID
does not exist in any patterns.
@ -252,7 +254,7 @@ does not exist in any patterns.
| ------------ | ----------------------------------- |
| `pattern_id` | The ID of the pattern rule. ~~str~~ |
## SpanRuler.clear {#clear tag="method"}
## SpanRuler.clear {id="clear",tag="method"}
Remove all patterns the span ruler.
@ -265,7 +267,7 @@ Remove all patterns the span ruler.
> ruler.clear()
> ```
## SpanRuler.to_disk {#to_disk tag="method"}
## SpanRuler.to_disk {id="to_disk",tag="method"}
Save the span ruler patterns to a directory. The patterns will be saved as
newline-delimited JSON (JSONL).
@ -281,7 +283,7 @@ newline-delimited JSON (JSONL).
| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ |
| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
## SpanRuler.from_disk {#from_disk tag="method"}
## SpanRuler.from_disk {id="from_disk",tag="method"}
Load the span ruler from a path.
@ -297,7 +299,7 @@ Load the span ruler from a path.
| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ |
## SpanRuler.to_bytes {#to_bytes tag="method"}
## SpanRuler.to_bytes {id="to_bytes",tag="method"}
Serialize the span ruler to a bytestring.
@ -312,7 +314,7 @@ Serialize the span ruler to a bytestring.
| ----------- | ---------------------------------- |
| **RETURNS** | The serialized patterns. ~~bytes~~ |
## SpanRuler.from_bytes {#from_bytes tag="method"}
## SpanRuler.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -329,7 +331,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `bytes_data` | The bytestring to load. ~~bytes~~ |
| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ |
## SpanRuler.labels {#labels tag="property"}
## SpanRuler.labels {id="labels",tag="property"}
All labels present in the match patterns.
@ -337,7 +339,7 @@ All labels present in the match patterns.
| ----------- | -------------------------------------- |
| **RETURNS** | The string labels. ~~Tuple[str, ...]~~ |
## SpanRuler.ids {#ids tag="property"}
## SpanRuler.ids {id="ids",tag="property"}
All IDs present in the `id` property of the match patterns.
@ -345,7 +347,7 @@ All IDs present in the `id` property of the match patterns.
| ----------- | ----------------------------------- |
| **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ |
## SpanRuler.patterns {#patterns tag="property"}
## SpanRuler.patterns {id="patterns",tag="property"}
All patterns that were added to the span ruler.
@ -353,7 +355,7 @@ All patterns that were added to the span ruler.
| ----------- | ---------------------------------------------------------------------------------------- |
| **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| ---------------- | -------------------------------------------------------------------------------- |

30
website/docs/api/stringstore.md → website/docs/api/stringstore.mdx
View File

@ -8,7 +8,7 @@ Look up strings by 64-bit hashes. As of v2.0, spaCy uses hash values instead of
integer IDs. This ensures that strings always map to the same ID, even from
different `StringStores`.
## StringStore.\_\_init\_\_ {#init tag="method"}
## StringStore.\_\_init\_\_ {id="init",tag="method"}
Create the `StringStore`.
@ -23,7 +23,7 @@ Create the `StringStore`.
| --------- | ---------------------------------------------------------------------- |
| `strings` | A sequence of strings to add to the store. ~~Optional[Iterable[str]]~~ |
## StringStore.\_\_len\_\_ {#len tag="method"}
## StringStore.\_\_len\_\_ {id="len",tag="method"}
Get the number of strings in the store.
@ -38,7 +38,7 @@ Get the number of strings in the store.
| ----------- | ------------------------------------------- |
| **RETURNS** | The number of strings in the store. ~~int~~ |
## StringStore.\_\_getitem\_\_ {#getitem tag="method"}
## StringStore.\_\_getitem\_\_ {id="getitem",tag="method"}
Retrieve a string from a given hash. If a string is passed as the input, add it
to the store and return its hash.
@ -57,7 +57,7 @@ to the store and return its hash.
| `string_or_hash` | The hash value to lookup or the string to store. ~~Union[str, int]~~ |
| **RETURNS** | The stored string or the hash of the newly added string. ~~Union[str, int]~~ |
## StringStore.\_\_contains\_\_ {#contains tag="method"}
## StringStore.\_\_contains\_\_ {id="contains",tag="method"}
Check whether a string or a hash is in the store.
@ -74,7 +74,7 @@ Check whether a string or a hash is in the store.
| `string_or_hash` | The string or hash to check. ~~Union[str, int]~~ |
| **RETURNS** | Whether the store contains the string or hash. ~~bool~~ |
## StringStore.\_\_iter\_\_ {#iter tag="method"}
## StringStore.\_\_iter\_\_ {id="iter",tag="method"}
Iterate over the stored strings in insertion order.
@ -90,7 +90,7 @@ Iterate over the stored strings in insertion order.
| ----------- | ------------------------------ |
| **RETURNS** | A string in the store. ~~str~~ |
## StringStore.items {#iter tag="method" new="4"}
## StringStore.items {id="iter", tag="method", version="4"}
Iterate over the stored string-hash pairs in insertion order.
@ -106,7 +106,7 @@ Iterate over the stored string-hash pairs in insertion order.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | A list of string-hash pairs. ~~List[Tuple[str, int]]~~ |
## StringStore.keys {#iter tag="method" new="4"}
## StringStore.keys {id="iter", tag="method", version="4"}
Iterate over the stored strings in insertion order.
@ -122,7 +122,7 @@ Iterate over the stored strings in insertion order.
| ----------- | -------------------------------- |
| **RETURNS** | A list of strings. ~~List[str]~~ |
## StringStore.values {#iter tag="method" new="4"}
## StringStore.values {id="iter", tag="method", version="4"}
Iterate over the stored string hashes in insertion order.
@ -138,7 +138,7 @@ Iterate over the stored string hashes in insertion order.
| ----------- | -------------------------------------- |
| **RETURNS** | A list of string hashes. ~~List[int]~~ |
## StringStore.add {#add tag="method"}
## StringStore.add {id="add", tag="method"}
Add a string to the `StringStore`.
@ -158,7 +158,7 @@ Add a string to the `StringStore`.
| `string` | The string to add. ~~str~~ |
| **RETURNS** | The string's hash value. ~~int~~ |
## StringStore.to_disk {#to_disk tag="method"}
## StringStore.to_disk {id="to_disk",tag="method"}
Save the current state to a directory.
@ -172,7 +172,7 @@ Save the current state to a directory.
| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ |
| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
## StringStore.from_disk {#from_disk tag="method" new="2"}
## StringStore.from_disk {id="from_disk",tag="method"}
Loads state from a directory. Modifies the object in place and returns it.
@ -188,7 +188,7 @@ Loads state from a directory. Modifies the object in place and returns it.
| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
| **RETURNS** | The modified `StringStore` object. ~~StringStore~~ |
## StringStore.to_bytes {#to_bytes tag="method"}
## StringStore.to_bytes {id="to_bytes",tag="method"}
Serialize the current state to a binary string.
@ -202,7 +202,7 @@ Serialize the current state to a binary string.
| ----------- | ---------------------------------------------------------- |
| **RETURNS** | The serialized form of the `StringStore` object. ~~bytes~~ |
## StringStore.from_bytes {#from_bytes tag="method"}
## StringStore.from_bytes {id="from_bytes",tag="method"}
Load state from a binary string.
@ -219,9 +219,9 @@ Load state from a binary string.
| `bytes_data` | The data to load from. ~~bytes~~ |
| **RETURNS** | The `StringStore` object. ~~StringStore~~ |
## Utilities {#util}
## Utilities {id="util"}
### strings.hash_string {#hash_string tag="function"}
### strings.hash_string {id="hash_string",tag="function"}
Get a 64-bit hash for a given string.

42
website/docs/api/tagger.md → website/docs/api/tagger.mdx
View File

@ -14,7 +14,7 @@ part-of-speech tag set.
In the pre-trained pipelines, the tag schemas vary by language; see the
[individual model pages](/models) for details.
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predictions are assigned to `Token.tag`.
@ -23,7 +23,7 @@ Predictions are assigned to `Token.tag`.
| `Token.tag` | The part of speech (hash). ~~int~~ |
| `Token.tag_` | The part of speech. ~~str~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -52,7 +52,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/tagger.pyx
```
## Tagger.\_\_init\_\_ {#init tag="method"}
## Tagger.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -82,7 +82,7 @@ shortcut for this and instantiate the component using its string name and
| `overwrite` <Tag variant="new">3.2</Tag> | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ |
| `scorer` <Tag variant="new">3.2</Tag> | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"tag"`. ~~Optional[Callable]~~ |
## Tagger.\_\_call\_\_ {#call tag="method"}
## Tagger.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -105,7 +105,7 @@ and all pipeline components are applied to the `Doc` in order. Both
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## Tagger.pipe {#pipe tag="method"}
## Tagger.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -128,7 +128,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/tagger#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## Tagger.initialize {#initialize tag="method" new="3"}
## Tagger.initialize {id="initialize",tag="method",version="3"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -165,7 +165,7 @@ config.
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ |
## Tagger.predict {#predict tag="method"}
## Tagger.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects, without
modifying them.
@ -182,7 +182,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The model's prediction for each document. |
## Tagger.set_annotations {#set_annotations tag="method"}
## Tagger.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
@ -199,7 +199,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `Tagger.predict`. |
## Tagger.update {#update tag="method"}
## Tagger.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects containing the
predictions and gold-standard annotations, and update the component's model.
@ -223,7 +223,7 @@ Delegates to [`predict`](/api/tagger#predict) and
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## Tagger.rehearse {#rehearse tag="method,experimental" new="3"}
## Tagger.rehearse {id="rehearse",tag="method,experimental",version="3"}
Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the
current model to make predictions similar to an initial model, to try to address
@ -246,7 +246,7 @@ the "catastrophic forgetting" problem. This feature is experimental.
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## Tagger.get_loss {#get_loss tag="method"}
## Tagger.get_loss {id="get_loss",tag="method"}
Find the loss and gradient of loss for the batch of documents and their
predicted scores.
@ -265,7 +265,7 @@ predicted scores.
| `scores` | Scores representing the model's predictions. |
| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ |
## Tagger.create_optimizer {#create_optimizer tag="method"}
## Tagger.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -280,7 +280,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## Tagger.use_params {#use_params tag="method, contextmanager"}
## Tagger.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model, to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -297,7 +297,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## Tagger.add_label {#add_label tag="method"}
## Tagger.add_label {id="add_label",tag="method"}
Add a new label to the pipe. Raises an error if the output dimension is already
set, or if the model has already been fully [initialized](#initialize). Note
@ -319,7 +319,7 @@ automatically.
| `label` | The label to add. ~~str~~ |
| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ |
## Tagger.to_disk {#to_disk tag="method"}
## Tagger.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -336,7 +336,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## Tagger.from_disk {#from_disk tag="method"}
## Tagger.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -354,7 +354,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `Tagger` object. ~~Tagger~~ |
## Tagger.to_bytes {#to_bytes tag="method"}
## Tagger.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -371,7 +371,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `Tagger` object. ~~bytes~~ |
## Tagger.from_bytes {#from_bytes tag="method"}
## Tagger.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -390,7 +390,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Tagger` object. ~~Tagger~~ |
## Tagger.labels {#labels tag="property"}
## Tagger.labels {id="labels",tag="property"}
The labels currently added to the component.
@ -405,7 +405,7 @@ The labels currently added to the component.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ |
## Tagger.label_data {#label_data tag="property" new="3"}
## Tagger.label_data {id="label_data",tag="property",version="3"}
The labels currently added to the component and their internal meta information.
This is the data generated by [`init labels`](/api/cli#init-labels) and used by
@ -423,7 +423,7 @@ pre-defined label set.
| ----------- | ---------------------------------------------------------- |
| **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

46
website/docs/api/textcategorizer.md → website/docs/api/textcategorizer.mdx
View File

@ -2,7 +2,7 @@
title: TextCategorizer
tag: class
source: spacy/pipeline/textcat.py
new: 2
version: 2
teaser: 'Pipeline component for text classification'
api_base_class: /api/pipe
api_string_name: textcat
@ -29,7 +29,7 @@ only.
</Infobox>
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
Predictions will be saved to `doc.cats` as a dictionary, where the key is the
name of the category and the value is a score between 0 and 1 (inclusive). For
@ -49,7 +49,7 @@ supported.
| ---------- | ------------------------------------- |
| `Doc.cats` | Category scores. ~~Dict[str, float]~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -93,7 +93,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/textcat_multilabel.py
```
## TextCategorizer.\_\_init\_\_ {#init tag="method"}
## TextCategorizer.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -126,7 +126,7 @@ shortcut for this and instantiate the component using its string name and
| `scorer` | The scoring method. Defaults to [`Scorer.score_cats`](/api/scorer#score_cats) for the attribute `"cats"`. ~~Optional[Callable]~~ |
| `save_activations` <Tag variant="new">4.0</Tag> | Save activations in `Doc` when annotating. The supported activations is `"probabilities"`. ~~Union[bool, list[str]]~~ |
## TextCategorizer.\_\_call\_\_ {#call tag="method"}
## TextCategorizer.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -149,7 +149,7 @@ delegate to the [`predict`](/api/textcategorizer#predict) and
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## TextCategorizer.pipe {#pipe tag="method"}
## TextCategorizer.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -173,7 +173,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/textcategorizer#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## TextCategorizer.initialize {#initialize tag="method" new="3"}
## TextCategorizer.initialize {id="initialize",tag="method",version="3"}
Initialize the component for training. `get_examples` should be a function that
returns an iterable of [`Example`](/api/example) objects. **At least one example
@ -212,7 +212,7 @@ config.
| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ |
| `positive_label` | The positive label for a binary task with exclusive classes, `None` otherwise and by default. This parameter is only used during scoring. It is not available when using the `textcat_multilabel` component. ~~Optional[str]~~ |
## TextCategorizer.predict {#predict tag="method"}
## TextCategorizer.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects without
modifying them.
@ -229,7 +229,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The model's prediction for each document. |
## TextCategorizer.set_annotations {#set_annotations tag="method"}
## TextCategorizer.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores.
@ -246,7 +246,7 @@ Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `TextCategorizer.predict`. |
## TextCategorizer.update {#update tag="method"}
## TextCategorizer.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects containing the
predictions and gold-standard annotations, and update the component's model.
@ -270,7 +270,7 @@ Delegates to [`predict`](/api/textcategorizer#predict) and
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## TextCategorizer.rehearse {#rehearse tag="method,experimental" new="3"}
## TextCategorizer.rehearse {id="rehearse",tag="method,experimental",version="3"}
Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the
current model to make predictions similar to an initial model to try to address
@ -293,7 +293,7 @@ the "catastrophic forgetting" problem. This feature is experimental.
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## TextCategorizer.get_loss {#get_loss tag="method"}
## TextCategorizer.get_loss {id="get_loss",tag="method"}
Find the loss and gradient of loss for the batch of documents and their
predicted scores.
@ -312,7 +312,7 @@ predicted scores.
| `scores` | Scores representing the model's predictions. |
| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ |
## TextCategorizer.score {#score tag="method" new="3"}
## TextCategorizer.score {id="score",tag="method",version="3"}
Score a batch of examples.
@ -328,7 +328,7 @@ Score a batch of examples.
| _keyword-only_ | |
| **RETURNS** | The scores, produced by [`Scorer.score_cats`](/api/scorer#score_cats). ~~Dict[str, Union[float, Dict[str, float]]]~~ |
## TextCategorizer.create_optimizer {#create_optimizer tag="method"}
## TextCategorizer.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -343,7 +343,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## TextCategorizer.use_params {#use_params tag="method, contextmanager"}
## TextCategorizer.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model to use the given parameter values.
@ -359,7 +359,7 @@ Modify the pipe's model to use the given parameter values.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## TextCategorizer.add_label {#add_label tag="method"}
## TextCategorizer.add_label {id="add_label",tag="method"}
Add a new label to the pipe. Raises an error if the output dimension is already
set, or if the model has already been fully [initialized](#initialize). Note
@ -381,7 +381,7 @@ automatically.
| `label` | The label to add. ~~str~~ |
| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ |
## TextCategorizer.to_disk {#to_disk tag="method"}
## TextCategorizer.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -398,7 +398,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## TextCategorizer.from_disk {#from_disk tag="method"}
## TextCategorizer.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -416,7 +416,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `TextCategorizer` object. ~~TextCategorizer~~ |
## TextCategorizer.to_bytes {#to_bytes tag="method"}
## TextCategorizer.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -433,7 +433,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `TextCategorizer` object. ~~bytes~~ |
## TextCategorizer.from_bytes {#from_bytes tag="method"}
## TextCategorizer.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -452,7 +452,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `TextCategorizer` object. ~~TextCategorizer~~ |
## TextCategorizer.labels {#labels tag="property"}
## TextCategorizer.labels {id="labels",tag="property"}
The labels currently added to the component.
@ -467,7 +467,7 @@ The labels currently added to the component.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ |
## TextCategorizer.label_data {#label_data tag="property" new="3"}
## TextCategorizer.label_data {id="label_data",tag="property",version="3"}
The labels currently added to the component and their internal meta information.
This is the data generated by [`init labels`](/api/cli#init-labels) and used by
@ -485,7 +485,7 @@ the model with a pre-defined label set.
| ----------- | ---------------------------------------------------------- |
| **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

32
website/docs/api/tok2vec.md → website/docs/api/tok2vec.mdx
View File

@ -1,7 +1,7 @@
---
title: Tok2Vec
source: spacy/pipeline/tok2vec.py
new: 3
version: 3
teaser: null
api_base_class: /api/pipe
api_string_name: tok2vec
@ -23,7 +23,7 @@ components can backpropagate to the shared weights. This implementation is used
because it allows us to avoid relying on object identity within the models to
achieve the parameter sharing.
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -48,7 +48,7 @@ architectures and their arguments and hyperparameters.
%%GITHUB_SPACY/spacy/pipeline/tok2vec.py
```
## Tok2Vec.\_\_init\_\_ {#init tag="method"}
## Tok2Vec.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -75,7 +75,7 @@ shortcut for this and instantiate the component using its string name and
| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]~~ |
| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ |
## Tok2Vec.\_\_call\_\_ {#call tag="method"}
## Tok2Vec.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document and add context-sensitive embeddings to the
`Doc.tensor` attribute, allowing them to be used as features by downstream
@ -100,7 +100,7 @@ pipeline components are applied to the `Doc` in order. Both
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## Tok2Vec.pipe {#pipe tag="method"}
## Tok2Vec.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -123,7 +123,7 @@ and [`set_annotations`](/api/tok2vec#set_annotations) methods.
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## Tok2Vec.initialize {#initialize tag="method"}
## Tok2Vec.initialize {id="initialize",tag="method"}
Initialize the component for training and return an
[`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a
@ -148,7 +148,7 @@ by [`Language.initialize`](/api/language#initialize).
| _keyword-only_ | |
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
## Tok2Vec.predict {#predict tag="method"}
## Tok2Vec.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects without
modifying them.
@ -165,7 +165,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The model's prediction for each document. |
## Tok2Vec.set_annotations {#set_annotations tag="method"}
## Tok2Vec.set_annotations {id="set_annotations",tag="method"}
Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
@ -182,7 +182,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `Tok2Vec.predict`. |
## Tok2Vec.update {#update tag="method"}
## Tok2Vec.update {id="update",tag="method"}
Learn from a batch of [`Example`](/api/example) objects containing the
predictions and gold-standard annotations, and update the component's model.
@ -205,7 +205,7 @@ Delegates to [`predict`](/api/tok2vec#predict).
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## Tok2Vec.create_optimizer {#create_optimizer tag="method"}
## Tok2Vec.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -220,7 +220,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## Tok2Vec.use_params {#use_params tag="method, contextmanager"}
## Tok2Vec.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -237,7 +237,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## Tok2Vec.to_disk {#to_disk tag="method"}
## Tok2Vec.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -254,7 +254,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## Tok2Vec.from_disk {#from_disk tag="method"}
## Tok2Vec.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -272,7 +272,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `Tok2Vec` object. ~~Tok2Vec~~ |
## Tok2Vec.to_bytes {#to_bytes tag="method"}
## Tok2Vec.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -289,7 +289,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `Tok2Vec` object. ~~bytes~~ |
## Tok2Vec.from_bytes {#from_bytes tag="method"}
## Tok2Vec.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -308,7 +308,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Tok2Vec` object. ~~Tok2Vec~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

48
website/docs/api/token.md → website/docs/api/token.mdx
View File

@ -5,7 +5,7 @@ tag: class
source: spacy/tokens/token.pyx
---
## Token.\_\_init\_\_ {#init tag="method"}
## Token.\_\_init\_\_ {id="init",tag="method"}
Construct a `Token` object.
@ -23,7 +23,7 @@ Construct a `Token` object.
| `doc` | The parent document. ~~Doc~~ |
| `offset` | The index of the token within the document. ~~int~~ |
## Token.\_\_len\_\_ {#len tag="method"}
## Token.\_\_len\_\_ {id="len",tag="method"}
The number of unicode characters in the token, i.e. `token.text`.
@ -39,7 +39,7 @@ The number of unicode characters in the token, i.e. `token.text`.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | The number of unicode characters in the token. ~~int~~ |
## Token.set_extension {#set_extension tag="classmethod" new="2"}
## Token.set_extension {id="set_extension",tag="classmethod",version="2"}
Define a custom attribute on the `Token` which becomes available via `Token._`.
For details, see the documentation on
@ -64,7 +64,7 @@ For details, see the documentation on
| `setter` | Setter function that takes the `Token` and a value, and modifies the object. Is called when the user writes to the `Token._` attribute. ~~Optional[Callable[[Token, Any], None]]~~ |
| `force` | Force overwriting existing attribute. ~~bool~~ |
## Token.get_extension {#get_extension tag="classmethod" new="2"}
## Token.get_extension {id="get_extension",tag="classmethod",version="2"}
Look up a previously registered extension by name. Returns a 4-tuple
`(default, method, getter, setter)` if the extension is registered. Raises a
@ -84,7 +84,7 @@ Look up a previously registered extension by name. Returns a 4-tuple
| `name` | Name of the extension. ~~str~~ |
| **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ |
## Token.has_extension {#has_extension tag="classmethod" new="2"}
## Token.has_extension {id="has_extension",tag="classmethod",version="2"}
Check whether an extension has been registered on the `Token` class.
@ -101,7 +101,7 @@ Check whether an extension has been registered on the `Token` class.
| `name` | Name of the extension to check. ~~str~~ |
| **RETURNS** | Whether the extension has been registered. ~~bool~~ |
## Token.remove_extension {#remove_extension tag="classmethod" new=""2.0.11""}
## Token.remove_extension {id="remove_extension",tag="classmethod",version="2.0.11"}
Remove a previously registered extension.
@ -119,7 +119,7 @@ Remove a previously registered extension.
| `name` | Name of the extension. ~~str~~ |
| **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ |
## Token.check_flag {#check_flag tag="method"}
## Token.check_flag {id="check_flag",tag="method"}
Check the value of a boolean flag.
@ -137,7 +137,7 @@ Check the value of a boolean flag.
| `flag_id` | The attribute ID of the flag to check. ~~int~~ |
| **RETURNS** | Whether the flag is set. ~~bool~~ |
## Token.similarity {#similarity tag="method" model="vectors"}
## Token.similarity {id="similarity",tag="method",model="vectors"}
Compute a semantic similarity estimate. Defaults to cosine over vectors.
@ -155,7 +155,7 @@ Compute a semantic similarity estimate. Defaults to cosine over vectors.
| other | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ |
| **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ |
## Token.nbor {#nbor tag="method"}
## Token.nbor {id="nbor",tag="method"}
Get a neighboring token.
@ -172,7 +172,7 @@ Get a neighboring token.
| `i` | The relative position of the token to get. Defaults to `1`. ~~int~~ |
| **RETURNS** | The token at position `self.doc[self.i+i]`. ~~Token~~ |
## Token.set_morph {#set_morph tag="method"}
## Token.set_morph {id="set_morph",tag="method"}
Set the morphological analysis from a UD FEATS string, hash value of a UD FEATS
string, features dict or `MorphAnalysis`. The value `None` can be used to reset
@ -191,7 +191,7 @@ the morph to an unset state.
| -------- | --------------------------------------------------------------------------------- |
| features | The morphological features to set. ~~Union[int, dict, str, MorphAnalysis, None]~~ |
## Token.has_morph {#has_morph tag="method"}
## Token.has_morph {id="has_morph",tag="method"}
Check whether the token has annotated morph information. Return `False` when the
morph annotation is unset/missing.
@ -200,7 +200,7 @@ morph annotation is unset/missing.
| ----------- | --------------------------------------------- |
| **RETURNS** | Whether the morph annotation is set. ~~bool~~ |
## Token.is_ancestor {#is_ancestor tag="method" model="parser"}
## Token.is_ancestor {id="is_ancestor",tag="method",model="parser"}
Check whether this token is a parent, grandparent, etc. of another in the
dependency tree.
@ -219,7 +219,7 @@ dependency tree.
| descendant | Another token. ~~Token~~ |
| **RETURNS** | Whether this token is the ancestor of the descendant. ~~bool~~ |
## Token.ancestors {#ancestors tag="property" model="parser"}
## Token.ancestors {id="ancestors",tag="property",model="parser"}
A sequence of the token's syntactic ancestors (parents, grandparents, etc).
@ -237,7 +237,7 @@ A sequence of the token's syntactic ancestors (parents, grandparents, etc).
| ---------- | ------------------------------------------------------------------------------- |
| **YIELDS** | A sequence of ancestor tokens such that `ancestor.is_ancestor(self)`. ~~Token~~ |
## Token.conjuncts {#conjuncts tag="property" model="parser"}
## Token.conjuncts {id="conjuncts",tag="property",model="parser"}
A tuple of coordinated tokens, not including the token itself.
@ -253,7 +253,7 @@ A tuple of coordinated tokens, not including the token itself.
| ----------- | --------------------------------------------- |
| **RETURNS** | The coordinated tokens. ~~Tuple[Token, ...]~~ |
## Token.children {#children tag="property" model="parser"}
## Token.children {id="children",tag="property",model="parser"}
A sequence of the token's immediate syntactic children.
@ -269,7 +269,7 @@ A sequence of the token's immediate syntactic children.
| ---------- | ------------------------------------------------------- |
| **YIELDS** | A child token such that `child.head == self`. ~~Token~~ |
## Token.lefts {#lefts tag="property" model="parser"}
## Token.lefts {id="lefts",tag="property",model="parser"}
The leftward immediate children of the word in the syntactic dependency parse.
@ -285,7 +285,7 @@ The leftward immediate children of the word in the syntactic dependency parse.
| ---------- | ------------------------------------ |
| **YIELDS** | A left-child of the token. ~~Token~~ |
## Token.rights {#rights tag="property" model="parser"}
## Token.rights {id="rights",tag="property",model="parser"}
The rightward immediate children of the word in the syntactic dependency parse.
@ -301,7 +301,7 @@ The rightward immediate children of the word in the syntactic dependency parse.
| ---------- | ------------------------------------- |
| **YIELDS** | A right-child of the token. ~~Token~~ |
## Token.n_lefts {#n_lefts tag="property" model="parser"}
## Token.n_lefts {id="n_lefts",tag="property",model="parser"}
The number of leftward immediate children of the word in the syntactic
dependency parse.
@ -317,7 +317,7 @@ dependency parse.
| ----------- | ---------------------------------------- |
| **RETURNS** | The number of left-child tokens. ~~int~~ |
## Token.n_rights {#n_rights tag="property" model="parser"}
## Token.n_rights {id="n_rights",tag="property",model="parser"}
The number of rightward immediate children of the word in the syntactic
dependency parse.
@ -333,7 +333,7 @@ dependency parse.
| ----------- | ----------------------------------------- |
| **RETURNS** | The number of right-child tokens. ~~int~~ |
## Token.subtree {#subtree tag="property" model="parser"}
## Token.subtree {id="subtree",tag="property",model="parser"}
A sequence containing the token and all the token's syntactic descendants.
@ -349,7 +349,7 @@ A sequence containing the token and all the token's syntactic descendants.
| ---------- | ------------------------------------------------------------------------------------ |
| **YIELDS** | A descendant token such that `self.is_ancestor(token)` or `token == self`. ~~Token~~ |
## Token.has_vector {#has_vector tag="property" model="vectors"}
## Token.has_vector {id="has_vector",tag="property",model="vectors"}
A boolean value indicating whether a word vector is associated with the token.
@ -365,7 +365,7 @@ A boolean value indicating whether a word vector is associated with the token.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | Whether the token has a vector data attached. ~~bool~~ |
## Token.vector {#vector tag="property" model="vectors"}
## Token.vector {id="vector",tag="property",model="vectors"}
A real-valued meaning representation.
@ -382,7 +382,7 @@ A real-valued meaning representation.
| ----------- | ----------------------------------------------------------------------------------------------- |
| **RETURNS** | A 1-dimensional array representing the token's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
## Token.vector_norm {#vector_norm tag="property" model="vectors"}
## Token.vector_norm {id="vector_norm",tag="property",model="vectors"}
The L2 norm of the token's vector representation.
@ -401,7 +401,7 @@ The L2 norm of the token's vector representation.
| ----------- | --------------------------------------------------- |
| **RETURNS** | The L2 norm of the vector representation. ~~float~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |

28
website/docs/api/tokenizer.md → website/docs/api/tokenizer.mdx
View File

@ -20,7 +20,7 @@ The tokenizer is typically created automatically when a
like punctuation and special case rules from the
[`Language.Defaults`](/api/language#defaults) provided by the language subclass.
## Tokenizer.\_\_init\_\_ {#init tag="method"}
## Tokenizer.\_\_init\_\_ {id="init",tag="method"}
Create a `Tokenizer` to create `Doc` objects given unicode text. For examples of
how to construct a custom tokenizer with different tokenization rules, see the
@ -55,7 +55,7 @@ how to construct a custom tokenizer with different tokenization rules, see the
| `url_match` | A function matching the signature of `re.compile(string).match` to find token matches after considering prefixes and suffixes. ~~Optional[Callable[[str], Optional[Match]]]~~ |
| `faster_heuristics` <Tag variant="new">3.3.0</Tag> | Whether to restrict the final `Matcher`-based pass for rules to those containing affixes or space. Defaults to `True`. ~~bool~~ |
## Tokenizer.\_\_call\_\_ {#call tag="method"}
## Tokenizer.\_\_call\_\_ {id="call",tag="method"}
Tokenize a string.
@ -71,7 +71,7 @@ Tokenize a string.
| `string` | The string to tokenize. ~~str~~ |
| **RETURNS** | A container for linguistic annotations. ~~Doc~~ |
## Tokenizer.pipe {#pipe tag="method"}
## Tokenizer.pipe {id="pipe",tag="method"}
Tokenize a stream of texts.
@ -89,7 +89,7 @@ Tokenize a stream of texts.
| `batch_size` | The number of texts to accumulate in an internal buffer. Defaults to `1000`. ~~int~~ |
| **YIELDS** | The tokenized `Doc` objects, in order. ~~Doc~~ |
## Tokenizer.find_infix {#find_infix tag="method"}
## Tokenizer.find_infix {id="find_infix",tag="method"}
Find internal split points of the string.
@ -98,7 +98,7 @@ Find internal split points of the string.
| `string` | The string to split. ~~str~~ |
| **RETURNS** | A list of `re.MatchObject` objects that have `.start()` and `.end()` methods, denoting the placement of internal segment separators, e.g. hyphens. ~~List[Match]~~ |
## Tokenizer.find_prefix {#find_prefix tag="method"}
## Tokenizer.find_prefix {id="find_prefix",tag="method"}
Find the length of a prefix that should be segmented from the string, or `None`
if no prefix rules match.
@ -108,7 +108,7 @@ if no prefix rules match.
| `string` | The string to segment. ~~str~~ |
| **RETURNS** | The length of the prefix if present, otherwise `None`. ~~Optional[int]~~ |
## Tokenizer.find_suffix {#find_suffix tag="method"}
## Tokenizer.find_suffix {id="find_suffix",tag="method"}
Find the length of a suffix that should be segmented from the string, or `None`
if no suffix rules match.
@ -118,7 +118,7 @@ if no suffix rules match.
| `string` | The string to segment. ~~str~~ |
| **RETURNS** | The length of the suffix if present, otherwise `None`. ~~Optional[int]~~ |
## Tokenizer.add_special_case {#add_special_case tag="method"}
## Tokenizer.add_special_case {id="add_special_case",tag="method"}
Add a special-case tokenization rule. This mechanism is also used to add custom
tokenizer exceptions to the language data. See the usage guide on the
@ -139,7 +139,7 @@ details and examples.
| `string` | The string to specially tokenize. ~~str~~ |
| `token_attrs` | A sequence of dicts, where each dict describes a token and its attributes. The `ORTH` fields of the attributes must exactly match the string when they are concatenated. ~~Iterable[Dict[int, str]]~~ |
## Tokenizer.explain {#explain tag="method"}
## Tokenizer.explain {id="explain",tag="method"}
Tokenize a string with a slow debugging tokenizer that provides information
about which tokenizer rule or pattern was matched for each token. The tokens
@ -158,7 +158,7 @@ produced are identical to `Tokenizer.__call__` except for whitespace tokens.
| `string` | The string to tokenize with the debugging tokenizer. ~~str~~ |
| **RETURNS** | A list of `(pattern_string, token_string)` tuples. ~~List[Tuple[str, str]]~~ |
## Tokenizer.to_disk {#to_disk tag="method"}
## Tokenizer.to_disk {id="to_disk",tag="method"}
Serialize the tokenizer to disk.
@ -175,7 +175,7 @@ Serialize the tokenizer to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## Tokenizer.from_disk {#from_disk tag="method"}
## Tokenizer.from_disk {id="from_disk",tag="method"}
Load the tokenizer from disk. Modifies the object in place and returns it.
@ -193,7 +193,7 @@ Load the tokenizer from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `Tokenizer` object. ~~Tokenizer~~ |
## Tokenizer.to_bytes {#to_bytes tag="method"}
## Tokenizer.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -210,7 +210,7 @@ Serialize the tokenizer to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `Tokenizer` object. ~~bytes~~ |
## Tokenizer.from_bytes {#from_bytes tag="method"}
## Tokenizer.from_bytes {id="from_bytes",tag="method"}
Load the tokenizer from a bytestring. Modifies the object in place and returns
it.
@ -230,7 +230,7 @@ it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Tokenizer` object. ~~Tokenizer~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
@ -241,7 +241,7 @@ it.
| `token_match` | A function matching the signature of `re.compile(string).match` to find token matches. Returns an `re.MatchObject` or `None`. ~~Optional[Callable[[str], Optional[Match]]]~~ |
| `rules` | A dictionary of tokenizer exceptions and special cases. ~~Optional[Dict[str, List[Dict[int, str]]]]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

200
website/docs/api/top-level.md → website/docs/api/top-level.mdx
View File

@ -13,9 +13,9 @@ menu:
- ['Utility Functions', 'util']
---
## spaCy {#spacy hidden="true"}
## spaCy {id="spacy",hidden="true"}
### spacy.load {#spacy.load tag="function"}
### spacy.load {id="spacy.load",tag="function"}
Load a pipeline using the name of an installed
[package](/usage/saving-loading#models), a string path or a `Path`-like object.
@ -61,8 +61,7 @@ Essentially, `spacy.load()` is a convenience wrapper that reads the pipeline's
information to construct a `Language` object, loads in the model data and
weights, and returns it.
```python
### Abstract example
```python {title="Abstract example"}
cls = spacy.util.get_lang_class(lang) # 1. Get Language class, e.g. English
nlp = cls() # 2. Initialize it
for name in pipeline:
@ -70,7 +69,7 @@ for name in pipeline:
nlp.from_disk(data_path) # 4. Load in the binary data
```
### spacy.blank {#spacy.blank tag="function" new="2"}
### spacy.blank {id="spacy.blank",tag="function",version="2"}
Create a blank pipeline of a given language class. This function is the twin of
`spacy.load()`.
@ -91,7 +90,7 @@ Create a blank pipeline of a given language class. This function is the twin of
| `meta` | Optional meta overrides for [`nlp.meta`](/api/language#meta). ~~Dict[str, Any]~~ |
| **RETURNS** | An empty `Language` object of the appropriate subclass. ~~Language~~ |
### spacy.info {#spacy.info tag="function"}
### spacy.info {id="spacy.info",tag="function"}
The same as the [`info` command](/api/cli#info). Pretty-print information about
your installation, installed pipelines and local setup from within spaCy.
@ -111,7 +110,7 @@ your installation, installed pipelines and local setup from within spaCy.
| `markdown` | Print information as Markdown. ~~bool~~ |
| `silent` | Don't print anything, just return. ~~bool~~ |
### spacy.explain {#spacy.explain tag="function"}
### spacy.explain {id="spacy.explain",tag="function"}
Get a description for a given POS tag, dependency label or entity type. For a
list of available terms, see [`glossary.py`](%%GITHUB_SPACY/spacy/glossary.py).
@ -134,7 +133,7 @@ list of available terms, see [`glossary.py`](%%GITHUB_SPACY/spacy/glossary.py).
| `term` | Term to explain. ~~str~~ |
| **RETURNS** | The explanation, or `None` if not found in the glossary. ~~Optional[str]~~ |
### spacy.prefer_gpu {#spacy.prefer_gpu tag="function" new="2.0.14"}
### spacy.prefer_gpu {id="spacy.prefer_gpu",tag="function",version="2.0.14"}
Allocate data and perform operations on [GPU](/usage/#gpu), if available. If
data has already been allocated on CPU, it will not be moved. Ideally, this
@ -162,7 +161,7 @@ ensure that the model is loaded on the correct device. See
| `gpu_id` | Device index to select. Defaults to `0`. ~~int~~ |
| **RETURNS** | Whether the GPU was activated. ~~bool~~ |
### spacy.require_gpu {#spacy.require_gpu tag="function" new="2.0.14"}
### spacy.require_gpu {id="spacy.require_gpu",tag="function",version="2.0.14"}
Allocate data and perform operations on [GPU](/usage/#gpu). Will raise an error
if no GPU is available. If data has already been allocated on CPU, it will not
@ -190,7 +189,7 @@ ensure that the model is loaded on the correct device. See
| `gpu_id` | Device index to select. Defaults to `0`. ~~int~~ |
| **RETURNS** | `True` ~~bool~~ |
### spacy.require_cpu {#spacy.require_cpu tag="function" new="3.0.0"}
### spacy.require_cpu {id="spacy.require_cpu",tag="function",version="3.0.0"}
Allocate data and perform operations on CPU. If data has already been allocated
on GPU, it will not be moved. Ideally, this function should be called right
@ -216,12 +215,12 @@ ensure that the model is loaded on the correct device. See
| ----------- | --------------- |
| **RETURNS** | `True` ~~bool~~ |
## displaCy {#displacy source="spacy/displacy"}
## displaCy {id="displacy",source="spacy/displacy"}
As of v2.0, spaCy comes with a built-in visualization suite. For more info and
examples, see the usage guide on [visualizing spaCy](/usage/visualizers).
### displacy.serve {#displacy.serve tag="method" new="2"}
### displacy.serve {id="displacy.serve",tag="method",version="2"}
Serve a dependency parse tree or named entity visualization to view it in your
browser. Will run a simple web server.
@ -237,18 +236,19 @@ browser. Will run a simple web server.
> displacy.serve([doc1, doc2], style="dep")
> ```
| Name | Description |
| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ |
| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` <Tag variant="new">3.3</Tag>. Defaults to `"dep"`. ~~str~~ |
| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ |
| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ |
| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ |
| `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ |
| `port` | Port to serve visualization. Defaults to `5000`. ~~int~~ |
| `host` | Host to serve visualization. Defaults to `"0.0.0.0"`. ~~str~~ |
| Name | Description |
| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ |
| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` <Tag variant="new">3.3</Tag>. Defaults to `"dep"`. ~~str~~ |
| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ |
| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ |
| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ |
| `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ |
| `port` | Port to serve visualization. Defaults to `5000`. ~~int~~ |
| `host` | Host to serve visualization. Defaults to `"0.0.0.0"`. ~~str~~ |
| `auto_select_port` | If `True`, automatically switch to a different port if the specified port is already in use. Defaults to `False`. ~~bool~~ |
### displacy.render {#displacy.render tag="method" new="2"}
### displacy.render {id="displacy.render",tag="method",version="2"}
Render a dependency parse tree or named entity visualization.
@ -266,14 +266,14 @@ Render a dependency parse tree or named entity visualization.
| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span, dict]], Doc, Span, dict]~~ |
| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` <Tag variant="new">3.3</Tag>. Defaults to `"dep"`. ~~str~~ |
| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ |
| `page` | Render markup as full HTML page. Defaults to `False`. ~~bool~~ |
| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ |
| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ |
| `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ |
| `jupyter` | Explicitly enable or disable "[Jupyter](http://jupyter.org/) mode" to return markup ready to be rendered in a notebook. Detected automatically if `None` (default). ~~Optional[bool]~~ |
| **RETURNS** | The rendered HTML markup. ~~str~~ |
### displacy.parse_deps {#displacy.parse_deps tag="method" new="2"}
### displacy.parse_deps {id="displacy.parse_deps",tag="method",version="2"}
Generate dependency parse in `{'words': [], 'arcs': []}` format. For use with
the `manual=True` argument in `displacy.render`.
@ -295,7 +295,7 @@ the `manual=True` argument in `displacy.render`.
| `options` | Dependency parse specific visualisation options. ~~Dict[str, Any]~~ |
| **RETURNS** | Generated dependency parse keyed by words and arcs. ~~dict~~ |
### displacy.parse_ents {#displacy.parse_ents tag="method" new="2"}
### displacy.parse_ents {id="displacy.parse_ents",tag="method",version="2"}
Generate named entities in `[{start: i, end: i, label: 'label'}]` format. For
use with the `manual=True` argument in `displacy.render`.
@ -317,7 +317,7 @@ use with the `manual=True` argument in `displacy.render`.
| `options` | NER-specific visualisation options. ~~Dict[str, Any]~~ |
| **RETURNS** | Generated entities keyed by text (original text) and ents. ~~dict~~ |
### displacy.parse_spans {#displacy.parse_spans tag="method" new="2"}
### displacy.parse_spans {id="displacy.parse_spans",tag="method",version="2"}
Generate spans in `[{start_token: i, end_token: i, label: 'label'}]` format. For
use with the `manual=True` argument in `displacy.render`.
@ -340,12 +340,12 @@ use with the `manual=True` argument in `displacy.render`.
| `options` | Span-specific visualisation options. ~~Dict[str, Any]~~ |
| **RETURNS** | Generated entities keyed by text (original text) and ents. ~~dict~~ |
### Visualizer options {#displacy_options}
### Visualizer options {id="displacy_options"}
The `options` argument lets you specify additional settings for each visualizer.
If a setting is not present in the options, the default value will be used.
#### Dependency Visualizer options {#options-dep}
#### Dependency Visualizer options {id="options-dep"}
> #### Example
>
@ -371,7 +371,7 @@ If a setting is not present in the options, the default value will be used.
| `word_spacing` | Vertical spacing between words and arcs in px. Defaults to `45`. ~~int~~ |
| `distance` | Distance between words in px. Defaults to `175` in regular mode and `150` in compact mode. ~~int~~ |
#### Named Entity Visualizer options {#displacy_options-ent}
#### Named Entity Visualizer options {id="displacy_options-ent"}
> #### Example
>
@ -388,7 +388,7 @@ If a setting is not present in the options, the default value will be used.
| `template` | Optional template to overwrite the HTML used to render entity spans. Should be a format string and can use `{bg}`, `{text}` and `{label}`. See [`templates.py`](%%GITHUB_SPACY/spacy/displacy/templates.py) for examples. ~~Optional[str]~~ |
| `kb_url_template` <Tag variant="new">3.2.1</Tag> | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in. ~~Optional[str]~~ |
#### Span Visualizer options {#displacy_options-span}
#### Span Visualizer options {id="displacy_options-span"}
> #### Example
>
@ -419,7 +419,7 @@ span. If you wish to link an entity to their URL then consider using the
should redirect you to their Wikidata page, in this case
`https://www.wikidata.org/wiki/Q95`.
## registry {#registry source="spacy/util.py" new="3"}
## registry {id="registry",source="spacy/util.py",version="3"}
spaCy's function registry extends
[Thinc's `registry`](https://thinc.ai/docs/api-config#registry) and allows you
@ -469,7 +469,7 @@ factories.
| `scorers` | Registry for functions that create scoring methods for user with the [`Scorer`](/api/scorer). Scoring methods are called with `Iterable[Example]` and arbitrary `\*\*kwargs` and return scores as `Dict[str, Any]`. |
| `tokenizers` | Registry for tokenizer factories. Registered functions should return a callback that receives the `nlp` object and returns a [`Tokenizer`](/api/tokenizer) or a custom callable. |
### spacy-transformers registry {#registry-transformers}
### spacy-transformers registry {id="registry-transformers"}
The following registries are added by the
[`spacy-transformers`](https://github.com/explosion/spacy-transformers) package.
@ -494,7 +494,7 @@ See the [`Transformer`](/api/transformer) API reference and
| [`span_getters`](/api/transformer#span_getters) | Registry for functions that take a batch of `Doc` objects and return a list of `Span` objects to process by the transformer, e.g. sentences. |
| [`annotation_setters`](/api/transformer#annotation_setters) | Registry for functions that create annotation setters. Annotation setters are functions that take a batch of `Doc` objects and a [`FullTransformerBatch`](/api/transformer#fulltransformerbatch) and can set additional annotations on the `Doc`. |
## Loggers {#loggers source="spacy/training/loggers.py" new="3"}
## Loggers {id="loggers",source="spacy/training/loggers.py",version="3"}
A logger records the training results. When a logger is created, two functions
are returned: one for logging the information for each training step, and a
@ -530,7 +530,7 @@ saves them to a `jsonl` file.
<Accordion title="Example console output" spaced>
```cli
```bash
$ python -m spacy train config.cfg
```
@ -570,7 +570,7 @@ start decreasing across epochs.
| `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ |
| `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ |
#### spacy.ConsoleLogger.v3 {#ConsoleLogger tag="registered function"}
#### spacy.ConsoleLogger.v3 {id="ConsoleLogger",tag="registered function"}
> #### Example config
>
@ -592,9 +592,9 @@ optionally saves them to a `jsonl` file.
| `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ |
| `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ |
## Readers {#readers}
## Readers {id="readers"}
### File readers {#file-readers source="github.com/explosion/srsly" new="3"}
### File readers {id="file-readers",source="github.com/explosion/srsly",version="3"}
The following file readers are provided by our serialization library
[`srsly`](https://github.com/explosion/srsly). All registered functions take one
@ -624,7 +624,7 @@ blocks that are **not executed at runtime** – for example, in `[training]` and
</Infobox>
#### spacy.read_labels.v1 {#read_labels tag="registered function"}
#### spacy.read_labels.v1 {id="read_labels",tag="registered function"}
Read a JSON-formatted labels file generated with
[`init labels`](/api/cli#init-labels). Typically used in the
@ -650,7 +650,7 @@ label sets.
| `require` | Whether to require the file to exist. If set to `False` and the labels file doesn't exist, the loader will return `None` and the `initialize` method will extract the labels from the data. Defaults to `False`. ~~bool~~ |
| **CREATES** | The list of labels. ~~List[str]~~ |
### Corpus readers {#corpus-readers source="spacy/training/corpus.py" new="3"}
### Corpus readers {id="corpus-readers",source="spacy/training/corpus.py",version="3"}
Corpus readers are registered functions that load data and return a function
that takes the current `nlp` object and yields [`Example`](/api/example) objects
@ -660,7 +660,7 @@ with your own registered function in the
[`@readers` registry](/api/top-level#registry) to customize the data loading and
streaming.
#### spacy.Corpus.v1 {#corpus tag="registered function"}
#### spacy.Corpus.v1 {id="corpus",tag="registered function"}
The `Corpus` reader manages annotated corpora and can be used for training and
development datasets in the [DocBin](/api/docbin) (`.spacy`) format. Also see
@ -689,7 +689,7 @@ the [`Corpus`](/api/corpus) class.
| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ |
| **CREATES** | The corpus reader. ~~Corpus~~ |
#### spacy.JsonlCorpus.v1 {#jsonlcorpus tag="registered function"}
#### spacy.JsonlCorpus.v1 {id="jsonlcorpus",tag="registered function"}
Create [`Example`](/api/example) objects from a JSONL (newline-delimited JSON)
file of texts keyed by `"text"`. Can be used to read the raw text corpus for
@ -718,7 +718,7 @@ JSONL file. Also see the [`JsonlCorpus`](/api/corpus#jsonlcorpus) class.
| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ |
| **CREATES** | The corpus reader. ~~JsonlCorpus~~ |
## Batchers {#batchers source="spacy/training/batchers.py" new="3"}
## Batchers {id="batchers",source="spacy/training/batchers.py",version="3"}
A data batcher implements a batching strategy that essentially turns a stream of
items into a stream of batches, with each batch consisting of one item or a list
@ -732,7 +732,7 @@ Instead of using one of the built-in batchers listed here, you can also
[implement your own](/usage/training#custom-code-readers-batchers), which may or
may not use a custom schedule.
### spacy.batch_by_words.v1 {#batch_by_words tag="registered function"}
### spacy.batch_by_words.v1 {id="batch_by_words",tag="registered function"}
Create minibatches of roughly a given number of words. If any examples are
longer than the specified batch length, they will appear in a batch by
@ -760,7 +760,7 @@ themselves, or be discarded if `discard_oversize` is set to `True`. The argument
| `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ |
| **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ |
### spacy.batch_by_sequence.v1 {#batch_by_sequence tag="registered function"}
### spacy.batch_by_sequence.v1 {id="batch_by_sequence",tag="registered function"}
> #### Example config
>
@ -779,7 +779,7 @@ Create a batcher that creates batches of the specified size.
| `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ |
| **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ |
### spacy.batch_by_padded.v1 {#batch_by_padded tag="registered function"}
### spacy.batch_by_padded.v1 {id="batch_by_padded",tag="registered function"}
> #### Example config
>
@ -805,7 +805,7 @@ sequences in the batch.
| `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ |
| **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ |
## Augmenters {#augmenters source="spacy/training/augment.py" new="3"}
## Augmenters {id="augmenters",source="spacy/training/augment.py",version="3"}
Data augmentation is the process of applying small modifications to the training
data. It can be especially useful for punctuation and case replacement – for
@ -814,7 +814,7 @@ variations using regular quotes, or to make the model less sensitive to
capitalization by including a mix of capitalized and lowercase examples. See the
[usage guide](/usage/training#data-augmentation) for details and examples.
### spacy.orth_variants.v1 {#orth_variants tag="registered function"}
### spacy.orth_variants.v1 {id="orth_variants",tag="registered function"}
> #### Example config
>
@ -841,7 +841,7 @@ beyond corpora that don't have smart quotes, or only have smart quotes etc.
| `orth_variants` | A dictionary containing the single and paired orth variants. Typically loaded from a JSON file. See [`en_orth_variants.json`](https://github.com/explosion/spacy-lookups-data/blob/master/spacy_lookups_data/data/en_orth_variants.json) for an example. ~~Dict[str, Dict[List[Union[str, List[str]]]]]~~ |
| **CREATES** | A function that takes the current `nlp` object and an [`Example`](/api/example) and yields augmented `Example` objects. ~~Callable[[Language, Example], Iterator[Example]]~~ |
### spacy.lower_case.v1 {#lower_case tag="registered function"}
### spacy.lower_case.v1 {id="lower_case",tag="registered function"}
> #### Example config
>
@ -860,12 +860,12 @@ useful for making the model less sensitive to capitalization.
| `level` | The percentage of texts that will be augmented. ~~float~~ |
| **CREATES** | A function that takes the current `nlp` object and an [`Example`](/api/example) and yields augmented `Example` objects. ~~Callable[[Language, Example], Iterator[Example]]~~ |
## Callbacks {#callbacks source="spacy/training/callbacks.py" new="3"}
## Callbacks {id="callbacks",source="spacy/training/callbacks.py",version="3"}
The config supports [callbacks](/usage/training#custom-code-nlp-callbacks) at
several points in the lifecycle that can be used modify the `nlp` object.
### spacy.copy_from_base_model.v1 {#copy_from_base_model tag="registered function"}
### spacy.copy_from_base_model.v1 {id="copy_from_base_model",tag="registered function"}
> #### Example config
>
@ -889,7 +889,7 @@ from the specified model. Intended for use in `[initialize.before_init]`.
| `vocab` | The pipeline to copy the vocab from. The vocab includes the lookups and vectors. Defaults to `None`. ~~Optional[str]~~ |
| **CREATES** | A function that takes the current `nlp` object and modifies its `tokenizer` and `vocab`. ~~Callable[[Language], None]~~ |
### spacy.models_with_nvtx_range.v1 {#models_with_nvtx_range tag="registered function"}
### spacy.models_with_nvtx_range.v1 {id="models_with_nvtx_range",tag="registered function"}
> #### Example config
>
@ -909,7 +909,7 @@ backprop passes.
| `backprop_color` | Color identifier for backpropagation passes. Defaults to `-1`. ~~int~~ |
| **CREATES** | A function that takes the current `nlp` and wraps forward/backprop passes in NVTX ranges. ~~Callable[[Language], Language]~~ |
### spacy.models_and_pipes_with_nvtx_range.v1 {#models_and_pipes_with_nvtx_range tag="registered function" new="3.4"}
### spacy.models_and_pipes_with_nvtx_range.v1 {id="models_and_pipes_with_nvtx_range",tag="registered function",version="3.4"}
> #### Example config
>
@ -930,9 +930,9 @@ methods are wrapped: `pipe`, `predict`, `set_annotations`, `update`, `rehearse`,
| `additional_pipe_functions` | Additional pipeline methods to wrap. Keys are pipeline names and values are lists of method identifiers. Defaults to `None`. ~~Optional[Dict[str, List[str]]]~~ |
| **CREATES** | A function that takes the current `nlp` and wraps pipe models and methods in NVTX ranges. ~~Callable[[Language], Language]~~ |
## Training data and alignment {#gold source="spacy/training"}
## Training data and alignment {id="gold",source="spacy/training"}
### training.offsets_to_biluo_tags {#offsets_to_biluo_tags tag="function"}
### training.offsets_to_biluo_tags {id="offsets_to_biluo_tags",tag="function"}
Encode labelled spans into per-token tags, using the
[BILUO scheme](/usage/linguistic-features#accessing-ner) (Begin, In, Last, Unit,
@ -969,7 +969,7 @@ This method was previously available as `spacy.gold.biluo_tags_from_offsets`.
| `missing` | The label used for missing values, e.g. if tokenization doesn't align with the entity offsets. Defaults to `"O"`. ~~str~~ |
| **RETURNS** | A list of strings, describing the [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ |
### training.biluo_tags_to_offsets {#biluo_tags_to_offsets tag="function"}
### training.biluo_tags_to_offsets {id="biluo_tags_to_offsets",tag="function"}
Encode per-token tags following the
[BILUO scheme](/usage/linguistic-features#accessing-ner) into entity offsets.
@ -997,7 +997,7 @@ This method was previously available as `spacy.gold.offsets_from_biluo_tags`.
| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags with each tag describing one token. Each tag string will be of the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of `"B"`, `"I"`, `"L"`, `"U"`. ~~List[str]~~ |
| **RETURNS** | A sequence of `(start, end, label)` triples. `start` and `end` will be character-offset integers denoting the slice into the original string. ~~List[Tuple[int, int, str]]~~ |
### training.biluo_tags_to_spans {#biluo_tags_to_spans tag="function" new="2.1"}
### training.biluo_tags_to_spans {id="biluo_tags_to_spans",tag="function",version="2.1"}
Encode per-token tags following the
[BILUO scheme](/usage/linguistic-features#accessing-ner) into
@ -1026,7 +1026,7 @@ This method was previously available as `spacy.gold.spans_from_biluo_tags`.
| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags with each tag describing one token. Each tag string will be of the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of `"B"`, `"I"`, `"L"`, `"U"`. ~~List[str]~~ |
| **RETURNS** | A sequence of `Span` objects with added entity labels. ~~List[Span]~~ |
### training.biluo_to_iob {#biluo_to_iob tag="function"}
### training.biluo_to_iob {id="biluo_to_iob",tag="function"}
Convert a sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags to
[IOB](/usage/linguistic-features#accessing-ner) tags. This is useful if you want
@ -1047,7 +1047,7 @@ use the BILUO tags with a model that only supports IOB tags.
| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ |
| **RETURNS** | A list of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ |
### training.iob_to_biluo {#iob_to_biluo tag="function"}
### training.iob_to_biluo {id="iob_to_biluo",tag="function"}
Convert a sequence of [IOB](/usage/linguistic-features#accessing-ner) tags to
[BILUO](/usage/linguistic-features#accessing-ner) tags. This is useful if you
@ -1074,7 +1074,55 @@ This method was previously available as `spacy.gold.iob_to_biluo`.
| `tags` | A sequence of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ |
| **RETURNS** | A list of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ |
## Utility functions {#util source="spacy/util.py"}
### training.biluo_to_iob {id="biluo_to_iob",tag="function"}
Convert a sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags to
[IOB](/usage/linguistic-features#accessing-ner) tags. This is useful if you want
use the BILUO tags with a model that only supports IOB tags.
> #### Example
>
> ```python
> from spacy.training import biluo_to_iob
>
> tags = ["O", "O", "B-LOC", "I-LOC", "L-LOC", "O"]
> iob_tags = biluo_to_iob(tags)
> assert iob_tags == ["O", "O", "B-LOC", "I-LOC", "I-LOC", "O"]
> ```
| Name | Description |
| ----------- | --------------------------------------------------------------------------------------- |
| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ |
| **RETURNS** | A list of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ |
### training.iob_to_biluo {id="iob_to_biluo",tag="function"}
Convert a sequence of [IOB](/usage/linguistic-features#accessing-ner) tags to
[BILUO](/usage/linguistic-features#accessing-ner) tags. This is useful if you
want use the IOB tags with a model that only supports BILUO tags.
<Infobox title="Changed in v3.0" variant="warning" id="iob_to_biluo">
This method was previously available as `spacy.gold.iob_to_biluo`.
</Infobox>
> #### Example
>
> ```python
> from spacy.training import iob_to_biluo
>
> tags = ["O", "O", "B-LOC", "I-LOC", "O"]
> biluo_tags = iob_to_biluo(tags)
> assert biluo_tags == ["O", "O", "B-LOC", "L-LOC", "O"]
> ```
| Name | Description |
| ----------- | ------------------------------------------------------------------------------------- |
| `tags` | A sequence of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ |
| **RETURNS** | A list of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ |
## Utility functions {id="util",source="spacy/util.py"}
spaCy comes with a small collection of utility functions located in
[`spacy/util.py`](%%GITHUB_SPACY/spacy/util.py). Because utility functions are
@ -1084,7 +1132,7 @@ use and we'll try to ensure backwards compatibility. However, we recommend
having additional tests in place if your application depends on any of spaCy's
utilities.
### util.get_lang_class {#util.get_lang_class tag="function"}
### util.get_lang_class {id="util.get_lang_class",tag="function"}
Import and load a `Language` class. Allows lazy-loading
[language data](/usage/linguistic-features#language-data) and importing
@ -1105,7 +1153,7 @@ custom language class, you can register it using the
| `lang` | Two-letter language code, e.g. `"en"`. ~~str~~ |
| **RETURNS** | The respective subclass. ~~Language~~ |
### util.lang_class_is_loaded {#util.lang_class_is_loaded tag="function" new="2.1"}
### util.lang_class_is_loaded {id="util.lang_class_is_loaded",tag="function",version="2.1"}
Check whether a `Language` subclass is already loaded. `Language` subclasses are
loaded lazily to avoid expensive setup code associated with the language data.
@ -1123,7 +1171,7 @@ loaded lazily to avoid expensive setup code associated with the language data.
| `name` | Two-letter language code, e.g. `"en"`. ~~str~~ |
| **RETURNS** | Whether the class has been loaded. ~~bool~~ |
### util.load_model {#util.load_model tag="function" new="2"}
### util.load_model {id="util.load_model",tag="function",version="2"}
Load a pipeline from a package or data path. If called with a string name, spaCy
will assume the pipeline is a Python package and import and call its `load()`
@ -1151,7 +1199,7 @@ and create a `Language` object. The model data will then be loaded in via
| `config` <Tag variant="new">3</Tag> | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ |
| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ |
### util.load_model_from_init_py {#util.load_model_from_init_py tag="function" new="2"}
### util.load_model_from_init_py {id="util.load_model_from_init_py",tag="function",version="2"}
A helper function to use in the `load()` method of a pipeline package's
[`__init__.py`](https://github.com/explosion/spacy-models/tree/master/template/model/xx_model_name/__init__.py).
@ -1176,7 +1224,7 @@ A helper function to use in the `load()` method of a pipeline package's
| `config` <Tag variant="new">3</Tag> | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ |
| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ |
### util.load_config {#util.load_config tag="function" new="3"}
### util.load_config {id="util.load_config",tag="function",version="3"}
Load a pipeline's [`config.cfg`](/api/data-formats#config) from a file path. The
config typically includes details about the components and how they're created,
@ -1196,7 +1244,7 @@ as well as all training settings and hyperparameters.
| `interpolate` | Whether to interpolate the config and replace variables like `${paths.train}` with their values. Defaults to `False`. ~~bool~~ |
| **RETURNS** | The pipeline's config. ~~Config~~ |
### util.load_meta {#util.load_meta tag="function" new="3"}
### util.load_meta {id="util.load_meta",tag="function",version="3"}
Get a pipeline's [`meta.json`](/api/data-formats#meta) from a file path and
validate its contents. The meta typically includes details about author,
@ -1213,7 +1261,7 @@ licensing, data sources and version.
| `path` | Path to the pipeline's `meta.json`. ~~Union[str, Path]~~ |
| **RETURNS** | The pipeline's meta data. ~~Dict[str, Any]~~ |
### util.get_installed_models {#util.get_installed_models tag="function" new="3"}
### util.get_installed_models {id="util.get_installed_models",tag="function",version="3"}
List all pipeline packages installed in the current environment. This will
include any spaCy pipeline that was packaged with
@ -1231,7 +1279,7 @@ object.
| ----------- | ------------------------------------------------------------------------------------- |
| **RETURNS** | The string names of the pipelines installed in the current environment. ~~List[str]~~ |
### util.is_package {#util.is_package tag="function"}
### util.is_package {id="util.is_package",tag="function"}
Check if string maps to a package installed via pip. Mainly used to validate
[pipeline packages](/usage/models).
@ -1248,7 +1296,7 @@ Check if string maps to a package installed via pip. Mainly used to validate
| `name` | Name of package. ~~str~~ |
| **RETURNS** | `True` if installed package, `False` if not. ~~bool~~ |
### util.get_package_path {#util.get_package_path tag="function" new="2"}
### util.get_package_path {id="util.get_package_path",tag="function",version="2"}
Get path to an installed package. Mainly used to resolve the location of
[pipeline packages](/usage/models). Currently imports the package to find its
@ -1266,7 +1314,7 @@ path.
| `package_name` | Name of installed package. ~~str~~ |
| **RETURNS** | Path to pipeline package directory. ~~Path~~ |
### util.is_in_jupyter {#util.is_in_jupyter tag="function" new="2"}
### util.is_in_jupyter {id="util.is_in_jupyter",tag="function",version="2"}
Check if user is running spaCy from a [Jupyter](https://jupyter.org) notebook by
detecting the IPython kernel. Mainly used for the
@ -1285,7 +1333,7 @@ detecting the IPython kernel. Mainly used for the
| ----------- | ---------------------------------------------- |
| **RETURNS** | `True` if in Jupyter, `False` if not. ~~bool~~ |
### util.compile_prefix_regex {#util.compile_prefix_regex tag="function"}
### util.compile_prefix_regex {id="util.compile_prefix_regex",tag="function"}
Compile a sequence of prefix rules into a regex object.
@ -1302,7 +1350,7 @@ Compile a sequence of prefix rules into a regex object.
| `entries` | The prefix rules, e.g. [`lang.punctuation.TOKENIZER_PREFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ |
| **RETURNS** | The regex object to be used for [`Tokenizer.prefix_search`](/api/tokenizer#attributes). ~~Pattern~~ |
### util.compile_suffix_regex {#util.compile_suffix_regex tag="function"}
### util.compile_suffix_regex {id="util.compile_suffix_regex",tag="function"}
Compile a sequence of suffix rules into a regex object.
@ -1319,7 +1367,7 @@ Compile a sequence of suffix rules into a regex object.
| `entries` | The suffix rules, e.g. [`lang.punctuation.TOKENIZER_SUFFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ |
| **RETURNS** | The regex object to be used for [`Tokenizer.suffix_search`](/api/tokenizer#attributes). ~~Pattern~~ |
### util.compile_infix_regex {#util.compile_infix_regex tag="function"}
### util.compile_infix_regex {id="util.compile_infix_regex",tag="function"}
Compile a sequence of infix rules into a regex object.
@ -1336,7 +1384,7 @@ Compile a sequence of infix rules into a regex object.
| `entries` | The infix rules, e.g. [`lang.punctuation.TOKENIZER_INFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ |
| **RETURNS** | The regex object to be used for [`Tokenizer.infix_finditer`](/api/tokenizer#attributes). ~~Pattern~~ |
### util.minibatch {#util.minibatch tag="function" new="2"}
### util.minibatch {id="util.minibatch",tag="function",version="2"}
Iterate over batches of items. `size` may be an iterator, so that batch-size can
vary on each step.
@ -1355,7 +1403,7 @@ vary on each step.
| `size` | The batch size(s). ~~Union[int, Sequence[int]]~~ |
| **YIELDS** | The batches. |
### util.filter_spans {#util.filter_spans tag="function" new="2.1.4"}
### util.filter_spans {id="util.filter_spans",tag="function",version="2.1.4"}
Filter a sequence of [`Span`](/api/span) objects and remove duplicates or
overlaps. Useful for creating named entities (where one token can only be part
@ -1376,7 +1424,7 @@ of one entity) or when merging spans with
| `spans` | The spans to filter. ~~Iterable[Span]~~ |
| **RETURNS** | The filtered spans. ~~List[Span]~~ |
### util.get_words_and_spaces {#get_words_and_spaces tag="function" new="3"}
### util.get_words_and_spaces {id="get_words_and_spaces",tag="function",version="3"}
Given a list of words and a text, reconstruct the original tokens and return a
list of words and spaces that can be used to create a [`Doc`](/api/doc#init).

54
website/docs/api/transformer.md → website/docs/api/transformer.mdx
View File

@ -3,7 +3,7 @@ title: Transformer
teaser: Pipeline component for multi-task learning with transformer models
tag: class
source: github.com/explosion/spacy-transformers/blob/master/spacy_transformers/pipeline_component.py
new: 3
version: 3
api_base_class: /api/pipe
api_string_name: transformer
---
@ -44,7 +44,7 @@ package also adds the function registries [`@span_getters`](#span_getters) and
functions. For more details, see the
[usage documentation](/usage/embeddings-transformers).
## Assigned Attributes {#assigned-attributes}
## Assigned Attributes {id="assigned-attributes"}
The component sets the following
[custom extension attribute](/usage/processing-pipeline#custom-components-attributes):
@ -53,7 +53,7 @@ The component sets the following
| ---------------- | ------------------------------------------------------------------------ |
| `Doc._.trf_data` | Transformer tokens and outputs for the `Doc` object. ~~TransformerData~~ |
## Config and implementation {#config}
## Config and implementation {id="config"}
The default config is defined by the pipeline component factory and describes
how the component should be configured. You can override its settings via the
@ -81,7 +81,7 @@ on the transformer architectures and their arguments and hyperparameters.
https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/pipeline_component.py
```
## Transformer.\_\_init\_\_ {#init tag="method"}
## Transformer.\_\_init\_\_ {id="init",tag="method"}
> #### Example
>
@ -124,7 +124,7 @@ component using its string name and [`nlp.add_pipe`](/api/language#create_pipe).
| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ |
| `max_batch_items` | Maximum size of a padded batch. Defaults to `128*32`. ~~int~~ |
## Transformer.\_\_call\_\_ {#call tag="method"}
## Transformer.\_\_call\_\_ {id="call",tag="method"}
Apply the pipe to one document. The document is modified in place, and returned.
This usually happens under the hood when the `nlp` object is called on a text
@ -147,7 +147,7 @@ to the [`predict`](/api/transformer#predict) and
| `doc` | The document to process. ~~Doc~~ |
| **RETURNS** | The processed document. ~~Doc~~ |
## Transformer.pipe {#pipe tag="method"}
## Transformer.pipe {id="pipe",tag="method"}
Apply the pipe to a stream of documents. This usually happens under the hood
when the `nlp` object is called on a text and all pipeline components are
@ -171,7 +171,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/transformer#call) and
| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ |
| **YIELDS** | The processed documents in order. ~~Doc~~ |
## Transformer.initialize {#initialize tag="method"}
## Transformer.initialize {id="initialize",tag="method"}
Initialize the component for training and return an
[`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a
@ -196,7 +196,7 @@ by [`Language.initialize`](/api/language#initialize).
| _keyword-only_ | |
| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ |
## Transformer.predict {#predict tag="method"}
## Transformer.predict {id="predict",tag="method"}
Apply the component's model to a batch of [`Doc`](/api/doc) objects without
modifying them.
@ -213,7 +213,7 @@ modifying them.
| `docs` | The documents to predict. ~~Iterable[Doc]~~ |
| **RETURNS** | The model's prediction for each document. |
## Transformer.set_annotations {#set_annotations tag="method"}
## Transformer.set_annotations {id="set_annotations",tag="method"}
Assign the extracted features to the `Doc` objects. By default, the
[`TransformerData`](/api/transformer#transformerdata) object is written to the
@ -233,7 +233,7 @@ callback is then called, if provided.
| `docs` | The documents to modify. ~~Iterable[Doc]~~ |
| `scores` | The scores to set, produced by `Transformer.predict`. |
## Transformer.update {#update tag="method"}
## Transformer.update {id="update",tag="method"}
Prepare for an update to the transformer. Like the [`Tok2Vec`](/api/tok2vec)
component, the `Transformer` component is unusual in that it does not receive
@ -266,7 +266,7 @@ and call the optimizer, while the others simply increment the gradients.
| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ |
| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ |
## Transformer.create_optimizer {#create_optimizer tag="method"}
## Transformer.create_optimizer {id="create_optimizer",tag="method"}
Create an optimizer for the pipeline component.
@ -281,7 +281,7 @@ Create an optimizer for the pipeline component.
| ----------- | ---------------------------- |
| **RETURNS** | The optimizer. ~~Optimizer~~ |
## Transformer.use_params {#use_params tag="method, contextmanager"}
## Transformer.use_params {id="use_params",tag="method, contextmanager"}
Modify the pipe's model to use the given parameter values. At the end of the
context, the original parameters are restored.
@ -298,7 +298,7 @@ context, the original parameters are restored.
| -------- | -------------------------------------------------- |
| `params` | The parameter values to use in the model. ~~dict~~ |
## Transformer.to_disk {#to_disk tag="method"}
## Transformer.to_disk {id="to_disk",tag="method"}
Serialize the pipe to disk.
@ -315,7 +315,7 @@ Serialize the pipe to disk.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## Transformer.from_disk {#from_disk tag="method"}
## Transformer.from_disk {id="from_disk",tag="method"}
Load the pipe from disk. Modifies the object in place and returns it.
@ -333,7 +333,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `Transformer` object. ~~Transformer~~ |
## Transformer.to_bytes {#to_bytes tag="method"}
## Transformer.to_bytes {id="to_bytes",tag="method"}
> #### Example
>
@ -350,7 +350,7 @@ Serialize the pipe to a bytestring.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `Transformer` object. ~~bytes~~ |
## Transformer.from_bytes {#from_bytes tag="method"}
## Transformer.from_bytes {id="from_bytes",tag="method"}
Load the pipe from a bytestring. Modifies the object in place and returns it.
@ -369,7 +369,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Transformer` object. ~~Transformer~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from
@ -387,7 +387,7 @@ serialization by passing in the string names via the `exclude` argument.
| `cfg` | The config file. You usually don't want to exclude this. |
| `model` | The binary model data. You usually don't want to exclude this. |
## TransformerData {#transformerdata tag="dataclass"}
## TransformerData {id="transformerdata",tag="dataclass"}
Transformer tokens and outputs for one `Doc` object. The transformer models
return tensors that refer to a whole padded batch of documents. These tensors
@ -405,7 +405,7 @@ by this class. Instances of this class are typically assigned to the
| `align` | Alignment from the `Doc`'s tokenization to the wordpieces. This is a ragged array, where `align.lengths[i]` indicates the number of wordpiece tokens that token `i` aligns against. The actual indices are provided at `align[i].dataXd`. ~~Ragged~~ |
| `width` | The width of the last hidden layer. ~~int~~ |
### TransformerData.empty {#transformerdata-emoty tag="classmethod"}
### TransformerData.empty {id="transformerdata-emoty",tag="classmethod"}
Create an empty `TransformerData` container.
@ -425,7 +425,7 @@ model.
</Accordion>
## FullTransformerBatch {#fulltransformerbatch tag="dataclass"}
## FullTransformerBatch {id="fulltransformerbatch",tag="dataclass"}
Holds a batch of input and output objects for a transformer model. The data can
then be split to a list of [`TransformerData`](/api/transformer#transformerdata)
@ -440,7 +440,7 @@ objects to associate the outputs to each [`Doc`](/api/doc) in the batch.
| `align` | Alignment from the spaCy tokenization to the wordpieces. This is a ragged array, where `align.lengths[i]` indicates the number of wordpiece tokens that token `i` aligns against. The actual indices are provided at `align[i].dataXd`. ~~Ragged~~ |
| `doc_data` | The outputs, split per `Doc` object. ~~List[TransformerData]~~ |
### FullTransformerBatch.unsplit_by_doc {#fulltransformerbatch-unsplit_by_doc tag="method"}
### FullTransformerBatch.unsplit_by_doc {id="fulltransformerbatch-unsplit_by_doc",tag="method"}
Return a new `FullTransformerBatch` from a split batch of activations, using the
current object's spans, tokens and alignment. This is used during the backward
@ -452,7 +452,7 @@ model.
| `arrays` | The split batch of activations. ~~List[List[Floats3d]]~~ |
| **RETURNS** | The transformer batch. ~~FullTransformerBatch~~ |
### FullTransformerBatch.split_by_doc {#fulltransformerbatch-split_by_doc tag="method"}
### FullTransformerBatch.split_by_doc {id="fulltransformerbatch-split_by_doc",tag="method"}
Split a `TransformerData` object that represents a batch into a list with one
`TransformerData` per `Doc`.
@ -468,7 +468,7 @@ In `spacy-transformers` v1.0, the model output is stored in
</Accordion>
## Span getters {#span_getters source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/span_getters.py"}
## Span getters {id="span_getters",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/span_getters.py"}
Span getters are functions that take a batch of [`Doc`](/api/doc) objects and
return a lists of [`Span`](/api/span) objects for each doc to be processed by
@ -498,7 +498,7 @@ using the `@spacy.registry.span_getters` decorator.
| `docs` | A batch of `Doc` objects. ~~Iterable[Doc]~~ |
| **RETURNS** | The spans to process by the transformer. ~~List[List[Span]]~~ |
### doc_spans.v1 {#doc_spans tag="registered function"}
### doc_spans.v1 {id="doc_spans",tag="registered function"}
> #### Example config
>
@ -511,7 +511,7 @@ Create a span getter that uses the whole document as its spans. This is the best
approach if your [`Doc`](/api/doc) objects already refer to relatively short
texts.
### sent_spans.v1 {#sent_spans tag="registered function"}
### sent_spans.v1 {id="sent_spans",tag="registered function"}
> #### Example config
>
@ -531,7 +531,7 @@ To set sentence boundaries with the `sentencizer` during training, add a
[`[training.annotating_components]`](/usage/training#annotating-components) to
have it set the sentence boundaries before the `transformer` component runs.
### strided_spans.v1 {#strided_spans tag="registered function"}
### strided_spans.v1 {id="strided_spans",tag="registered function"}
> #### Example config
>
@ -553,7 +553,7 @@ right context.
| `window` | The window size. ~~int~~ |
| `stride` | The stride size. ~~int~~ |
## Annotation setters {#annotation_setters tag="registered functions" source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/annotation_setters.py"}
## Annotation setters {id="annotation_setters",tag="registered functions",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/annotation_setters.py"}
Annotation setters are functions that take a batch of `Doc` objects and a
[`FullTransformerBatch`](/api/transformer#fulltransformerbatch) and can set

50
website/docs/api/vectors.md → website/docs/api/vectors.mdx
View File

@ -3,7 +3,7 @@ title: Vectors
teaser: Store, save and load word vectors
tag: class
source: spacy/vectors.pyx
new: 2
version: 2
---
Vectors data is kept in the `Vectors.data` attribute, which should be an
@ -25,7 +25,7 @@ As of spaCy v3.2, `Vectors` supports two types of vector tables:
the sum of one or more rows as determined by the settings related to character
ngrams and the hash table.
## Vectors.\_\_init\_\_ {#init tag="method"}
## Vectors.\_\_init\_\_ {id="init",tag="method"}
Create a new vector store. With the default mode, you can set the vector values
and keys directly on initialization, or supply a `shape` keyword argument to
@ -61,7 +61,7 @@ modified later.
| `bow` <Tag variant="new">3.2</Tag> | The floret BOW string (default: `"<"`). ~~str~~ |
| `eow` <Tag variant="new">3.2</Tag> | The floret EOW string (default: `">"`). ~~str~~ |
## Vectors.\_\_getitem\_\_ {#getitem tag="method"}
## Vectors.\_\_getitem\_\_ {id="getitem",tag="method"}
Get a vector by key. If the key is not found in the table, a `KeyError` is
raised.
@ -79,7 +79,7 @@ raised.
| `key` | The key to get the vector for. ~~Union[int, str]~~ |
| **RETURNS** | The vector for the key. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
## Vectors.\_\_setitem\_\_ {#setitem tag="method"}
## Vectors.\_\_setitem\_\_ {id="setitem",tag="method"}
Set a vector for the given key. Not supported for `floret` mode.
@ -96,7 +96,7 @@ Set a vector for the given key. Not supported for `floret` mode.
| `key` | The key to set the vector for. ~~int~~ |
| `vector` | The vector to set. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
## Vectors.\_\_iter\_\_ {#iter tag="method"}
## Vectors.\_\_iter\_\_ {id="iter",tag="method"}
Iterate over the keys in the table. In `floret` mode, the keys table is not
used.
@ -112,7 +112,7 @@ used.
| ---------- | --------------------------- |
| **YIELDS** | A key in the table. ~~int~~ |
## Vectors.\_\_len\_\_ {#len tag="method"}
## Vectors.\_\_len\_\_ {id="len",tag="method"}
Return the number of vectors in the table.
@ -127,7 +127,7 @@ Return the number of vectors in the table.
| ----------- | ------------------------------------------- |
| **RETURNS** | The number of vectors in the table. ~~int~~ |
## Vectors.\_\_contains\_\_ {#contains tag="method"}
## Vectors.\_\_contains\_\_ {id="contains",tag="method"}
Check whether a key has been mapped to a vector entry in the table. In `floret`
mode, returns `True` for all keys.
@ -145,7 +145,7 @@ mode, returns `True` for all keys.
| `key` | The key to check. ~~int~~ |
| **RETURNS** | Whether the key has a vector entry. ~~bool~~ |
## Vectors.add {#add tag="method"}
## Vectors.add {id="add",tag="method"}
Add a key to the table, optionally setting a vector value as well. Keys can be
mapped to an existing vector by setting `row`, or a new vector can be added. Not
@ -168,7 +168,7 @@ supported for `floret` mode.
| `row` | An optional row number of a vector to map the key to. ~~int~~ |
| **RETURNS** | The row the vector was added to. ~~int~~ |
## Vectors.resize {#resize tag="method"}
## Vectors.resize {id="resize",tag="method"}
Resize the underlying vectors array. If `inplace=True`, the memory is
reallocated. This may cause other references to the data to become invalid, so
@ -189,7 +189,7 @@ for `floret` mode.
| `inplace` | Reallocate the memory. ~~bool~~ |
| **RETURNS** | The removed items as a list of `(key, row)` tuples. ~~List[Tuple[int, int]]~~ |
## Vectors.keys {#keys tag="method"}
## Vectors.keys {id="keys",tag="method"}
A sequence of the keys in the table. In `floret` mode, the keys table is not
used.
@ -205,7 +205,7 @@ used.
| ----------- | --------------------------- |
| **RETURNS** | The keys. ~~Iterable[int]~~ |
## Vectors.values {#values tag="method"}
## Vectors.values {id="values",tag="method"}
Iterate over vectors that have been assigned to at least one key. Note that some
vectors may be unassigned, so the number of vectors returned may be less than
@ -222,7 +222,7 @@ the length of the vectors table. In `floret` mode, the keys table is not used.
| ---------- | --------------------------------------------------------------- |
| **YIELDS** | A vector in the table. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
## Vectors.items {#items tag="method"}
## Vectors.items {id="items",tag="method"}
Iterate over `(key, vector)` pairs, in order. In `floret` mode, the keys table
is empty.
@ -238,7 +238,7 @@ is empty.
| ---------- | ------------------------------------------------------------------------------------- |
| **YIELDS** | `(key, vector)` pairs, in order. ~~Tuple[int, numpy.ndarray[ndim=1, dtype=float32]]~~ |
## Vectors.find {#find tag="method"}
## Vectors.find {id="find",tag="method"}
Look up one or more keys by row, or vice versa. Not supported for `floret` mode.
@ -260,7 +260,7 @@ Look up one or more keys by row, or vice versa. Not supported for `floret` mode.
| `rows` | Find the keys that point to the rows. Returns `numpy.ndarray`. ~~Iterable[int]~~ |
| **RETURNS** | The requested key, keys, row or rows. ~~Union[int, numpy.ndarray[ndim=1, dtype=float32]]~~ |
## Vectors.shape {#shape tag="property"}
## Vectors.shape {id="shape",tag="property"}
Get `(rows, dims)` tuples of number of rows and number of dimensions in the
vector table.
@ -279,7 +279,7 @@ vector table.
| ----------- | ------------------------------------------ |
| **RETURNS** | A `(rows, dims)` pair. ~~Tuple[int, int]~~ |
## Vectors.size {#size tag="property"}
## Vectors.size {id="size",tag="property"}
The vector size, i.e. `rows * dims`.
@ -294,7 +294,7 @@ The vector size, i.e. `rows * dims`.
| ----------- | ------------------------ |
| **RETURNS** | The vector size. ~~int~~ |
## Vectors.is_full {#is_full tag="property"}
## Vectors.is_full {id="is_full",tag="property"}
Whether the vectors table is full and has no slots are available for new keys.
If a table is full, it can be resized using
@ -313,7 +313,7 @@ full and cannot be resized.
| ----------- | ------------------------------------------- |
| **RETURNS** | Whether the vectors table is full. ~~bool~~ |
## Vectors.n_keys {#n_keys tag="property"}
## Vectors.n_keys {id="n_keys",tag="property"}
Get the number of keys in the table. Note that this is the number of _all_ keys,
not just unique vectors. If several keys are mapped to the same vectors, they
@ -331,7 +331,7 @@ will be counted individually. In `floret` mode, the keys table is not used.
| ----------- | ----------------------------------------------------------------------------- |
| **RETURNS** | The number of all keys in the table. Returns `-1` for floret vectors. ~~int~~ |
## Vectors.most_similar {#most_similar tag="method"}
## Vectors.most_similar {id="most_similar",tag="method"}
For each of the given vectors, find the `n` most similar entries to it by
cosine. Queries are by vector. Results are returned as a
@ -356,7 +356,7 @@ supported for `floret` mode.
| `sort` | Whether to sort the entries returned by score. Defaults to `True`. ~~bool~~ |
| **RETURNS** | The most similar entries as a `(keys, best_rows, scores)` tuple. ~~Tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray]~~ |
## Vectors.get_batch {#get_batch tag="method" new="3.2"}
## Vectors.get_batch {id="get_batch",tag="method",version="3.2"}
Get the vectors for the provided keys efficiently as a batch.
@ -371,7 +371,7 @@ Get the vectors for the provided keys efficiently as a batch.
| ------ | --------------------------------------- |
| `keys` | The keys. ~~Iterable[Union[int, str]]~~ |
## Vectors.to_ops {#to_ops tag="method"}
## Vectors.to_ops {id="to_ops",tag="method"}
Change the embedding matrix to use different Thinc ops.
@ -388,7 +388,7 @@ Change the embedding matrix to use different Thinc ops.
| ----- | -------------------------------------------------------- |
| `ops` | The Thinc ops to switch the embedding matrix to. ~~Ops~~ |
## Vectors.to_disk {#to_disk tag="method"}
## Vectors.to_disk {id="to_disk",tag="method"}
Save the current state to a directory.
@ -403,7 +403,7 @@ Save the current state to a directory.
| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ |
| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
## Vectors.from_disk {#from_disk tag="method"}
## Vectors.from_disk {id="from_disk",tag="method"}
Loads state from a directory. Modifies the object in place and returns it.
@ -419,7 +419,7 @@ Loads state from a directory. Modifies the object in place and returns it.
| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ |
| **RETURNS** | The modified `Vectors` object. ~~Vectors~~ |
## Vectors.to_bytes {#to_bytes tag="method"}
## Vectors.to_bytes {id="to_bytes",tag="method"}
Serialize the current state to a binary string.
@ -433,7 +433,7 @@ Serialize the current state to a binary string.
| ----------- | ------------------------------------------------------ |
| **RETURNS** | The serialized form of the `Vectors` object. ~~bytes~~ |
## Vectors.from_bytes {#from_bytes tag="method"}
## Vectors.from_bytes {id="from_bytes",tag="method"}
Load state from a binary string.
@ -451,7 +451,7 @@ Load state from a binary string.
| `data` | The data to load from. ~~bytes~~ |
| **RETURNS** | The `Vectors` object. ~~Vectors~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
| Name | Description |
| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |

36
website/docs/api/vocab.md → website/docs/api/vocab.mdx
View File

@ -10,7 +10,7 @@ The `Vocab` object provides a lookup table that allows you to access
[`StringStore`](/api/stringstore). It also owns underlying C-data that is shared
between `Doc` objects.
## Vocab.\_\_init\_\_ {#init tag="method"}
## Vocab.\_\_init\_\_ {id="init",tag="method"}
Create the vocabulary.
@ -31,7 +31,7 @@ Create the vocabulary.
| `writing_system` | A dictionary describing the language's writing system. Typically provided by [`Language.Defaults`](/api/language#defaults). ~~Dict[str, Any]~~ |
| `get_noun_chunks` | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ |
## Vocab.\_\_len\_\_ {#len tag="method"}
## Vocab.\_\_len\_\_ {id="len",tag="method"}
Get the current number of lexemes in the vocabulary.
@ -46,7 +46,7 @@ Get the current number of lexemes in the vocabulary.
| ----------- | ------------------------------------------------ |
| **RETURNS** | The number of lexemes in the vocabulary. ~~int~~ |
## Vocab.\_\_getitem\_\_ {#getitem tag="method"}
## Vocab.\_\_getitem\_\_ {id="getitem",tag="method"}
Retrieve a lexeme, given an int ID or a string. If a previously unseen string is
given, a new lexeme is created and stored.
@ -63,7 +63,7 @@ given, a new lexeme is created and stored.
| `id_or_string` | The hash value of a word, or its string. ~~Union[int, str]~~ |
| **RETURNS** | The lexeme indicated by the given ID. ~~Lexeme~~ |
## Vocab.\_\_iter\_\_ {#iter tag="method"}
## Vocab.\_\_iter\_\_ {id="iter",tag="method"}
Iterate over the lexemes in the vocabulary.
@ -77,7 +77,7 @@ Iterate over the lexemes in the vocabulary.
| ---------- | -------------------------------------- |
| **YIELDS** | An entry in the vocabulary. ~~Lexeme~~ |
## Vocab.\_\_contains\_\_ {#contains tag="method"}
## Vocab.\_\_contains\_\_ {id="contains",tag="method"}
Check whether the string has an entry in the vocabulary. To get the ID for a
given string, you need to look it up in
@ -97,7 +97,7 @@ given string, you need to look it up in
| `string` | The ID string. ~~str~~ |
| **RETURNS** | Whether the string has an entry in the vocabulary. ~~bool~~ |
## Vocab.add_flag {#add_flag tag="method"}
## Vocab.add_flag {id="add_flag",tag="method"}
Set a new boolean flag to words in the vocabulary. The `flag_getter` function
will be called over the words currently in the vocab, and then applied to new
@ -122,7 +122,7 @@ using `token.check_flag(flag_id)`.
| `flag_id` | An integer between `1` and `63` (inclusive), specifying the bit at which the flag will be stored. If `-1`, the lowest available bit will be chosen. ~~int~~ |
| **RETURNS** | The integer ID by which the flag value can be checked. ~~int~~ |
## Vocab.reset_vectors {#reset_vectors tag="method" new="2"}
## Vocab.reset_vectors {id="reset_vectors",tag="method",version="2"}
Drop the current vector table. Because all vectors must be the same width, you
have to call this to change the size of the vectors. Only one of the `width` and
@ -140,7 +140,7 @@ have to call this to change the size of the vectors. Only one of the `width` and
| `width` | The new width. ~~int~~ |
| `shape` | The new shape. ~~int~~ |
## Vocab.prune_vectors {#prune_vectors tag="method" new="2"}
## Vocab.prune_vectors {id="prune_vectors",tag="method",version="2"}
Reduce the current vector table to `nr_row` unique entries. Words mapped to the
discarded vectors will be remapped to the closest vector among those remaining.
@ -165,7 +165,7 @@ cosines are calculated in minibatches to reduce memory usage.
| `batch_size` | Batch of vectors for calculating the similarities. Larger batch sizes might be faster, while temporarily requiring more memory. ~~int~~ |
| **RETURNS** | A dictionary keyed by removed words mapped to `(string, score)` tuples, where `string` is the entry the removed word was mapped to, and `score` the similarity score between the two words. ~~Dict[str, Tuple[str, float]]~~ |
## Vocab.deduplicate_vectors {#deduplicate_vectors tag="method" new="3.3"}
## Vocab.deduplicate_vectors {id="deduplicate_vectors",tag="method",version="3.3"}
> #### Example
>
@ -176,7 +176,7 @@ cosines are calculated in minibatches to reduce memory usage.
Remove any duplicate rows from the current vector table, maintaining the
mappings for all words in the vectors.
## Vocab.get_vector {#get_vector tag="method" new="2"}
## Vocab.get_vector {id="get_vector",tag="method",version="2"}
Retrieve a vector for a word in the vocabulary. Words can be looked up by string
or hash value. If the current vectors do not contain an entry for the word, a
@ -194,7 +194,7 @@ or hash value. If the current vectors do not contain an entry for the word, a
| `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ |
| **RETURNS** | A word vector. Size and shape are determined by the `Vocab.vectors` instance. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
## Vocab.set_vector {#set_vector tag="method" new="2"}
## Vocab.set_vector {id="set_vector",tag="method",version="2"}
Set a vector for a word in the vocabulary. Words can be referenced by string or
hash value.
@ -210,7 +210,7 @@ hash value.
| `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ |
| `vector` | The vector to set. ~~numpy.ndarray[ndim=1, dtype=float32]~~ |
## Vocab.has_vector {#has_vector tag="method" new="2"}
## Vocab.has_vector {id="has_vector",tag="method",version="2"}
Check whether a word has a vector. Returns `False` if no vectors are loaded.
Words can be looked up by string or hash value.
@ -227,7 +227,7 @@ Words can be looked up by string or hash value.
| `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ |
| **RETURNS** | Whether the word has a vector. ~~bool~~ |
## Vocab.to_disk {#to_disk tag="method" new="2"}
## Vocab.to_disk {id="to_disk",tag="method",version="2"}
Save the current state to a directory.
@ -243,7 +243,7 @@ Save the current state to a directory.
| _keyword-only_ | |
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
## Vocab.from_disk {#from_disk tag="method" new="2"}
## Vocab.from_disk {id="from_disk",tag="method",version="2"}
Loads state from a directory. Modifies the object in place and returns it.
@ -261,7 +261,7 @@ Loads state from a directory. Modifies the object in place and returns it.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The modified `Vocab` object. ~~Vocab~~ |
## Vocab.to_bytes {#to_bytes tag="method"}
## Vocab.to_bytes {id="to_bytes",tag="method"}
Serialize the current state to a binary string.
@ -277,7 +277,7 @@ Serialize the current state to a binary string.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The serialized form of the `Vocab` object. ~~Vocab~~ |
## Vocab.from_bytes {#from_bytes tag="method"}
## Vocab.from_bytes {id="from_bytes",tag="method"}
Load state from a binary string.
@ -297,7 +297,7 @@ Load state from a binary string.
| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ |
| **RETURNS** | The `Vocab` object. ~~Vocab~~ |
## Attributes {#attributes}
## Attributes {id="attributes"}
> #### Example
>
@ -317,7 +317,7 @@ Load state from a binary string.
| `writing_system` | A dict with information about the language's writing system. ~~Dict[str, Any]~~ |
| `get_noun_chunks` <Tag variant="new">3.0</Tag> | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ |
## Serialization fields {#serialization-fields}
## Serialization fields {id="serialization-fields"}
During serialization, spaCy will export several data fields used to restore
different aspects of the object. If needed, you can exclude them from

58
website/docs/images/displacy-dep-founded.html
View File

@ -1,58 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" xlink="http://www.w3.org/1999/xlink" xml:lang="en" id="c3124cc3e661444cb9d4175a5b7c09d1-0" class="displacy" width="925" height="399.5" direction="ltr" style="max-width: none; height: 399.5px; color: #000000; background: #ffffff; font-family: Arial; direction: ltr">
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="50">Smith</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="50"></tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="225">founded</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="225"></tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="400">a</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="400"></tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="575">healthcare</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="575"></tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="750">company</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="750"></tspan>
</text>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-c3124cc3e661444cb9d4175a5b7c09d1-0-0" stroke-width="2px" d="M70,264.5 C70,177.0 215.0,177.0 215.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textPath xlink:href="#arrow-c3124cc3e661444cb9d4175a5b7c09d1-0-0" class="displacy-label" startOffset="50%" side="left" fill="currentColor" text-anchor="middle">nsubj</textPath>
</text>
<path class="displacy-arrowhead" d="M70,266.5 L62,254.5 78,254.5" fill="currentColor"></path>
</g>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-c3124cc3e661444cb9d4175a5b7c09d1-0-1" stroke-width="2px" d="M420,264.5 C420,89.5 745.0,89.5 745.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textPath xlink:href="#arrow-c3124cc3e661444cb9d4175a5b7c09d1-0-1" class="displacy-label" startOffset="50%" side="left" fill="currentColor" text-anchor="middle">det</textPath>
</text>
<path class="displacy-arrowhead" d="M420,266.5 L412,254.5 428,254.5" fill="currentColor"></path>
</g>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-c3124cc3e661444cb9d4175a5b7c09d1-0-2" stroke-width="2px" d="M595,264.5 C595,177.0 740.0,177.0 740.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textPath xlink:href="#arrow-c3124cc3e661444cb9d4175a5b7c09d1-0-2" class="displacy-label" startOffset="50%" side="left" fill="currentColor" text-anchor="middle">compound</textPath>
</text>
<path class="displacy-arrowhead" d="M595,266.5 L587,254.5 603,254.5" fill="currentColor"></path>
</g>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-c3124cc3e661444cb9d4175a5b7c09d1-0-3" stroke-width="2px" d="M245,264.5 C245,2.0 750.0,2.0 750.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textPath xlink:href="#arrow-c3124cc3e661444cb9d4175a5b7c09d1-0-3" class="displacy-label" startOffset="50%" side="left" fill="currentColor" text-anchor="middle">dobj</textPath>
</text>
<path class="displacy-arrowhead" d="M750.0,266.5 L758.0,254.5 742.0,254.5" fill="currentColor"></path>
</g>
</svg>
Side by Side

Before

Width:  |  Height:  |  Size: 3.8 KiB

33
website/docs/images/displacy-ent-custom.html
View File

@ -1,33 +0,0 @@
<div
class="entities"
style="line-height: 2.5; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; font-size: 18px"
>But
<mark
class="entity"
style="background: linear-gradient(90deg, #AA9CFC, #FC9CE7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>Google
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>ORG</span
></mark
>is starting from behind. The company made a late push into hardware, and
<mark
class="entity"
style="background: linear-gradient(90deg, #AA9CFC, #FC9CE7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>Apple
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>ORG</span
></mark
>’s Siri, available on iPhones, and
<mark
class="entity"
style="background: linear-gradient(90deg, #AA9CFC, #FC9CE7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>Amazon
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>ORG</span
></mark
>’s Alexa software, which runs on its Echo and Dot devices, have clear leads in consumer
adoption.</div
>

26
website/docs/images/displacy-ent-snek.html
View File

@ -1,26 +0,0 @@
<div
class="entities"
style="line-height: 2.5; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; font-size: 16px"
>
🌱🌿
<mark
class="entity"
style="background: #3dff74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>🐍
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>SNEK</span
></mark
>
____ 🌳🌲 ____
<mark
class="entity"
style="background: #cfc5ff; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>👨‍🌾
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>HUMAN</span
></mark
>
🏘️
</div>

37
website/docs/images/displacy-ent1.html
View File

@ -1,37 +0,0 @@
<div
class="entities"
style="line-height: 2.5; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; font-size: 16px"
>
<mark
class="entity"
style="background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>
Apple
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>ORG</span
>
</mark>
is looking at buying
<mark
class="entity"
style="background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>
U.K.
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>GPE</span
>
</mark>
startup for
<mark
class="entity"
style="background: #e4e7d2; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>
$1 billion
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>MONEY</span
>
</mark>
</div>

39
website/docs/images/displacy-ent2.html
View File

@ -1,39 +0,0 @@
<div
class="entities"
style="line-height: 2.5; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; font-size: 18px"
>
When
<mark
class="entity"
style="background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>
Sebastian Thrun
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>PERSON</span
>
</mark>
started working on self-driving cars at
<mark
class="entity"
style="background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>
Google
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>ORG</span
>
</mark>
in
<mark
class="entity"
style="background: #bfe1d9; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em"
>
2007
<span
style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem"
>DATE</span
>
</mark>
, few people outside of the company took him seriously.
</div>

84
website/docs/images/displacy-long2.html
View File

@ -1,84 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="0" class="displacy" width="1275" height="399.5" style="max-width: none; height: 399.5px; color: #000000; background: #ffffff; font-family: Arial">
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="50">Autonomous</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="50">ADJ</tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="225">cars</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="225">NOUN</tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="400">shift</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="400">VERB</tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="575">insurance</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="575">NOUN</tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="750">liability</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="750">NOUN</tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="925">toward</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="925">ADP</tspan>
</text>
<text class="displacy-token" fill="currentColor" text-anchor="middle" y="309.5">
<tspan class="displacy-word" fill="currentColor" x="1100">manufacturers</tspan>
<tspan class="displacy-tag" dy="2em" fill="currentColor" x="1100">NOUN</tspan>
</text>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-0-0" stroke-width="2px" d="M70,264.5 C70,177.0 215.0,177.0 215.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textpath xlink:href="#arrow-0-0" class="displacy-label" startOffset="50%" fill="currentColor" text-anchor="middle">amod</textpath>
</text>
<path class="displacy-arrowhead" d="M70,266.5 L62,254.5 78,254.5" fill="currentColor"></path>
</g>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-0-1" stroke-width="2px" d="M245,264.5 C245,177.0 390.0,177.0 390.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textpath xlink:href="#arrow-0-1" class="displacy-label" startOffset="50%" fill="currentColor" text-anchor="middle">nsubj</textpath>
</text>
<path class="displacy-arrowhead" d="M245,266.5 L237,254.5 253,254.5" fill="currentColor"></path>
</g>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-0-2" stroke-width="2px" d="M595,264.5 C595,177.0 740.0,177.0 740.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textpath xlink:href="#arrow-0-2" class="displacy-label" startOffset="50%" fill="currentColor" text-anchor="middle">compound</textpath>
</text>
<path class="displacy-arrowhead" d="M595,266.5 L587,254.5 603,254.5" fill="currentColor"></path>
</g>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-0-3" stroke-width="2px" d="M420,264.5 C420,89.5 745.0,89.5 745.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textpath xlink:href="#arrow-0-3" class="displacy-label" startOffset="50%" fill="currentColor" text-anchor="middle">dobj</textpath>
</text>
<path class="displacy-arrowhead" d="M745.0,266.5 L753.0,254.5 737.0,254.5" fill="currentColor"></path>
</g>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-0-4" stroke-width="2px" d="M420,264.5 C420,2.0 925.0,2.0 925.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textpath xlink:href="#arrow-0-4" class="displacy-label" startOffset="50%" fill="currentColor" text-anchor="middle">prep</textpath>
</text>
<path class="displacy-arrowhead" d="M925.0,266.5 L933.0,254.5 917.0,254.5" fill="currentColor"></path>
</g>
<g class="displacy-arrow">
<path class="displacy-arc" id="arrow-0-5" stroke-width="2px" d="M945,264.5 C945,177.0 1090.0,177.0 1090.0,264.5" fill="none" stroke="currentColor"></path>
<text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
<textpath xlink:href="#arrow-0-5" class="displacy-label" startOffset="50%" fill="currentColor" text-anchor="middle">pobj</textpath>
</text>
<path class="displacy-arrowhead" d="M1090.0,266.5 L1098.0,254.5 1082.0,254.5" fill="currentColor"></path>
</g>
</svg>
Side by Side

Before

Width:  |  Height:  |  Size: 5.0 KiB

31
website/docs/images/displacy-span-custom.html
View File

@ -1,31 +0,0 @@
<div class="spans"
style="line-height: 2.5; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; font-size: 18px; direction: ltr">
Welcome to the
<span style="font-weight: bold; display: inline-block; position: relative;">
Bank
<span
style="background: #ddd; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
</span>
<span
style="background: #ddd; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;">
<span
style="background: #ddd; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px">
BANK
</span>
</span>
</span>
<span style="font-weight: bold; display: inline-block; position: relative;">
of
<span
style="background: #ddd; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
</span>
</span>
<span style="font-weight: bold; display: inline-block; position: relative;">
China
<span
style="background: #ddd; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
</span>
</span>
.
</div>

41
website/docs/images/displacy-span.html
View File

@ -1,41 +0,0 @@
<div class="spans"
style="line-height: 2.5; direction: ltr; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; font-size: 18px">
Welcome to the
<span style="font-weight: bold; display: inline-block; position: relative;">
Bank
<span
style="background: #7aecec; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
</span>
<span
style="background: #7aecec; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;">
<span
style="background: #7aecec; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px">
ORG
</span>
</span>
</span>
<span style="font-weight: bold; display: inline-block; position: relative;">
of
<span
style="background: #7aecec; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
</span>
</span>
<span style="font-weight: bold; display: inline-block; position: relative;">
China
<span
style="background: #7aecec; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
</span>
<span
style="background: #feca74; top: 57px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;">
</span>
<span
style="background: #feca74; top: 57px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;">
<span
style="background: #feca74; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px">
GPE
</span>
</span>
</span>
.
</div>

6
website/docs/index.md
View File

@ -1,6 +0,0 @@
---
---
import Landing from 'widgets/landing.js'
<Landing />

22
website/docs/models/index.md → website/docs/models/index.mdx
View File

@ -7,7 +7,7 @@ menu:
- ['Pipeline Design', 'design']
---
<!-- TODO: include interactive demo -->
{/* TODO: include interactive demo */}
### Quickstart {hidden="true"}
@ -16,11 +16,9 @@ menu:
> For more details on how to use trained pipelines with spaCy, see the
> [usage guide](/usage/models).
import QuickstartModels from 'widgets/quickstart-models.js'
<QuickstartModels id="quickstart" />
## Package naming conventions {#conventions}
## Package naming conventions {id="conventions"}
In general, spaCy expects all pipeline packages to follow the naming convention
of `[lang]\_[name]`. For spaCy's pipelines, we also chose to divide the name
@ -45,7 +43,7 @@ For example, [`en_core_web_sm`](/models/en#en_core_web_sm) is a small English
pipeline trained on written web text (blogs, news, comments), that includes
vocabulary, syntax and entities.
### Package versioning {#model-versioning}
### Package versioning {id="model-versioning"}
Additionally, the pipeline package versioning reflects both the compatibility
with spaCy, as well as the model version. A package version `a.b.c` translates
@ -62,7 +60,7 @@ For a detailed compatibility overview, see the
This is also the source of spaCy's internal compatibility check, performed when
you run the [`download`](/api/cli#download) command.
## Trained pipeline design {#design}
## Trained pipeline design {id="design"}
The spaCy v3 trained pipelines are designed to be efficient and configurable.
For example, multiple components can share a common "token-to-vector" model and
@ -89,9 +87,9 @@ Main changes from spaCy v2 models:
- The lemmatizer tables and processing move from the vocab and tagger to a
separate `lemmatizer` component.
### CNN/CPU pipeline design {#design-cnn}
### CNN/CPU pipeline design {id="design-cnn"}
![Components and their dependencies in the CNN pipelines](../images/pipeline-design.svg)
![Components and their dependencies in the CNN pipelines](/images/pipeline-design.svg)
In the `sm`/`md`/`lg` models:
@ -132,13 +130,13 @@ vector keys for default vectors.
- [`Vectors.most_similar`](/api/vectors#most_similar) is not supported because
there's no fixed list of vectors to compare your vectors to.
### Transformer pipeline design {#design-trf}
### Transformer pipeline design {id="design-trf"}
In the transformer (`trf`) models, the `tagger`, `parser` and `ner` (if present)
all listen to the `transformer` component. The `attribute_ruler` and
`lemmatizer` have the same configuration as in the CNN models.
### Modifying the default pipeline {#design-modify}
### Modifying the default pipeline {id="design-modify"}
For faster processing, you may only want to run a subset of the components in a
trained pipeline. The `disable` and `exclude` arguments to
@ -189,8 +187,8 @@ than the rule-based `sentencizer`.
#### Switch from trainable lemmatizer to default lemmatizer
Since v3.3, a number of pipelines use a trainable lemmatizer. You can check whether
the lemmatizer is trainable:
Since v3.3, a number of pipelines use a trainable lemmatizer. You can check
whether the lemmatizer is trainable:
```python
nlp = spacy.load("de_core_web_sm")

165
website/docs/styleguide.md → website/docs/styleguide.mdx
View File

@ -42,9 +42,7 @@ enough, JSX components can be used.
> For more details on editing the site locally, see the installation
> instructions and markdown reference below.
## Logo {#logo source="website/src/images/logo.svg"}
import { Logos } from 'widgets/styleguide'
## Logo {id="logo",source="website/src/images/logo.svg"}
If you would like to use the spaCy logo on your site, please get in touch and
ask us first. However, if you want to show support and tell others that your
@ -53,9 +51,7 @@ project is using spaCy, you can grab one of our
<Logos />
## Colors {#colors}
import { Colors, Patterns } from 'widgets/styleguide'
## Colors {id="colors"}
<Colors />
@ -63,17 +59,16 @@ import { Colors, Patterns } from 'widgets/styleguide'
<Patterns />
## Typography {#typography}
import { H1, H2, H3, H4, H5, Label, InlineList, Comment } from
'components/typography'
## Typography {id="typography"}
> #### Markdown
>
> ```markdown_
> ```markdown
> ## Headline 2
> ## Headline 2 {#some_id}
> ## Headline 2 {#some_id tag="method"}
>
> ## Headline 2 {id="some_id"}
>
> ## Headline 2 {id="some_id" tag="method"}
> ```
>
> #### JSX
@ -101,12 +96,11 @@ in the sidebar menu.
</Infobox>
<div>
<H1>Headline 1</H1>
<H2>Headline 2</H2>
<H3>Headline 3</H3>
<H4>Headline 4</H4>
<H5>Headline 5</H5>
<Label>Label</Label>
<H2>Headline 2</H2>
<H3>Headline 3</H3>
<H4>Headline 4</H4>
<H5>Headline 5</H5>
<Label>Label</Label>
</div>
---
@ -116,16 +110,16 @@ example, to add a tag for the documented type or mark features that have been
introduced in a specific version or require statistical models to be loaded.
Tags are also available as standalone `<Tag />` components.
| Argument | Example | Result |
| -------- | -------------------------- | ----------------------------------------- |
| `tag` | `{tag="method"}` | <Tag>method</Tag> |
| `new` | `{new="3"}` | <Tag variant="new">3</Tag> |
| `model` | `{model="tagger, parser"}` | <Tag variant="model">tagger, parser</Tag> |
| `hidden` | `{hidden="true"}` | |
| Argument | Example | Result |
| --------- | -------------------------- | ----------------------------------------- |
| `tag` | `{tag="method"}` | <Tag>method</Tag> |
| `version` | `{version="3"}` | <Tag variant="new">3</Tag> |
| `model` | `{model="tagger, parser"}` | <Tag variant="model">tagger, parser</Tag> |
| `hidden` | `{hidden="true"}` | |
## Elements {#elements}
## Elements {id="elements"}
### Links {#links}
### Links {id="links"}
> #### Markdown
>
@ -147,9 +141,7 @@ Special link styles are used depending on the link URL.
- [I am a link to a model](/models/en#en_core_web_sm)
- [I am a link to GitHub](https://github.com/explosion/spaCy)
### Abbreviations {#abbr}
import { Abbr } from 'components/typography'
### Abbreviations {id="abbr"}
> #### JSX
>
@ -161,13 +153,11 @@ Some text with <Abbr title="Explanation here">an abbreviation</Abbr>. On small
screens, I collapse and the explanation text is displayed next to the
abbreviation.
### Tags {#tags}
import Tag from 'components/tag'
### Tags {id="tags"}
> ```jsx
> <Tag>method</Tag>
> <Tag variant="new">4</Tag>
> <Tag variant="version">4</Tag>
> <Tag variant="model">tagger, parser</Tag>
> ```
@ -180,16 +170,13 @@ new anymore. Setting `variant="model"` takes a description of model capabilities
and can be used to mark features that require a respective model to be
installed.
<InlineList>
<p>
<Tag>method</Tag>
<Tag variant="new">4</Tag>
<Tag variant="model">tagger, parser</Tag>
</p>
<Tag>method</Tag> <Tag variant="new">4</Tag> <Tag variant="model">tagger,
parser</Tag>
</InlineList>
### Buttons {#buttons}
import Button from 'components/button'
### Buttons {id="buttons"}
> ```jsx
> <Button to="#" variant="primary">Primary small</Button>
@ -200,21 +187,29 @@ Link buttons come in two variants, `primary` and `secondary` and two sizes, with
an optional `large` size modifier. Since they're mostly used as enhanced links,
the buttons are implemented as styled links instead of native button elements.
<InlineList><Button to="#" variant="primary">Primary small</Button>
<Button to="#" variant="secondary">Secondary small</Button></InlineList>
<p>
<Button to="#" variant="primary">Primary small</Button>
<br />
{' '}
<InlineList><Button to="#" variant="primary" large>Primary large</Button>
<Button to="#" variant="secondary" large>Secondary large</Button></InlineList>
<Button to="#" variant="secondary">Secondary small</Button>
</p>
<p>
<Button to="#" variant="primary">Primary small</Button>
{' '}
<Button to="#" variant="secondary">Secondary small</Button>
</p>
## Components
### Table {#table}
### Table {id="table"}
> #### Markdown
>
> ```markdown_
> ```markdown
> | Header 1 | Header 2 |
> | -------- | -------- |
> | Column 1 | Column 2 |
@ -248,7 +243,7 @@ be italicized:
> #### Markdown
>
> ```markdown_
> ```markdown
> | Header 1 | Header 2 | Header 3 |
> | -------- | -------- | -------- |
> | Column 1 | Column 2 | Column 3 |
@ -262,11 +257,11 @@ be italicized:
| _Hello_ | | |
| Column 1 | Column 2 | Column 3 |
### Type Annotations {#type-annotations}
### Type Annotations {id="type-annotations"}
> #### Markdown
>
> ```markdown_
> ```markdown
> ~~Model[List[Doc], Floats2d]~~
> ```
>
@ -295,9 +290,9 @@ always be the **last element** in the row.
> #### Markdown
>
> ```markdown_
> | Header 1 | Header 2 |
> | -------- | ----------------------- |
> ```markdown
> | Header 1 | Header 2 |
> | -------- | ---------------------- |
> | Column 1 | Column 2 ~~List[Doc]~~ |
> ```
@ -307,11 +302,11 @@ always be the **last element** in the row.
| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) wrapping the transformer. ~~Model[List[Doc], FullTransformerBatch]~~ |
| `set_extra_annotations` | Function that takes a batch of `Doc` objects and transformer outputs and can set additional annotations on the `Doc`. ~~Callable[[List[Doc], FullTransformerBatch], None]~~ |
### List {#list}
### List {id="list"}
> #### Markdown
>
> ```markdown_
> ```markdown
> 1. One
> 2. Two
> ```
@ -338,12 +333,13 @@ automatically.
3. Lorem ipsum dolor
4. consectetur adipiscing elit
### Aside {#aside}
### Aside {id="aside"}
> #### Markdown
>
> ```markdown_
> ```markdown
> > #### Aside title
> >
> > This is aside text.
> ```
>
@ -363,11 +359,11 @@ To make them easier to use in Markdown, paragraphs formatted as blockquotes will
turn into asides by default. Level 4 headlines (with a leading `####`) will
become aside titles.
### Code Block {#code-block}
### Code Block {id="code-block"}
> #### Markdown
>
> ````markdown_
> ````markdown
> ```python
> ### This is a title
> import spacy
@ -388,8 +384,7 @@ to raw text with no highlighting. An optional label can be added as the first
line with the prefix `####` (Python-like) and `///` (JavaScript-like). the
indented block as plain text and preserve whitespace.
```python
### Using spaCy
```python {title="Using spaCy"}
import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("This is a sentence.")
@ -403,7 +398,7 @@ adding `{highlight="..."}` to the headline. Acceptable ranges are spans like
> #### Markdown
>
> ````markdown_
> ````markdown
> ```python
> ### This is a title {highlight="1-2"}
> import spacy
@ -411,8 +406,7 @@ adding `{highlight="..."}` to the headline. Acceptable ranges are spans like
> ```
> ````
```python
### Using the matcher {highlight="5-7"}
```python {title="Using the matcher",highlight="5-7"}
import spacy
from spacy.matcher import Matcher
@ -431,7 +425,7 @@ interactive widget defaults to a regular code block.
> #### Markdown
>
> ````markdown_
> ````markdown
> ```python
> ### {executable="true"}
> import spacy
@ -439,8 +433,7 @@ interactive widget defaults to a regular code block.
> ```
> ````
```python
### {executable="true"}
```python {executable="true"}
import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("This is a sentence.")
@ -454,7 +447,7 @@ original file is shown at the top of the widget.
> #### Markdown
>
> ````markdown_
> ````markdown
> ```python
> https://github.com/...
> ```
@ -470,9 +463,7 @@ original file is shown at the top of the widget.
https://github.com/explosion/spaCy/tree/master/spacy/language.py
```
### Infobox {#infobox}
import Infobox from 'components/infobox'
### Infobox {id="infobox"}
> #### JSX
>
@ -508,9 +499,7 @@ blocks.
</Infobox>
### Accordion {#accordion}
import Accordion from 'components/accordion'
### Accordion {id="accordion"}
> #### JSX
>
@ -537,9 +526,9 @@ sit amet dignissim justo congue.
</Accordion>
## Markdown reference {#markdown}
## Markdown reference {id="markdown"}
All page content and page meta lives in the `.md` files in the `/docs`
All page content and page meta lives in the `.mdx` files in the `/docs`
directory. The frontmatter block at the top of each file defines the page title
and other settings like the sidebar menu.
@ -548,7 +537,7 @@ and other settings like the sidebar menu.
title: Page title
---
## Headline starting a section {#some_id}
## Headline starting a section {id="some_id"}
This is a regular paragraph with a [link](https://spacy.io) and **bold text**.
@ -562,8 +551,7 @@ This is a regular paragraph with a [link](https://spacy.io) and **bold text**.
| -------- | -------- |
| Column 1 | Column 2 |
```python
### Code block title {highlight="2-3"}
```python {title="Code block title",highlight="2-3"}
import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("Hello world")
@ -585,7 +573,7 @@ In addition to the native markdown elements, you can use the components
[abbr]: https://spacy.io/styleguide#abbr
[tag]: https://spacy.io/styleguide#tag
## Editorial {#editorial}
## Editorial {id="editorial"}
- "spaCy" should always be spelled with a lowercase "s" and a capital "C",
unless it specifically refers to the Python package or Python import `spacy`
@ -609,21 +597,16 @@ In addition to the native markdown elements, you can use the components
- ❌ The [`Span`](/api/span) and [`Token`](/api/token) objects are views of a
[`Doc`](/api/doc). [`Span.as_doc`](/api/span#as_doc) creates a
[`Doc`](/api/doc) object from a [`Span`](/api/span).
* Other things we format as code are: references to trained pipeline packages
- Other things we format as code are: references to trained pipeline packages
like `en_core_web_sm` or file names like `code.py` or `meta.json`.
- ✅ After training, the `config.cfg` is saved to disk.
* [Type annotations](#type-annotations) are a special type of code formatting,
- [Type annotations](#type-annotations) are a special type of code formatting,
expressed by wrapping the text in `~~` instead of backticks. The result looks
like this: ~~List[Doc]~~. All references to known types will be linked
automatically.
- ✅ The model has the input type ~~List[Doc]~~ and it outputs a
~~List[Array2d]~~.
* We try to keep links meaningful but short.
- We try to keep links meaningful but short.
- ✅ For details, see the usage guide on
[training with custom code](/usage/training#custom-code).
- ❌ For details, see

12
website/docs/usage/101/_architecture.md → website/docs/usage/101/_architecture.mdx
View File

@ -14,9 +14,9 @@ of the pipeline. The `Language` object coordinates these components. It takes
raw text and sends it through the pipeline, returning an **annotated document**.
It also orchestrates training and serialization.
![Library architecture](../../images/architecture.svg)
![Library architecture {{w:1080, h:1254}}](/images/architecture.svg)
### Container objects {#architecture-containers}
### Container objects {id="architecture-containers"}
| Name | Description |
| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
@ -29,7 +29,7 @@ It also orchestrates training and serialization.
| [`SpanGroup`](/api/spangroup) | A named collection of spans belonging to a `Doc`. |
| [`Token`](/api/token) | An individual token — i.e. a word, punctuation symbol, whitespace, etc. |
### Processing pipeline {#architecture-pipeline}
### Processing pipeline {id="architecture-pipeline"}
The processing pipeline consists of one or more **pipeline components** that are
called on the `Doc` in order. The tokenizer runs before the components. Pipeline
@ -39,7 +39,7 @@ rule-based modifications to the `Doc`. spaCy provides a range of built-in
components for different language processing tasks and also allows adding
[custom components](/usage/processing-pipelines#custom-components).
![The processing pipeline](../../images/pipeline.svg)
![The processing pipeline](/images/pipeline.svg)
| Component name | Component class | Description |
| ---------------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------- |
@ -63,7 +63,7 @@ components for different language processing tasks and also allows adding
| - | [`TrainablePipe`](/api/pipe) | Class that all trainable pipeline components inherit from. |
| - | [Other functions](/api/pipeline-functions) | Automatically apply something to the `Doc`, e.g. to merge spans of tokens. |
### Matchers {#architecture-matchers}
### Matchers {id="architecture-matchers"}
Matchers help you find and extract information from [`Doc`](/api/doc) objects
based on match patterns describing the sequences you're looking for. A matcher
@ -75,7 +75,7 @@ operates on a `Doc` and gives you access to the matched tokens **in context**.
| [`Matcher`](/api/matcher) | Match sequences of tokens, based on pattern rules, similar to regular expressions. |
| [`PhraseMatcher`](/api/phrasematcher) | Match sequences of tokens based on phrases. |
### Other classes {#architecture-other}
### Other classes {id="architecture-other"}
| Name | Description |
| ------------------------------------------------ | -------------------------------------------------------------------------------------------------- |

0
website/docs/usage/101/_language-data.md → website/docs/usage/101/_language-data.mdx
View File

20
website/docs/usage/101/_named-entities.md → website/docs/usage/101/_named-entities.mdx
View File

@ -1,14 +1,13 @@
A named entity is a "real-world object" that's assigned a name – for example, a
person, a country, a product or a book title. spaCy can **recognize various
types of named entities in a document, by asking the model for a
prediction**. Because models are statistical and strongly depend on the
examples they were trained on, this doesn't always work _perfectly_ and might
need some tuning later, depending on your use case.
types of named entities in a document, by asking the model for a prediction**.
Because models are statistical and strongly depend on the examples they were
trained on, this doesn't always work _perfectly_ and might need some tuning
later, depending on your use case.
Named entities are available as the `ents` property of a `Doc`:
```python
### {executable="true"}
```python {executable="true"}
import spacy
nlp = spacy.load("en_core_web_sm")
@ -32,7 +31,8 @@ for ent in doc.ents:
Using spaCy's built-in [displaCy visualizer](/usage/visualizers), here's what
our example sentence and its named entities look like:
import DisplaCyEntHtml from 'images/displacy-ent1.html'; import { Iframe } from
'components/embed'
<Iframe title="displaCy visualization of entities" html={DisplaCyEntHtml} height={100} />
<Iframe
title="displaCy visualization of entities"
src="/images/displacy-ent1.html"
height={100}
/>

4
website/docs/usage/101/_pipelines.md → website/docs/usage/101/_pipelines.mdx
View File

@ -5,7 +5,7 @@ referred to as the **processing pipeline**. The pipeline used by the
and an entity recognizer. Each pipeline component returns the processed `Doc`,
which is then passed on to the next component.
![The processing pipeline](../../images/pipeline.svg)
![The processing pipeline](/images/pipeline.svg)
> - **Name**: ID of the pipeline component.
> - **Component:** spaCy's implementation of the component.
@ -35,8 +35,6 @@ the [config](/usage/training#config):
pipeline = ["tok2vec", "tagger", "parser", "ner"]
```
import Accordion from 'components/accordion.js'
<Accordion title="Does the order of pipeline components matter?" id="pipeline-components-order">
The statistical components like the tagger or parser are typically independent

12
website/docs/usage/101/_pos-deps.md → website/docs/usage/101/_pos-deps.mdx
View File

@ -11,8 +11,7 @@ Linguistic annotations are available as
efficiency. So to get the readable string representation of an attribute, we
need to add an underscore `_` to its name:
```python
### {executable="true"}
```python {executable="true"}
import spacy
nlp = spacy.load("en_core_web_sm")
@ -57,7 +56,8 @@ for token in doc:
Using spaCy's built-in [displaCy visualizer](/usage/visualizers), here's what
our example sentence and its dependencies look like:
import DisplaCyLongHtml from 'images/displacy-long.html'; import { Iframe } from
'components/embed'
<Iframe title="displaCy visualization of dependencies and entities" html={DisplaCyLongHtml} height={450} />
<Iframe
title="displaCy visualization of dependencies and entities"
src="/images/displacy-long.html"
height={450}
/>

Some files were not shown because too many files have changed in this diff Show More