| Get a custom spaCy pipeline, tailor-made for your NLP problem by spaCy's core developers. Streamlined, production-ready, predictable and maintainable. Start by completing our 5-minute questionnaire to tell us what you need and we'll be in touch! **[Learn more →](https://explosion.ai/spacy-tailored-pipelines)** |
+| 
| Bespoke advice for problem solving, strategy and analysis for applied NLP projects. Services include data strategy, code reviews, pipeline design and annotation coaching. Curious? Fill in our 5-minute questionnaire to tell us what you need and we'll be in touch! **[Learn more →](https://explosion.ai/spacy-tailored-analysis)** |
[spacy 101]: https://spacy.io/usage/spacy-101
[new in v3.0]: https://spacy.io/usage/v3
@@ -59,6 +60,7 @@ open-source software, released under the MIT license.
[changelog]: https://spacy.io/usage#changelog
[contribute]: https://github.com/explosion/spaCy/blob/master/CONTRIBUTING.md
+
## 💬 Where to ask questions
The spaCy project is maintained by the [spaCy team](https://explosion.ai/about).
@@ -103,7 +105,7 @@ For detailed installation instructions, see the
- **Operating system**: macOS / OS X · Linux · Windows (Cygwin, MinGW, Visual
Studio)
-- **Python version**: Python 3.6+ (only 64 bit)
+- **Python version**: Python 3.8+ (only 64 bit)
- **Package managers**: [pip] · [conda] (via `conda-forge`)
[pip]: https://pypi.org/project/spacy/
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 3499042cb..a6a575315 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -11,25 +11,39 @@ trigger:
exclude:
- "website/*"
- "*.md"
+ - "*.mdx"
- ".github/workflows/*"
pr:
paths:
exclude:
- "*.md"
+ - "*.mdx"
- "website/docs/*"
- "website/src/*"
+ - "website/meta/*.tsx"
+ - "website/meta/*.mjs"
+ - "website/meta/languages.json"
+ - "website/meta/site.json"
+ - "website/meta/sidebars.json"
+ - "website/meta/type-annotations.json"
+ - "website/pages/*"
- ".github/workflows/*"
jobs:
- # Perform basic checks for most important errors (syntax etc.) Uses the config
- # defined in .flake8 and overwrites the selected codes.
+ # Check formatting and linting. Perform basic checks for most important errors
+ # (syntax etc.) Uses the config defined in setup.cfg and overwrites the
+ # selected codes.
- job: "Validate"
pool:
vmImage: "ubuntu-latest"
steps:
- task: UsePythonVersion@0
inputs:
- versionSpec: "3.7"
+ versionSpec: "3.8"
+ - script: |
+ pip install black==22.3.0
+ python -m black spacy --check
+ displayName: "black"
- script: |
pip install flake8==5.0.4
python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics
@@ -40,24 +54,6 @@ jobs:
strategy:
matrix:
# We're only running one platform per Python version to speed up builds
- Python36Linux:
- imageName: "ubuntu-latest"
- python.version: "3.6"
- # Python36Windows:
- # imageName: "windows-latest"
- # python.version: "3.6"
- # Python36Mac:
- # imageName: "macos-latest"
- # python.version: "3.6"
- # Python37Linux:
- # imageName: "ubuntu-latest"
- # python.version: "3.7"
- Python37Windows:
- imageName: "windows-latest"
- python.version: "3.7"
- # Python37Mac:
- # imageName: "macos-latest"
- # python.version: "3.7"
# Python38Linux:
# imageName: "ubuntu-latest"
# python.version: "3.8"
@@ -87,13 +83,13 @@ jobs:
# python.version: "3.10"
Python311Linux:
imageName: 'ubuntu-latest'
- python.version: '3.11.0'
+ python.version: '3.11'
Python311Windows:
imageName: 'windows-latest'
- python.version: '3.11.0'
+ python.version: '3.11'
Python311Mac:
imageName: 'macos-latest'
- python.version: '3.11.0'
+ python.version: '3.11'
maxParallel: 4
pool:
vmImage: $(imageName)
diff --git a/build-constraints.txt b/build-constraints.txt
index 956973abf..c1e82f1b0 100644
--- a/build-constraints.txt
+++ b/build-constraints.txt
@@ -5,4 +5,5 @@ numpy==1.17.3; python_version=='3.8' and platform_machine!='aarch64'
numpy==1.19.2; python_version=='3.8' and platform_machine=='aarch64'
numpy==1.19.3; python_version=='3.9'
numpy==1.21.3; python_version=='3.10'
-numpy; python_version>='3.11'
+numpy==1.23.2; python_version=='3.11'
+numpy; python_version>='3.12'
diff --git a/pyproject.toml b/pyproject.toml
index 7abd7a96f..837cf1fd8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,7 +5,7 @@ requires = [
"cymem>=2.0.2,<2.1.0",
"preshed>=3.0.2,<3.1.0",
"murmurhash>=0.28.0,<1.1.0",
- "thinc>=8.1.0,<8.2.0",
+ "thinc>=9.0.0.dev2,<9.1.0",
"numpy>=1.15.0",
]
build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
index 9d6bbb2c4..78cccfbf1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,16 +1,17 @@
# Our libraries
-spacy-legacy>=3.0.10,<3.1.0
+spacy-legacy>=4.0.0.dev0,<4.1.0
spacy-loggers>=1.0.0,<2.0.0
cymem>=2.0.2,<2.1.0
preshed>=3.0.2,<3.1.0
-thinc>=8.1.0,<8.2.0
+thinc>=9.0.0.dev2,<9.1.0
ml_datasets>=0.2.0,<0.3.0
murmurhash>=0.28.0,<1.1.0
-wasabi>=0.9.1,<1.1.0
+wasabi>=0.9.1,<1.2.0
srsly>=2.4.3,<3.0.0
catalogue>=2.0.6,<2.1.0
-typer>=0.3.0,<0.5.0
-pathy>=0.3.5
+typer>=0.3.0,<0.8.0
+pathy>=0.10.0
+smart-open>=5.2.1,<7.0.0
# Third party dependencies
numpy>=1.15.0
requests>=2.13.0,<3.0.0
@@ -21,7 +22,6 @@ langcodes>=3.2.0,<4.0.0
# Official Python utilities
setuptools
packaging>=20.0
-typing_extensions>=3.7.4.1,<4.2.0; python_version < "3.8"
# Development dependencies
pre-commit>=2.13.0
cython>=0.25,<3.0
@@ -30,8 +30,7 @@ pytest-timeout>=1.3.0,<2.0.0
mock>=2.0.0,<3.0.0
flake8>=3.8.0,<6.0.0
hypothesis>=3.27.0,<7.0.0
-mypy>=0.980,<0.990; platform_machine != "aarch64" and python_version >= "3.7"
-types-dataclasses>=0.1.3; python_version < "3.7"
+mypy>=0.990,<0.1000; platform_machine != "aarch64"
types-mock>=0.1.1
types-setuptools>=57.0.0
types-requests
diff --git a/setup.cfg b/setup.cfg
index 587af7e64..975ec03ce 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -17,11 +17,10 @@ classifiers =
Operating System :: Microsoft :: Windows
Programming Language :: Cython
Programming Language :: Python :: 3
- Programming Language :: Python :: 3.6
- Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
+ Programming Language :: Python :: 3.11
Topic :: Scientific/Engineering
project_urls =
Release notes = https://github.com/explosion/spaCy/releases
@@ -30,21 +29,22 @@ project_urls =
[options]
zip_safe = false
include_package_data = true
-python_requires = >=3.6
+python_requires = >=3.8
install_requires =
# Our libraries
- spacy-legacy>=3.0.10,<3.1.0
+ spacy-legacy>=4.0.0.dev0,<4.1.0
spacy-loggers>=1.0.0,<2.0.0
murmurhash>=0.28.0,<1.1.0
cymem>=2.0.2,<2.1.0
preshed>=3.0.2,<3.1.0
- thinc>=8.1.0,<8.2.0
- wasabi>=0.9.1,<1.1.0
+ thinc>=9.0.0.dev2,<9.1.0
+ wasabi>=0.9.1,<1.2.0
srsly>=2.4.3,<3.0.0
catalogue>=2.0.6,<2.1.0
# Third-party dependencies
- typer>=0.3.0,<0.5.0
- pathy>=0.3.5
+ typer>=0.3.0,<0.8.0
+ pathy>=0.10.0
+ smart-open>=5.2.1,<7.0.0
tqdm>=4.38.0,<5.0.0
numpy>=1.15.0
requests>=2.13.0,<3.0.0
@@ -53,7 +53,6 @@ install_requires =
# Official Python utilities
setuptools
packaging>=20.0
- typing_extensions>=3.7.4,<4.2.0; python_version < "3.8"
langcodes>=3.2.0,<4.0.0
[options.entry_points]
@@ -64,7 +63,7 @@ console_scripts =
lookups =
spacy_lookups_data>=1.0.3,<1.1.0
transformers =
- spacy_transformers>=1.1.2,<1.2.0
+ spacy_transformers>=1.1.2,<1.3.0
ray =
spacy_ray>=0.1.0,<1.0.0
cuda =
diff --git a/setup.py b/setup.py
index bc911276f..d5b82ec68 100755
--- a/setup.py
+++ b/setup.py
@@ -33,13 +33,10 @@ MOD_NAMES = [
"spacy.kb.candidate",
"spacy.kb.kb",
"spacy.kb.kb_in_memory",
- "spacy.ml.parser_model",
+ "spacy.ml.tb_framework",
"spacy.morphology",
- "spacy.pipeline.dep_parser",
"spacy.pipeline._edit_tree_internals.edit_trees",
"spacy.pipeline.morphologizer",
- "spacy.pipeline.multitask",
- "spacy.pipeline.ner",
"spacy.pipeline.pipe",
"spacy.pipeline.trainable_pipe",
"spacy.pipeline.sentencizer",
@@ -47,12 +44,15 @@ MOD_NAMES = [
"spacy.pipeline.tagger",
"spacy.pipeline.transition_parser",
"spacy.pipeline._parser_internals.arc_eager",
+ "spacy.pipeline._parser_internals.batch",
"spacy.pipeline._parser_internals.ner",
"spacy.pipeline._parser_internals.nonproj",
+ "spacy.pipeline._parser_internals.search",
"spacy.pipeline._parser_internals._state",
"spacy.pipeline._parser_internals.stateclass",
"spacy.pipeline._parser_internals.transition_system",
"spacy.pipeline._parser_internals._beam_utils",
+ "spacy.pipeline._parser_internals._parser_utils",
"spacy.tokenizer",
"spacy.training.align",
"spacy.training.gold_io",
@@ -68,6 +68,7 @@ MOD_NAMES = [
"spacy.matcher.dependencymatcher",
"spacy.symbols",
"spacy.vectors",
+ "spacy.tests.parser._search",
]
COMPILE_OPTIONS = {
"msvc": ["/Ox", "/EHsc"],
diff --git a/spacy/about.py b/spacy/about.py
index ce86e6294..eddbeea09 100644
--- a/spacy/about.py
+++ b/spacy/about.py
@@ -1,6 +1,6 @@
# fmt: off
__title__ = "spacy"
-__version__ = "3.4.2"
+__version__ = "4.0.0.dev0"
__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
__projects__ = "https://github.com/explosion/projects"
diff --git a/spacy/cli/__init__.py b/spacy/cli/__init__.py
index ce76ef9a9..868526b42 100644
--- a/spacy/cli/__init__.py
+++ b/spacy/cli/__init__.py
@@ -4,6 +4,7 @@ from ._util import app, setup_cli # noqa: F401
# These are the actual functions, NOT the wrapped CLI commands. The CLI commands
# are registered automatically and won't have to be imported here.
+from .benchmark_speed import benchmark_speed_cli # noqa: F401
from .download import download # noqa: F401
from .info import info # noqa: F401
from .package import package # noqa: F401
@@ -16,6 +17,7 @@ from .debug_config import debug_config # noqa: F401
from .debug_model import debug_model # noqa: F401
from .debug_diff import debug_diff # noqa: F401
from .evaluate import evaluate # noqa: F401
+from .apply import apply # noqa: F401
from .convert import convert # noqa: F401
from .init_pipeline import init_pipeline_cli # noqa: F401
from .init_config import init_config, fill_config # noqa: F401
@@ -27,6 +29,7 @@ from .project.dvc import project_update_dvc # noqa: F401
from .project.push import project_push # noqa: F401
from .project.pull import project_pull # noqa: F401
from .project.document import project_document # noqa: F401
+from .find_threshold import find_threshold # noqa: F401
@app.command("link", no_args_is_help=True, deprecated=True, hidden=True)
diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py
index 897964a88..eb4869666 100644
--- a/spacy/cli/_util.py
+++ b/spacy/cli/_util.py
@@ -1,4 +1,4 @@
-from typing import Dict, Any, Union, List, Optional, Tuple, Iterable
+from typing import Dict, Any, Union, List, Optional, Tuple, Iterable, Literal
from typing import TYPE_CHECKING, overload
import sys
import shutil
@@ -16,14 +16,13 @@ from thinc.util import gpu_is_available
from configparser import InterpolationError
import os
-from ..compat import Literal
from ..schemas import ProjectConfigSchema, validate
from ..util import import_file, run_command, make_tempdir, registry, logger
from ..util import is_compatible_version, SimpleFrozenDict, ENV_VARS
from .. import about
if TYPE_CHECKING:
- from pathy import Pathy # noqa: F401
+ from pathy import FluidPath # noqa: F401
SDIST_SUFFIX = ".tar.gz"
@@ -46,6 +45,7 @@ DEBUG_HELP = """Suite of helpful commands for debugging and profiling. Includes
commands to check and validate your config files, training and evaluation data,
and custom model implementations.
"""
+BENCHMARK_HELP = """Commands for benchmarking pipelines."""
INIT_HELP = """Commands for initializing configs and pipeline packages."""
# Wrappers for Typer's annotations. Initially created to set defaults and to
@@ -54,12 +54,14 @@ Arg = typer.Argument
Opt = typer.Option
app = typer.Typer(name=NAME, help=HELP)
+benchmark_cli = typer.Typer(name="benchmark", help=BENCHMARK_HELP, no_args_is_help=True)
project_cli = typer.Typer(name="project", help=PROJECT_HELP, no_args_is_help=True)
debug_cli = typer.Typer(name="debug", help=DEBUG_HELP, no_args_is_help=True)
init_cli = typer.Typer(name="init", help=INIT_HELP, no_args_is_help=True)
app.add_typer(project_cli)
app.add_typer(debug_cli)
+app.add_typer(benchmark_cli)
app.add_typer(init_cli)
@@ -158,15 +160,15 @@ def load_project_config(
sys.exit(1)
validate_project_version(config)
validate_project_commands(config)
+ if interpolate:
+ err = f"{PROJECT_FILE} validation error"
+ with show_validation_error(title=err, hint_fill=False):
+ config = substitute_project_variables(config, overrides)
# Make sure directories defined in config exist
for subdir in config.get("directories", []):
dir_path = path / subdir
if not dir_path.exists():
dir_path.mkdir(parents=True)
- if interpolate:
- err = f"{PROJECT_FILE} validation error"
- with show_validation_error(title=err, hint_fill=False):
- config = substitute_project_variables(config, overrides)
return config
@@ -331,7 +333,7 @@ def import_code(code_path: Optional[Union[Path, str]]) -> None:
msg.fail(f"Couldn't load Python code: {code_path}", e, exits=1)
-def upload_file(src: Path, dest: Union[str, "Pathy"]) -> None:
+def upload_file(src: Path, dest: Union[str, "FluidPath"]) -> None:
"""Upload a file.
src (Path): The source path.
@@ -339,13 +341,20 @@ def upload_file(src: Path, dest: Union[str, "Pathy"]) -> None:
"""
import smart_open
+ # Create parent directories for local paths
+ if isinstance(dest, Path):
+ if not dest.parent.exists():
+ dest.parent.mkdir(parents=True)
+
dest = str(dest)
with smart_open.open(dest, mode="wb") as output_file:
with src.open(mode="rb") as input_file:
output_file.write(input_file.read())
-def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False) -> None:
+def download_file(
+ src: Union[str, "FluidPath"], dest: Path, *, force: bool = False
+) -> None:
"""Download a file using smart_open.
url (str): The URL of the file.
@@ -358,7 +367,7 @@ def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False)
if dest.exists() and not force:
return None
src = str(src)
- with smart_open.open(src, mode="rb", ignore_ext=True) as input_file:
+ with smart_open.open(src, mode="rb", compression="disable") as input_file:
with dest.open(mode="wb") as output_file:
shutil.copyfileobj(input_file, output_file)
@@ -368,7 +377,7 @@ def ensure_pathy(path):
slow and annoying Google Cloud warning)."""
from pathy import Pathy # noqa: F811
- return Pathy(path)
+ return Pathy.fluid(path)
def git_checkout(
@@ -575,6 +584,33 @@ def setup_gpu(use_gpu: int, silent=None) -> None:
local_msg.info("To switch to GPU 0, use the option: --gpu-id 0")
+def walk_directory(path: Path, suffix: Optional[str] = None) -> List[Path]:
+ """Given a directory and a suffix, recursively find all files matching the suffix.
+ Directories or files with names beginning with a . are ignored, but hidden flags on
+ filesystems are not checked.
+ When provided with a suffix `None`, there is no suffix-based filtering."""
+ if not path.is_dir():
+ return [path]
+ paths = [path]
+ locs = []
+ seen = set()
+ for path in paths:
+ if str(path) in seen:
+ continue
+ seen.add(str(path))
+ if path.parts[-1].startswith("."):
+ continue
+ elif path.is_dir():
+ paths.extend(path.iterdir())
+ elif suffix is not None and not path.parts[-1].endswith(suffix):
+ continue
+ else:
+ locs.append(path)
+ # It's good to sort these, in case the ordering messes up cache.
+ locs.sort()
+ return locs
+
+
def _format_number(number: Union[int, float], ndigits: int = 2) -> str:
"""Formats a number (float or int) rounding to `ndigits`, without truncating trailing 0s,
as happens with `round(number, ndigits)`"""
diff --git a/spacy/cli/apply.py b/spacy/cli/apply.py
new file mode 100644
index 000000000..f0df4e757
--- /dev/null
+++ b/spacy/cli/apply.py
@@ -0,0 +1,143 @@
+import tqdm
+import srsly
+
+from itertools import chain
+from pathlib import Path
+from typing import Optional, List, Iterable, cast, Union
+
+from wasabi import msg
+
+from ._util import app, Arg, Opt, setup_gpu, import_code, walk_directory
+
+from ..tokens import Doc, DocBin
+from ..vocab import Vocab
+from ..util import ensure_path, load_model
+
+
+path_help = """Location of the documents to predict on.
+Can be a single file in .spacy format or a .jsonl file.
+Files with other extensions are treated as single plain text documents.
+If a directory is provided it is traversed recursively to grab
+all files to be processed.
+The files can be a mixture of .spacy, .jsonl and text files.
+If .jsonl is provided the specified field is going
+to be grabbed ("text" by default)."""
+
+out_help = "Path to save the resulting .spacy file"
+code_help = (
+ "Path to Python file with additional " "code (registered functions) to be imported"
+)
+gold_help = "Use gold preprocessing provided in the .spacy files"
+force_msg = (
+ "The provided output file already exists. "
+ "To force overwriting the output file, set the --force or -F flag."
+)
+
+
+DocOrStrStream = Union[Iterable[str], Iterable[Doc]]
+
+
+def _stream_docbin(path: Path, vocab: Vocab) -> Iterable[Doc]:
+ """
+ Stream Doc objects from DocBin.
+ """
+ docbin = DocBin().from_disk(path)
+ for doc in docbin.get_docs(vocab):
+ yield doc
+
+
+def _stream_jsonl(path: Path, field: str) -> Iterable[str]:
+ """
+ Stream "text" field from JSONL. If the field "text" is
+ not found it raises error.
+ """
+ for entry in srsly.read_jsonl(path):
+ if field not in entry:
+ msg.fail(f"{path} does not contain the required '{field}' field.", exits=1)
+ else:
+ yield entry[field]
+
+
+def _stream_texts(paths: Iterable[Path]) -> Iterable[str]:
+ """
+ Yields strings from text files in paths.
+ """
+ for path in paths:
+ with open(path, "r") as fin:
+ text = fin.read()
+ yield text
+
+
+@app.command("apply")
+def apply_cli(
+ # fmt: off
+ model: str = Arg(..., help="Model name or path"),
+ data_path: Path = Arg(..., help=path_help, exists=True),
+ output_file: Path = Arg(..., help=out_help, dir_okay=False),
+ code_path: Optional[Path] = Opt(None, "--code", "-c", help=code_help),
+ text_key: str = Opt("text", "--text-key", "-tk", help="Key containing text string for JSONL"),
+ force_overwrite: bool = Opt(False, "--force", "-F", help="Force overwriting the output file"),
+ use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU."),
+ batch_size: int = Opt(1, "--batch-size", "-b", help="Batch size."),
+ n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use.")
+):
+ """
+ Apply a trained pipeline to documents to get predictions.
+ Expects a loadable spaCy pipeline and path to the data, which
+ can be a directory or a file.
+ The data files can be provided in multiple formats:
+ 1. .spacy files
+ 2. .jsonl files with a specified "field" to read the text from.
+ 3. Files with any other extension are assumed to be containing
+ a single document.
+ DOCS: https://spacy.io/api/cli#apply
+ """
+ data_path = ensure_path(data_path)
+ output_file = ensure_path(output_file)
+ code_path = ensure_path(code_path)
+ if output_file.exists() and not force_overwrite:
+ msg.fail(force_msg, exits=1)
+ if not data_path.exists():
+ msg.fail(f"Couldn't find data path: {data_path}", exits=1)
+ import_code(code_path)
+ setup_gpu(use_gpu)
+ apply(data_path, output_file, model, text_key, batch_size, n_process)
+
+
+def apply(
+ data_path: Path,
+ output_file: Path,
+ model: str,
+ json_field: str,
+ batch_size: int,
+ n_process: int,
+):
+ docbin = DocBin(store_user_data=True)
+ paths = walk_directory(data_path)
+ if len(paths) == 0:
+ docbin.to_disk(output_file)
+ msg.warn(
+ "Did not find data to process,"
+ f" {data_path} seems to be an empty directory."
+ )
+ return
+ nlp = load_model(model)
+ msg.good(f"Loaded model {model}")
+ vocab = nlp.vocab
+ streams: List[DocOrStrStream] = []
+ text_files = []
+ for path in paths:
+ if path.suffix == ".spacy":
+ streams.append(_stream_docbin(path, vocab))
+ elif path.suffix == ".jsonl":
+ streams.append(_stream_jsonl(path, json_field))
+ else:
+ text_files.append(path)
+ if len(text_files) > 0:
+ streams.append(_stream_texts(text_files))
+ datagen = cast(DocOrStrStream, chain(*streams))
+ for doc in tqdm.tqdm(nlp.pipe(datagen, batch_size=batch_size, n_process=n_process)):
+ docbin.add(doc)
+ if output_file.suffix == "":
+ output_file = output_file.with_suffix(".spacy")
+ docbin.to_disk(output_file)
diff --git a/spacy/cli/benchmark_speed.py b/spacy/cli/benchmark_speed.py
new file mode 100644
index 000000000..4eb20a5fa
--- /dev/null
+++ b/spacy/cli/benchmark_speed.py
@@ -0,0 +1,174 @@
+from typing import Iterable, List, Optional
+import random
+from itertools import islice
+import numpy
+from pathlib import Path
+import time
+from tqdm import tqdm
+import typer
+from wasabi import msg
+
+from .. import util
+from ..language import Language
+from ..tokens import Doc
+from ..training import Corpus
+from ._util import Arg, Opt, benchmark_cli, setup_gpu
+
+
+@benchmark_cli.command(
+ "speed",
+ context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+)
+def benchmark_speed_cli(
+ # fmt: off
+ ctx: typer.Context,
+ model: str = Arg(..., help="Model name or path"),
+ data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
+ batch_size: Optional[int] = Opt(None, "--batch-size", "-b", min=1, help="Override the pipeline batch size"),
+ no_shuffle: bool = Opt(False, "--no-shuffle", help="Do not shuffle benchmark data"),
+ use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
+ n_batches: int = Opt(50, "--batches", help="Minimum number of batches to benchmark", min=30,),
+ warmup_epochs: int = Opt(3, "--warmup", "-w", min=0, help="Number of iterations over the data for warmup"),
+ # fmt: on
+):
+ """
+ Benchmark a pipeline. Expects a loadable spaCy pipeline and benchmark
+ data in the binary .spacy format.
+ """
+ setup_gpu(use_gpu=use_gpu, silent=False)
+
+ nlp = util.load_model(model)
+ batch_size = batch_size if batch_size is not None else nlp.batch_size
+ corpus = Corpus(data_path)
+ docs = [eg.predicted for eg in corpus(nlp)]
+
+ if len(docs) == 0:
+ msg.fail("Cannot benchmark speed using an empty corpus.", exits=1)
+
+ print(f"Warming up for {warmup_epochs} epochs...")
+ warmup(nlp, docs, warmup_epochs, batch_size)
+
+ print()
+ print(f"Benchmarking {n_batches} batches...")
+ wps = benchmark(nlp, docs, n_batches, batch_size, not no_shuffle)
+
+ print()
+ print_outliers(wps)
+ print_mean_with_ci(wps)
+
+
+# Lowercased, behaves as a context manager function.
+class time_context:
+ """Register the running time of a context."""
+
+ def __enter__(self):
+ self.start = time.perf_counter()
+ return self
+
+ def __exit__(self, type, value, traceback):
+ self.elapsed = time.perf_counter() - self.start
+
+
+class Quartiles:
+ """Calculate the q1, q2, q3 quartiles and the inter-quartile range (iqr)
+ of a sample."""
+
+ q1: float
+ q2: float
+ q3: float
+ iqr: float
+
+ def __init__(self, sample: numpy.ndarray) -> None:
+ self.q1 = numpy.quantile(sample, 0.25)
+ self.q2 = numpy.quantile(sample, 0.5)
+ self.q3 = numpy.quantile(sample, 0.75)
+ self.iqr = self.q3 - self.q1
+
+
+def annotate(
+ nlp: Language, docs: List[Doc], batch_size: Optional[int]
+) -> numpy.ndarray:
+ docs = nlp.pipe(tqdm(docs, unit="doc"), batch_size=batch_size)
+ wps = []
+ while True:
+ with time_context() as elapsed:
+ batch_docs = list(
+ islice(docs, batch_size if batch_size else nlp.batch_size)
+ )
+ if len(batch_docs) == 0:
+ break
+ n_tokens = count_tokens(batch_docs)
+ wps.append(n_tokens / elapsed.elapsed)
+
+ return numpy.array(wps)
+
+
+def benchmark(
+ nlp: Language,
+ docs: List[Doc],
+ n_batches: int,
+ batch_size: int,
+ shuffle: bool,
+) -> numpy.ndarray:
+ if shuffle:
+ bench_docs = [
+ nlp.make_doc(random.choice(docs).text)
+ for _ in range(n_batches * batch_size)
+ ]
+ else:
+ bench_docs = [
+ nlp.make_doc(docs[i % len(docs)].text)
+ for i in range(n_batches * batch_size)
+ ]
+
+ return annotate(nlp, bench_docs, batch_size)
+
+
+def bootstrap(x, statistic=numpy.mean, iterations=10000) -> numpy.ndarray:
+ """Apply a statistic to repeated random samples of an array."""
+ return numpy.fromiter(
+ (
+ statistic(numpy.random.choice(x, len(x), replace=True))
+ for _ in range(iterations)
+ ),
+ numpy.float64,
+ )
+
+
+def count_tokens(docs: Iterable[Doc]) -> int:
+ return sum(len(doc) for doc in docs)
+
+
+def print_mean_with_ci(sample: numpy.ndarray):
+ mean = numpy.mean(sample)
+ bootstrap_means = bootstrap(sample)
+ bootstrap_means.sort()
+
+ # 95% confidence interval
+ low = bootstrap_means[int(len(bootstrap_means) * 0.025)]
+ high = bootstrap_means[int(len(bootstrap_means) * 0.975)]
+
+ print(f"Mean: {mean:.1f} words/s (95% CI: {low-mean:.1f} +{high-mean:.1f})")
+
+
+def print_outliers(sample: numpy.ndarray):
+ quartiles = Quartiles(sample)
+
+ n_outliers = numpy.sum(
+ (sample < (quartiles.q1 - 1.5 * quartiles.iqr))
+ | (sample > (quartiles.q3 + 1.5 * quartiles.iqr))
+ )
+ n_extreme_outliers = numpy.sum(
+ (sample < (quartiles.q1 - 3.0 * quartiles.iqr))
+ | (sample > (quartiles.q3 + 3.0 * quartiles.iqr))
+ )
+ print(
+ f"Outliers: {(100 * n_outliers) / len(sample):.1f}%, extreme outliers: {(100 * n_extreme_outliers) / len(sample)}%"
+ )
+
+
+def warmup(
+ nlp: Language, docs: List[Doc], warmup_epochs: int, batch_size: Optional[int]
+) -> numpy.ndarray:
+ docs = warmup_epochs * docs
+ return annotate(nlp, docs, batch_size)
diff --git a/spacy/cli/convert.py b/spacy/cli/convert.py
index 04eb7078f..68d454b3e 100644
--- a/spacy/cli/convert.py
+++ b/spacy/cli/convert.py
@@ -1,4 +1,4 @@
-from typing import Callable, Iterable, Mapping, Optional, Any, List, Union
+from typing import Callable, Iterable, Mapping, Optional, Any, Union
from enum import Enum
from pathlib import Path
from wasabi import Printer
@@ -7,7 +7,7 @@ import re
import sys
import itertools
-from ._util import app, Arg, Opt
+from ._util import app, Arg, Opt, walk_directory
from ..training import docs_to_json
from ..tokens import Doc, DocBin
from ..training.converters import iob_to_docs, conll_ner_to_docs, json_to_docs
@@ -28,6 +28,8 @@ CONVERTERS: Mapping[str, Callable[..., Iterable[Doc]]] = {
"json": json_to_docs,
}
+AUTO = "auto"
+
# File types that can be written to stdout
FILE_TYPES_STDOUT = ("json",)
@@ -49,7 +51,7 @@ def convert_cli(
model: Optional[str] = Opt(None, "--model", "--base", "-b", help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)"),
morphology: bool = Opt(False, "--morphology", "-m", help="Enable appending morphology to tags"),
merge_subtokens: bool = Opt(False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"),
- converter: str = Opt("auto", "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"),
+ converter: str = Opt(AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"),
ner_map: Optional[Path] = Opt(None, "--ner-map", "-nm", help="NER tag mapping (as JSON-encoded dict of entity types)", exists=True),
lang: Optional[str] = Opt(None, "--lang", "-l", help="Language (if tokenizer required)"),
concatenate: bool = Opt(None, "--concatenate", "-C", help="Concatenate output to a single file"),
@@ -70,8 +72,8 @@ def convert_cli(
output_dir: Union[str, Path] = "-" if output_dir == Path("-") else output_dir
silent = output_dir == "-"
msg = Printer(no_print=silent)
- verify_cli_args(msg, input_path, output_dir, file_type.value, converter, ner_map)
converter = _get_converter(msg, converter, input_path)
+ verify_cli_args(msg, input_path, output_dir, file_type.value, converter, ner_map)
convert(
input_path,
output_dir,
@@ -100,7 +102,7 @@ def convert(
model: Optional[str] = None,
morphology: bool = False,
merge_subtokens: bool = False,
- converter: str = "auto",
+ converter: str,
ner_map: Optional[Path] = None,
lang: Optional[str] = None,
concatenate: bool = False,
@@ -189,33 +191,6 @@ def autodetect_ner_format(input_data: str) -> Optional[str]:
return None
-def walk_directory(path: Path, converter: str) -> List[Path]:
- if not path.is_dir():
- return [path]
- paths = [path]
- locs = []
- seen = set()
- for path in paths:
- if str(path) in seen:
- continue
- seen.add(str(path))
- if path.parts[-1].startswith("."):
- continue
- elif path.is_dir():
- paths.extend(path.iterdir())
- elif converter == "json" and not path.parts[-1].endswith("json"):
- continue
- elif converter == "conll" and not path.parts[-1].endswith("conll"):
- continue
- elif converter == "iob" and not path.parts[-1].endswith("iob"):
- continue
- else:
- locs.append(path)
- # It's good to sort these, in case the ordering messes up cache.
- locs.sort()
- return locs
-
-
def verify_cli_args(
msg: Printer,
input_path: Path,
@@ -239,18 +214,22 @@ def verify_cli_args(
input_locs = walk_directory(input_path, converter)
if len(input_locs) == 0:
msg.fail("No input files in directory", input_path, exits=1)
- file_types = list(set([loc.suffix[1:] for loc in input_locs]))
- if converter == "auto" and len(file_types) >= 2:
- file_types_str = ",".join(file_types)
- msg.fail("All input files must be same type", file_types_str, exits=1)
- if converter != "auto" and converter not in CONVERTERS:
+ if converter not in CONVERTERS:
msg.fail(f"Can't find converter for {converter}", exits=1)
def _get_converter(msg, converter, input_path: Path):
if input_path.is_dir():
- input_path = walk_directory(input_path, converter)[0]
- if converter == "auto":
+ if converter == AUTO:
+ input_locs = walk_directory(input_path, suffix=None)
+ file_types = list(set([loc.suffix[1:] for loc in input_locs]))
+ if len(file_types) >= 2:
+ file_types_str = ",".join(file_types)
+ msg.fail("All input files must be same type", file_types_str, exits=1)
+ input_path = input_locs[0]
+ else:
+ input_path = walk_directory(input_path, suffix=converter)[0]
+ if converter == AUTO:
converter = input_path.suffix[1:]
if converter == "ner" or converter == "iob":
with input_path.open(encoding="utf8") as file_:
diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py
index 963d5b926..1c242cec8 100644
--- a/spacy/cli/debug_data.py
+++ b/spacy/cli/debug_data.py
@@ -1,5 +1,5 @@
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
-from typing import cast, overload
+from typing import Literal, cast, overload
from pathlib import Path
from collections import Counter
import sys
@@ -13,13 +13,14 @@ from ._util import import_code, debug_cli, _format_number
from ..training import Example, remove_bilu_prefix
from ..training.initialize import get_sourced_components
from ..schemas import ConfigSchemaTraining
+from ..pipeline import TrainablePipe
from ..pipeline._parser_internals import nonproj
from ..pipeline._parser_internals.nonproj import DELIMITER
from ..pipeline import Morphologizer, SpanCategorizer
+from ..pipeline._edit_tree_internals.edit_trees import EditTrees
from ..morphology import Morphology
from ..language import Language
from ..util import registry, resolve_dot_names
-from ..compat import Literal
from ..vectors import Mode as VectorsMode
from .. import util
@@ -670,6 +671,59 @@ def debug_data(
f"Found {gold_train_data['n_cycles']} projectivized train sentence(s) with cycles"
)
+ if "trainable_lemmatizer" in factory_names:
+ msg.divider("Trainable Lemmatizer")
+ trees_train: Set[str] = gold_train_data["lemmatizer_trees"]
+ trees_dev: Set[str] = gold_dev_data["lemmatizer_trees"]
+ # This is necessary context when someone is attempting to interpret whether the
+ # number of trees exclusively in the dev set is meaningful.
+ msg.info(f"{len(trees_train)} lemmatizer trees generated from training data")
+ msg.info(f"{len(trees_dev)} lemmatizer trees generated from dev data")
+ dev_not_train = trees_dev - trees_train
+
+ if len(dev_not_train) != 0:
+ pct = len(dev_not_train) / len(trees_dev)
+ msg.info(
+ f"{len(dev_not_train)} lemmatizer trees ({pct*100:.1f}% of dev trees)"
+ " were found exclusively in the dev data."
+ )
+ else:
+ # Would we ever expect this case? It seems like it would be pretty rare,
+ # and we might actually want a warning?
+ msg.info("All trees in dev data present in training data.")
+
+ if gold_train_data["n_low_cardinality_lemmas"] > 0:
+ n = gold_train_data["n_low_cardinality_lemmas"]
+ msg.warn(f"{n} training docs with 0 or 1 unique lemmas.")
+
+ if gold_dev_data["n_low_cardinality_lemmas"] > 0:
+ n = gold_dev_data["n_low_cardinality_lemmas"]
+ msg.warn(f"{n} dev docs with 0 or 1 unique lemmas.")
+
+ if gold_train_data["no_lemma_annotations"] > 0:
+ n = gold_train_data["no_lemma_annotations"]
+ msg.warn(f"{n} training docs with no lemma annotations.")
+ else:
+ msg.good("All training docs have lemma annotations.")
+
+ if gold_dev_data["no_lemma_annotations"] > 0:
+ n = gold_dev_data["no_lemma_annotations"]
+ msg.warn(f"{n} dev docs with no lemma annotations.")
+ else:
+ msg.good("All dev docs have lemma annotations.")
+
+ if gold_train_data["partial_lemma_annotations"] > 0:
+ n = gold_train_data["partial_lemma_annotations"]
+ msg.info(f"{n} training docs with partial lemma annotations.")
+ else:
+ msg.good("All training docs have complete lemma annotations.")
+
+ if gold_dev_data["partial_lemma_annotations"] > 0:
+ n = gold_dev_data["partial_lemma_annotations"]
+ msg.info(f"{n} dev docs with partial lemma annotations.")
+ else:
+ msg.good("All dev docs have complete lemma annotations.")
+
msg.divider("Summary")
good_counts = msg.counts[MESSAGES.GOOD]
warn_counts = msg.counts[MESSAGES.WARN]
@@ -731,7 +785,13 @@ def _compile_gold(
"n_cats_multilabel": 0,
"n_cats_bad_values": 0,
"texts": set(),
+ "lemmatizer_trees": set(),
+ "no_lemma_annotations": 0,
+ "partial_lemma_annotations": 0,
+ "n_low_cardinality_lemmas": 0,
}
+ if "trainable_lemmatizer" in factory_names:
+ trees = EditTrees(nlp.vocab.strings)
for eg in examples:
gold = eg.reference
doc = eg.predicted
@@ -861,6 +921,25 @@ def _compile_gold(
data["n_nonproj"] += 1
if nonproj.contains_cycle(aligned_heads):
data["n_cycles"] += 1
+ if "trainable_lemmatizer" in factory_names:
+ # from EditTreeLemmatizer._labels_from_data
+ if all(token.lemma == 0 for token in gold):
+ data["no_lemma_annotations"] += 1
+ continue
+ if any(token.lemma == 0 for token in gold):
+ data["partial_lemma_annotations"] += 1
+ lemma_set = set()
+ for token in gold:
+ if token.lemma != 0:
+ lemma_set.add(token.lemma)
+ tree_id = trees.add(token.text, token.lemma_)
+ tree_str = trees.tree_to_str(tree_id)
+ data["lemmatizer_trees"].add(tree_str)
+ # We want to identify cases where lemmas aren't assigned
+ # or are all assigned the same value, as this would indicate
+ # an issue since we're expecting a large set of lemmas
+ if len(lemma_set) < 2 and len(gold) > 1:
+ data["n_low_cardinality_lemmas"] += 1
return data
@@ -934,6 +1013,7 @@ def _get_labels_from_model(nlp: Language, factory_name: str) -> Set[str]:
labels: Set[str] = set()
for pipe_name in pipe_names:
pipe = nlp.get_pipe(pipe_name)
+ assert isinstance(pipe, TrainablePipe)
labels.update(pipe.labels)
return labels
diff --git a/spacy/cli/download.py b/spacy/cli/download.py
index 0c9a32b93..4c998a6e0 100644
--- a/spacy/cli/download.py
+++ b/spacy/cli/download.py
@@ -8,7 +8,6 @@ from ._util import app, Arg, Opt, WHEEL_SUFFIX, SDIST_SUFFIX
from .. import about
from ..util import is_package, get_minor_version, run_command
from ..util import is_prerelease_version
-from ..errors import OLD_MODEL_SHORTCUTS
@app.command(
@@ -61,12 +60,6 @@ def download(
version = components[-1]
else:
model_name = model
- if model in OLD_MODEL_SHORTCUTS:
- msg.warn(
- f"As of spaCy v3.0, shortcuts like '{model}' are deprecated. Please "
- f"use the full pipeline package name '{OLD_MODEL_SHORTCUTS[model]}' instead."
- )
- model_name = OLD_MODEL_SHORTCUTS[model]
compatibility = get_compatibility()
version = get_version(model_name, compatibility)
diff --git a/spacy/cli/evaluate.py b/spacy/cli/evaluate.py
index 0d08d2c5e..8f3d6b859 100644
--- a/spacy/cli/evaluate.py
+++ b/spacy/cli/evaluate.py
@@ -7,12 +7,15 @@ from thinc.api import fix_random_seed
from ..training import Corpus
from ..tokens import Doc
-from ._util import app, Arg, Opt, setup_gpu, import_code
+from ._util import app, Arg, Opt, setup_gpu, import_code, benchmark_cli
from ..scorer import Scorer
from .. import util
from .. import displacy
+@benchmark_cli.command(
+ "accuracy",
+)
@app.command("evaluate")
def evaluate_cli(
# fmt: off
@@ -36,7 +39,7 @@ def evaluate_cli(
dependency parses in a HTML file, set as output directory as the
displacy_path argument.
- DOCS: https://spacy.io/api/cli#evaluate
+ DOCS: https://spacy.io/api/cli#benchmark-accuracy
"""
import_code(code_path)
evaluate(
diff --git a/spacy/cli/find_threshold.py b/spacy/cli/find_threshold.py
new file mode 100644
index 000000000..efa664832
--- /dev/null
+++ b/spacy/cli/find_threshold.py
@@ -0,0 +1,233 @@
+import functools
+import operator
+from pathlib import Path
+import logging
+from typing import Optional, Tuple, Any, Dict, List
+
+import numpy
+import wasabi.tables
+
+from ..pipeline import TextCategorizer, MultiLabel_TextCategorizer
+from ..errors import Errors
+from ..training import Corpus
+from ._util import app, Arg, Opt, import_code, setup_gpu
+from .. import util
+
+_DEFAULTS = {
+ "n_trials": 11,
+ "use_gpu": -1,
+ "gold_preproc": False,
+}
+
+
+@app.command(
+ "find-threshold",
+ context_settings={"allow_extra_args": False, "ignore_unknown_options": True},
+)
+def find_threshold_cli(
+ # fmt: off
+ model: str = Arg(..., help="Model name or path"),
+ data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
+ pipe_name: str = Arg(..., help="Name of pipe to examine thresholds for"),
+ threshold_key: str = Arg(..., help="Key of threshold attribute in component's configuration"),
+ scores_key: str = Arg(..., help="Metric to optimize"),
+ n_trials: int = Opt(_DEFAULTS["n_trials"], "--n_trials", "-n", help="Number of trials to determine optimal thresholds"),
+ code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
+ use_gpu: int = Opt(_DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
+ gold_preproc: bool = Opt(_DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing"),
+ verbose: bool = Opt(False, "--silent", "-V", "-VV", help="Display more information for debugging purposes"),
+ # fmt: on
+):
+ """
+ Runs prediction trials for a trained model with varying tresholds to maximize
+ the specified metric. The search space for the threshold is traversed linearly
+ from 0 to 1 in `n_trials` steps. Results are displayed in a table on `stdout`
+ (the corresponding API call to `spacy.cli.find_threshold.find_threshold()`
+ returns all results).
+
+ This is applicable only for components whose predictions are influenced by
+ thresholds - e.g. `textcat_multilabel` and `spancat`, but not `textcat`. Note
+ that the full path to the corresponding threshold attribute in the config has to
+ be provided.
+
+ DOCS: https://spacy.io/api/cli#find-threshold
+ """
+
+ util.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+ import_code(code_path)
+ find_threshold(
+ model=model,
+ data_path=data_path,
+ pipe_name=pipe_name,
+ threshold_key=threshold_key,
+ scores_key=scores_key,
+ n_trials=n_trials,
+ use_gpu=use_gpu,
+ gold_preproc=gold_preproc,
+ silent=False,
+ )
+
+
+def find_threshold(
+ model: str,
+ data_path: Path,
+ pipe_name: str,
+ threshold_key: str,
+ scores_key: str,
+ *,
+ n_trials: int = _DEFAULTS["n_trials"], # type: ignore
+ use_gpu: int = _DEFAULTS["use_gpu"], # type: ignore
+ gold_preproc: bool = _DEFAULTS["gold_preproc"], # type: ignore
+ silent: bool = True,
+) -> Tuple[float, float, Dict[float, float]]:
+ """
+ Runs prediction trials for models with varying tresholds to maximize the specified metric.
+ model (Union[str, Path]): Pipeline to evaluate. Can be a package or a path to a data directory.
+ data_path (Path): Path to file with DocBin with docs to use for threshold search.
+ pipe_name (str): Name of pipe to examine thresholds for.
+ threshold_key (str): Key of threshold attribute in component's configuration.
+ scores_key (str): Name of score to metric to optimize.
+ n_trials (int): Number of trials to determine optimal thresholds.
+ use_gpu (int): GPU ID or -1 for CPU.
+ gold_preproc (bool): Whether to use gold preprocessing. Gold preprocessing helps the annotations align to the
+ tokenization, and may result in sequences of more consistent length. However, it may reduce runtime accuracy due
+ to train/test skew.
+ silent (bool): Whether to print non-error-related output to stdout.
+ RETURNS (Tuple[float, float, Dict[float, float]]): Best found threshold, the corresponding score, scores for all
+ evaluated thresholds.
+ """
+
+ setup_gpu(use_gpu, silent=silent)
+ data_path = util.ensure_path(data_path)
+ if not data_path.exists():
+ wasabi.msg.fail("Evaluation data not found", data_path, exits=1)
+ nlp = util.load_model(model)
+
+ if pipe_name not in nlp.component_names:
+ raise AttributeError(
+ Errors.E001.format(name=pipe_name, opts=nlp.component_names)
+ )
+ pipe = nlp.get_pipe(pipe_name)
+ if not hasattr(pipe, "scorer"):
+ raise AttributeError(Errors.E1045)
+
+ if type(pipe) == TextCategorizer:
+ wasabi.msg.warn(
+ "The `textcat` component doesn't use a threshold as it's not applicable to the concept of "
+ "exclusive classes. All thresholds will yield the same results."
+ )
+
+ if not silent:
+ wasabi.msg.info(
+ title=f"Optimizing for {scores_key} for component '{pipe_name}' with {n_trials} "
+ f"trials."
+ )
+
+ # Load evaluation corpus.
+ corpus = Corpus(data_path, gold_preproc=gold_preproc)
+ dev_dataset = list(corpus(nlp))
+ config_keys = threshold_key.split(".")
+
+ def set_nested_item(
+ config: Dict[str, Any], keys: List[str], value: float
+ ) -> Dict[str, Any]:
+ """Set item in nested dictionary. Adapted from https://stackoverflow.com/a/54138200.
+ config (Dict[str, Any]): Configuration dictionary.
+ keys (List[Any]): Path to value to set.
+ value (float): Value to set.
+ RETURNS (Dict[str, Any]): Updated dictionary.
+ """
+ functools.reduce(operator.getitem, keys[:-1], config)[keys[-1]] = value
+ return config
+
+ def filter_config(
+ config: Dict[str, Any], keys: List[str], full_key: str
+ ) -> Dict[str, Any]:
+ """Filters provided config dictionary so that only the specified keys path remains.
+ config (Dict[str, Any]): Configuration dictionary.
+ keys (List[Any]): Path to value to set.
+ full_key (str): Full user-specified key.
+ RETURNS (Dict[str, Any]): Filtered dictionary.
+ """
+ if keys[0] not in config:
+ wasabi.msg.fail(
+ title=f"Failed to look up `{full_key}` in config: sub-key {[keys[0]]} not found.",
+ text=f"Make sure you specified {[keys[0]]} correctly. The following sub-keys are available instead: "
+ f"{list(config.keys())}",
+ exits=1,
+ )
+ return {
+ keys[0]: filter_config(config[keys[0]], keys[1:], full_key)
+ if len(keys) > 1
+ else config[keys[0]]
+ }
+
+ # Evaluate with varying threshold values.
+ scores: Dict[float, float] = {}
+ config_keys_full = ["components", pipe_name, *config_keys]
+ table_col_widths = (10, 10)
+ thresholds = numpy.linspace(0, 1, n_trials)
+ print(wasabi.tables.row(["Threshold", f"{scores_key}"], widths=table_col_widths))
+ for threshold in thresholds:
+ # Reload pipeline with overrides specifying the new threshold.
+ nlp = util.load_model(
+ model,
+ config=set_nested_item(
+ filter_config(
+ nlp.config, config_keys_full, ".".join(config_keys_full)
+ ).copy(),
+ config_keys_full,
+ threshold,
+ ),
+ )
+ if hasattr(pipe, "cfg"):
+ setattr(
+ nlp.get_pipe(pipe_name),
+ "cfg",
+ set_nested_item(getattr(pipe, "cfg"), config_keys, threshold),
+ )
+
+ eval_scores = nlp.evaluate(dev_dataset)
+ if scores_key not in eval_scores:
+ wasabi.msg.fail(
+ title=f"Failed to look up score `{scores_key}` in evaluation results.",
+ text=f"Make sure you specified the correct value for `scores_key`. The following scores are "
+ f"available: {list(eval_scores.keys())}",
+ exits=1,
+ )
+ scores[threshold] = eval_scores[scores_key]
+
+ if not isinstance(scores[threshold], (float, int)):
+ wasabi.msg.fail(
+ f"Returned score for key '{scores_key}' is not numeric. Threshold optimization only works for numeric "
+ f"scores.",
+ exits=1,
+ )
+ print(
+ wasabi.row(
+ [round(threshold, 3), round(scores[threshold], 3)],
+ widths=table_col_widths,
+ )
+ )
+
+ best_threshold = max(scores.keys(), key=(lambda key: scores[key]))
+
+ # If all scores are identical, emit warning.
+ if len(set(scores.values())) == 1:
+ wasabi.msg.warn(
+ title="All scores are identical. Verify that all settings are correct.",
+ text=""
+ if (
+ not isinstance(pipe, MultiLabel_TextCategorizer)
+ or scores_key in ("cats_macro_f", "cats_micro_f")
+ )
+ else "Use `cats_macro_f` or `cats_micro_f` when optimizing the threshold for `textcat_multilabel`.",
+ )
+
+ else:
+ if not silent:
+ print(
+ f"\nBest threshold: {round(best_threshold, ndigits=4)} with {scores_key} value of {scores[best_threshold]}."
+ )
+
+ return best_threshold, scores[best_threshold], scores
diff --git a/spacy/cli/project/assets.py b/spacy/cli/project/assets.py
index 61438d1a8..8f35b2d23 100644
--- a/spacy/cli/project/assets.py
+++ b/spacy/cli/project/assets.py
@@ -189,7 +189,11 @@ def convert_asset_url(url: str) -> str:
RETURNS (str): The converted URL.
"""
# If the asset URL is a regular GitHub URL it's likely a mistake
- if re.match(r"(http(s?)):\/\/github.com", url) and "releases/download" not in url:
+ if (
+ re.match(r"(http(s?)):\/\/github.com", url)
+ and "releases/download" not in url
+ and "/raw/" not in url
+ ):
converted = url.replace("github.com", "raw.githubusercontent.com")
converted = re.sub(r"/(tree|blob)/", "/", converted)
msg.warn(
diff --git a/spacy/cli/project/remote_storage.py b/spacy/cli/project/remote_storage.py
index 336a4bcb3..076541580 100644
--- a/spacy/cli/project/remote_storage.py
+++ b/spacy/cli/project/remote_storage.py
@@ -5,14 +5,17 @@ import hashlib
import urllib.parse
import tarfile
from pathlib import Path
+from wasabi import msg
-from .._util import get_hash, get_checksum, download_file, ensure_pathy
-from ...util import make_tempdir, get_minor_version, ENV_VARS, check_bool_env_var
+from .._util import get_hash, get_checksum, upload_file, download_file
+from .._util import ensure_pathy, make_tempdir
+from ...util import get_minor_version, ENV_VARS, check_bool_env_var
from ...git_info import GIT_VERSION
from ... import about
+from ...errors import Errors
if TYPE_CHECKING:
- from pathy import Pathy # noqa: F401
+ from pathy import FluidPath # noqa: F401
class RemoteStorage:
@@ -27,7 +30,7 @@ class RemoteStorage:
self.url = ensure_pathy(url)
self.compression = compression
- def push(self, path: Path, command_hash: str, content_hash: str) -> "Pathy":
+ def push(self, path: Path, command_hash: str, content_hash: str) -> "FluidPath":
"""Compress a file or directory within a project and upload it to a remote
storage. If an object exists at the full URL, nothing is done.
@@ -48,9 +51,7 @@ class RemoteStorage:
mode_string = f"w:{self.compression}" if self.compression else "w"
with tarfile.open(tar_loc, mode=mode_string) as tar_file:
tar_file.add(str(loc), arcname=str(path))
- with tar_loc.open(mode="rb") as input_file:
- with url.open(mode="wb") as output_file:
- output_file.write(input_file.read())
+ upload_file(tar_loc, url)
return url
def pull(
@@ -59,7 +60,7 @@ class RemoteStorage:
*,
command_hash: Optional[str] = None,
content_hash: Optional[str] = None,
- ) -> Optional["Pathy"]:
+ ) -> Optional["FluidPath"]:
"""Retrieve a file from the remote cache. If the file already exists,
nothing is done.
@@ -84,7 +85,23 @@ class RemoteStorage:
with tarfile.open(tar_loc, mode=mode_string) as tar_file:
# This requires that the path is added correctly, relative
# to root. This is how we set things up in push()
- tar_file.extractall(self.root)
+
+ # Disallow paths outside the current directory for the tar
+ # file (CVE-2007-4559, directory traversal vulnerability)
+ def is_within_directory(directory, target):
+ abs_directory = os.path.abspath(directory)
+ abs_target = os.path.abspath(target)
+ prefix = os.path.commonprefix([abs_directory, abs_target])
+ return prefix == abs_directory
+
+ def safe_extract(tar, path):
+ for member in tar.getmembers():
+ member_path = os.path.join(path, member.name)
+ if not is_within_directory(path, member_path):
+ raise ValueError(Errors.E852)
+ tar.extractall(path)
+
+ safe_extract(tar_file, self.root)
return url
def find(
@@ -93,25 +110,37 @@ class RemoteStorage:
*,
command_hash: Optional[str] = None,
content_hash: Optional[str] = None,
- ) -> Optional["Pathy"]:
+ ) -> Optional["FluidPath"]:
"""Find the best matching version of a file within the storage,
or `None` if no match can be found. If both the creation and content hash
are specified, only exact matches will be returned. Otherwise, the most
recent matching file is preferred.
"""
name = self.encode_name(str(path))
+ urls = []
if command_hash is not None and content_hash is not None:
- url = self.make_url(path, command_hash, content_hash)
+ url = self.url / name / command_hash / content_hash
urls = [url] if url.exists() else []
elif command_hash is not None:
- urls = list((self.url / name / command_hash).iterdir())
+ if (self.url / name / command_hash).exists():
+ urls = list((self.url / name / command_hash).iterdir())
else:
- urls = list((self.url / name).iterdir())
- if content_hash is not None:
- urls = [url for url in urls if url.parts[-1] == content_hash]
+ if (self.url / name).exists():
+ for sub_dir in (self.url / name).iterdir():
+ urls.extend(sub_dir.iterdir())
+ if content_hash is not None:
+ urls = [url for url in urls if url.parts[-1] == content_hash]
+ if len(urls) >= 2:
+ try:
+ urls.sort(key=lambda x: x.stat().last_modified) # type: ignore
+ except Exception:
+ msg.warn(
+ "Unable to sort remote files by last modified. The file(s) "
+ "pulled from the cache may not be the most recent."
+ )
return urls[-1] if urls else None
- def make_url(self, path: Path, command_hash: str, content_hash: str) -> "Pathy":
+ def make_url(self, path: Path, command_hash: str, content_hash: str) -> "FluidPath":
"""Construct a URL from a subpath, a creation hash and a content hash."""
return self.url / self.encode_name(str(path)) / command_hash / content_hash
diff --git a/spacy/cli/project/run.py b/spacy/cli/project/run.py
index ebab7471e..6dd174902 100644
--- a/spacy/cli/project/run.py
+++ b/spacy/cli/project/run.py
@@ -53,6 +53,7 @@ def project_run(
force: bool = False,
dry: bool = False,
capture: bool = False,
+ skip_requirements_check: bool = False,
) -> None:
"""Run a named script defined in the project.yml. If the script is part
of the default pipeline (defined in the "run" section), DVC is used to
@@ -69,6 +70,7 @@ def project_run(
sys.exit will be called with the return code. You should use capture=False
when you want to turn over execution to the command, and capture=True
when you want to run the command more like a function.
+ skip_requirements_check (bool): Whether to skip the requirements check.
"""
config = load_project_config(project_dir, overrides=overrides)
commands = {cmd["name"]: cmd for cmd in config.get("commands", [])}
@@ -76,9 +78,10 @@ def project_run(
validate_subcommand(list(commands.keys()), list(workflows.keys()), subcommand)
req_path = project_dir / "requirements.txt"
- if config.get("check_requirements", True) and os.path.exists(req_path):
- with req_path.open() as requirements_file:
- _check_requirements([req.replace("\n", "") for req in requirements_file])
+ if not skip_requirements_check:
+ if config.get("check_requirements", True) and os.path.exists(req_path):
+ with req_path.open() as requirements_file:
+ _check_requirements([req.strip() for req in requirements_file])
if subcommand in workflows:
msg.info(f"Running workflow '{subcommand}'")
@@ -90,6 +93,7 @@ def project_run(
force=force,
dry=dry,
capture=capture,
+ skip_requirements_check=True,
)
else:
cmd = commands[subcommand]
@@ -97,8 +101,8 @@ def project_run(
if not (project_dir / dep).exists():
err = f"Missing dependency specified by command '{subcommand}': {dep}"
err_help = "Maybe you forgot to run the 'project assets' command or a previous step?"
- err_kwargs = {"exits": 1} if not dry else {}
- msg.fail(err, err_help, **err_kwargs)
+ err_exits = 1 if not dry else None
+ msg.fail(err, err_help, exits=err_exits)
check_spacy_commit = check_bool_env_var(ENV_VARS.PROJECT_USE_GIT_VERSION)
with working_dir(project_dir) as current_dir:
msg.divider(subcommand)
@@ -338,6 +342,12 @@ def _check_requirements(requirements: List[str]) -> Tuple[bool, bool]:
failed_pkgs_msgs.append(dnf.report())
except pkg_resources.VersionConflict as vc:
conflicting_pkgs_msgs.append(vc.report())
+ except Exception:
+ msg.warn(
+ f"Unable to check requirement: {req} "
+ "Checks are currently limited to requirement specifiers "
+ "(PEP 508)"
+ )
if len(failed_pkgs_msgs) or len(conflicting_pkgs_msgs):
msg.warn(
diff --git a/spacy/cli/templates/quickstart_training.jinja b/spacy/cli/templates/quickstart_training.jinja
index 58864883a..eb48d1de5 100644
--- a/spacy/cli/templates/quickstart_training.jinja
+++ b/spacy/cli/templates/quickstart_training.jinja
@@ -1,7 +1,7 @@
{# This is a template for training configs used for the quickstart widget in
the docs and the init config command. It encodes various best practices and
can help generate the best possible configuration, given a user's requirements. #}
-{%- set use_transformer = hardware != "cpu" -%}
+{%- set use_transformer = hardware != "cpu" and transformer_data -%}
{%- set transformer = transformer_data[optimize] if use_transformer else {} -%}
{%- set listener_components = ["tagger", "morphologizer", "parser", "ner", "textcat", "textcat_multilabel", "entity_linker", "spancat", "trainable_lemmatizer"] -%}
[paths]
@@ -87,12 +87,11 @@ grad_factor = 1.0
factory = "parser"
[components.parser.model]
-@architectures = "spacy.TransitionBasedParser.v2"
+@architectures = "spacy.TransitionBasedParser.v3"
state_type = "parser"
extra_state_tokens = false
hidden_width = 128
maxout_pieces = 3
-use_upper = false
nO = null
[components.parser.model.tok2vec]
@@ -108,12 +107,11 @@ grad_factor = 1.0
factory = "ner"
[components.ner.model]
-@architectures = "spacy.TransitionBasedParser.v2"
+@architectures = "spacy.TransitionBasedParser.v3"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
-use_upper = false
nO = null
[components.ner.model.tok2vec]
@@ -314,12 +312,11 @@ width = ${components.tok2vec.model.encode.width}
factory = "parser"
[components.parser.model]
-@architectures = "spacy.TransitionBasedParser.v2"
+@architectures = "spacy.TransitionBasedParser.v3"
state_type = "parser"
extra_state_tokens = false
hidden_width = 128
maxout_pieces = 3
-use_upper = true
nO = null
[components.parser.model.tok2vec]
@@ -332,12 +329,11 @@ width = ${components.tok2vec.model.encode.width}
factory = "ner"
[components.ner.model]
-@architectures = "spacy.TransitionBasedParser.v2"
+@architectures = "spacy.TransitionBasedParser.v3"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
-use_upper = true
nO = null
[components.ner.model.tok2vec]
diff --git a/spacy/cli/templates/quickstart_training_recommendations.yml b/spacy/cli/templates/quickstart_training_recommendations.yml
index 27945e27a..4f214d22d 100644
--- a/spacy/cli/templates/quickstart_training_recommendations.yml
+++ b/spacy/cli/templates/quickstart_training_recommendations.yml
@@ -37,6 +37,15 @@ bn:
accuracy:
name: sagorsarker/bangla-bert-base
size_factor: 3
+ca:
+ word_vectors: null
+ transformer:
+ efficiency:
+ name: projecte-aina/roberta-base-ca-v2
+ size_factor: 3
+ accuracy:
+ name: projecte-aina/roberta-base-ca-v2
+ size_factor: 3
da:
word_vectors: da_core_news_lg
transformer:
diff --git a/spacy/compat.py b/spacy/compat.py
index 89132735d..5344b7cd4 100644
--- a/spacy/compat.py
+++ b/spacy/compat.py
@@ -22,19 +22,6 @@ try:
except ImportError:
cupy = None
-if sys.version_info[:2] >= (3, 8): # Python 3.8+
- from typing import Literal, Protocol, runtime_checkable
-else:
- from typing_extensions import Literal, Protocol, runtime_checkable # noqa: F401
-
-# Important note: The importlib_metadata "backport" includes functionality
-# that's not part of the built-in importlib.metadata. We should treat this
-# import like the built-in and only use what's available there.
-try: # Python 3.8+
- import importlib.metadata as importlib_metadata
-except ImportError:
- from catalogue import _importlib_metadata as importlib_metadata # type: ignore[no-redef] # noqa: F401
-
from thinc.api import Optimizer # noqa: F401
pickle = pickle
diff --git a/spacy/default_config.cfg b/spacy/default_config.cfg
index 86a72926e..694fb732f 100644
--- a/spacy/default_config.cfg
+++ b/spacy/default_config.cfg
@@ -90,6 +90,8 @@ dev_corpus = "corpora.dev"
train_corpus = "corpora.train"
# Optional callback before nlp object is saved to disk after training
before_to_disk = null
+# Optional callback that is invoked at the start of each training step
+before_update = null
[training.logger]
@loggers = "spacy.ConsoleLogger.v1"
diff --git a/spacy/displacy/__init__.py b/spacy/displacy/__init__.py
index 7bb300afa..ea6bba2c9 100644
--- a/spacy/displacy/__init__.py
+++ b/spacy/displacy/__init__.py
@@ -11,6 +11,7 @@ from .render import DependencyRenderer, EntityRenderer, SpanRenderer
from ..tokens import Doc, Span
from ..errors import Errors, Warnings
from ..util import is_in_jupyter
+from ..util import find_available_port
_html = {}
@@ -36,7 +37,7 @@ def render(
jupyter (bool): Override Jupyter auto-detection.
options (dict): Visualiser-specific options, e.g. colors.
manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts.
- RETURNS (str): Rendered HTML markup.
+ RETURNS (str): Rendered SVG or HTML markup.
DOCS: https://spacy.io/api/top-level#displacy.render
USAGE: https://spacy.io/usage/visualizers
@@ -82,6 +83,7 @@ def serve(
manual: bool = False,
port: int = 5000,
host: str = "0.0.0.0",
+ auto_select_port: bool = False,
) -> None:
"""Serve displaCy visualisation.
@@ -93,12 +95,15 @@ def serve(
manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts.
port (int): Port to serve visualisation.
host (str): Host to serve visualisation.
+ auto_select_port (bool): Automatically select a port if the specified port is in use.
DOCS: https://spacy.io/api/top-level#displacy.serve
USAGE: https://spacy.io/usage/visualizers
"""
from wsgiref import simple_server
+ port = find_available_port(port, host, auto_select_port)
+
if is_in_jupyter():
warnings.warn(Warnings.W011)
render(docs, style=style, page=page, minify=minify, options=options, manual=manual)
@@ -228,12 +233,13 @@ def parse_spans(doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]:
"kb_id": span.kb_id_ if span.kb_id_ else "",
"kb_url": kb_url_template.format(span.kb_id_) if kb_url_template else "#",
}
- for span in doc.spans[spans_key]
+ for span in doc.spans.get(spans_key, [])
]
tokens = [token.text for token in doc]
if not spans:
- warnings.warn(Warnings.W117.format(spans_key=spans_key))
+ keys = list(doc.spans.keys())
+ warnings.warn(Warnings.W117.format(spans_key=spans_key, keys=keys))
title = doc.user_data.get("title", None) if hasattr(doc, "user_data") else None
settings = get_doc_settings(doc)
return {
diff --git a/spacy/displacy/render.py b/spacy/displacy/render.py
index 50dc3466c..f74222dc2 100644
--- a/spacy/displacy/render.py
+++ b/spacy/displacy/render.py
@@ -94,7 +94,7 @@ class SpanRenderer:
parsed (list): Dependency parses to render.
page (bool): Render parses wrapped as full HTML page.
minify (bool): Minify HTML markup.
- RETURNS (str): Rendered HTML markup.
+ RETURNS (str): Rendered SVG or HTML markup.
"""
rendered = []
for i, p in enumerate(parsed):
@@ -510,7 +510,7 @@ class EntityRenderer:
parsed (list): Dependency parses to render.
page (bool): Render parses wrapped as full HTML page.
minify (bool): Minify HTML markup.
- RETURNS (str): Rendered HTML markup.
+ RETURNS (str): Rendered SVG or HTML markup.
"""
rendered = []
for i, p in enumerate(parsed):
diff --git a/spacy/errors.py b/spacy/errors.py
index 820f7352e..5f480c16c 100644
--- a/spacy/errors.py
+++ b/spacy/errors.py
@@ -1,5 +1,5 @@
+from typing import Literal
import warnings
-from .compat import Literal
class ErrorsWithCodes(type):
@@ -131,13 +131,6 @@ class Warnings(metaclass=ErrorsWithCodes):
"and make it independent. For example, `replace_listeners = "
"[\"model.tok2vec\"]` See the documentation for details: "
"https://spacy.io/usage/training#config-components-listeners")
- W088 = ("The pipeline component {name} implements a `begin_training` "
- "method, which won't be called by spaCy. As of v3.0, `begin_training` "
- "has been renamed to `initialize`, so you likely want to rename the "
- "component method. See the documentation for details: "
- "https://spacy.io/api/language#initialize")
- W089 = ("As of spaCy v3.0, the `nlp.begin_training` method has been renamed "
- "to `nlp.initialize`.")
W090 = ("Could not locate any {format} files in path '{path}'.")
W091 = ("Could not clean/remove the temp directory at {dir}: {msg}.")
W092 = ("Ignoring annotations for sentence starts, as dependency heads are set.")
@@ -199,7 +192,7 @@ class Warnings(metaclass=ErrorsWithCodes):
W117 = ("No spans to visualize found in Doc object with spans_key: '{spans_key}'. If this is "
"surprising to you, make sure the Doc was processed using a model "
"that supports span categorization, and check the `doc.spans[spans_key]` "
- "property manually if necessary.")
+ "property manually if necessary.\n\nAvailable keys: {keys}")
W118 = ("Term '{term}' not found in glossary. It may however be explained in documentation "
"for the corpora used to train the language. Please check "
"`nlp.meta[\"sources\"]` for any relevant links.")
@@ -212,8 +205,11 @@ class Warnings(metaclass=ErrorsWithCodes):
W121 = ("Attempting to trace non-existent method '{method}' in pipe '{pipe}'")
W122 = ("Couldn't trace method '{method}' in pipe '{pipe}'. This can happen if the pipe class "
"is a Cython extension type.")
- W123 = ("Argument {arg} with value {arg_value} is used instead of {config_value} as specified in the config. Be "
- "aware that this might affect other components in your pipeline.")
+ W123 = ("Argument `enable` with value {enable} does not contain all values specified in the config option "
+ "`enabled` ({enabled}). Be aware that this might affect other components in your pipeline.")
+ W124 = ("{host}:{port} is already in use, using the nearest available port {serve_port} as an alternative.")
+
+ W400 = ("`use_upper=False` is ignored, the upper layer is always enabled")
class Errors(metaclass=ErrorsWithCodes):
@@ -250,8 +246,6 @@ class Errors(metaclass=ErrorsWithCodes):
"https://spacy.io/usage/models")
E011 = ("Unknown operator: '{op}'. Options: {opts}")
E012 = ("Cannot add pattern for zero tokens to matcher.\nKey: {key}")
- E016 = ("MultitaskObjective target should be function or one of: dep, "
- "tag, ent, dep_tag_offset, ent_tag.")
E017 = ("Can only add 'str' inputs to StringStore. Got type: {value_type}")
E018 = ("Can't retrieve string for hash '{hash_value}'. This usually "
"refers to an issue with the `Vocab` or `StringStore`.")
@@ -345,6 +339,11 @@ class Errors(metaclass=ErrorsWithCodes):
"clear the existing vectors and resize the table.")
E074 = ("Error interpreting compiled match pattern: patterns are expected "
"to end with the attribute {attr}. Got: {bad_attr}.")
+ E079 = ("Error computing states in beam: number of predicted beams "
+ "({pbeams}) does not equal number of gold beams ({gbeams}).")
+ E080 = ("Duplicate state found in beam: {key}.")
+ E081 = ("Error getting gradient in beam: number of histories ({n_hist}) "
+ "does not equal number of losses ({losses}).")
E082 = ("Error deprojectivizing parse: number of heads ({n_heads}), "
"projective heads ({n_proj_heads}) and labels ({n_labels}) do not "
"match.")
@@ -544,6 +543,10 @@ class Errors(metaclass=ErrorsWithCodes):
"during training, make sure to include it in 'annotating components'")
# New errors added in v3.x
+ E851 = ("The 'textcat' component labels should only have values of 0 or 1, "
+ "but found value of '{val}'.")
+ E852 = ("The tar file pulled from the remote attempted an unsafe path "
+ "traversal.")
E853 = ("Unsupported component factory name '{name}'. The character '.' is "
"not permitted in factory names.")
E854 = ("Unable to set doc.ents. Check that the 'ents_filter' does not "
@@ -723,13 +726,6 @@ class Errors(metaclass=ErrorsWithCodes):
"method in component '{name}'. If you want to use this "
"method, make sure it's overwritten on the subclass.")
E940 = ("Found NaN values in scores.")
- E941 = ("Can't find model '{name}'. It looks like you're trying to load a "
- "model from a shortcut, which is obsolete as of spaCy v3.0. To "
- "load the model, use its full name instead:\n\n"
- "nlp = spacy.load(\"{full}\")\n\nFor more details on the available "
- "models, see the models directory: https://spacy.io/models. If you "
- "want to create a blank model, use spacy.blank: "
- "nlp = spacy.blank(\"{name}\")")
E942 = ("Executing `after_{name}` callback failed. Expected the function to "
"return an initialized nlp object but got: {value}. Maybe "
"you forgot to return the modified object in your function?")
@@ -950,20 +946,21 @@ class Errors(metaclass=ErrorsWithCodes):
"sure it's overwritten on the subclass.")
E1046 = ("{cls_name} is an abstract class and cannot be instantiated. If you are looking for spaCy's default "
"knowledge base, use `InMemoryLookupKB`.")
+ E1047 = ("`find_threshold()` only supports components with a `scorer` attribute.")
+ E1048 = ("Got '{unexpected}' as console progress bar type, but expected one of the following: {expected}")
+ E1049 = ("No available port found for displaCy on host {host}. Please specify an available port "
+ "with `displacy.serve(doc, port=port)`")
+ E1050 = ("Port {port} is already in use. Please specify an available port with `displacy.serve(doc, port=port)` "
+ "or use `auto_switch_port=True` to pick an available port automatically.")
# v4 error strings
E4000 = ("Expected a Doc as input, but got: '{type}'")
E4001 = ("Expected input to be one of the following types: ({expected_types}), "
"but got '{received_type}'")
-
-
-# Deprecated model shortcuts, only used in errors and warnings
-OLD_MODEL_SHORTCUTS = {
- "en": "en_core_web_sm", "de": "de_core_news_sm", "es": "es_core_news_sm",
- "pt": "pt_core_news_sm", "fr": "fr_core_news_sm", "it": "it_core_news_sm",
- "nl": "nl_core_news_sm", "el": "el_core_news_sm", "nb": "nb_core_news_sm",
- "lt": "lt_core_news_sm", "xx": "xx_ent_wiki_sm"
-}
+ E4002 = ("Pipe '{name}' requires a teacher pipe for distillation.")
+ E4003 = ("Training examples for distillation must have the exact same tokens in the "
+ "reference and predicted docs.")
+ E4004 = ("Backprop is not supported when is_train is not set.")
# fmt: on
diff --git a/spacy/kb/kb_in_memory.pyx b/spacy/kb/kb_in_memory.pyx
index 485e52c2f..edba523cf 100644
--- a/spacy/kb/kb_in_memory.pyx
+++ b/spacy/kb/kb_in_memory.pyx
@@ -25,7 +25,7 @@ cdef class InMemoryLookupKB(KnowledgeBase):
"""An `InMemoryLookupKB` instance stores unique identifiers for entities and their textual aliases,
to support entity linking of named entities to real-world concepts.
- DOCS: https://spacy.io/api/kb_in_memory
+ DOCS: https://spacy.io/api/inmemorylookupkb
"""
def __init__(self, Vocab vocab, entity_vector_length):
diff --git a/spacy/lang/nl/stop_words.py b/spacy/lang/nl/stop_words.py
index a2c6198e7..cd4fdefdf 100644
--- a/spacy/lang/nl/stop_words.py
+++ b/spacy/lang/nl/stop_words.py
@@ -15,7 +15,7 @@
STOP_WORDS = set(
"""
-aan af al alle alles allebei alleen allen als altijd ander anders andere anderen aangaangde aangezien achter achterna
+aan af al alle alles allebei alleen allen als altijd ander anders andere anderen aangaande aangezien achter achterna
afgelopen aldus alhoewel anderzijds
ben bij bijna bijvoorbeeld behalve beide beiden beneden bent bepaald beter betere betreffende binnen binnenin boven
diff --git a/spacy/lang/ru/lemmatizer.py b/spacy/lang/ru/lemmatizer.py
index c37a3a91a..f4a35de38 100644
--- a/spacy/lang/ru/lemmatizer.py
+++ b/spacy/lang/ru/lemmatizer.py
@@ -28,34 +28,39 @@ class RussianLemmatizer(Lemmatizer):
from pymorphy2 import MorphAnalyzer
except ImportError:
raise ImportError(
- "The Russian lemmatizer mode 'pymorphy2' requires the "
- "pymorphy2 library. Install it with: pip install pymorphy2"
+ "The lemmatizer mode 'pymorphy2' requires the "
+ "pymorphy2 library and dictionaries. Install them with: "
+ "pip install pymorphy2"
+ "# for Ukrainian dictionaries:"
+ "pip install pymorphy2-dicts-uk"
) from None
if getattr(self, "_morph", None) is None:
- self._morph = MorphAnalyzer()
- elif mode == "pymorphy3":
+ self._morph = MorphAnalyzer(lang="ru")
+ elif mode in {"pymorphy3", "pymorphy3_lookup"}:
try:
from pymorphy3 import MorphAnalyzer
except ImportError:
raise ImportError(
- "The Russian lemmatizer mode 'pymorphy3' requires the "
- "pymorphy3 library. Install it with: pip install pymorphy3"
+ "The lemmatizer mode 'pymorphy3' requires the "
+ "pymorphy3 library and dictionaries. Install them with: "
+ "pip install pymorphy3"
+ "# for Ukrainian dictionaries:"
+ "pip install pymorphy3-dicts-uk"
) from None
if getattr(self, "_morph", None) is None:
- self._morph = MorphAnalyzer()
+ self._morph = MorphAnalyzer(lang="ru")
super().__init__(
vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer
)
- def pymorphy2_lemmatize(self, token: Token) -> List[str]:
+ def _pymorphy_lemmatize(self, token: Token) -> List[str]:
string = token.text
univ_pos = token.pos_
morphology = token.morph.to_dict()
if univ_pos == "PUNCT":
return [PUNCT_RULES.get(string, string)]
if univ_pos not in ("ADJ", "DET", "NOUN", "NUM", "PRON", "PROPN", "VERB"):
- # Skip unchangeable pos
- return [string.lower()]
+ return self._pymorphy_lookup_lemmatize(token)
analyses = self._morph.parse(string)
filtered_analyses = []
for analysis in analyses:
@@ -63,8 +68,10 @@ class RussianLemmatizer(Lemmatizer):
# Skip suggested parse variant for unknown word for pymorphy
continue
analysis_pos, _ = oc2ud(str(analysis.tag))
- if analysis_pos == univ_pos or (
- analysis_pos in ("NOUN", "PROPN") and univ_pos in ("NOUN", "PROPN")
+ if (
+ analysis_pos == univ_pos
+ or (analysis_pos in ("NOUN", "PROPN") and univ_pos in ("NOUN", "PROPN"))
+ or ((analysis_pos == "PRON") and (univ_pos == "DET"))
):
filtered_analyses.append(analysis)
if not len(filtered_analyses):
@@ -107,15 +114,27 @@ class RussianLemmatizer(Lemmatizer):
dict.fromkeys([analysis.normal_form for analysis in filtered_analyses])
)
- def pymorphy2_lookup_lemmatize(self, token: Token) -> List[str]:
+ def _pymorphy_lookup_lemmatize(self, token: Token) -> List[str]:
string = token.text
analyses = self._morph.parse(string)
- if len(analyses) == 1:
- return [analyses[0].normal_form]
+ # often multiple forms would derive from the same normal form
+ # thus check _unique_ normal forms
+ normal_forms = set([an.normal_form for an in analyses])
+ if len(normal_forms) == 1:
+ return [next(iter(normal_forms))]
return [string]
+ def pymorphy2_lemmatize(self, token: Token) -> List[str]:
+ return self._pymorphy_lemmatize(token)
+
+ def pymorphy2_lookup_lemmatize(self, token: Token) -> List[str]:
+ return self._pymorphy_lookup_lemmatize(token)
+
def pymorphy3_lemmatize(self, token: Token) -> List[str]:
- return self.pymorphy2_lemmatize(token)
+ return self._pymorphy_lemmatize(token)
+
+ def pymorphy3_lookup_lemmatize(self, token: Token) -> List[str]:
+ return self._pymorphy_lookup_lemmatize(token)
def oc2ud(oc_tag: str) -> Tuple[str, Dict[str, str]]:
diff --git a/spacy/lang/ru/tokenizer_exceptions.py b/spacy/lang/ru/tokenizer_exceptions.py
index f3756e26c..e1889f785 100644
--- a/spacy/lang/ru/tokenizer_exceptions.py
+++ b/spacy/lang/ru/tokenizer_exceptions.py
@@ -61,6 +61,11 @@ for abbr in [
{ORTH: "2к23", NORM: "2023"},
{ORTH: "2к24", NORM: "2024"},
{ORTH: "2к25", NORM: "2025"},
+ {ORTH: "2к26", NORM: "2026"},
+ {ORTH: "2к27", NORM: "2027"},
+ {ORTH: "2к28", NORM: "2028"},
+ {ORTH: "2к29", NORM: "2029"},
+ {ORTH: "2к30", NORM: "2030"},
]:
_exc[abbr[ORTH]] = [abbr]
@@ -268,8 +273,8 @@ for abbr in [
{ORTH: "з-ка", NORM: "заимка"},
{ORTH: "п-к", NORM: "починок"},
{ORTH: "киш.", NORM: "кишлак"},
- {ORTH: "п. ст. ", NORM: "поселок станция"},
- {ORTH: "п. ж/д ст. ", NORM: "поселок при железнодорожной станции"},
+ {ORTH: "п. ст.", NORM: "поселок станция"},
+ {ORTH: "п. ж/д ст.", NORM: "поселок при железнодорожной станции"},
{ORTH: "ж/д бл-ст", NORM: "железнодорожный блокпост"},
{ORTH: "ж/д б-ка", NORM: "железнодорожная будка"},
{ORTH: "ж/д в-ка", NORM: "железнодорожная ветка"},
@@ -280,12 +285,12 @@ for abbr in [
{ORTH: "ж/д п.п.", NORM: "железнодорожный путевой пост"},
{ORTH: "ж/д о.п.", NORM: "железнодорожный остановочный пункт"},
{ORTH: "ж/д рзд.", NORM: "железнодорожный разъезд"},
- {ORTH: "ж/д ст. ", NORM: "железнодорожная станция"},
+ {ORTH: "ж/д ст.", NORM: "железнодорожная станция"},
{ORTH: "м-ко", NORM: "местечко"},
{ORTH: "д.", NORM: "деревня"},
{ORTH: "с.", NORM: "село"},
{ORTH: "сл.", NORM: "слобода"},
- {ORTH: "ст. ", NORM: "станция"},
+ {ORTH: "ст.", NORM: "станция"},
{ORTH: "ст-ца", NORM: "станица"},
{ORTH: "у.", NORM: "улус"},
{ORTH: "х.", NORM: "хутор"},
@@ -388,8 +393,9 @@ for abbr in [
{ORTH: "прим.", NORM: "примечание"},
{ORTH: "прим.ред.", NORM: "примечание редакции"},
{ORTH: "см. также", NORM: "смотри также"},
- {ORTH: "кв.м.", NORM: "квадрантный метр"},
- {ORTH: "м2", NORM: "квадрантный метр"},
+ {ORTH: "см.", NORM: "смотри"},
+ {ORTH: "кв.м.", NORM: "квадратный метр"},
+ {ORTH: "м2", NORM: "квадратный метр"},
{ORTH: "б/у", NORM: "бывший в употреблении"},
{ORTH: "сокр.", NORM: "сокращение"},
{ORTH: "чел.", NORM: "человек"},
diff --git a/spacy/lang/uk/lemmatizer.py b/spacy/lang/uk/lemmatizer.py
index 8337e7328..37015cc2a 100644
--- a/spacy/lang/uk/lemmatizer.py
+++ b/spacy/lang/uk/lemmatizer.py
@@ -29,7 +29,7 @@ class UkrainianLemmatizer(RussianLemmatizer):
) from None
if getattr(self, "_morph", None) is None:
self._morph = MorphAnalyzer(lang="uk")
- elif mode == "pymorphy3":
+ elif mode in {"pymorphy3", "pymorphy3_lookup"}:
try:
from pymorphy3 import MorphAnalyzer
except ImportError:
diff --git a/spacy/language.py b/spacy/language.py
index d391f15ab..2e3c6d2a2 100644
--- a/spacy/language.py
+++ b/spacy/language.py
@@ -1,4 +1,4 @@
-from typing import Iterator, Optional, Any, Dict, Callable, Iterable
+from typing import Iterator, Optional, Any, Dict, Callable, Iterable, Literal
from typing import Union, Tuple, List, Set, Pattern, Sequence
from typing import NoReturn, TYPE_CHECKING, TypeVar, cast, overload
@@ -22,7 +22,7 @@ from . import ty
from .tokens.underscore import Underscore
from .vocab import Vocab, create_vocab
from .pipe_analysis import validate_attrs, analyze_pipes, print_pipe_analysis
-from .training import Example, validate_examples
+from .training import Example, validate_examples, validate_distillation_examples
from .training.initialize import init_vocab, init_tok2vec
from .scorer import Scorer
from .util import registry, SimpleFrozenList, _pipe, raise_error, _DEFAULT_EMPTY_PIPES
@@ -40,11 +40,9 @@ from .git_info import GIT_VERSION
from . import util
from . import about
from .lookups import load_lookups
-from .compat import Literal
-if TYPE_CHECKING:
- from .pipeline import Pipe # noqa: F401
+PipeCallable = Callable[[Doc], Doc]
# This is the base config will all settings (training etc.)
@@ -181,7 +179,7 @@ class Language:
self.vocab: Vocab = vocab
if self.lang is None:
self.lang = self.vocab.lang
- self._components: List[Tuple[str, "Pipe"]] = []
+ self._components: List[Tuple[str, PipeCallable]] = []
self._disabled: Set[str] = set()
self.max_length = max_length
# Create the default tokenizer from the default config
@@ -303,7 +301,7 @@ class Language:
return SimpleFrozenList(names)
@property
- def components(self) -> List[Tuple[str, "Pipe"]]:
+ def components(self) -> List[Tuple[str, PipeCallable]]:
"""Get all (name, component) tuples in the pipeline, including the
currently disabled components.
"""
@@ -322,12 +320,12 @@ class Language:
return SimpleFrozenList(names, error=Errors.E926.format(attr="component_names"))
@property
- def pipeline(self) -> List[Tuple[str, "Pipe"]]:
+ def pipeline(self) -> List[Tuple[str, PipeCallable]]:
"""The processing pipeline consisting of (name, component) tuples. The
components are called on the Doc in order as it passes through the
pipeline.
- RETURNS (List[Tuple[str, Pipe]]): The pipeline.
+ RETURNS (List[Tuple[str, Callable[[Doc], Doc]]]): The pipeline.
"""
pipes = [(n, p) for n, p in self._components if n not in self._disabled]
return SimpleFrozenList(pipes, error=Errors.E926.format(attr="pipeline"))
@@ -527,7 +525,7 @@ class Language:
assigns: Iterable[str] = SimpleFrozenList(),
requires: Iterable[str] = SimpleFrozenList(),
retokenizes: bool = False,
- func: Optional["Pipe"] = None,
+ func: Optional[PipeCallable] = None,
) -> Callable[..., Any]:
"""Register a new pipeline component. Can be used for stateless function
components that don't require a separate factory. Can be used as a
@@ -542,7 +540,7 @@ class Language:
e.g. "token.ent_id". Used for pipeline analysis.
retokenizes (bool): Whether the component changes the tokenization.
Used for pipeline analysis.
- func (Optional[Callable]): Factory function if not used as a decorator.
+ func (Optional[Callable[[Doc], Doc]): Factory function if not used as a decorator.
DOCS: https://spacy.io/api/language#component
"""
@@ -553,11 +551,11 @@ class Language:
raise ValueError(Errors.E853.format(name=name))
component_name = name if name is not None else util.get_object_name(func)
- def add_component(component_func: "Pipe") -> Callable:
+ def add_component(component_func: PipeCallable) -> Callable:
if isinstance(func, type): # function is a class
raise ValueError(Errors.E965.format(name=component_name))
- def factory_func(nlp, name: str) -> "Pipe":
+ def factory_func(nlp, name: str) -> PipeCallable:
return component_func
internal_name = cls.get_factory_name(name)
@@ -607,7 +605,7 @@ class Language:
print_pipe_analysis(analysis, keys=keys)
return analysis
- def get_pipe(self, name: str) -> "Pipe":
+ def get_pipe(self, name: str) -> PipeCallable:
"""Get a pipeline component for a given component name.
name (str): Name of pipeline component to get.
@@ -628,7 +626,7 @@ class Language:
config: Dict[str, Any] = SimpleFrozenDict(),
raw_config: Optional[Config] = None,
validate: bool = True,
- ) -> "Pipe":
+ ) -> PipeCallable:
"""Create a pipeline component. Mostly used internally. To create and
add a component to the pipeline, you can use nlp.add_pipe.
@@ -640,7 +638,7 @@ class Language:
raw_config (Optional[Config]): Internals: the non-interpolated config.
validate (bool): Whether to validate the component config against the
arguments and types expected by the factory.
- RETURNS (Pipe): The pipeline component.
+ RETURNS (Callable[[Doc], Doc]): The pipeline component.
DOCS: https://spacy.io/api/language#create_pipe
"""
@@ -695,24 +693,18 @@ class Language:
def create_pipe_from_source(
self, source_name: str, source: "Language", *, name: str
- ) -> Tuple["Pipe", str]:
+ ) -> Tuple[PipeCallable, str]:
"""Create a pipeline component by copying it from an existing model.
source_name (str): Name of the component in the source pipeline.
source (Language): The source nlp object to copy from.
name (str): Optional alternative name to use in current pipeline.
- RETURNS (Tuple[Callable, str]): The component and its factory name.
+ RETURNS (Tuple[Callable[[Doc], Doc], str]): The component and its factory name.
"""
# Check source type
if not isinstance(source, Language):
raise ValueError(Errors.E945.format(name=source_name, source=type(source)))
- # Check vectors, with faster checks first
- if (
- self.vocab.vectors.shape != source.vocab.vectors.shape
- or self.vocab.vectors.key2row != source.vocab.vectors.key2row
- or self.vocab.vectors.to_bytes(exclude=["strings"])
- != source.vocab.vectors.to_bytes(exclude=["strings"])
- ):
+ if self.vocab.vectors != source.vocab.vectors:
warnings.warn(Warnings.W113.format(name=source_name))
if source_name not in source.component_names:
raise KeyError(
@@ -746,7 +738,7 @@ class Language:
config: Dict[str, Any] = SimpleFrozenDict(),
raw_config: Optional[Config] = None,
validate: bool = True,
- ) -> "Pipe":
+ ) -> PipeCallable:
"""Add a component to the processing pipeline. Valid components are
callables that take a `Doc` object, modify it and return it. Only one
of before/after/first/last can be set. Default behaviour is "last".
@@ -769,7 +761,7 @@ class Language:
raw_config (Optional[Config]): Internals: the non-interpolated config.
validate (bool): Whether to validate the component config against the
arguments and types expected by the factory.
- RETURNS (Pipe): The pipeline component.
+ RETURNS (Callable[[Doc], Doc]): The pipeline component.
DOCS: https://spacy.io/api/language#add_pipe
"""
@@ -790,14 +782,6 @@ class Language:
factory_name, source, name=name
)
else:
- if not self.has_factory(factory_name):
- err = Errors.E002.format(
- name=factory_name,
- opts=", ".join(self.factory_names),
- method="add_pipe",
- lang=util.get_object_name(self),
- lang_code=self.lang,
- )
pipe_component = self.create_pipe(
factory_name,
name=name,
@@ -883,7 +867,7 @@ class Language:
*,
config: Dict[str, Any] = SimpleFrozenDict(),
validate: bool = True,
- ) -> "Pipe":
+ ) -> PipeCallable:
"""Replace a component in the pipeline.
name (str): Name of the component to replace.
@@ -892,7 +876,7 @@ class Language:
component. Will be merged with default config, if available.
validate (bool): Whether to validate the component config against the
arguments and types expected by the factory.
- RETURNS (Pipe): The new pipeline component.
+ RETURNS (Callable[[Doc], Doc]): The new pipeline component.
DOCS: https://spacy.io/api/language#replace_pipe
"""
@@ -944,11 +928,11 @@ class Language:
init_cfg = self._config["initialize"]["components"].pop(old_name)
self._config["initialize"]["components"][new_name] = init_cfg
- def remove_pipe(self, name: str) -> Tuple[str, "Pipe"]:
+ def remove_pipe(self, name: str) -> Tuple[str, PipeCallable]:
"""Remove a component from the pipeline.
name (str): Name of the component to remove.
- RETURNS (tuple): A `(name, component)` tuple of the removed component.
+ RETURNS (Tuple[str, Callable[[Doc], Doc]]): A `(name, component)` tuple of the removed component.
DOCS: https://spacy.io/api/language#remove_pipe
"""
@@ -1033,6 +1017,102 @@ class Language:
raise ValueError(Errors.E005.format(name=name, returned_type=type(doc)))
return doc
+ def distill(
+ self,
+ teacher: "Language",
+ examples: Iterable[Example],
+ *,
+ drop: float = 0.0,
+ sgd: Optional[Optimizer] = None,
+ losses: Optional[Dict[str, float]] = None,
+ component_cfg: Optional[Dict[str, Dict[str, Any]]] = None,
+ exclude: Iterable[str] = SimpleFrozenList(),
+ annotates: Iterable[str] = SimpleFrozenList(),
+ student_to_teacher: Optional[Dict[str, str]] = None,
+ ):
+ """Distill the models in a student pipeline from a teacher pipeline.
+ teacher (Language): Teacher to distill from.
+ examples (Iterable[Example]): Distillation examples. The reference
+ (teacher) and predicted (student) docs must have the same number of
+ tokens and the same orthography.
+ drop (float): The dropout rate.
+ sgd (Optional[Optimizer]): An optimizer.
+ losses (Optional(Dict[str, float])): Dictionary to update with the loss,
+ keyed by component.
+ component_cfg (Optional[Dict[str, Dict[str, Any]]]): Config parameters
+ for specific pipeline components, keyed by component name.
+ exclude (Iterable[str]): Names of components that shouldn't be updated.
+ annotates (Iterable[str]): Names of components that should set
+ annotations on the predicted examples after updating.
+ student_to_teacher (Optional[Dict[str, str]]): Map student pipe name to
+ teacher pipe name, only needed for pipes where the student pipe
+ name does not match the teacher pipe name.
+ RETURNS (Dict[str, float]): The updated losses dictionary
+
+ DOCS: https://spacy.io/api/language#distill
+ """
+ if student_to_teacher is None:
+ student_to_teacher = {}
+ if losses is None:
+ losses = {}
+ if isinstance(examples, list) and len(examples) == 0:
+ return losses
+
+ validate_distillation_examples(examples, "Language.distill")
+ examples = _copy_examples(examples)
+
+ if sgd is None:
+ if self._optimizer is None:
+ self._optimizer = self.create_optimizer()
+ sgd = self._optimizer
+
+ if component_cfg is None:
+ component_cfg = {}
+ pipe_kwargs = {}
+ for student_name, student_proc in self.pipeline:
+ component_cfg.setdefault(student_name, {})
+ pipe_kwargs[student_name] = deepcopy(component_cfg[student_name])
+ component_cfg[student_name].setdefault("drop", drop)
+ pipe_kwargs[student_name].setdefault("batch_size", self.batch_size)
+
+ teacher_pipes = dict(teacher.pipeline)
+ for student_name, student_proc in self.pipeline:
+ if student_name in annotates:
+ for doc, eg in zip(
+ _pipe(
+ (eg.predicted for eg in examples),
+ proc=student_proc,
+ name=student_name,
+ default_error_handler=self.default_error_handler,
+ kwargs=pipe_kwargs[student_name],
+ ),
+ examples,
+ ):
+ eg.predicted = doc
+
+ if (
+ student_name not in exclude
+ and isinstance(student_proc, ty.DistillableComponent)
+ and student_proc.is_distillable
+ ):
+ # A missing teacher pipe is not an error, some student pipes
+ # do not need a teacher, such as tok2vec layer losses.
+ teacher_name = (
+ student_to_teacher[student_name]
+ if student_name in student_to_teacher
+ else student_name
+ )
+ teacher_pipe = teacher_pipes.get(teacher_name, None)
+ student_proc.distill(
+ teacher_pipe,
+ examples,
+ sgd=sgd,
+ losses=losses,
+ **component_cfg[student_name],
+ )
+
+ return losses
+
def disable_pipes(self, *names) -> "DisabledPipes":
"""Disable one or more pipeline components. If used as a context
manager, the pipeline will be restored to the initial state at the end
@@ -1254,25 +1334,20 @@ class Language:
sgd(key, W, dW) # type: ignore[call-arg, misc]
return losses
- def begin_training(
- self,
- get_examples: Optional[Callable[[], Iterable[Example]]] = None,
- *,
- sgd: Optional[Optimizer] = None,
- ) -> Optimizer:
- warnings.warn(Warnings.W089, DeprecationWarning)
- return self.initialize(get_examples, sgd=sgd)
-
def initialize(
self,
get_examples: Optional[Callable[[], Iterable[Example]]] = None,
*,
+ labels: Optional[Dict[str, Any]] = None,
sgd: Optional[Optimizer] = None,
) -> Optimizer:
"""Initialize the pipe for training, using data examples if available.
get_examples (Callable[[], Iterable[Example]]): Optional function that
returns gold-standard Example objects.
+ labels (Optional[Dict[str, Any]]): Labels to pass to pipe initialization,
+ using the names of the pipes as keys. Overrides labels that are in
+ the model configuration.
sgd (Optional[Optimizer]): An optimizer to use for updates. If not
provided, will be created using the .create_optimizer() method.
RETURNS (thinc.api.Optimizer): The optimizer.
@@ -1317,6 +1392,8 @@ class Language:
for name, proc in self.pipeline:
if isinstance(proc, ty.InitializableComponent):
p_settings = I["components"].get(name, {})
+ if labels is not None and name in labels:
+ p_settings["labels"] = labels[name]
p_settings = validate_init_settings(
proc.initialize, p_settings, section="components", name=name
)
@@ -1363,15 +1440,15 @@ class Language:
def set_error_handler(
self,
- error_handler: Callable[[str, "Pipe", List[Doc], Exception], NoReturn],
+ error_handler: Callable[[str, PipeCallable, List[Doc], Exception], NoReturn],
):
- """Set an error handler object for all the components in the pipeline that implement
- a set_error_handler function.
+ """Set an error handler object for all the components in the pipeline
+ that implement a set_error_handler function.
- error_handler (Callable[[str, Pipe, List[Doc], Exception], NoReturn]):
- Function that deals with a failing batch of documents. This callable function should take in
- the component's name, the component itself, the offending batch of documents, and the exception
- that was thrown.
+ error_handler (Callable[[str, Callable[[Doc], Doc], List[Doc], Exception], NoReturn]):
+ Function that deals with a failing batch of documents. This callable
+ function should take in the component's name, the component itself,
+ the offending batch of documents, and the exception that was thrown.
DOCS: https://spacy.io/api/language#set_error_handler
"""
self.default_error_handler = error_handler
@@ -1750,6 +1827,7 @@ class Language:
# using the nlp.config with all defaults.
config = util.copy_config(config)
orig_pipeline = config.pop("components", {})
+ orig_distill = config.pop("distill", None)
orig_pretraining = config.pop("pretraining", None)
config["components"] = {}
if auto_fill:
@@ -1758,6 +1836,9 @@ class Language:
filled = config
filled["components"] = orig_pipeline
config["components"] = orig_pipeline
+ if orig_distill is not None:
+ filled["distill"] = orig_distill
+ config["distill"] = orig_distill
if orig_pretraining is not None:
filled["pretraining"] = orig_pretraining
config["pretraining"] = orig_pretraining
@@ -1879,31 +1960,22 @@ class Language:
if isinstance(exclude, str):
exclude = [exclude]
- def fetch_pipes_status(value: Iterable[str], key: str) -> Iterable[str]:
- """Fetch value for `enable` or `disable` w.r.t. the specified config and passed arguments passed to
- .load(). If both arguments and config specified values for this field, the passed arguments take precedence
- and a warning is printed.
- value (Iterable[str]): Passed value for `enable` or `disable`.
- key (str): Key for field in config (either "enabled" or "disabled").
- RETURN (Iterable[str]):
- """
- # We assume that no argument was passed if the value is the specified default value.
- if id(value) == id(_DEFAULT_EMPTY_PIPES):
- return config["nlp"].get(key, [])
- else:
- if len(config["nlp"].get(key, [])):
- warnings.warn(
- Warnings.W123.format(
- arg=key[:-1],
- arg_value=value,
- config_value=config["nlp"][key],
- )
+ # `enable` should not be merged with `enabled` (the opposite is true for `disable`/`disabled`). If the config
+ # specifies values for `enabled` not included in `enable`, emit warning.
+ if id(enable) != id(_DEFAULT_EMPTY_PIPES):
+ enabled = config["nlp"].get("enabled", [])
+ if len(enabled) and not set(enabled).issubset(enable):
+ warnings.warn(
+ Warnings.W123.format(
+ enable=enable,
+ enabled=enabled,
)
- return value
+ )
+ # Ensure sets of disabled/enabled pipe names are not contradictory.
disabled_pipes = cls._resolve_component_status(
- fetch_pipes_status(disable, "disabled"),
- fetch_pipes_status(enable, "enabled"),
+ list({*disable, *config["nlp"].get("disabled", [])}),
+ enable,
config["nlp"]["pipeline"],
)
nlp._disabled = set(p for p in disabled_pipes if p not in exclude)
@@ -2084,10 +2156,12 @@ class Language:
if enable:
if isinstance(enable, str):
enable = [enable]
- to_disable = [
- pipe_name for pipe_name in pipe_names if pipe_name not in enable
- ]
- if disable and disable != to_disable:
+ to_disable = {
+ *[pipe_name for pipe_name in pipe_names if pipe_name not in enable],
+ *disable,
+ }
+ # If any pipe to be enabled is in to_disable, the specification is inconsistent.
+ if len(set(enable) & to_disable):
raise ValueError(Errors.E1042.format(enable=enable, disable=disable))
return tuple(to_disable)
diff --git a/spacy/lexeme.pxd b/spacy/lexeme.pxd
index 8dea0d6a2..2d14edcd6 100644
--- a/spacy/lexeme.pxd
+++ b/spacy/lexeme.pxd
@@ -5,7 +5,6 @@ from .attrs cimport attr_id_t
from .attrs cimport ID, ORTH, LOWER, NORM, SHAPE, PREFIX, SUFFIX, LENGTH, LANG
from .structs cimport LexemeC
-from .strings cimport StringStore
from .vocab cimport Vocab
diff --git a/spacy/lexeme.pyx b/spacy/lexeme.pyx
index 73bf28dc2..e57098f17 100644
--- a/spacy/lexeme.pyx
+++ b/spacy/lexeme.pyx
@@ -41,7 +41,7 @@ cdef class Lexeme:
"""
self.vocab = vocab
self.orth = orth
- self.c = | Header 1 | Header 2 |
|---|---|
| Column 1 | Column 2 |
+
+ + +{' '} + + +
+ ++ + +{' '} + + +
+ +## Components + +### Table {id="table"} + +> #### Markdown +> +> ```markdown +> | Header 1 | Header 2 | +> | -------- | -------- | +> | Column 1 | Column 2 | +> ``` +> +> #### JSX +> +> ```markup +>| Header 1 | Header 2 |
|---|---|
| Column 1 | Column 2 |
+
```markdown
[](https://spacy.io)
```
-
```markdown
-[](https://spacy.io)
+[](https://spacy.io)
```
diff --git a/website/docs/usage/training.md b/website/docs/usage/training.mdx
similarity index 95%
rename from website/docs/usage/training.md
rename to website/docs/usage/training.mdx
index 5ee148224..c5ff323c7 100644
--- a/website/docs/usage/training.md
+++ b/website/docs/usage/training.mdx
@@ -15,15 +15,17 @@ menu:
- ['Internal API', 'api']
---
-## Introduction to training {#basics hidden="true"}
-
-import Training101 from 'usage/101/\_training.md'
+## Introduction to training {id="basics",hidden="true"}
+
+
+ + Get a custom spaCy pipeline, tailor-made for your NLP problem by + spaCy's core developers. + +
+
+
+ Prodigy is an annotation tool so efficient that data + scientists can do the annotation themselves, enabling a new level of rapid + iteration. Whether you're working on entity recognition, intent + detection or image classification, Prodigy can help you{' '} + train and evaluate your models faster. +
- spaCy's new project system gives you a smooth path from prototype to + spaCy's new project system gives you a smooth path from prototype to production. It lets you keep track of all those{' '} data transformation, preprocessing and{' '} training steps, so you can make sure your project is always @@ -236,13 +246,15 @@ const Landing = ({ data }) => { button="See what's new" small > - spaCy v3.0 features all new transformer-based pipelines that - bring spaCy's accuracy right up to the current state-of-the-art - . You can use any pretrained transformer to train your own pipelines, and even - share one transformer between multiple components with{' '} - multi-task learning. Training is now fully configurable and - extensible, and you can define your own custom models using{' '} - PyTorch, TensorFlow and other frameworks. +
+ spaCy v3.0 features all new transformer-based pipelines{' '} + that bring spaCy's accuracy right up to the current{' '} + state-of-the-art. You can use any pretrained transformer to + train your own pipelines, and even share one transformer between multiple + components with multi-task learning. Training is now fully + configurable and extensible, and you can define your own custom models using{' '} + PyTorch, TensorFlow and other frameworks. +
+
+
+ In this free and interactive online course you’ll learn how + to use spaCy to build advanced natural language understanding systems, using + both rule-based and machine learning approaches. It includes{' '} + 55 exercises featuring videos, slide decks, multiple-choice + questions and interactive coding practice in the browser. +
- spaCy v3.0 introduces transformer-based pipelines that bring spaCy's
+ spaCy v3.0 introduces transformer-based pipelines that bring spaCy's
accuracy right up to the current state-of-the-art. You can
also use a CPU-optimized pipeline, which is less accurate but much cheaper
to run.
@@ -285,33 +299,8 @@ const Landing = ({ data }) => {
-
-)
+const splitLines = (children) => {
+ const listChildrenPerLine = []
-export const Pre = props => {
- return {props.children}
-}
+ if (typeof children === 'string') {
+ listChildrenPerLine.push(...children.split('\n'))
+ } else {
+ listChildrenPerLine.push([])
+ let indexLine = 0
+ if (Array.isArray(children)) {
+ children.forEach((child) => {
+ if (typeof child === 'string' && child.includes('\n')) {
+ const listString = child.split('\n')
+ listString.forEach((string, index) => {
+ listChildrenPerLine[indexLine].push(string)
-export const InlineCode = ({ wrap = false, className, children, ...props }) => {
- const codeClassNames = classNames(classes.inlineCode, className, {
- [classes.wrap]: wrap || (isString(children) && children.length >= WRAP_THRESHOLD),
- })
- return (
-
- {children}
-
- )
-}
-
-InlineCode.propTypes = {
- wrap: PropTypes.bool,
- className: PropTypes.string,
- children: PropTypes.node,
-}
-
-function linkType(el, showLink = true) {
- if (!isString(el) || !el.length) return el
- const elStr = el.trim()
- if (!elStr) return el
- const typeUrl = CUSTOM_TYPES[elStr]
- const url = typeUrl == true ? DEFAULT_TYPE_URL : typeUrl
- const ws = el[0] == ' '
- return url && showLink ? (
-
- {elements.map((el, i) => (
- {linkType(el, !!link)}
- ))}
- {meta && {meta}}
-
- )
-}
+ }
-function replacePrompt(line, prompt, isFirst = false) {
- let result = line
- const hasPrompt = result.startsWith(`${prompt} `)
- const showPrompt = hasPrompt || isFirst
- if (hasPrompt) result = result.slice(2)
- return result && showPrompt ? `${result}` : result
+ const listLine = listChildrenPerLine[listChildrenPerLine.length - 1]
+ if (listLine === '' || (listLine.length === 1 && listLine[0] === '')) {
+ listChildrenPerLine.pop()
+ }
+
+ return listChildrenPerLine.map((childrenPerLine, index) => (
+ <>
+ {childrenPerLine}
+ {index !== listChildrenPerLine.length - 1 && '\n'}
+
+ ))
}
function parseArgs(raw) {
- let args = raw.split(' ').filter(arg => arg)
+ let args = raw.split(' ').filter((arg) => arg)
const result = {}
while (args.length) {
let opt = args.shift()
@@ -120,208 +89,221 @@ function parseArgs(raw) {
return result
}
-function convertLine(line, i) {
- const cliRegex = /^(\$ )?python -m spacy/
- if (cliRegex.test(line)) {
- const text = line.replace(cliRegex, '')
- const args = parseArgs(text)
- const cmd = Object.keys(args).map((key, i) => {
- const value = args[key]
- return value === null || value === true || i === 0 ? key : `${key} ${value}`
- })
+const flattenReact = (children) => {
+ if (children === null || children === undefined || children === false) {
+ return []
+ }
+
+ if (typeof children === 'string') {
+ return [children]
+ }
+
+ if (children.props) {
+ return flattenReact(children.props.children)
+ }
+
+ return children.flatMap(flattenReact)
+}
+
+const checkoutForComment = (line) => {
+ const lineParts = line.split(' # ')
+
+ if (lineParts.length !== 2) {
+ return line
+ }
+
+ return (
+ <>
+ {lineParts[0]}
+ {` `}
+
+ {`# `}
+ {lineParts[1]}
+
+
+ )
+}
+
+const handlePromot = ({ lineFlat, prompt }) => {
+ const lineWithoutPrompt = lineFlat.slice(prompt.length + 1)
+
+ const cliRegex = /^python -m spacy/
+
+ if (!cliRegex.test(lineWithoutPrompt)) {
+ return {checkoutForComment(lineWithoutPrompt)}
+ }
+
+ const text = lineWithoutPrompt.replace(cliRegex, '')
+ const args = parseArgs(text)
+ const cmd = Object.keys(args).map((key, i) => {
+ const value = args[key]
+ return value === null || value === true || i === 0 ? key : `${key} ${value}`
+ })
+ return (
+
+ python -m spacy{' '}
+ {cmd.map((item, j) => {
+ const isCmd = j === 0
+ const url = isCmd ? `/api/cli#${item.replace(' ', '-')}` : null
+ const isAbstract = isString(item) && /^\[(.+)\]$/.test(item)
+ const itemClassNames = classNames(classes['cli-arg'], {
+ [classes['cli-arg-highlight']]: isCmd,
+ [classes['cli-arg-emphasis']]: isAbstract,
+ })
+ const text = isAbstract ? item.slice(1, -1) : item
+ return (
+ {html}
+ {title &&
+
+ {children}
+
+
)
}
}
-const JuniperWrapper = ({ Juniper, title, lang, children }) => (
- {props.children}
+}
+
+const CodeBlock = (props) => (
+
+
+)
+export default CodeBlock
diff --git a/website/src/components/codeDynamic.js b/website/src/components/codeDynamic.js
new file mode 100644
index 000000000..8c9483567
--- /dev/null
+++ b/website/src/components/codeDynamic.js
@@ -0,0 +1,5 @@
+import dynamic from 'next/dynamic'
+
+export default dynamic(() => import('./code'), {
+ loading: () =>